def test_base_model_validator_ptransform(self): with pipeline.TestPipeline(runner=direct_runner.DirectRunner()) as p: invalid_id = MockModel(id='123@?!*', deleted=False, created_on=self.year_ago, last_updated=self.now) invalid_timestamp = MockModel(id='124', deleted=False, created_on=self.now, last_updated=self.year_later) expired_model = MockModel(id='125', deleted=True, created_on=self.year_ago, last_updated=self.year_ago) valid_model = MockModel(id='126', deleted=False, created_on=self.year_ago, last_updated=self.now) pcoll = (p | beam.Create([ invalid_id, invalid_timestamp, expired_model, valid_model ])) output = pcoll | base_model_validator.BaseModelValidator() beam_testing_util.assert_that( output, beam_testing_util.equal_to([ errors.ModelInvalidIdError(invalid_id), errors.ModelMutatedDuringJobError(invalid_timestamp), errors.ModelExpiredError(expired_model) ]))
def test_base_model_validator_ptransform(self): model_with_invalid_id = base_models.BaseModel(id='123@?!*', deleted=False, created_on=self.YEAR_AGO, last_updated=self.NOW) model_with_invalid_timestamp = base_models.BaseModel( id='124', deleted=False, created_on=self.NOW, last_updated=self.YEAR_LATER) expired_model = base_models.BaseModel(id='125', deleted=True, created_on=self.YEAR_AGO, last_updated=self.YEAR_AGO) valid_model = base_models.BaseModel(id='126', deleted=False, created_on=self.YEAR_AGO, last_updated=self.NOW) output = (self.pipeline | beam.Create([ model_with_invalid_id, model_with_invalid_timestamp, expired_model, valid_model, ]) | base_model_validator.BaseModelValidator()) self.assert_pcoll_equal(output, [ errors.InvalidIdError( model_with_invalid_id, base_model_validator.DEFAULT_ID_REGEX_STRING), errors.ModelMutatedDuringJobError(model_with_invalid_timestamp), errors.ModelExpiredError(expired_model), ])
def test_model_mutated_during_job_error(self): model = MockModel(id='124', created_on=self.now, last_updated=self.year_later) error = errors.ModelMutatedDuringJobError(model) msg = ('Entity id %s: The last_updated field has a value %s which ' 'is greater than the time when the job was run' % (model.id, model.last_updated)) self.assertEqual(error.message, msg)
def test_message(self): model = base_models.BaseModel(id='123', created_on=self.NOW, last_updated=self.YEAR_LATER) error = errors.ModelMutatedDuringJobError(model) self.assertEqual( error.message, 'ModelMutatedDuringJobError in BaseModel(id="123"): ' 'last_updated=%r is later than the validation job\'s start time' % (model.last_updated))
def test_process_reports_model_mutated_during_job_error(self): invalid_timestamp = base_models.BaseModel(id='124', created_on=self.NOW, last_updated=self.YEAR_LATER) output = (self.pipeline | beam.Create([invalid_timestamp]) | beam.ParDo(base_model_validator.ValidateModelTimestamps())) self.assert_pcoll_equal(output, [ errors.ModelMutatedDuringJobError(invalid_timestamp), ])
def test_process_reports_model_mutated_during_job_error(self): with pipeline.TestPipeline(runner=direct_runner.DirectRunner()) as p: invalid_timestamp = MockModel(id='124', created_on=self.now, last_updated=self.year_later) pcoll = p | beam.Create([invalid_timestamp]) output = (pcoll | beam.ParDo( base_model_validator.ValidateModelTimeFields())) beam_testing_util.assert_that( output, beam_testing_util.equal_to( [errors.ModelMutatedDuringJobError(invalid_timestamp)]))
def process(self, input_model): """Function that defines how to process each element in a pipeline of models. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for timestamp validation. ModelTimestampRelationshipError. Error for timestamp validation. """ model = jobs_utils.clone_model(input_model) if model.created_on > (model.last_updated + MAX_CLOCK_SKEW_SECS): yield errors.ModelTimestampRelationshipError(model) current_datetime = datetime.datetime.utcnow() if (model.last_updated - MAX_CLOCK_SKEW_SECS) > current_datetime: yield errors.ModelMutatedDuringJobError(model)