def test_base_validation(self): base_model_with_invalid_id = self.create_model( base_models.BaseModel, id='123@?!*', deleted=False) base_model_with_invalid_timestamps = self.create_model( base_models.BaseModel, id='124', deleted=False, created_on=self.NOW, last_updated=self.YEAR_LATER) base_model_with_inconsistent_timestamps = self.create_model( base_models.BaseModel, id='125', deleted=False, created_on=self.YEAR_LATER, last_updated=self.YEAR_AGO) expired_base_model = self.create_model( base_models.BaseModel, id='126', deleted=True) valid_base_model = self.create_model( base_models.BaseModel, id='127', deleted=False) self.put_multi([ base_model_with_invalid_id, base_model_with_invalid_timestamps, base_model_with_inconsistent_timestamps, expired_base_model, valid_base_model, ]) self.assert_job_output_is([ base_validation_errors.ModelIdRegexError( base_model_with_invalid_id, base_validation.BASE_MODEL_ID_PATTERN), base_validation_errors.ModelMutatedDuringJobError( base_model_with_invalid_timestamps), base_validation_errors.InconsistentTimestampsError( base_model_with_inconsistent_timestamps), base_validation_errors.ModelExpiredError(expired_base_model), ])
def process(self, entity): """Function that defines how to process each entity in a pipeline of models. Args: entity: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for models mutated during the job. InconsistentTimestampsError. Error for models with inconsistent timestamps. """ cloned_entity = job_utils.clone_model(entity) last_updated_corrected = (cloned_entity.last_updated + MAX_CLOCK_SKEW_SECS) if cloned_entity.created_on > last_updated_corrected: yield base_validation_errors.InconsistentTimestampsError( cloned_entity) current_datetime = datetime.datetime.utcnow() last_updated_corrected = (cloned_entity.last_updated - MAX_CLOCK_SKEW_SECS) if last_updated_corrected > current_datetime: yield base_validation_errors.ModelMutatedDuringJobError( cloned_entity)
def test_message(self): model = base_models.BaseModel(id='123', created_on=self.NOW, last_updated=self.YEAR_LATER) error = base_validation_errors.ModelMutatedDuringJobError(model) self.assertEqual( error.stderr, 'ModelMutatedDuringJobError in BaseModel(id="123"): ' 'last_updated=%r is later than the audit job\'s start time' % (model.last_updated))
def test_process_reports_model_mutated_during_job_error(self): invalid_timestamp = base_models.BaseModel(id='124', created_on=self.NOW, last_updated=self.YEAR_LATER) output = (self.pipeline | beam.Create([invalid_timestamp]) | beam.ParDo(base_validation.ValidateModelTimestamps())) self.assert_pcoll_equal(output, [ base_validation_errors.ModelMutatedDuringJobError( invalid_timestamp), ])
def process(self, input_model): """Function that defines how to process each element in a pipeline of models. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for models mutated during the job. InconsistentTimestampsError. Error for models with inconsistent timestamps. """ model = job_utils.clone_model(input_model) if model.created_on > (model.last_updated + MAX_CLOCK_SKEW_SECS): yield base_validation_errors.InconsistentTimestampsError(model) current_datetime = datetime.datetime.utcnow() if (model.last_updated - MAX_CLOCK_SKEW_SECS) > current_datetime: yield base_validation_errors.ModelMutatedDuringJobError(model)