def process(self, entity):
"""Function that defines how to process each entity in a pipeline of
models.
Args:
entity: datastore_services.Model. Entity to validate.
Yields:
ModelMutatedDuringJobError. Error for models mutated during the job.
InconsistentTimestampsError. Error for models with inconsistent
timestamps.
"""
cloned_entity = job_utils.clone_model(entity)
last_updated_corrected = (cloned_entity.last_updated +
MAX_CLOCK_SKEW_SECS)
if cloned_entity.created_on > last_updated_corrected:
yield base_validation_errors.InconsistentTimestampsError(
cloned_entity)
current_datetime = datetime.datetime.utcnow()
last_updated_corrected = (cloned_entity.last_updated -
MAX_CLOCK_SKEW_SECS)
if last_updated_corrected > current_datetime:
yield base_validation_errors.ModelMutatedDuringJobError(
cloned_entity)
def test_message(self) -> None:
model = base_models.BaseModel(id='123',
created_on=self.NOW,
last_updated=self.YEAR_AGO)
error = base_validation_errors.InconsistentTimestampsError(model)
self.assertEqual(
error.stderr,
'InconsistentTimestampsError in BaseModel(id="123"): '
'created_on=%r is later than last_updated=%r' %
(self.NOW, self.YEAR_AGO))
def test_process_reports_model_timestamp_relationship_error(self):
invalid_timestamp = base_models.BaseModel(id='123',
created_on=self.NOW,
last_updated=self.YEAR_AGO)
output = (self.pipeline
| beam.Create([invalid_timestamp])
| beam.ParDo(base_validation.ValidateModelTimestamps()))
self.assert_pcoll_equal(output, [
base_validation_errors.InconsistentTimestampsError(
invalid_timestamp),
])
def test_base_validation(self):
base_model_with_invalid_id = self.create_model(base_models.BaseModel,
id='123@?!*',
deleted=False)
base_model_with_invalid_timestamps = self.create_model(
base_models.BaseModel,
id='124',
deleted=False,
created_on=self.NOW,
last_updated=self.YEAR_LATER)
base_model_with_inconsistent_timestamps = self.create_model(
base_models.BaseModel,
id='125',
deleted=False,
created_on=self.YEAR_LATER,
last_updated=self.YEAR_AGO)
expired_base_model = self.create_model(base_models.BaseModel,
id='126',
deleted=True)
valid_base_model = self.create_model(base_models.BaseModel,
id='127',
deleted=False)
self.put_multi([
base_model_with_invalid_id,
base_model_with_invalid_timestamps,
base_model_with_inconsistent_timestamps,
expired_base_model,
valid_base_model,
])
self.assert_job_output_is([
base_validation_errors.ModelIdRegexError(
base_model_with_invalid_id,
base_validation.BASE_MODEL_ID_PATTERN),
base_validation_errors.ModelMutatedDuringJobError(
base_model_with_invalid_timestamps),
base_validation_errors.InconsistentTimestampsError(
base_model_with_inconsistent_timestamps),
base_validation_errors.ModelExpiredError(expired_base_model),
])
