def process(self, input_model):
"""Function that validates that the published timestamp of the blog post
models is either None or is greater than created on time, is less than
current datetime and is equal to or greater than the last updated
timestamp.
Args:
input_model: datastore_services.Model. Entity to validate.
Yields:
ModelMutatedDuringJobError. Error for models mutated during the job.
InconsistentTimestampsError. Error for models with inconsistent
timestamps.
"""
model = job_utils.clone_model(input_model)
if model.published_on is None:
return
if model.created_on > (model.published_on +
base_validation.MAX_CLOCK_SKEW_SECS):
yield blog_validation_errors.InconsistentPublishTimestampsError(
model)
current_datetime = datetime.datetime.utcnow()
if (model.published_on -
base_validation.MAX_CLOCK_SKEW_SECS) > (current_datetime):
yield blog_validation_errors.ModelMutatedDuringJobError(model)
if (model.published_on -
base_validation.MAX_CLOCK_SKEW_SECS) > (model.last_updated):
yield blog_validation_errors.InconsistentPublishLastUpdatedTimestampsError(model) # pylint: disable=line-too-long
def process(self, entity):
"""Function that defines how to process each entity in a pipeline of
models.
Args:
entity: datastore_services.Model. Entity to validate.
Yields:
ModelMutatedDuringJobError. Error for models mutated during the job.
InconsistentTimestampsError. Error for models with inconsistent
timestamps.
"""
cloned_entity = job_utils.clone_model(entity)
last_updated_corrected = (cloned_entity.last_updated +
MAX_CLOCK_SKEW_SECS)
if cloned_entity.created_on > last_updated_corrected:
yield base_validation_errors.InconsistentTimestampsError(
cloned_entity)
current_datetime = datetime.datetime.utcnow()
last_updated_corrected = (cloned_entity.last_updated -
MAX_CLOCK_SKEW_SECS)
if last_updated_corrected > current_datetime:
yield base_validation_errors.ModelMutatedDuringJobError(
cloned_entity)
def process(
self, user_stats_model: user_models.UserStatsModel
) -> Iterable[user_models.UserStatsModel]:
"""Updates weekly dashboard stats with the current values.
Args:
user_stats_model: UserStatsModel. Model for which to update
the weekly dashboard stats.
Yields:
UserStatsModel. The updated user stats model.
"""
model = job_utils.clone_model(user_stats_model)
schema_version = model.schema_version
if schema_version != feconf.CURRENT_DASHBOARD_STATS_SCHEMA_VERSION:
user_services.migrate_dashboard_stats_to_latest_schema(
model) # type: ignore[no-untyped-call]
weekly_creator_stats = {
user_services.get_current_date_as_string():
{ # type: ignore[no-untyped-call]
'num_ratings': model.num_ratings or 0,
'average_ratings': model.average_ratings,
'total_plays': model.total_plays or 0
}
}
model.weekly_creator_stats_list.append(weekly_creator_stats)
model.update_timestamps()
yield model
def test_clone_model(self):
model = base_models.BaseModel(id='123', deleted=True)
clone = job_utils.clone_model(model)
self.assertEqual(model.id, clone.id)
self.assertEqual(model, clone)
self.assertIsNot(model, clone)
self.assertIsInstance(clone, base_models.BaseModel)
def test_clone_with_changes(self):
model = base_models.BaseModel(id='123', deleted=True)
clone = job_utils.clone_model(model, deleted=False)
self.assertNotEqual(model, clone)
self.assertIsNot(model, clone)
self.assertIsInstance(clone, base_models.BaseModel)
self.assertTrue(model.deleted)
self.assertFalse(clone.deleted)
def test_clone_with_changes_to_id(self):
model = base_models.BaseModel(id='123')
clone = job_utils.clone_model(model, id='124')
self.assertNotEqual(model, clone)
self.assertIsNot(model, clone)
self.assertIsInstance(clone, base_models.BaseModel)
self.assertEqual(model.id, '123')
self.assertEqual(clone.id, '124')
def test_clone_sub_class_with_changes(self):
model = FooModel(prop='original')
clone = job_utils.clone_model(model, prop='updated')
self.assertNotEqual(model, clone)
self.assertIsNot(model, clone)
self.assertIsInstance(clone, FooModel)
self.assertEqual(model.prop, 'original')
self.assertEqual(clone.prop, 'updated')
def test_clone_sub_class(self) -> None:
model = FooModel(prop='original')
clone = job_utils.clone_model(model)
self.assertEqual(model, clone)
self.assertIsNot(model, clone)
self.assertIsInstance(clone, FooModel)
self.assertEqual(model.prop, 'original')
self.assertEqual(clone.prop, 'original')
def process(self, input_model):
"""Function that checks if a model is old enough to mark them deleted.
Args:
input_model: user_models.UserQueryModel. Entity to validate.
Yields:
ModelExpiringError. An error class for expiring models.
"""
model = job_utils.clone_model(input_model)
expiration_date = (datetime.datetime.utcnow() -
feconf.PERIOD_TO_MARK_MODELS_AS_DELETED)
if expiration_date > model.last_updated:
yield audit_errors.ModelExpiringError(model)
def process(self, input_model):
"""Function that defines how to process each element in a pipeline of
models.
Args:
input_model: datastore_services.Model. Entity to validate.
Yields:
ModelIdRegexError. An error class for models with invalid IDs.
"""
model = job_utils.clone_model(input_model)
if not re.match(self._pattern, model.id):
yield audit_errors.ModelIdRegexError(model, self._pattern)
def process(self, input_model):
"""Function that checks if the entity type is valid
Args:
input_model: feedback_models.GeneralFeedbackThreadModel.
Entity to validate.
Yields:
InvalidEntityTypeError. Error for models with invalid entity type.
"""
model = job_utils.clone_model(input_model)
if (model.entity_type
not in feedback_services.TARGET_TYPE_TO_TARGET_MODEL):
yield feedback_validation_errors.InvalidEntityTypeError(model)
def process(self, input_model):
"""Function that defines how to process each element in a pipeline of
models.
Args:
input_model: datastore_services.Model. Entity to validate.
Yields:
ModelCommitTypeError. Error for commit_type validation.
"""
model = job_utils.clone_model(input_model)
if (model.commit_type
not in base_models.VersionedModel.COMMIT_TYPE_CHOICES):
yield base_validation_errors.InvalidCommitTypeError(model)
def process(self, entity):
"""Function that defines how to process each entity in a pipeline of
models.
Args:
entity: datastore_services.Model. Entity to validate.
Yields:
ModelIdRegexError. An error class for models with invalid IDs.
"""
cloned_entity = job_utils.clone_model(entity)
if not re.match(self._pattern, cloned_entity.id):
yield base_validation_errors.ModelIdRegexError(
cloned_entity, self._pattern)
def process(self, input_model):
"""Function that validate that canonical name of the model is same as
name of the model in lowercase.
Args:
input_model: datastore_services.Model. TopicModel to validate.
Yields:
ModelCanonicalNameMismatchError. An error class for
name mismatched models.
"""
model = job_utils.clone_model(input_model)
name = model.name
if name.lower() != model.canonical_name:
yield topic_validation_errors.ModelCanonicalNameMismatchError(model)
def process(self, input_model):
"""Function that checks if archived model is marked deleted.
Args:
input_model: user_models.UserQueryModel.
Entity to validate.
Yields:
ArchivedModelNotMarkedDeletedError. Error for models marked
archived but not deleted.
"""
model = job_utils.clone_model(input_model)
if model.query_status == feconf.USER_QUERY_STATUS_ARCHIVED:
yield user_validation_errors.ArchivedModelNotMarkedDeletedError(
model)
def _get_change_domain_class(self, input_model):
"""Returns a change domain class.
Args:
input_model: datastore_services.Model. Entity to validate.
Returns:
subtopic_page_domain.SubtopicPageChange. A domain object class for
the changes made by commit commands of the model.
"""
model = job_utils.clone_model(input_model)
if model.id.startswith('subtopicpage'):
return subtopic_page_domain.SubtopicPageChange
else:
return None
def _get_change_domain_class(self, input_model): # pylint: disable=unused-argument
"""Returns a change domain class.
Args:
input_model: datastore_services.Model. Entity to validate.
Returns:
question_domain.QuestionChange. A domain object class for the
changes made by commit commands of the model.
"""
model = job_utils.clone_model(input_model)
if model.id.startswith('question'):
return question_domain.QuestionChange
else:
return None
def process(self, input_model):
"""Function validates that post_commit_status is either public or
private
Args:
input_model: base_models.BaseCommitLogEntryModel.
Entity to validate.
Yields:
InvalidCommitStatusError. Error for commit_type validation.
"""
model = job_utils.clone_model(input_model)
if model.post_commit_status not in [
feconf.POST_COMMIT_STATUS_PUBLIC,
feconf.POST_COMMIT_STATUS_PRIVATE]:
yield audit_errors.InvalidCommitStatusError(model)
def _get_change_domain_class(self, input_model): # pylint: disable=unused-argument
# type: (Any) -> Optional[Type[skill_domain.SkillChange]]
"""Returns a change domain class.
Args:
input_model: datastore_services.Model. Entity to validate.
Returns:
skill_domain.SkillChange. A domain object class for the
changes made by commit commands of the model.
"""
model = job_utils.clone_model(input_model) # type: ignore[no-untyped-call]
if model.id.startswith('skill'):
return skill_domain.SkillChange
else:
return None
def process(self, entity):
"""Function validates that post_commit_status is either public or
private
Args:
entity: base_models.BaseCommitLogEntryModel. Entity to validate.
Yields:
InvalidCommitStatusError. Error for commit_type validation.
"""
cloned_entity = job_utils.clone_model(entity)
if cloned_entity.post_commit_status not in [
feconf.POST_COMMIT_STATUS_PUBLIC,
feconf.POST_COMMIT_STATUS_PRIVATE
]:
yield base_validation_errors.InvalidCommitStatusError(
cloned_entity)
def process(self, input_model):
"""Yields audit errors that are discovered in the input model.
Args:
input_model: datastore_services.Model. Entity to validate.
Yields:
ModelExpiredError. An error class for expired models.
"""
model = job_utils.clone_model(input_model)
expiration_date = (
datetime.datetime.utcnow() -
feconf.PERIOD_TO_HARD_DELETE_MODELS_MARKED_AS_DELETED)
if model.last_updated < expiration_date:
yield base_validation_errors.ModelExpiredError(model)
def process(self, input_model):
"""Function validates that post_commit_is_private is true iff
post_commit_status is private
Args:
input_model: base_models.BaseCommitLogEntryModel.
Entity to validate.
Yields:
ModelInvalidCommitStatus. Error for commit_type validation.
"""
model = job_utils.clone_model(input_model)
expected_post_commit_is_private = (
model.post_commit_status == feconf.POST_COMMIT_STATUS_PRIVATE)
if model.post_commit_is_private != expected_post_commit_is_private:
yield audit_errors.InvalidCommitStatusError(model)
def process(self, input_model):
"""Function validates that post_commit_is_public is true iff
post_commit_status is public.
Args:
input_model: base_models.BaseCommitLogEntryModel.
Entity to validate.
Yields:
InvalidPublicCommitStatusError. Error for public commit_type
validation.
"""
model = job_utils.clone_model(input_model)
expected_post_commit_is_public = (
model.post_commit_status == feconf.POST_COMMIT_STATUS_PUBLIC)
if model.post_commit_community_owned != expected_post_commit_is_public:
yield base_validation_errors.InvalidPublicCommitStatusError(model)
def _get_change_domain_class(self, input_model):
"""Returns a change domain class.
Args:
input_model: datastore_services.Model. Entity to validate.
Returns:
rights_domain.ExplorationRightsChange|exp_domain.ExplorationChange.
A domain object class for the changes made by commit commands of
the model.
"""
model = job_utils.clone_model(input_model)
if model.id.startswith('rights'):
return rights_domain.ExplorationRightsChange
elif model.id.startswith('exploration'):
return exp_domain.ExplorationChange
else:
return None
def process(self, input_model):
"""Function that defines how to process each element in a pipeline of
models.
Args:
input_model: datastore_services.Model. Entity to validate.
Yields:
ModelMutatedDuringJobError. Error for models mutated during the job.
InconsistentTimestampsError. Error for models with inconsistent
timestamps.
"""
model = job_utils.clone_model(input_model)
if model.created_on > (model.last_updated + MAX_CLOCK_SKEW_SECS):
yield base_validation_errors.InconsistentTimestampsError(model)
current_datetime = datetime.datetime.utcnow()
if (model.last_updated - MAX_CLOCK_SKEW_SECS) > current_datetime:
yield base_validation_errors.ModelMutatedDuringJobError(model)
def process(self, input_model):
"""Function that checks if the composite entity id is valid
Args:
input_model: improvements_models.TaskEntryModel.
Entity to validate.
Yields:
InvalidCompositeEntityError. Error for models with
invalid composite entity.
"""
model = job_utils.clone_model(input_model)
expected_composite_entity_id = (
improvements_models.TaskEntryModel.generate_composite_entity_id(
model.entity_type, model.entity_id, model.entity_version))
if model.composite_entity_id != expected_composite_entity_id:
yield improvements_validation_errors.InvalidCompositeEntityError(
model)
def process(self, entity):
"""Function validates that post_commit_is_private is true iff
post_commit_status is private
Args:
entity: base_models.BaseCommitLogEntryModel.
Entity to validate.
Yields:
InvalidPrivateCommitStatusError. Error for private commit_type
validation.
"""
cloned_entity = job_utils.clone_model(entity)
expected_post_commit_is_private = (cloned_entity.post_commit_status ==
feconf.POST_COMMIT_STATUS_PRIVATE)
if (cloned_entity.post_commit_is_private !=
expected_post_commit_is_private):
yield base_validation_errors.InvalidPrivateCommitStatusError(
cloned_entity)
def process(self, input_model):
"""Function that checks if last_updated for draft change list is valid.
Args:
input_model: user_models.ExplorationUserDataModel.
Entity to validate.
Yields:
DraftChangeListLastUpdatedNoneError. Error for models with
draft change list but no draft_change_list_last_updated
DraftChangeListLastUpdatedInvalidError. Error for models with
draft_change_list_last_updated greater than current time.
"""
model = job_utils.clone_model(input_model)
if (model.draft_change_list
and not model.draft_change_list_last_updated):
yield audit_errors.DraftChangeListLastUpdatedNoneError(model)
current_time = datetime.datetime.utcnow()
if (model.draft_change_list_last_updated
and model.draft_change_list_last_updated > current_time):
yield audit_errors.DraftChangeListLastUpdatedInvalidError(model)
def process(self, input_model):
"""Function that check for incorrect key in model.
Args:
input_model: user_models.PendingDeletionRequestModel. Entity to
validate.
Yields:
ModelIncorrectkeyError. An error class for incorrect key.
"""
model = job_utils.clone_model(input_model)
allowed_keys = [
name.value for name in models.MODULES_WITH_PSEUDONYMIZABLE_CLASSES
]
incorrect_keys = [
key for key in model.pseudonymizable_entity_mappings.keys()
if key not in allowed_keys
]
if incorrect_keys:
yield audit_errors.ModelIncorrectKeyError(model, incorrect_keys)
def test_hashable(self):
set_of_errors = {
FooError(self.model),
FooError(job_utils.clone_model(self.model)),
}
self.assertEqual(len(set_of_errors), 1)
