def process(self, input_model): """Function that validates that the published timestamp of the blog post models is either None or is greater than created on time, is less than current datetime and is equal to or greater than the last updated timestamp. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for models mutated during the job. InconsistentTimestampsError. Error for models with inconsistent timestamps. """ model = job_utils.clone_model(input_model) if model.published_on is None: return if model.created_on > (model.published_on + base_validation.MAX_CLOCK_SKEW_SECS): yield blog_validation_errors.InconsistentPublishTimestampsError( model) current_datetime = datetime.datetime.utcnow() if (model.published_on - base_validation.MAX_CLOCK_SKEW_SECS) > (current_datetime): yield blog_validation_errors.ModelMutatedDuringJobError(model) if (model.published_on - base_validation.MAX_CLOCK_SKEW_SECS) > (model.last_updated): yield blog_validation_errors.InconsistentPublishLastUpdatedTimestampsError(model) # pylint: disable=line-too-long
def process(self, entity): """Function that defines how to process each entity in a pipeline of models. Args: entity: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for models mutated during the job. InconsistentTimestampsError. Error for models with inconsistent timestamps. """ cloned_entity = job_utils.clone_model(entity) last_updated_corrected = (cloned_entity.last_updated + MAX_CLOCK_SKEW_SECS) if cloned_entity.created_on > last_updated_corrected: yield base_validation_errors.InconsistentTimestampsError( cloned_entity) current_datetime = datetime.datetime.utcnow() last_updated_corrected = (cloned_entity.last_updated - MAX_CLOCK_SKEW_SECS) if last_updated_corrected > current_datetime: yield base_validation_errors.ModelMutatedDuringJobError( cloned_entity)
def process( self, user_stats_model: user_models.UserStatsModel ) -> Iterable[user_models.UserStatsModel]: """Updates weekly dashboard stats with the current values. Args: user_stats_model: UserStatsModel. Model for which to update the weekly dashboard stats. Yields: UserStatsModel. The updated user stats model. """ model = job_utils.clone_model(user_stats_model) schema_version = model.schema_version if schema_version != feconf.CURRENT_DASHBOARD_STATS_SCHEMA_VERSION: user_services.migrate_dashboard_stats_to_latest_schema( model) # type: ignore[no-untyped-call] weekly_creator_stats = { user_services.get_current_date_as_string(): { # type: ignore[no-untyped-call] 'num_ratings': model.num_ratings or 0, 'average_ratings': model.average_ratings, 'total_plays': model.total_plays or 0 } } model.weekly_creator_stats_list.append(weekly_creator_stats) model.update_timestamps() yield model
def test_clone_model(self): model = base_models.BaseModel(id='123', deleted=True) clone = job_utils.clone_model(model) self.assertEqual(model.id, clone.id) self.assertEqual(model, clone) self.assertIsNot(model, clone) self.assertIsInstance(clone, base_models.BaseModel)
def test_clone_with_changes(self): model = base_models.BaseModel(id='123', deleted=True) clone = job_utils.clone_model(model, deleted=False) self.assertNotEqual(model, clone) self.assertIsNot(model, clone) self.assertIsInstance(clone, base_models.BaseModel) self.assertTrue(model.deleted) self.assertFalse(clone.deleted)
def test_clone_with_changes_to_id(self): model = base_models.BaseModel(id='123') clone = job_utils.clone_model(model, id='124') self.assertNotEqual(model, clone) self.assertIsNot(model, clone) self.assertIsInstance(clone, base_models.BaseModel) self.assertEqual(model.id, '123') self.assertEqual(clone.id, '124')
def test_clone_sub_class_with_changes(self): model = FooModel(prop='original') clone = job_utils.clone_model(model, prop='updated') self.assertNotEqual(model, clone) self.assertIsNot(model, clone) self.assertIsInstance(clone, FooModel) self.assertEqual(model.prop, 'original') self.assertEqual(clone.prop, 'updated')
def test_clone_sub_class(self) -> None: model = FooModel(prop='original') clone = job_utils.clone_model(model) self.assertEqual(model, clone) self.assertIsNot(model, clone) self.assertIsInstance(clone, FooModel) self.assertEqual(model.prop, 'original') self.assertEqual(clone.prop, 'original')
def process(self, input_model): """Function that checks if a model is old enough to mark them deleted. Args: input_model: user_models.UserQueryModel. Entity to validate. Yields: ModelExpiringError. An error class for expiring models. """ model = job_utils.clone_model(input_model) expiration_date = (datetime.datetime.utcnow() - feconf.PERIOD_TO_MARK_MODELS_AS_DELETED) if expiration_date > model.last_updated: yield audit_errors.ModelExpiringError(model)
def process(self, input_model): """Function that defines how to process each element in a pipeline of models. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelIdRegexError. An error class for models with invalid IDs. """ model = job_utils.clone_model(input_model) if not re.match(self._pattern, model.id): yield audit_errors.ModelIdRegexError(model, self._pattern)
def process(self, input_model): """Function that checks if the entity type is valid Args: input_model: feedback_models.GeneralFeedbackThreadModel. Entity to validate. Yields: InvalidEntityTypeError. Error for models with invalid entity type. """ model = job_utils.clone_model(input_model) if (model.entity_type not in feedback_services.TARGET_TYPE_TO_TARGET_MODEL): yield feedback_validation_errors.InvalidEntityTypeError(model)
def process(self, input_model): """Function that defines how to process each element in a pipeline of models. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelCommitTypeError. Error for commit_type validation. """ model = job_utils.clone_model(input_model) if (model.commit_type not in base_models.VersionedModel.COMMIT_TYPE_CHOICES): yield base_validation_errors.InvalidCommitTypeError(model)
def process(self, entity): """Function that defines how to process each entity in a pipeline of models. Args: entity: datastore_services.Model. Entity to validate. Yields: ModelIdRegexError. An error class for models with invalid IDs. """ cloned_entity = job_utils.clone_model(entity) if not re.match(self._pattern, cloned_entity.id): yield base_validation_errors.ModelIdRegexError( cloned_entity, self._pattern)
def process(self, input_model): """Function that validate that canonical name of the model is same as name of the model in lowercase. Args: input_model: datastore_services.Model. TopicModel to validate. Yields: ModelCanonicalNameMismatchError. An error class for name mismatched models. """ model = job_utils.clone_model(input_model) name = model.name if name.lower() != model.canonical_name: yield topic_validation_errors.ModelCanonicalNameMismatchError(model)
def process(self, input_model): """Function that checks if archived model is marked deleted. Args: input_model: user_models.UserQueryModel. Entity to validate. Yields: ArchivedModelNotMarkedDeletedError. Error for models marked archived but not deleted. """ model = job_utils.clone_model(input_model) if model.query_status == feconf.USER_QUERY_STATUS_ARCHIVED: yield user_validation_errors.ArchivedModelNotMarkedDeletedError( model)
def _get_change_domain_class(self, input_model): """Returns a change domain class. Args: input_model: datastore_services.Model. Entity to validate. Returns: subtopic_page_domain.SubtopicPageChange. A domain object class for the changes made by commit commands of the model. """ model = job_utils.clone_model(input_model) if model.id.startswith('subtopicpage'): return subtopic_page_domain.SubtopicPageChange else: return None
def _get_change_domain_class(self, input_model): # pylint: disable=unused-argument """Returns a change domain class. Args: input_model: datastore_services.Model. Entity to validate. Returns: question_domain.QuestionChange. A domain object class for the changes made by commit commands of the model. """ model = job_utils.clone_model(input_model) if model.id.startswith('question'): return question_domain.QuestionChange else: return None
def process(self, input_model): """Function validates that post_commit_status is either public or private Args: input_model: base_models.BaseCommitLogEntryModel. Entity to validate. Yields: InvalidCommitStatusError. Error for commit_type validation. """ model = job_utils.clone_model(input_model) if model.post_commit_status not in [ feconf.POST_COMMIT_STATUS_PUBLIC, feconf.POST_COMMIT_STATUS_PRIVATE]: yield audit_errors.InvalidCommitStatusError(model)
def _get_change_domain_class(self, input_model): # pylint: disable=unused-argument # type: (Any) -> Optional[Type[skill_domain.SkillChange]] """Returns a change domain class. Args: input_model: datastore_services.Model. Entity to validate. Returns: skill_domain.SkillChange. A domain object class for the changes made by commit commands of the model. """ model = job_utils.clone_model(input_model) # type: ignore[no-untyped-call] if model.id.startswith('skill'): return skill_domain.SkillChange else: return None
def process(self, entity): """Function validates that post_commit_status is either public or private Args: entity: base_models.BaseCommitLogEntryModel. Entity to validate. Yields: InvalidCommitStatusError. Error for commit_type validation. """ cloned_entity = job_utils.clone_model(entity) if cloned_entity.post_commit_status not in [ feconf.POST_COMMIT_STATUS_PUBLIC, feconf.POST_COMMIT_STATUS_PRIVATE ]: yield base_validation_errors.InvalidCommitStatusError( cloned_entity)
def process(self, input_model): """Yields audit errors that are discovered in the input model. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelExpiredError. An error class for expired models. """ model = job_utils.clone_model(input_model) expiration_date = ( datetime.datetime.utcnow() - feconf.PERIOD_TO_HARD_DELETE_MODELS_MARKED_AS_DELETED) if model.last_updated < expiration_date: yield base_validation_errors.ModelExpiredError(model)
def process(self, input_model): """Function validates that post_commit_is_private is true iff post_commit_status is private Args: input_model: base_models.BaseCommitLogEntryModel. Entity to validate. Yields: ModelInvalidCommitStatus. Error for commit_type validation. """ model = job_utils.clone_model(input_model) expected_post_commit_is_private = ( model.post_commit_status == feconf.POST_COMMIT_STATUS_PRIVATE) if model.post_commit_is_private != expected_post_commit_is_private: yield audit_errors.InvalidCommitStatusError(model)
def process(self, input_model): """Function validates that post_commit_is_public is true iff post_commit_status is public. Args: input_model: base_models.BaseCommitLogEntryModel. Entity to validate. Yields: InvalidPublicCommitStatusError. Error for public commit_type validation. """ model = job_utils.clone_model(input_model) expected_post_commit_is_public = ( model.post_commit_status == feconf.POST_COMMIT_STATUS_PUBLIC) if model.post_commit_community_owned != expected_post_commit_is_public: yield base_validation_errors.InvalidPublicCommitStatusError(model)
def _get_change_domain_class(self, input_model): """Returns a change domain class. Args: input_model: datastore_services.Model. Entity to validate. Returns: rights_domain.ExplorationRightsChange|exp_domain.ExplorationChange. A domain object class for the changes made by commit commands of the model. """ model = job_utils.clone_model(input_model) if model.id.startswith('rights'): return rights_domain.ExplorationRightsChange elif model.id.startswith('exploration'): return exp_domain.ExplorationChange else: return None
def process(self, input_model): """Function that defines how to process each element in a pipeline of models. Args: input_model: datastore_services.Model. Entity to validate. Yields: ModelMutatedDuringJobError. Error for models mutated during the job. InconsistentTimestampsError. Error for models with inconsistent timestamps. """ model = job_utils.clone_model(input_model) if model.created_on > (model.last_updated + MAX_CLOCK_SKEW_SECS): yield base_validation_errors.InconsistentTimestampsError(model) current_datetime = datetime.datetime.utcnow() if (model.last_updated - MAX_CLOCK_SKEW_SECS) > current_datetime: yield base_validation_errors.ModelMutatedDuringJobError(model)
def process(self, input_model): """Function that checks if the composite entity id is valid Args: input_model: improvements_models.TaskEntryModel. Entity to validate. Yields: InvalidCompositeEntityError. Error for models with invalid composite entity. """ model = job_utils.clone_model(input_model) expected_composite_entity_id = ( improvements_models.TaskEntryModel.generate_composite_entity_id( model.entity_type, model.entity_id, model.entity_version)) if model.composite_entity_id != expected_composite_entity_id: yield improvements_validation_errors.InvalidCompositeEntityError( model)
def process(self, entity): """Function validates that post_commit_is_private is true iff post_commit_status is private Args: entity: base_models.BaseCommitLogEntryModel. Entity to validate. Yields: InvalidPrivateCommitStatusError. Error for private commit_type validation. """ cloned_entity = job_utils.clone_model(entity) expected_post_commit_is_private = (cloned_entity.post_commit_status == feconf.POST_COMMIT_STATUS_PRIVATE) if (cloned_entity.post_commit_is_private != expected_post_commit_is_private): yield base_validation_errors.InvalidPrivateCommitStatusError( cloned_entity)
def process(self, input_model): """Function that checks if last_updated for draft change list is valid. Args: input_model: user_models.ExplorationUserDataModel. Entity to validate. Yields: DraftChangeListLastUpdatedNoneError. Error for models with draft change list but no draft_change_list_last_updated DraftChangeListLastUpdatedInvalidError. Error for models with draft_change_list_last_updated greater than current time. """ model = job_utils.clone_model(input_model) if (model.draft_change_list and not model.draft_change_list_last_updated): yield audit_errors.DraftChangeListLastUpdatedNoneError(model) current_time = datetime.datetime.utcnow() if (model.draft_change_list_last_updated and model.draft_change_list_last_updated > current_time): yield audit_errors.DraftChangeListLastUpdatedInvalidError(model)
def process(self, input_model): """Function that check for incorrect key in model. Args: input_model: user_models.PendingDeletionRequestModel. Entity to validate. Yields: ModelIncorrectkeyError. An error class for incorrect key. """ model = job_utils.clone_model(input_model) allowed_keys = [ name.value for name in models.MODULES_WITH_PSEUDONYMIZABLE_CLASSES ] incorrect_keys = [ key for key in model.pseudonymizable_entity_mappings.keys() if key not in allowed_keys ] if incorrect_keys: yield audit_errors.ModelIncorrectKeyError(model, incorrect_keys)
def test_hashable(self): set_of_errors = { FooError(self.model), FooError(job_utils.clone_model(self.model)), } self.assertEqual(len(set_of_errors), 1)