class TaskEntryModel(base_models.BaseModel): """Model representation of an actionable task from the improvements tab. The ID of a task has the form: "[entity_type].[entity_id].[entity_version]. [task_type].[target_type].[target_id]". """ # Utility field which results in a 20% speedup compared to querying by each # of the invididual fields used to compose it. # Value has the form: "[entity_type].[entity_id].[entity_version]". composite_entity_id = datastore_services.StringProperty(required=True, indexed=True) # The type of entity a task entry refers to. entity_type = datastore_services.StringProperty(required=True, indexed=True, choices=TASK_ENTITY_TYPES) # The ID of the entity a task entry refers to. entity_id = datastore_services.StringProperty(required=True, indexed=True) # The version of the entity a task entry refers to. entity_version = datastore_services.IntegerProperty(required=True, indexed=True) # The type of task a task entry tracks. task_type = datastore_services.StringProperty(required=True, indexed=True, choices=TASK_TYPES) # The type of sub-entity a task entry focuses on. Value is None when an # entity does not have any meaningful sub-entities to target. target_type = datastore_services.StringProperty(required=True, indexed=True, choices=TASK_TARGET_TYPES) # Uniquely identifies the sub-entity a task entry focuses on. Value is None # when an entity does not have any meaningful sub-entities to target. target_id = datastore_services.StringProperty(required=True, indexed=True) # A sentence generated by Oppia to describe why the task was created. issue_description = datastore_services.StringProperty(default=None, required=False, indexed=True) # Tracks the state/progress of a task entry. status = datastore_services.StringProperty(required=True, indexed=True, choices=TASK_STATUS_CHOICES) # ID of the user who closed the task, if any. resolver_id = datastore_services.StringProperty(default=None, required=False, indexed=True) # The date and time at which a task was closed or deprecated. resolved_on = datastore_services.DateTimeProperty(default=None, required=False, indexed=True) @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether any TaskEntryModel references the given user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return cls.query(cls.resolver_id == user_id).get() is not None @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data to delete corresponding to a user: resolver_id field. It is okay to delete task entries since, after they are resolved, they only act as a historical record. The removal just removes the historical record. """ return base_models.DELETION_POLICY.DELETE @classmethod def apply_deletion_policy(cls, user_id: str) -> None: """Delete instances of TaskEntryModel for the user. Args: user_id: str. The ID of the user whose data should be deleted. """ task_entry_keys = (cls.query(cls.resolver_id == user_id).fetch( keys_only=True)) datastore_services.delete_multi(task_entry_keys) @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model is exported as one instance shared across users since multiple users resolve tasks. """ return (base_models.MODEL_ASSOCIATION_TO_USER. ONE_INSTANCE_SHARED_ACROSS_USERS) @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data to export corresponding to a user: TaskEntryModel contains the ID of the user that acted on a task. """ return dict( super(cls, cls).get_export_policy(), **{ 'composite_entity_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entity_type': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entity_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entity_version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'task_type': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'target_type': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'target_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'issue_description': base_models.EXPORT_POLICY.EXPORTED, 'status': base_models.EXPORT_POLICY.EXPORTED, 'resolver_id': base_models.EXPORT_POLICY.EXPORTED, 'resolved_on': base_models.EXPORT_POLICY.EXPORTED }) @classmethod def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]: """Defines the mapping of field names to takeout keys since this model is exported as one instance shared across users. """ return { 'resolver_id': 'task_ids_resolved_by_user', 'issue_description': 'issue_descriptions', 'status': 'statuses', 'resolved_on': 'resolution_msecs' } @staticmethod def export_data(user_id: str) -> Dict[str, List[str]]: """Returns the user-relevant properties of TaskEntryModels. Args: user_id: str. The ID of the user whose data should be exported. Returns: dict. The user-relevant properties of TaskEntryModel in a dict format. In this case, we are returning all the ids of the tasks which were closed by this user. """ task_ids_resolved_by_user = TaskEntryModel.query( TaskEntryModel.resolver_id == user_id) return { 'task_ids_resolved_by_user': ([t.id for t in task_ids_resolved_by_user]), 'issue_descriptions': ([t.issue_description for t in task_ids_resolved_by_user]), 'statuses': ([t.status for t in task_ids_resolved_by_user]), 'resolution_msecs': ([t.resolved_on for t in task_ids_resolved_by_user]), } @classmethod def generate_task_id(cls, entity_type: str, entity_id: str, entity_version: int, task_type: str, target_type: str, target_id: str) -> str: """Generates a new task entry ID. Args: entity_type: str. The type of entity a task entry refers to. entity_id: str. The ID of the entity a task entry refers to. entity_version: int. The version of the entity a task entry refers to. task_type: str. The type of task a task entry tracks. target_type: str. The type of sub-entity a task entry refers to. target_id: str. The ID of the sub-entity a task entry refers to. Returns: str. The ID for the given task. """ return feconf.TASK_ENTRY_ID_TEMPLATE % (entity_type, entity_id, entity_version, task_type, target_type, target_id) @classmethod def generate_composite_entity_id(cls, entity_type: str, entity_id: str, entity_version: int) -> str: """Generates a new composite_entity_id value. Args: entity_type: str. The type of entity a task entry refers to. entity_id: str. The ID of the entity a task entry refers to. entity_version: int. The version of the entity a task entry refers to. Returns: str. The composite_entity_id for the given task. """ return feconf.COMPOSITE_ENTITY_ID_TEMPLATE % (entity_type, entity_id, entity_version) @classmethod def create(cls, entity_type: str, entity_id: str, entity_version: int, task_type: str, target_type: str, target_id: str, issue_description: Optional[str] = None, status: str = constants.TASK_STATUS_OBSOLETE, resolver_id: Optional[str] = None, resolved_on: Optional[str] = None) -> str: """Creates a new task entry and puts it in storage. Args: entity_type: str. The type of entity a task entry refers to. entity_id: str. The ID of the entity a task entry refers to. entity_version: int. The version of the entity a task entry refers to. task_type: str. The type of task a task entry tracks. target_type: str. The type of sub-entity a task entry refers to. target_id: str. The ID of the sub-entity a task entry refers to. issue_description: str. Sentence generated by Oppia to describe why the task was created. status: str. Tracks the state/progress of a task entry. resolver_id: str. ID of the user who closed the task, if any. resolved_on: str. The date and time at which a task was closed or deprecated. Returns: str. The ID of the new task. Raises: Exception. A task corresponding to the provided identifier values (entity_type, entity_id, entity_version, task_type, target_type, target_id) already exists in storage. """ task_id = cls.generate_task_id(entity_type, entity_id, entity_version, task_type, target_type, target_id) if cls.get_by_id(task_id) is not None: raise Exception('Task id %s already exists' % task_id) composite_entity_id = cls.generate_composite_entity_id( entity_type, entity_id, entity_version) task_entry = cls(id=task_id, composite_entity_id=composite_entity_id, entity_type=entity_type, entity_id=entity_id, entity_version=entity_version, task_type=task_type, target_type=target_type, target_id=target_id, issue_description=issue_description, status=status, resolver_id=resolver_id, resolved_on=resolved_on) task_entry.update_timestamps() task_entry.put() return task_id
class StorySummaryModel(base_models.BaseModel): """Summary model for an Oppia Story. This should be used whenever the content blob of the story is not needed (e.g. search results, etc). A StorySummaryModel instance stores the following information: id, description, language_code, last_updated, created_on, version. The key of each instance is the story id. """ # The title of the story. title = datastore_services.StringProperty(required=True, indexed=True) # The ISO 639-1 code for the language this story is written in. language_code = (datastore_services.StringProperty(required=True, indexed=True)) # A high-level description of the story. description = datastore_services.TextProperty(required=True, indexed=False) # Time when the story model was last updated (not to be # confused with last_updated, which is the time when the # story *summary* model was last updated). story_model_last_updated = (datastore_services.DateTimeProperty( required=True, indexed=True)) # Time when the story model was created (not to be confused # with created_on, which is the time when the story *summary* # model was created). story_model_created_on = (datastore_services.DateTimeProperty( required=True, indexed=True)) # The titles of the nodes in the story, in the same order as present there. node_titles = (datastore_services.StringProperty(repeated=True, indexed=True)) # The thumbnail filename of the story. thumbnail_filename = datastore_services.StringProperty(indexed=True) # The thumbnail background color of the story. thumbnail_bg_color = datastore_services.StringProperty(indexed=True) version = datastore_services.IntegerProperty(required=True) # The url fragment for the story. url_fragment = (datastore_services.StringProperty(required=True, indexed=True)) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any data directly corresponding to a user.""" return base_models.DELETION_POLICY.NOT_APPLICABLE @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model does not contain user data.""" return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model doesn't contain any data directly corresponding to a user.""" return dict( super(cls, cls).get_export_policy(), **{ 'title': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'description': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'story_model_last_updated': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'story_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'node_titles': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE })
class TopicSummaryModel(base_models.BaseModel): """Summary model for an Oppia Topic. This should be used whenever the content blob of the topic is not needed (e.g. search results, etc). A TopicSummaryModel instance stores the following information: id, description, language_code, last_updated, created_on, version, url_fragment. The key of each instance is the topic id. """ # The name of the topic. name = datastore_services.StringProperty(required=True, indexed=True) # The canonical name of the topic, created by making `name` lowercase. canonical_name = ( datastore_services.StringProperty(required=True, indexed=True)) # The ISO 639-1 code for the language this topic is written in. language_code = ( datastore_services.StringProperty(required=True, indexed=True)) # The description of the topic. description = datastore_services.TextProperty(indexed=False) # The url fragment of the topic. url_fragment = ( datastore_services.StringProperty(required=True, indexed=True)) # Time when the topic model was last updated (not to be # confused with last_updated, which is the time when the # topic *summary* model was last updated). topic_model_last_updated = ( datastore_services.DateTimeProperty(required=True, indexed=True)) # Time when the topic model was created (not to be confused # with created_on, which is the time when the topic *summary* # model was created). topic_model_created_on = ( datastore_services.DateTimeProperty(required=True, indexed=True)) # The number of canonical stories that are part of this topic. canonical_story_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The number of additional stories that are part of this topic. additional_story_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The total number of skills in the topic (including those that are # uncategorized). total_skill_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The total number of published chapters in the topic. total_published_node_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The number of skills that are not part of any subtopic. uncategorized_skill_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The number of subtopics of the topic. subtopic_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The thumbnail filename of the topic. thumbnail_filename = datastore_services.StringProperty(indexed=True) # The thumbnail background color of the topic. thumbnail_bg_color = datastore_services.StringProperty(indexed=True) version = datastore_services.IntegerProperty(required=True) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any data directly corresponding to a user.""" return base_models.DELETION_POLICY.NOT_APPLICABLE @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model does not contain user data.""" return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model doesn't contain any data directly corresponding to a user.""" return dict(super(cls, cls).get_export_policy(), **{ 'name': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'description': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'topic_model_last_updated': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'topic_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'canonical_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'additional_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'total_skill_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'total_published_node_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'uncategorized_skill_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'subtopic_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE })
class QuestionSummaryModel(base_models.BaseModel): """Summary model for an Oppia question. This should be used whenever the content blob of the question is not needed (e.g. in search results, etc). A QuestionSummaryModel instance stores the following information: question_model_last_updated, question_model_created_on, question_state_data. The key of each instance is the question id. """ # Time when the question model was last updated (not to be # confused with last_updated, which is the time when the # question *summary* model was last updated). question_model_last_updated = datastore_services.DateTimeProperty( indexed=True, required=True) # Time when the question model was created (not to be confused # with created_on, which is the time when the question *summary* # model was created). question_model_created_on = datastore_services.DateTimeProperty( indexed=True, required=True) # The html content for the question. question_content = ( datastore_services.TextProperty(indexed=False, required=True)) # The ID of the interaction. interaction_id = ( datastore_services.StringProperty(indexed=True, required=True)) # The misconception ids addressed in the question. This includes # tagged misconceptions ids as well as inapplicable misconception # ids in the question. misconception_ids = ( datastore_services.StringProperty(indexed=True, repeated=True)) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any data directly corresponding to a user.""" return base_models.DELETION_POLICY.NOT_APPLICABLE @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model data has already been exported as a part of the QuestionModel export_data function, and thus a new export_data function does not need to be defined here. """ return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user, but this isn't exported because because noteworthy details that belong to this model have already been exported as a part of the QuestionModel export_data function. """ return dict(super(cls, cls).get_export_policy(), **{ 'question_model_last_updated': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'question_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'question_content': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'misconception_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE })
class ExpSummaryModel(base_models.BaseModel): """Summary model for an Oppia exploration. This should be used whenever the content blob of the exploration is not needed (e.g. in search results, etc). A ExpSummaryModel instance stores the following information: id, title, category, objective, language_code, tags, last_updated, created_on, status (private, public), community_owned, owner_ids, editor_ids, viewer_ids, version. The key of each instance is the exploration id. """ # What this exploration is called. title = datastore_services.StringProperty(required=True) # The category this exploration belongs to. category = datastore_services.StringProperty(required=True, indexed=True) # The objective of this exploration. objective = datastore_services.TextProperty(required=True, indexed=False) # The ISO 639-1 code for the language this exploration is written in. language_code = ( datastore_services.StringProperty(required=True, indexed=True)) # Tags associated with this exploration. tags = datastore_services.StringProperty(repeated=True, indexed=True) # Aggregate user-assigned ratings of the exploration. ratings = datastore_services.JsonProperty(default=None, indexed=False) # Scaled average rating for the exploration. scaled_average_rating = datastore_services.FloatProperty(indexed=True) # Time when the exploration model was last updated (not to be # confused with last_updated, which is the time when the # exploration *summary* model was last updated). exploration_model_last_updated = ( datastore_services.DateTimeProperty(indexed=True)) # Time when the exploration model was created (not to be confused # with created_on, which is the time when the exploration *summary* # model was created). exploration_model_created_on = ( datastore_services.DateTimeProperty(indexed=True)) # Time when the exploration was first published. first_published_msec = datastore_services.FloatProperty(indexed=True) # The publication status of this exploration. status = datastore_services.StringProperty( default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True, choices=[ constants.ACTIVITY_STATUS_PRIVATE, constants.ACTIVITY_STATUS_PUBLIC ] ) # Whether this exploration is owned by the community. community_owned = ( datastore_services.BooleanProperty(required=True, indexed=True)) # The user_ids of owners of this exploration. owner_ids = datastore_services.StringProperty(indexed=True, repeated=True) # The user_ids of users who are allowed to edit this exploration. editor_ids = datastore_services.StringProperty(indexed=True, repeated=True) # The user_ids of users who are allowed to voiceover this exploration. voice_artist_ids = ( datastore_services.StringProperty(indexed=True, repeated=True)) # The user_ids of users who are allowed to view this exploration. viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True) # The user_ids of users who have contributed (humans who have made a # positive (not just a revert) change to the exploration's content). # NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be # synchronized, meaning that the keys in contributors_summary need be # equal to the contributor_ids list. contributor_ids = ( datastore_services.StringProperty(indexed=True, repeated=True)) # A dict representing the contributors of non-trivial commits to this # exploration. Each key of this dict is a user_id, and the corresponding # value is the number of non-trivial commits that the user has made. contributors_summary = ( datastore_services.JsonProperty(default={}, indexed=False)) # The version number of the exploration after this commit. Only populated # for commits to an exploration (as opposed to its rights, etc.). version = datastore_services.IntegerProperty() @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data to pseudonymize or delete corresponding to a user: viewer_ids, voice_artist_ids, editor_ids, owner_ids, contributor_ids, and contributors_summary fields. """ return ( base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE ) @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether ExpSummaryModel references user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return cls.query(datastore_services.any_of( cls.owner_ids == user_id, cls.editor_ids == user_id, cls.voice_artist_ids == user_id, cls.viewer_ids == user_id, cls.contributor_ids == user_id )).get(keys_only=True) is not None @classmethod def get_non_private(cls) -> Sequence['ExpSummaryModel']: """Returns an iterable with non-private ExpSummary models. Returns: iterable. An iterable with non-private ExpSummary models. """ return ExpSummaryModel.query().filter( ExpSummaryModel.status != constants.ACTIVITY_STATUS_PRIVATE ).filter( ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison ).fetch(feconf.DEFAULT_QUERY_LIMIT) @classmethod def get_top_rated(cls, limit: int) -> Sequence['ExpSummaryModel']: """Fetches the top-rated exp summaries that are public in descending order of scaled_average_rating. Args: limit: int. The maximum number of results to return. Returns: iterable. An iterable with the top rated exp summaries that are public in descending order of scaled_average_rating. """ return ExpSummaryModel.query().filter( ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC ).filter( ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison ).order( -ExpSummaryModel.scaled_average_rating ).fetch(limit) @classmethod def get_private_at_least_viewable( cls, user_id: str ) -> Sequence['ExpSummaryModel']: """Fetches private exp summaries that are at least viewable by the given user. Args: user_id: str. The id of the given user. Returns: iterable. An iterable with private exp summaries that are at least viewable by the given user. """ return ExpSummaryModel.query().filter( ExpSummaryModel.status == constants.ACTIVITY_STATUS_PRIVATE ).filter( datastore_services.any_of( ExpSummaryModel.owner_ids == user_id, ExpSummaryModel.editor_ids == user_id, ExpSummaryModel.voice_artist_ids == user_id, ExpSummaryModel.viewer_ids == user_id) ).filter( ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison ).fetch(feconf.DEFAULT_QUERY_LIMIT) @classmethod def get_at_least_editable(cls, user_id: str) -> Sequence['ExpSummaryModel']: """Fetches exp summaries that are at least editable by the given user. Args: user_id: str. The id of the given user. Returns: iterable. An iterable with exp summaries that are at least editable by the given user. """ return ExpSummaryModel.query().filter( datastore_services.any_of( ExpSummaryModel.owner_ids == user_id, ExpSummaryModel.editor_ids == user_id) ).filter( ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison ).fetch(feconf.DEFAULT_QUERY_LIMIT) @classmethod def get_recently_published(cls, limit: int) -> Sequence['ExpSummaryModel']: """Fetches exp summaries that are recently published. Args: limit: int. The maximum number of results to return. Returns: iterable. An iterable with exp summaries that are recently published. The returned list is sorted by the time of publication with latest being first in the list. """ return ExpSummaryModel.query().filter( ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC ).filter( ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison ).order( -ExpSummaryModel.first_published_msec ).fetch(limit) @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model data has already been exported as a part of the ExplorationModel and thus does not need a separate export. """ return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user, but this isn't exported because because noteworthy details that belong to this model have already been exported as a part of the ExplorationModel. """ return dict(super(cls, cls).get_export_policy(), **{ 'title': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'category': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'scaled_average_rating': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'exploration_model_last_updated': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'exploration_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'first_published_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'status': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'voice_artist_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'contributors_summary': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'version': base_models.EXPORT_POLICY.NOT_APPLICABLE })
class SkillSummaryModel(base_models.BaseModel): """Summary model for an Oppia Skill. This should be used whenever the content blob of the skill is not needed (e.g. search results, etc). A SkillSummaryModel instance stores the following information: id, description, language_code, last_updated, created_on, version. The key of each instance is the skill id. """ # The description of the skill. description = datastore_services.StringProperty(required=True, indexed=True) # The number of misconceptions associated with the skill. misconception_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The number of worked examples in the skill. worked_examples_count = ( datastore_services.IntegerProperty(required=True, indexed=True)) # The ISO 639-1 code for the language this skill is written in. language_code = ( datastore_services.StringProperty(required=True, indexed=True)) # Time when the skill model was last updated (not to be # confused with last_updated, which is the time when the # skill *summary* model was last updated). skill_model_last_updated = ( datastore_services.DateTimeProperty(required=True, indexed=True)) # Time when the skill model was created (not to be confused # with created_on, which is the time when the skill *summary* # model was created). skill_model_created_on = ( datastore_services.DateTimeProperty(required=True, indexed=True)) version = datastore_services.IntegerProperty(required=True) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any data directly corresponding to a user.""" return base_models.DELETION_POLICY.NOT_APPLICABLE @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model does not contain user data.""" return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model doesn't contain any data directly corresponding to a user.""" return dict(super(cls, cls).get_export_policy(), **{ 'description': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'misconception_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'worked_examples_count': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'skill_model_last_updated': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'skill_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'version': base_models.EXPORT_POLICY.NOT_APPLICABLE }) # TODO(#13523): Change the return value of the function below from # tuple(list, str|None, bool) to a domain object. @classmethod def fetch_page( cls, page_size: int, urlsafe_start_cursor: Optional[str], sort_by: Optional[str] ) -> Tuple[Sequence[SkillSummaryModel], Optional[str], bool]: """Returns the models according to values specified. Args: page_size: int. Number of skills to fetch. urlsafe_start_cursor: str|None. The cursor to the next page or None. If None, this means that the search should start from the first page of results. sort_by: str|None. A string indicating how to sort the result. Returns: 3-tuple(query_models, urlsafe_start_cursor, more). where: query_models: list(SkillSummary). The list of summaries of skills starting at the given cursor. urlsafe_start_cursor: str or None. A query cursor pointing to the next batch of results. If there are no more results, this might be None. more: bool. If True, there are (probably) more results after this batch. If False, there are no further results after this batch. """ cursor = ( datastore_services.make_cursor(urlsafe_cursor=urlsafe_start_cursor)) sort = -cls.skill_model_created_on if sort_by == ( constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[ 'DecreasingCreatedOn']): sort = cls.skill_model_created_on elif sort_by == ( constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[ 'IncreasingUpdatedOn']): sort = -cls.skill_model_last_updated elif sort_by == ( constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[ 'DecreasingUpdatedOn']): sort = cls.skill_model_last_updated sort_query = cls.query().order(sort) fetch_result: Tuple[ Sequence[SkillSummaryModel], datastore_services.Cursor, bool ] = sort_query.fetch_page(page_size, start_cursor=cursor) query_models, next_cursor, _ = fetch_result # TODO(#13462): Refactor this so that we don't do the lookup. # Do a forward lookup so that we can know if there are more values. fetch_result = sort_query.fetch_page(page_size + 1, start_cursor=cursor) plus_one_query_models, _, _ = fetch_result # The urlsafe returns bytes and we need to decode them to string. more_results = len(plus_one_query_models) == page_size + 1 new_urlsafe_start_cursor = ( next_cursor.urlsafe().decode('utf-8') if (next_cursor and more_results) else None ) return ( query_models, new_urlsafe_start_cursor, more_results )
class BlogPostSummaryModel(base_models.BaseModel): """Summary model for blog posts. This should be used whenever the content of the blog post is not needed (e.g. in search results, displaying blog post cards etc). The key of each instance is the blog post id. """ # The ID of the user the blog post is authored by. author_id = datastore_services.StringProperty(indexed=True, required=True) # Title of the blog post. title = datastore_services.StringProperty(indexed=True, required=True) # Autogenerated summary of the blog post. summary = datastore_services.StringProperty(required=True, default='') # The unique url fragment of the blog post. url_fragment = (datastore_services.StringProperty(indexed=True, required=True)) # Tags associated with the blog post. tags = datastore_services.StringProperty(indexed=True, repeated=True) # The thumbnail filename of the blog post.It's value will be none until # a thumbnail is added to the blog post.It can be None only when blog # post is a draft. thumbnail_filename = datastore_services.StringProperty(indexed=True) # Time when the blog post model was last published. Value will be None # if the blog post has never been published. published_on = (datastore_services.DateTimeProperty(indexed=True)) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data to pseudonymize corresponding to a user: author_id field. """ return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether BlogPostSummaryModel exists for user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return cls.query(cls.author_id == user_id).get( keys_only=True) is not None @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model data has already been associated as a part of the BlogPostModel to the user and thus does not need a separate user association. """ return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user (author_id), but this isn't exported because noteworthy details that belong to this model have already been exported as a part of the BlogPostModel. """ return dict( super(BlogPostSummaryModel, cls).get_export_policy(), **{ 'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'title': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'summary': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'published_on': base_models.EXPORT_POLICY.NOT_APPLICABLE, })
class BlogPostModel(base_models.BaseModel): """Model to store blog post data. The id of instances of this class is in the form of random hash of 12 chars. """ # We use the model id as a key in the Takeout dict. ID_IS_USED_AS_TAKEOUT_KEY: Literal[True] = True # The ID of the user the blog post is authored by. author_id = datastore_services.StringProperty(indexed=True, required=True) # Title of the blog post. title = datastore_services.StringProperty(indexed=True, required=True) # Content of the blog post. content = datastore_services.TextProperty(indexed=False, required=True) # The unique url fragment of the blog post. If the user directly enters the # blog post's url in the editor or the homepage, the blogPostModel will be # queried using the url fragment to retrieve data for populating the editor # dashboard / blog post page. url_fragment = (datastore_services.StringProperty(indexed=True, required=True)) # Tags associated with the blog post. tags = datastore_services.StringProperty(indexed=True, repeated=True) # The thumbnail filename of the blog post. It's value will be None until # a thumbnail is added to the blog post. It can be None only when blog # post is a draft. thumbnail_filename = datastore_services.StringProperty(indexed=True) # Time when the blog post model was last published. Value will be None # if the blog has never been published. published_on = (datastore_services.DateTimeProperty(indexed=True)) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data to pseudonymize corresponding to a user: author_id field. """ return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether BlogPostModel references user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return cls.query(cls.author_id == user_id).get( keys_only=True) is not None @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model is exported as multiple instances per user since there can be multiple blog post models relevant to a user. """ return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user to export.""" return dict( super(BlogPostModel, cls).get_export_policy(), **{ # We do not export the author_id because we should not # export internal user ids. 'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'title': base_models.EXPORT_POLICY.EXPORTED, 'content': base_models.EXPORT_POLICY.EXPORTED, 'url_fragment': base_models.EXPORT_POLICY.EXPORTED, 'tags': base_models.EXPORT_POLICY.EXPORTED, 'thumbnail_filename': base_models.EXPORT_POLICY.EXPORTED, 'published_on': base_models.EXPORT_POLICY.EXPORTED, }) @classmethod def generate_new_blog_post_id(cls) -> str: """Generates a new blog post ID which is unique and is in the form of random hash of 12 chars. Returns: str. A blog post ID that is different from the IDs of all the existing blog posts. Raises: Exception. There were too many collisions with existing blog post IDs when attempting to generate a new blog post ID. """ for _ in range(base_models.MAX_RETRIES): blog_post_id = utils.convert_to_hash( str(utils.get_random_int(base_models.RAND_RANGE)), base_models.ID_LENGTH) if not cls.get_by_id(blog_post_id): return blog_post_id raise Exception( 'New blog post id generator is producing too many collisions.') @classmethod def create(cls, blog_post_id: str, author_id: str) -> BlogPostModel: """Creates a new BlogPostModel entry. Args: blog_post_id: str. Blog Post ID of the newly-created blog post. author_id: str. User ID of the author. Returns: BlogPostModel. The newly created BlogPostModel instance. Raises: Exception. A blog post with the given blog post ID exists already. """ if cls.get_by_id(blog_post_id): raise Exception( 'A blog post with the given blog post ID exists already.') entity = cls(id=blog_post_id, author_id=author_id, content='', title='', published_on=None, url_fragment='', tags=[], thumbnail_filename=None) entity.update_timestamps() entity.put() return entity @classmethod def get_by_url_fragment(cls, url_fragment: str) -> Optional[BlogPostModel]: """Gets BlogPostModel by url_fragment. Returns None if the blog post with the given url_fragment doesn't exist. Args: url_fragment: str. The url fragment of the blog post. Returns: BlogPostModel | None. The blog post model of the Blog or None if not found. """ return BlogPostModel.query( datastore_services.all_of(cls.url_fragment == url_fragment, cls.deleted == False) # pylint: disable=singleton-comparison ).get() @classmethod def export_data(cls, user_id: str) -> Dict[str, BlogPostModelDataDict]: """Exports the data from BlogPostModel into dict format for Takeout. Args: user_id: str. The ID of the user whose data should be exported. Returns: dict. Dictionary of the data from BlogPostModel. """ user_data: Dict[str, BlogPostModelDataDict] = {} blog_post_models: Sequence[BlogPostModel] = cls.get_all().filter( cls.author_id == user_id).fetch() for blog_post_model in blog_post_models: user_data[blog_post_model.id] = { 'title': blog_post_model.title, 'content': blog_post_model.content, 'url_fragment': blog_post_model.url_fragment, 'tags': blog_post_model.tags, 'thumbnail_filename': blog_post_model.thumbnail_filename, 'published_on': utils.get_time_in_millisecs(blog_post_model.published_on), } return user_data
class SentEmailModel(base_models.BaseModel): """Records the content and metadata of an email sent from Oppia. This model is read-only; entries cannot be modified once created. The id/key of instances of this class has the form '[intent].[random hash]'. """ # TODO(sll): Implement functionality to get all emails sent to a particular # user with a given intent within a given time period. # The user ID of the email recipient. recipient_id = (datastore_services.StringProperty(required=True, indexed=True)) # The email address of the recipient. recipient_email = datastore_services.StringProperty(required=True) # The user ID of the email sender. For site-generated emails this is equal # to SYSTEM_COMMITTER_ID. sender_id = datastore_services.StringProperty(required=True, indexed=True) # The email address used to send the notification. This should be either # the noreply address or the system address. sender_email = datastore_services.StringProperty(required=True) # The intent of the email. intent = datastore_services.StringProperty( required=True, indexed=True, choices=[ feconf.EMAIL_INTENT_SIGNUP, feconf.EMAIL_INTENT_MARKETING, feconf.EMAIL_INTENT_DAILY_BATCH, feconf.EMAIL_INTENT_EDITOR_ROLE_NOTIFICATION, feconf.EMAIL_INTENT_FEEDBACK_MESSAGE_NOTIFICATION, feconf.EMAIL_INTENT_SUBSCRIPTION_NOTIFICATION, feconf.EMAIL_INTENT_SUGGESTION_NOTIFICATION, feconf.EMAIL_INTENT_UNPUBLISH_EXPLORATION, feconf.EMAIL_INTENT_DELETE_EXPLORATION, feconf.EMAIL_INTENT_REPORT_BAD_CONTENT, feconf.EMAIL_INTENT_QUERY_STATUS_NOTIFICATION, feconf.EMAIL_INTENT_ONBOARD_REVIEWER, feconf.EMAIL_INTENT_REMOVE_REVIEWER, feconf.EMAIL_INTENT_ADDRESS_CONTRIBUTOR_DASHBOARD_SUGGESTIONS, feconf.EMAIL_INTENT_REVIEW_CREATOR_DASHBOARD_SUGGESTIONS, feconf.EMAIL_INTENT_REVIEW_CONTRIBUTOR_DASHBOARD_SUGGESTIONS, feconf.EMAIL_INTENT_ADD_CONTRIBUTOR_DASHBOARD_REVIEWERS, feconf.EMAIL_INTENT_VOICEOVER_APPLICATION_UPDATES, feconf.EMAIL_INTENT_ACCOUNT_DELETED, feconf.BULK_EMAIL_INTENT_TEST ]) # The subject line of the email. subject = datastore_services.TextProperty(required=True) # The HTML content of the email body. html_body = datastore_services.TextProperty(required=True) # The datetime the email was sent, in UTC. sent_datetime = (datastore_services.DateTimeProperty(required=True, indexed=True)) # The hash of the recipient id, email subject and message body. email_hash = datastore_services.StringProperty(indexed=True) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data corresponding to a user: recipient_id, recipient_email, sender_id, and sender_email, but this isn't deleted because this model is needed for auditing purposes. """ return base_models.DELETION_POLICY.KEEP @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Users already have access to this data since emails were sent to them. """ return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user, but this isn't exported because users already have access to noteworthy details of this data (since emails were sent to them). """ return dict( super(cls, cls).get_export_policy(), **{ 'recipient_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'recipient_email': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'email_hash': base_models.EXPORT_POLICY.NOT_APPLICABLE }) @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether SentEmailModel exists for user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return cls.query( datastore_services.any_of( cls.recipient_id == user_id, cls.sender_id == user_id, )).get(keys_only=True) is not None @classmethod def _generate_id(cls, intent: str) -> str: """Generates an ID for a new SentEmailModel instance. Args: intent: str. The intent string, i.e. the purpose of the email. Valid intent strings are defined in feconf.py. Returns: str. The newly-generated ID for the SentEmailModel instance. Raises: Exception. The id generator for SentEmailModel is producing too many collisions. """ id_prefix = '%s.' % intent for _ in python_utils.RANGE(base_models.MAX_RETRIES): new_id = '%s.%s' % (id_prefix, utils.convert_to_hash( python_utils.UNICODE( utils.get_random_int( base_models.RAND_RANGE)), base_models.ID_LENGTH)) if not cls.get_by_id(new_id): return new_id raise Exception( 'The id generator for SentEmailModel is producing too many ' 'collisions.') @classmethod def create(cls, recipient_id: str, recipient_email: str, sender_id: str, sender_email: str, intent: str, subject: str, html_body: str, sent_datetime: datetime.datetime) -> None: """Creates a new SentEmailModel entry. Args: recipient_id: str. The user ID of the email recipient. recipient_email: str. The email address of the recipient. sender_id: str. The user ID of the email sender. sender_email: str. The email address used to send the notification. intent: str. The intent string, i.e. the purpose of the email. subject: str. The subject line of the email. html_body: str. The HTML content of the email body. sent_datetime: datetime.datetime. The datetime the email was sent, in UTC. """ instance_id = cls._generate_id(intent) email_model_instance = cls(id=instance_id, recipient_id=recipient_id, recipient_email=recipient_email, sender_id=sender_id, sender_email=sender_email, intent=intent, subject=subject, html_body=html_body, sent_datetime=sent_datetime) email_model_instance.update_timestamps() email_model_instance.put() def _pre_put_hook(self) -> None: """Operations to perform just before the model is `put` into storage.""" super(SentEmailModel, self)._pre_put_hook() self.email_hash = self._generate_hash(self.recipient_id, self.subject, self.html_body) @classmethod def get_by_hash( cls, email_hash: str, sent_datetime_lower_bound: Optional[datetime.datetime] = None ) -> List['SentEmailModel']: """Returns all messages with a given email_hash. This also takes an optional sent_datetime_lower_bound argument, which is a datetime instance. If this is given, only SentEmailModel instances sent after sent_datetime_lower_bound should be returned. Args: email_hash: str. The hash value of the email. sent_datetime_lower_bound: datetime.datetime. The lower bound on sent_datetime of the email to be searched. Returns: list(SentEmailModel). A list of emails which have the given hash value and sent more recently than sent_datetime_lower_bound. Raises: Exception. The sent_datetime_lower_bound is not a valid datetime.datetime. """ if sent_datetime_lower_bound is not None: if not isinstance(sent_datetime_lower_bound, datetime.datetime): raise Exception('Expected datetime, received %s of type %s' % (sent_datetime_lower_bound, type(sent_datetime_lower_bound))) query = cls.query().filter(cls.email_hash == email_hash) if sent_datetime_lower_bound is not None: query = query.filter(cls.sent_datetime > sent_datetime_lower_bound) messages = cast(List[SentEmailModel], query.fetch()) return messages @classmethod def _generate_hash(cls, recipient_id: str, email_subject: str, email_body: str) -> str: """Generate hash for a given recipient_id, email_subject and cleaned email_body. Args: recipient_id: str. The user ID of the email recipient. email_subject: str. The subject line of the email. email_body: str. The HTML content of the email body. Returns: str. The generated hash value of the given email. """ hash_value = utils.convert_to_hash( recipient_id + email_subject + email_body, 100) return hash_value @classmethod def check_duplicate_message(cls, recipient_id: str, email_subject: str, email_body: str) -> bool: """Check for a given recipient_id, email_subject and cleaned email_body, whether a similar message has been sent in the last DUPLICATE_EMAIL_INTERVAL_MINS. Args: recipient_id: str. The user ID of the email recipient. email_subject: str. The subject line of the email. email_body: str. The HTML content of the email body. Returns: bool. Whether a similar message has been sent to the same recipient in the last DUPLICATE_EMAIL_INTERVAL_MINS. """ email_hash = cls._generate_hash(recipient_id, email_subject, email_body) datetime_now = datetime.datetime.utcnow() time_interval = datetime.timedelta( minutes=feconf.DUPLICATE_EMAIL_INTERVAL_MINS) sent_datetime_lower_bound = datetime_now - time_interval messages = cls.get_by_hash( email_hash, sent_datetime_lower_bound=sent_datetime_lower_bound) for message in messages: if (message.recipient_id == recipient_id and message.subject == email_subject and message.html_body == email_body): return True return False
class BulkEmailModel(base_models.BaseModel): """Records the content of an email sent from Oppia to multiple users. This model is read-only; entries cannot be modified once created. The id/key of instances of this model is randomly generated string of length 12. """ # The user IDs of the email recipients. recipient_ids = datastore_services.JsonProperty(default=[], compressed=True) # The user ID of the email sender. For site-generated emails this is equal # to SYSTEM_COMMITTER_ID. sender_id = datastore_services.StringProperty(required=True, indexed=True) # The email address used to send the notification. sender_email = datastore_services.StringProperty(required=True) # The intent of the email. intent = datastore_services.StringProperty( required=True, indexed=True, choices=[ feconf.BULK_EMAIL_INTENT_MARKETING, feconf.BULK_EMAIL_INTENT_IMPROVE_EXPLORATION, feconf.BULK_EMAIL_INTENT_CREATE_EXPLORATION, feconf.BULK_EMAIL_INTENT_CREATOR_REENGAGEMENT, feconf.BULK_EMAIL_INTENT_LEARNER_REENGAGEMENT, feconf.BULK_EMAIL_INTENT_ML_JOB_FAILURE ]) # The subject line of the email. subject = datastore_services.TextProperty(required=True) # The HTML content of the email body. html_body = datastore_services.TextProperty(required=True) # The datetime the email was sent, in UTC. sent_datetime = (datastore_services.DateTimeProperty(required=True, indexed=True)) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model contains data corresponding to a user: recipient_ids, sender_id, and sender_email, but this isn't deleted because this model is needed for auditing purposes. """ return base_models.DELETION_POLICY.KEEP @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Users already have access to this data since the emails were sent to them. """ return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data corresponding to a user, but this isn't exported because users already have access to noteworthy details of this data (since emails were sent to them). """ return dict( super(cls, cls).get_export_policy(), **{ 'recipient_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE }) @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether BulkEmailModel exists for user. Since recipient_ids can't be indexed it also can't be checked by this method, we can allow this because the deletion policy for this model is keep , thus even the deleted user's id will remain here. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether any models refer to the given user ID. """ return (cls.query(cls.sender_id == user_id).get(keys_only=True) is not None) @classmethod def create(cls, instance_id: str, recipient_ids: List[str], sender_id: str, sender_email: str, intent: str, subject: str, html_body: str, sent_datetime: datetime.datetime) -> None: """Creates a new BulkEmailModel entry. Args: instance_id: str. The ID of the instance. recipient_ids: list(str). The user IDs of the email recipients. sender_id: str. The user ID of the email sender. sender_email: str. The email address used to send the notification. intent: str. The intent string, i.e. the purpose of the email. subject: str. The subject line of the email. html_body: str. The HTML content of the email body. sent_datetime: datetime.datetime. The date and time the email was sent, in UTC. """ email_model_instance = cls(id=instance_id, recipient_ids=recipient_ids, sender_id=sender_id, sender_email=sender_email, intent=intent, subject=subject, html_body=html_body, sent_datetime=sent_datetime) email_model_instance.update_timestamps() email_model_instance.put()
class AppFeedbackReportModel(base_models.BaseModel): """Model for storing feedback reports sent from learners. Instances of this model contain information about learner's device and Oppia app settings, as well as information provided by the user in the feedback report. The id of each model instance is determined by concatenating the platform, the timestamp of the report's submission date (in sec since epoch, in UTC), and a hash of a string representation of a random int. """ # We use the model id as a key in the Takeout dict. ID_IS_USED_AS_TAKEOUT_KEY = True # The platform (web or Android) that the report is sent from and that the # feedback corresponds to. platform = datastore_services.StringProperty(required=True, indexed=True, choices=PLATFORM_CHOICES) # The ID of the user that scrubbed this report, if it has been scrubbed. scrubbed_by = datastore_services.StringProperty(required=False, indexed=True) # Unique ID for the ticket this report is assigned to (see # AppFeedbackReportTicketModel for how this is constructed). This defaults # to None since initially, new reports received will not be assigned to a # ticket. ticket_id = datastore_services.StringProperty(required=False, indexed=True) # The local datetime of when the report was submitted by the user on their # device. This may be much earlier than the model entity's creation date if # the report was locally cached for a long time on an Android device. submitted_on = datastore_services.DateTimeProperty(required=True, indexed=True) # The nuber of hours offset from UTC of the user's local timezone. local_timezone_offset_hrs = datastore_services.IntegerProperty( required=False, indexed=True) # The type of feedback for this report; this can be an arbitrary string # since future iterations of the report structure may introduce new types # and we cannot rely on the backend updates to fully sync with the frontend # report updates. report_type = datastore_services.StringProperty(required=True, indexed=True) # The category that this feedback is for. Possible categories include: # suggestion_feature, suggestion_language, suggestion_other, # issue_lesson_question, issue_general_language, issue_audio_language, # issue_text_language, issue_topics, issue_profile, issue_other, crash. category = datastore_services.StringProperty(required=True, indexed=True) # The version of the app; on Android this is the package version name (e.g. # 0.1-alpha-abcdef1234) and on web this is the release version (e.g. 3.0.8). platform_version = datastore_services.StringProperty(required=True, indexed=True) # The entry point location that the user is accessing the feedback report # from on both web & Android devices. Possible entry points include: # navigation_drawer, lesson_player, revision_card, or crash. entry_point = datastore_services.StringProperty(required=True, indexed=True) # Additional topic / story / exploration IDs that may be collected depending # on the entry_point used to send the report; a lesson player entry point # will have topic_id, story_id, and exploration_id, while revision cards # will have topic_id and subtopic_id. entry_point_topic_id = datastore_services.StringProperty(required=False, indexed=True) entry_point_story_id = datastore_services.StringProperty(required=False, indexed=True) entry_point_exploration_id = datastore_services.StringProperty( required=False, indexed=True) entry_point_subtopic_id = datastore_services.IntegerProperty( required=False, indexed=True) # The text language on Oppia set by the user in its ISO-639 language code; # this is set by the user in Oppia's app preferences on all platforms. text_language_code = datastore_services.StringProperty(required=True, indexed=True) # The audio language ISO-639 code on Oppia set by the user; this is set in # Oppia's app preferences on all platforms. audio_language_code = datastore_services.StringProperty(required=True, indexed=True) # The user's country locale represented as a ISO-3166 code; the locale is # determined by the user's Android device settings. android_device_country_locale_code = datastore_services.StringProperty( required=False, indexed=True) # The Android device model used to submit the report. android_device_model = datastore_services.StringProperty(required=False, indexed=True) # The Android SDK version on the user's device. android_sdk_version = datastore_services.IntegerProperty(required=False, indexed=True) # The feedback collected for Android reports; None if the platform is 'web'. android_report_info = datastore_services.JsonProperty(required=False, indexed=False) # The schema version for the feedback report info; None if the platform is # 'web'. android_report_info_schema_version = datastore_services.IntegerProperty( required=False, indexed=True) # The feedback collected for Web reports; None if the platform is 'android'. web_report_info = datastore_services.JsonProperty(required=False, indexed=False) # The schema version for the feedback report info; None if the platform is # 'android'. web_report_info_schema_version = datastore_services.IntegerProperty( required=False, indexed=True) # TODO(#13523): Change 'android_report_info' and 'web_report_info' to domain # objects/TypedDict to remove Any from type-annotation below. @classmethod def create(cls, entity_id: str, platform: str, submitted_on: datetime.datetime, local_timezone_offset_hrs: int, report_type: str, category: str, platform_version: str, android_device_country_locale_code: Optional[str], android_sdk_version: Optional[int], android_device_model: Optional[str], entry_point: str, entry_point_topic_id: Optional[str], entry_point_story_id: Optional[str], entry_point_exploration_id: Optional[str], entry_point_subtopic_id: Optional[str], text_language_code: str, audio_language_code: str, android_report_info: Optional[Dict[str, Any]], web_report_info: Optional[Dict[str, Any]]) -> str: """Creates a new AppFeedbackReportModel instance and returns its ID. Args: entity_id: str. The ID used for this entity. platform: str. The platform the report is submitted on. submitted_on: datetime.datetime. The date and time the report was submitted, in the user's local time zone. local_timezone_offset_hrs: int. The hours offset from UTC of the user's local time zone. report_type: str. The type of report. category: str. The category the report is providing feedback on. platform_version: str. The version of Oppia that the report was submitted on. android_device_country_locale_code: str|None. The ISO-3166 code for the user's country locale or None if it's a web report. android_sdk_version: int|None. The SDK version running when on the device or None if it's a web report. android_device_model: str|None. The device model of the Android device, or None if it's a web report. entry_point: str. The entry point used to start the report. entry_point_topic_id: str|None. The current topic ID depending on the type of entry point used. entry_point_story_id: str|None. The current story ID depending on the type of entry point used. entry_point_exploration_id: str|None. The current exploration ID depending on the type of entry point used. entry_point_subtopic_id: int|None. The current subtopic ID depending on the type of entry point used. text_language_code: str. The ISO-639 language code for the text language set by the user on the Oppia app. audio_language_code: str. The language code for the audio language set by the user on the Oppia app, as defined by Oppia (not necessarily an ISO-639 code). android_report_info: dict|None. The information collected as part of the Android-specific feedback report. web_report_info: dict|None. The information collected as part of the web-specific feedback report. Returns: AppFeedbackReportModel. The newly created AppFeedbackReportModel instance. """ android_schema_version = None web_schema_version = None if platform == PLATFORM_CHOICE_ANDROID: android_schema_version = ( feconf.CURRENT_ANDROID_REPORT_SCHEMA_VERSION) else: web_schema_version = (feconf.CURRENT_WEB_REPORT_SCHEMA_VERSION) report_entity = cls( id=entity_id, platform=platform, submitted_on=submitted_on, local_timezone_offset_hrs=local_timezone_offset_hrs, report_type=report_type, category=category, platform_version=platform_version, android_device_country_locale_code=( android_device_country_locale_code), android_sdk_version=android_sdk_version, android_device_model=android_device_model, entry_point=entry_point, entry_point_topic_id=entry_point_topic_id, entry_point_exploration_id=entry_point_exploration_id, entry_point_story_id=entry_point_story_id, entry_point_subtopic_id=entry_point_subtopic_id, text_language_code=text_language_code, audio_language_code=audio_language_code, android_report_info=android_report_info, android_report_info_schema_version=android_schema_version, web_report_info=web_report_info, web_report_info_schema_version=web_schema_version) report_entity.update_timestamps() report_entity.put() return entity_id @classmethod def generate_id(cls, platform: str, submitted_on_datetime: datetime.datetime) -> str: """Generates key for the instance of AppFeedbackReportModel class in the required format with the arguments provided. Args: platform: str. The platform the user is the report from. submitted_on_datetime: datetime.datetime. The datetime that the report was submitted on in UTC. Returns: str. The generated ID for this entity using platform, submitted_on_sec, and a random string, of the form '[platform].[submitted_on_msec].[random hash]'. Raises: Exception. If the id generator is producing too many collisions. """ submitted_datetime_in_msec = utils.get_time_in_millisecs( submitted_on_datetime) for _ in range(base_models.MAX_RETRIES): random_hash = utils.convert_to_hash( str(utils.get_random_int(base_models.RAND_RANGE)), base_models.ID_LENGTH) new_id = '%s.%s.%s' % (platform, int(submitted_datetime_in_msec), random_hash) if not cls.get_by_id(new_id): return new_id raise Exception( 'The id generator for AppFeedbackReportModel is producing too ' 'many collisions.') @classmethod def get_all_unscrubbed_expiring_report_models( cls) -> Sequence[AppFeedbackReportModel]: """Fetches the reports that are past their 90-days in storage and must be scrubbed. Returns: list(AppFeedbackReportModel). A list of AppFeedbackReportModel entities that need to be scrubbed. """ datetime_now = datetime.datetime.utcnow() datetime_before_which_to_scrub = datetime_now - ( feconf.APP_FEEDBACK_REPORT_MAXIMUM_LIFESPAN + datetime.timedelta(days=1)) # The below return checks for '== None' rather than 'is None' since # the latter throws "Cannot filter a non-Node argument; received False". report_models: Sequence[AppFeedbackReportModel] = cls.query( cls.created_on < datetime_before_which_to_scrub, cls.scrubbed_by == None # pylint: disable=singleton-comparison ).fetch() return report_models @classmethod def get_filter_options_for_field( cls, filter_field: FILTER_FIELD_NAMES) -> List[str]: """Fetches values that can be used to filter reports by. Args: filter_field: FILTER_FIELD_NAME. The enum type of the field we want to fetch all possible values for. Returns: list(str). The possible values that the field name can have. """ query = cls.query(projection=[filter_field.name], distinct=True) filter_values = [] if filter_field == FILTER_FIELD_NAMES.report_type: filter_values = [model.report_type for model in query] elif filter_field == FILTER_FIELD_NAMES.platform: filter_values = [model.platform for model in query] elif filter_field == FILTER_FIELD_NAMES.entry_point: filter_values = [model.entry_point for model in query] elif filter_field == FILTER_FIELD_NAMES.submitted_on: filter_values = [model.submitted_on.date() for model in query] elif filter_field == FILTER_FIELD_NAMES.android_device_model: filter_values = [model.android_device_model for model in query] elif filter_field == FILTER_FIELD_NAMES.android_sdk_version: filter_values = [model.android_sdk_version for model in query] elif filter_field == FILTER_FIELD_NAMES.text_language_code: filter_values = [model.text_language_code for model in query] elif filter_field == FILTER_FIELD_NAMES.audio_language_code: filter_values = [model.audio_language_code for model in query] elif filter_field == FILTER_FIELD_NAMES.platform_version: filter_values = [model.platform_version for model in query] elif filter_field == ( FILTER_FIELD_NAMES.android_device_country_locale_code): filter_values = [ model.android_device_country_locale_code for model in query ] else: raise utils.InvalidInputException( 'The field %s is not a valid field to filter reports on' % (filter_field.name)) return filter_values @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model stores the user ID of who has scrubbed this report for auditing purposes but otherwise does not contain data directly corresponding to the user themselves. """ return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model contains data referencing user and will be exported.""" return dict( super(cls, cls).get_export_policy(), **{ 'platform': base_models.EXPORT_POLICY.EXPORTED, 'scrubbed_by': base_models.EXPORT_POLICY.EXPORTED, 'ticket_id': base_models.EXPORT_POLICY.EXPORTED, 'submitted_on': base_models.EXPORT_POLICY.EXPORTED, 'local_timezone_offset_hrs': base_models.EXPORT_POLICY.EXPORTED, 'report_type': base_models.EXPORT_POLICY.EXPORTED, 'category': base_models.EXPORT_POLICY.EXPORTED, 'platform_version': base_models.EXPORT_POLICY.EXPORTED, 'android_device_country_locale_code': (base_models.EXPORT_POLICY.NOT_APPLICABLE), 'android_device_model': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'android_sdk_version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entry_point': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entry_point_topic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entry_point_story_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'entry_point_exploration_id': (base_models.EXPORT_POLICY.NOT_APPLICABLE), 'entry_point_subtopic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'text_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'audio_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'android_report_info': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'android_report_info_schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'web_report_info': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'web_report_info_schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE }) @classmethod def export_data(cls, user_id: str) -> Dict[str, Dict[str, str]]: """Exports the data from AppFeedbackReportModel into dict format for Takeout. Args: user_id: str. The ID of the user whose data should be exported; this would be the ID of the user who has scrubbed the report. Returns: dict. Dictionary of the data from AppFeedbackReportModel. """ user_data = {} report_models: Sequence[AppFeedbackReportModel] = ( cls.get_all().filter(cls.scrubbed_by == user_id).fetch()) for report_model in report_models: submitted_on_msec = utils.get_time_in_millisecs( report_model.submitted_on) user_data[report_model.id] = { 'scrubbed_by': report_model.scrubbed_by, 'platform': report_model.platform, 'ticket_id': report_model.ticket_id, 'submitted_on': utils.get_human_readable_time_string(submitted_on_msec), 'local_timezone_offset_hrs': (report_model.local_timezone_offset_hrs), 'report_type': report_model.report_type, 'category': report_model.category, 'platform_version': report_model.platform_version } return user_data @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model is exported as multiple instances per user since there are multiple reports relevant to a user. """ return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER @staticmethod def get_lowest_supported_role() -> str: """The lowest supported role for feedback reports will be moderator.""" return feconf.ROLE_ID_MODERATOR @classmethod def has_reference_to_user_id(cls, user_id: str) -> bool: """Check whether AppFeedbackReportModel exists for user. Args: user_id: str. The ID of the user whose data should be checked. Returns: bool. Whether a model is associated with the user. """ return cls.query(cls.scrubbed_by == user_id).get( keys_only=True) is not None
class AppFeedbackReportTicketModel(base_models.BaseModel): """Model for storing tickets created to triage feedback reports. Instances of this model contain information about ticket and associated reports. The id of each model instance is created by combining the entity's ticket_name hash, creation timestamp, and a random 16-character string. """ # A name for the ticket given by the maintainer, limited to 100 characters. ticket_name = datastore_services.StringProperty(required=True, indexed=True) # The platform that the reports in this ticket pertain to. platform = datastore_services.StringProperty(required=True, indexed=True, choices=PLATFORM_CHOICES) # The Github repository that has the associated issue for this ticket. The # possible values correspond to GITHUB_REPO_CHOICES. If None then the # ticket has not yet been assigned to a Github issue. github_issue_repo_name = datastore_services.StringProperty( required=False, indexed=True, choices=GITHUB_REPO_CHOICES) # The Github issue number that applies to this ticket. github_issue_number = datastore_services.IntegerProperty(required=False, indexed=True) # Whether this ticket has been archived. archived = datastore_services.BooleanProperty(required=True, indexed=True) # The datetime in UTC that the newest report in this ticket was created on, # to help with sorting tickets. If all reports assigned to this ticket have # been reassigned to a different ticket then this timestamp is None. newest_report_timestamp = datastore_services.DateTimeProperty( required=False, indexed=True) # A list of report IDs associated with this ticket. report_ids = datastore_services.StringProperty(indexed=True, repeated=True) @classmethod def create(cls, entity_id: str, ticket_name: str, platform: str, github_issue_repo_name: Optional[str], github_issue_number: Optional[int], newest_report_timestamp: datetime.datetime, report_ids: List[str]) -> str: """Creates a new AppFeedbackReportTicketModel instance and returns its ID. Args: entity_id: str. The ID used for this entity. ticket_name: str. The name assigned to the ticket by the moderator. platform: str. The platform that this ticket fixes an issue on, corresponding to one of PLATFORM_CHOICES. github_issue_repo_name: str. The name of the Github repo with the associated Github issue for this ticket. github_issue_number: int|None. The Github issue number associated with the ticket, if it has one. newest_report_timestamp: datetime.datetime. The date and time of the newest report that is a part of this ticket, by submission datetime. report_ids: list(str). The report_ids that are a part of this ticket. Returns: AppFeedbackReportModel. The newly created AppFeedbackReportModel instance. """ ticket_entity = cls(id=entity_id, ticket_name=ticket_name, platform=platform, github_issue_repo_name=github_issue_repo_name, github_issue_number=github_issue_number, archived=False, newest_report_timestamp=newest_report_timestamp, report_ids=report_ids) ticket_entity.update_timestamps() ticket_entity.put() return entity_id @classmethod def generate_id(cls, ticket_name: str) -> str: """Generates key for the instance of AppFeedbackReportTicketModel class in the required format with the arguments provided. Args: ticket_name: str. The name assigned to the ticket on creation. Returns: str. The generated ID for this entity using the current datetime in milliseconds (as the entity's creation timestamp), a SHA1 hash of the ticket_name, and a random string, of the form '[creation_datetime_msec]:[hash(ticket_name)]:[random hash]'. Raises: Exception. If the id generator is producing too many collisions. """ current_datetime_in_msec = utils.get_time_in_millisecs( datetime.datetime.utcnow()) for _ in range(base_models.MAX_RETRIES): name_hash = utils.convert_to_hash(ticket_name, base_models.ID_LENGTH) random_hash = utils.convert_to_hash( str(utils.get_random_int(base_models.RAND_RANGE)), base_models.ID_LENGTH) new_id = '%s.%s.%s' % (int(current_datetime_in_msec), name_hash, random_hash) if not cls.get_by_id(new_id): return new_id raise Exception( 'The id generator for AppFeedbackReportTicketModel is producing too' 'many collisions.') @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any information directly corresponding to a user. """ return base_models.DELETION_POLICY.NOT_APPLICABLE @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model doesn't contain any data directly corresponding to a user.""" return dict( super(cls, cls).get_export_policy(), **{ 'ticket_name': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'platform': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'github_issue_repo_name': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'github_issue_number': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'archived': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'newest_report_timestamp': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'report_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE }) @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model doesn't contain any data directly corresponding to a user.""" return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @staticmethod def get_lowest_supported_role() -> str: """The lowest supported role for feedback report tickets will be moderator. """ return feconf.ROLE_ID_MODERATOR
class ClassifierTrainingJobModel(base_models.BaseModel): """Model for storing classifier training jobs. The id of instances of this class has the form '[exp_id].[random hash of 12 chars]'. """ # The ID of the algorithm used to create the model. algorithm_id = datastore_services.StringProperty(required=True, indexed=True) # The ID of the interaction to which the algorithm belongs. interaction_id = (datastore_services.StringProperty(required=True, indexed=True)) # The exploration_id of the exploration to whose state the model belongs. exp_id = datastore_services.StringProperty(required=True, indexed=True) # The exploration version at the time this training job was created. exp_version = (datastore_services.IntegerProperty(required=True, indexed=True)) # The name of the state to which the model belongs. state_name = datastore_services.StringProperty(required=True, indexed=True) # The status of the training job. It can be either NEW, COMPLETE or PENDING. status = datastore_services.StringProperty( required=True, choices=feconf.ALLOWED_TRAINING_JOB_STATUSES, default=feconf.TRAINING_JOB_STATUS_PENDING, indexed=True) # The training data which is to be populated when retrieving the job. # The list contains dicts where each dict represents a single training # data group. The training data are computed from answers that have been # anonymized and that are not connected to any existing or deleted users. training_data = datastore_services.JsonProperty(default=None) # The time when the job's status should next be checked. # It is incremented by TTL when a job with status NEW is picked up by VM. next_scheduled_check_time = datastore_services.DateTimeProperty( required=True, indexed=True) # The algorithm version for the classifier. Algorithm version identifies # the format of the classifier_data as well as the prediction API to be # used. algorithm_version = datastore_services.IntegerProperty(required=True, indexed=True) @staticmethod def get_deletion_policy() -> base_models.DELETION_POLICY: """Model doesn't contain any data directly corresponding to a user.""" return base_models.DELETION_POLICY.NOT_APPLICABLE @staticmethod def get_model_association_to_user( ) -> base_models.MODEL_ASSOCIATION_TO_USER: """Model does not contain user data.""" return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER @classmethod def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]: """Model doesn't contain any data directly corresponding to a user.""" return dict( super(cls, cls).get_export_policy(), **{ 'algorithm_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'exp_id': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'exp_version': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'state_name': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'status': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'training_data': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'next_scheduled_check_time': base_models.EXPORT_POLICY.NOT_APPLICABLE, 'algorithm_version': base_models.EXPORT_POLICY.NOT_APPLICABLE }) @classmethod def _generate_id(cls, exp_id: str) -> str: """Generates a unique id for the training job of the form '[exp_id].[random hash of 16 chars]'. Args: exp_id: str. ID of the exploration. Returns: str. ID of the new ClassifierTrainingJobModel instance. Raises: Exception. The id generator for ClassifierTrainingJobModel is producing too many collisions. """ for _ in range(base_models.MAX_RETRIES): new_id = '%s.%s' % (exp_id, utils.convert_to_hash( python_utils.UNICODE( utils.get_random_int( base_models.RAND_RANGE)), base_models.ID_LENGTH)) if not cls.get_by_id(new_id): return new_id raise Exception( 'The id generator for ClassifierTrainingJobModel is producing ' 'too many collisions.') @classmethod def create(cls, algorithm_id: str, interaction_id: str, exp_id: str, exp_version: int, next_scheduled_check_time: datetime.datetime, training_data: TrainingDataUnionType, state_name: str, status: str, algorithm_version: int) -> str: """Creates a new ClassifierTrainingJobModel entry. Args: algorithm_id: str. ID of the algorithm used to generate the model. interaction_id: str. ID of the interaction to which the algorithm belongs. exp_id: str. ID of the exploration. exp_version: int. The exploration version at the time this training job was created. next_scheduled_check_time: datetime.datetime. The next scheduled time to check the job. training_data: dict. The data used in training phase. state_name: str. The name of the state to which the classifier belongs. status: str. The status of the training job. algorithm_version: int. The version of the classifier model to be trained. Returns: str. ID of the new ClassifierModel entry. Raises: Exception. A model with the same ID already exists. """ instance_id = cls._generate_id(exp_id) training_job_instance = cls( id=instance_id, algorithm_id=algorithm_id, interaction_id=interaction_id, exp_id=exp_id, exp_version=exp_version, next_scheduled_check_time=next_scheduled_check_time, state_name=state_name, status=status, training_data=training_data, algorithm_version=algorithm_version) training_job_instance.update_timestamps() training_job_instance.put() return instance_id @classmethod def query_new_and_pending_training_jobs( cls, offset: int) -> Tuple[Sequence['ClassifierTrainingJobModel'], int]: """Gets the next 10 jobs which are either in status "new" or "pending", ordered by their next_scheduled_check_time attribute. Args: offset: int. Number of query results to skip. Returns: tuple(list(ClassifierTrainingJobModel), int). A tuple containing the list of the ClassifierTrainingJobModels with status new or pending and the offset value. """ query = (cls.get_all().filter( datastore_services.all_of( cls.status.IN([ feconf.TRAINING_JOB_STATUS_NEW, feconf.TRAINING_JOB_STATUS_PENDING ]), cls.next_scheduled_check_time <= datetime.datetime.utcnow())).order( cls.next_scheduled_check_time)) classifier_job_models: Sequence[ClassifierTrainingJobModel] = ( query.fetch(NEW_AND_PENDING_TRAINING_JOBS_FETCH_LIMIT, offset=offset)) offset = offset + len(classifier_job_models) return classifier_job_models, offset # TODO(#13523): Change 'job_dict' to domain object/TypedDict to # remove Any from type-annotation below. @classmethod def create_multi(cls, job_dicts_list: List[Dict[str, Any]]) -> List[str]: """Creates multiple new ClassifierTrainingJobModel entries. Args: job_dicts_list: list(dict). The list of dicts where each dict represents the attributes of one ClassifierTrainingJobModel. Returns: list(str). List of job IDs. """ job_models = [] job_ids = [] for job_dict in job_dicts_list: instance_id = cls._generate_id(job_dict['exp_id']) training_job_instance = cls( id=instance_id, algorithm_id=job_dict['algorithm_id'], interaction_id=job_dict['interaction_id'], exp_id=job_dict['exp_id'], exp_version=job_dict['exp_version'], next_scheduled_check_time=job_dict[ 'next_scheduled_check_time'], state_name=job_dict['state_name'], status=job_dict['status'], training_data=job_dict['training_data'], algorithm_version=job_dict['algorithm_version']) job_models.append(training_job_instance) job_ids.append(instance_id) cls.update_timestamps_multi(job_models) cls.put_multi(job_models) return job_ids