class TaskEntryModel(base_models.BaseModel):
"""Model representation of an actionable task from the improvements tab.
The ID of a task has the form: "[entity_type].[entity_id].[entity_version].
[task_type].[target_type].[target_id]".
"""
# Utility field which results in a 20% speedup compared to querying by each
# of the invididual fields used to compose it.
# Value has the form: "[entity_type].[entity_id].[entity_version]".
composite_entity_id = datastore_services.StringProperty(required=True,
indexed=True)
# The type of entity a task entry refers to.
entity_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_ENTITY_TYPES)
# The ID of the entity a task entry refers to.
entity_id = datastore_services.StringProperty(required=True, indexed=True)
# The version of the entity a task entry refers to.
entity_version = datastore_services.IntegerProperty(required=True,
indexed=True)
# The type of task a task entry tracks.
task_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_TYPES)
# The type of sub-entity a task entry focuses on. Value is None when an
# entity does not have any meaningful sub-entities to target.
target_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_TARGET_TYPES)
# Uniquely identifies the sub-entity a task entry focuses on. Value is None
# when an entity does not have any meaningful sub-entities to target.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# A sentence generated by Oppia to describe why the task was created.
issue_description = datastore_services.StringProperty(default=None,
required=False,
indexed=True)
# Tracks the state/progress of a task entry.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_STATUS_CHOICES)
# ID of the user who closed the task, if any.
resolver_id = datastore_services.StringProperty(default=None,
required=False,
indexed=True)
# The date and time at which a task was closed or deprecated.
resolved_on = datastore_services.DateTimeProperty(default=None,
required=False,
indexed=True)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether any TaskEntryModel references the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.resolver_id == user_id).get() is not None
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to delete corresponding to a user:
resolver_id field.
It is okay to delete task entries since, after they are resolved, they
only act as a historical record. The removal just removes the historical
record.
"""
return base_models.DELETION_POLICY.DELETE
@classmethod
def apply_deletion_policy(cls, user_id: str) -> None:
"""Delete instances of TaskEntryModel for the user.
Args:
user_id: str. The ID of the user whose data should be deleted.
"""
task_entry_keys = (cls.query(cls.resolver_id == user_id).fetch(
keys_only=True))
datastore_services.delete_multi(task_entry_keys)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users resolve tasks.
"""
return (base_models.MODEL_ASSOCIATION_TO_USER.
ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user:
TaskEntryModel contains the ID of the user that acted on a task.
"""
return dict(
super(cls, cls).get_export_policy(), **{
'composite_entity_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'task_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'issue_description': base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
'resolver_id': base_models.EXPORT_POLICY.EXPORTED,
'resolved_on': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'resolver_id': 'task_ids_resolved_by_user',
'issue_description': 'issue_descriptions',
'status': 'statuses',
'resolved_on': 'resolution_msecs'
}
@staticmethod
def export_data(user_id: str) -> Dict[str, List[str]]:
"""Returns the user-relevant properties of TaskEntryModels.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. The user-relevant properties of TaskEntryModel in a dict
format. In this case, we are returning all the ids of the tasks
which were closed by this user.
"""
task_ids_resolved_by_user = TaskEntryModel.query(
TaskEntryModel.resolver_id == user_id)
return {
'task_ids_resolved_by_user':
([t.id for t in task_ids_resolved_by_user]),
'issue_descriptions':
([t.issue_description for t in task_ids_resolved_by_user]),
'statuses': ([t.status for t in task_ids_resolved_by_user]),
'resolution_msecs':
([t.resolved_on for t in task_ids_resolved_by_user]),
}
@classmethod
def generate_task_id(cls, entity_type: str, entity_id: str,
entity_version: int, task_type: str, target_type: str,
target_id: str) -> str:
"""Generates a new task entry ID.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
task_type: str. The type of task a task entry tracks.
target_type: str. The type of sub-entity a task entry refers to.
target_id: str. The ID of the sub-entity a task entry refers to.
Returns:
str. The ID for the given task.
"""
return feconf.TASK_ENTRY_ID_TEMPLATE % (entity_type, entity_id,
entity_version, task_type,
target_type, target_id)
@classmethod
def generate_composite_entity_id(cls, entity_type: str, entity_id: str,
entity_version: int) -> str:
"""Generates a new composite_entity_id value.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
Returns:
str. The composite_entity_id for the given task.
"""
return feconf.COMPOSITE_ENTITY_ID_TEMPLATE % (entity_type, entity_id,
entity_version)
@classmethod
def create(cls,
entity_type: str,
entity_id: str,
entity_version: int,
task_type: str,
target_type: str,
target_id: str,
issue_description: Optional[str] = None,
status: str = constants.TASK_STATUS_OBSOLETE,
resolver_id: Optional[str] = None,
resolved_on: Optional[str] = None) -> str:
"""Creates a new task entry and puts it in storage.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
task_type: str. The type of task a task entry tracks.
target_type: str. The type of sub-entity a task entry refers to.
target_id: str. The ID of the sub-entity a task entry refers to.
issue_description: str. Sentence generated by Oppia to describe why
the task was created.
status: str. Tracks the state/progress of a task entry.
resolver_id: str. ID of the user who closed the task, if any.
resolved_on: str. The date and time at which a task was closed or
deprecated.
Returns:
str. The ID of the new task.
Raises:
Exception. A task corresponding to the provided identifier values
(entity_type, entity_id, entity_version, task_type, target_type,
target_id) already exists in storage.
"""
task_id = cls.generate_task_id(entity_type, entity_id, entity_version,
task_type, target_type, target_id)
if cls.get_by_id(task_id) is not None:
raise Exception('Task id %s already exists' % task_id)
composite_entity_id = cls.generate_composite_entity_id(
entity_type, entity_id, entity_version)
task_entry = cls(id=task_id,
composite_entity_id=composite_entity_id,
entity_type=entity_type,
entity_id=entity_id,
entity_version=entity_version,
task_type=task_type,
target_type=target_type,
target_id=target_id,
issue_description=issue_description,
status=status,
resolver_id=resolver_id,
resolved_on=resolved_on)
task_entry.update_timestamps()
task_entry.put()
return task_id
class StorySummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Story.
This should be used whenever the content blob of the story is not
needed (e.g. search results, etc).
A StorySummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version.
The key of each instance is the story id.
"""
# The title of the story.
title = datastore_services.StringProperty(required=True, indexed=True)
# The ISO 639-1 code for the language this story is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# A high-level description of the story.
description = datastore_services.TextProperty(required=True, indexed=False)
# Time when the story model was last updated (not to be
# confused with last_updated, which is the time when the
# story *summary* model was last updated).
story_model_last_updated = (datastore_services.DateTimeProperty(
required=True, indexed=True))
# Time when the story model was created (not to be confused
# with created_on, which is the time when the story *summary*
# model was created).
story_model_created_on = (datastore_services.DateTimeProperty(
required=True, indexed=True))
# The titles of the nodes in the story, in the same order as present there.
node_titles = (datastore_services.StringProperty(repeated=True,
indexed=True))
# The thumbnail filename of the story.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the story.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
# The url fragment for the story.
url_fragment = (datastore_services.StringProperty(required=True,
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_titles': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class TopicSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Topic.
This should be used whenever the content blob of the topic is not
needed (e.g. search results, etc).
A TopicSummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version,
url_fragment.
The key of each instance is the topic id.
"""
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (
datastore_services.StringProperty(required=True, indexed=True))
# The ISO 639-1 code for the language this topic is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# The url fragment of the topic.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# Time when the topic model was last updated (not to be
# confused with last_updated, which is the time when the
# topic *summary* model was last updated).
topic_model_last_updated = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# Time when the topic model was created (not to be confused
# with created_on, which is the time when the topic *summary*
# model was created).
topic_model_created_on = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# The number of canonical stories that are part of this topic.
canonical_story_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of additional stories that are part of this topic.
additional_story_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The total number of skills in the topic (including those that are
# uncategorized).
total_skill_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The total number of published chapters in the topic.
total_published_node_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of skills that are not part of any subtopic.
uncategorized_skill_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of subtopics of the topic.
subtopic_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_skill_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_published_node_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class QuestionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia question.
This should be used whenever the content blob of the question is not
needed (e.g. in search results, etc).
A QuestionSummaryModel instance stores the following information:
question_model_last_updated, question_model_created_on,
question_state_data.
The key of each instance is the question id.
"""
# Time when the question model was last updated (not to be
# confused with last_updated, which is the time when the
# question *summary* model was last updated).
question_model_last_updated = datastore_services.DateTimeProperty(
indexed=True, required=True)
# Time when the question model was created (not to be confused
# with created_on, which is the time when the question *summary*
# model was created).
question_model_created_on = datastore_services.DateTimeProperty(
indexed=True, required=True)
# The html content for the question.
question_content = (
datastore_services.TextProperty(indexed=False, required=True))
# The ID of the interaction.
interaction_id = (
datastore_services.StringProperty(indexed=True, required=True))
# The misconception ids addressed in the question. This includes
# tagged misconceptions ids as well as inapplicable misconception
# ids in the question.
misconception_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the QuestionModel
export_data function, and thus a new export_data function does not
need to be defined here.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because because noteworthy details that belong to this model have
already been exported as a part of the QuestionModel export_data
function.
"""
return dict(super(cls, cls).get_export_policy(), **{
'question_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_content': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconception_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class ExpSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia exploration.
This should be used whenever the content blob of the exploration is not
needed (e.g. in search results, etc).
A ExpSummaryModel instance stores the following information:
id, title, category, objective, language_code, tags,
last_updated, created_on, status (private, public),
community_owned, owner_ids, editor_ids,
viewer_ids, version.
The key of each instance is the exploration id.
"""
# What this exploration is called.
title = datastore_services.StringProperty(required=True)
# The category this exploration belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this exploration.
objective = datastore_services.TextProperty(required=True, indexed=False)
# The ISO 639-1 code for the language this exploration is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# Tags associated with this exploration.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# Aggregate user-assigned ratings of the exploration.
ratings = datastore_services.JsonProperty(default=None, indexed=False)
# Scaled average rating for the exploration.
scaled_average_rating = datastore_services.FloatProperty(indexed=True)
# Time when the exploration model was last updated (not to be
# confused with last_updated, which is the time when the
# exploration *summary* model was last updated).
exploration_model_last_updated = (
datastore_services.DateTimeProperty(indexed=True))
# Time when the exploration model was created (not to be confused
# with created_on, which is the time when the exploration *summary*
# model was created).
exploration_model_created_on = (
datastore_services.DateTimeProperty(indexed=True))
# Time when the exploration was first published.
first_published_msec = datastore_services.FloatProperty(indexed=True)
# The publication status of this exploration.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE,
constants.ACTIVITY_STATUS_PUBLIC
]
)
# Whether this exploration is owned by the community.
community_owned = (
datastore_services.BooleanProperty(required=True, indexed=True))
# The user_ids of owners of this exploration.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this exploration.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to voiceover this exploration.
voice_artist_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# The user_ids of users who are allowed to view this exploration.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who have contributed (humans who have made a
# positive (not just a revert) change to the exploration's content).
# NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be
# synchronized, meaning that the keys in contributors_summary need be
# equal to the contributor_ids list.
contributor_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# A dict representing the contributors of non-trivial commits to this
# exploration. Each key of this dict is a user_id, and the corresponding
# value is the number of non-trivial commits that the user has made.
contributors_summary = (
datastore_services.JsonProperty(default={}, indexed=False))
# The version number of the exploration after this commit. Only populated
# for commits to an exploration (as opposed to its rights, etc.).
version = datastore_services.IntegerProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, voice_artist_ids, editor_ids, owner_ids,
contributor_ids, and contributors_summary fields.
"""
return (
base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE
)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether ExpSummaryModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.voice_artist_ids == user_id,
cls.viewer_ids == user_id,
cls.contributor_ids == user_id
)).get(keys_only=True) is not None
@classmethod
def get_non_private(cls) -> Sequence['ExpSummaryModel']:
"""Returns an iterable with non-private ExpSummary models.
Returns:
iterable. An iterable with non-private ExpSummary models.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status != constants.ACTIVITY_STATUS_PRIVATE
).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_top_rated(cls, limit: int) -> Sequence['ExpSummaryModel']:
"""Fetches the top-rated exp summaries that are public in descending
order of scaled_average_rating.
Args:
limit: int. The maximum number of results to return.
Returns:
iterable. An iterable with the top rated exp summaries that are
public in descending order of scaled_average_rating.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC
).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).order(
-ExpSummaryModel.scaled_average_rating
).fetch(limit)
@classmethod
def get_private_at_least_viewable(
cls,
user_id: str
) -> Sequence['ExpSummaryModel']:
"""Fetches private exp summaries that are at least viewable by the
given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with private exp summaries that are at least
viewable by the given user.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status == constants.ACTIVITY_STATUS_PRIVATE
).filter(
datastore_services.any_of(
ExpSummaryModel.owner_ids == user_id,
ExpSummaryModel.editor_ids == user_id,
ExpSummaryModel.voice_artist_ids == user_id,
ExpSummaryModel.viewer_ids == user_id)
).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_at_least_editable(cls, user_id: str) -> Sequence['ExpSummaryModel']:
"""Fetches exp summaries that are at least editable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with exp summaries that are at least
editable by the given user.
"""
return ExpSummaryModel.query().filter(
datastore_services.any_of(
ExpSummaryModel.owner_ids == user_id,
ExpSummaryModel.editor_ids == user_id)
).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_recently_published(cls, limit: int) -> Sequence['ExpSummaryModel']:
"""Fetches exp summaries that are recently published.
Args:
limit: int. The maximum number of results to return.
Returns:
iterable. An iterable with exp summaries that are
recently published. The returned list is sorted by the time of
publication with latest being first in the list.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC
).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).order(
-ExpSummaryModel.first_published_msec
).fetch(limit)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the
ExplorationModel and thus does not need a separate export.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because because noteworthy details that belong to this model have
already been exported as a part of the ExplorationModel.
"""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'scaled_average_rating': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exploration_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exploration_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'first_published_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'voice_artist_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributors_summary': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class SkillSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Skill.
This should be used whenever the content blob of the skill is not
needed (e.g. search results, etc).
A SkillSummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version.
The key of each instance is the skill id.
"""
# The description of the skill.
description = datastore_services.StringProperty(required=True, indexed=True)
# The number of misconceptions associated with the skill.
misconception_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of worked examples in the skill.
worked_examples_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The ISO 639-1 code for the language this skill is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# Time when the skill model was last updated (not to be
# confused with last_updated, which is the time when the
# skill *summary* model was last updated).
skill_model_last_updated = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# Time when the skill model was created (not to be confused
# with created_on, which is the time when the skill *summary*
# model was created).
skill_model_created_on = (
datastore_services.DateTimeProperty(required=True, indexed=True))
version = datastore_services.IntegerProperty(required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconception_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'worked_examples_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def fetch_page(
cls,
page_size: int,
urlsafe_start_cursor: Optional[str],
sort_by: Optional[str]
) -> Tuple[Sequence[SkillSummaryModel], Optional[str], bool]:
"""Returns the models according to values specified.
Args:
page_size: int. Number of skills to fetch.
urlsafe_start_cursor: str|None. The cursor to the next page or
None. If None, this means that the search should start from the
first page of results.
sort_by: str|None. A string indicating how to sort the result.
Returns:
3-tuple(query_models, urlsafe_start_cursor, more). where:
query_models: list(SkillSummary). The list of summaries
of skills starting at the given cursor.
urlsafe_start_cursor: str or None. A query cursor pointing to
the next batch of results. If there are no more results,
this might be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results
after this batch.
"""
cursor = (
datastore_services.make_cursor(urlsafe_cursor=urlsafe_start_cursor))
sort = -cls.skill_model_created_on
if sort_by == (
constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[
'DecreasingCreatedOn']):
sort = cls.skill_model_created_on
elif sort_by == (
constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[
'IncreasingUpdatedOn']):
sort = -cls.skill_model_last_updated
elif sort_by == (
constants.TOPIC_SKILL_DASHBOARD_SORT_OPTIONS[
'DecreasingUpdatedOn']):
sort = cls.skill_model_last_updated
sort_query = cls.query().order(sort)
fetch_result: Tuple[
Sequence[SkillSummaryModel], datastore_services.Cursor, bool
] = sort_query.fetch_page(page_size, start_cursor=cursor)
query_models, next_cursor, _ = fetch_result
# TODO(#13462): Refactor this so that we don't do the lookup.
# Do a forward lookup so that we can know if there are more values.
fetch_result = sort_query.fetch_page(page_size + 1, start_cursor=cursor)
plus_one_query_models, _, _ = fetch_result
# The urlsafe returns bytes and we need to decode them to string.
more_results = len(plus_one_query_models) == page_size + 1
new_urlsafe_start_cursor = (
next_cursor.urlsafe().decode('utf-8')
if (next_cursor and more_results) else None
)
return (
query_models,
new_urlsafe_start_cursor,
more_results
)
class BlogPostSummaryModel(base_models.BaseModel):
"""Summary model for blog posts.
This should be used whenever the content of the blog post is not
needed (e.g. in search results, displaying blog post cards etc).
The key of each instance is the blog post id.
"""
# The ID of the user the blog post is authored by.
author_id = datastore_services.StringProperty(indexed=True, required=True)
# Title of the blog post.
title = datastore_services.StringProperty(indexed=True, required=True)
# Autogenerated summary of the blog post.
summary = datastore_services.StringProperty(required=True, default='')
# The unique url fragment of the blog post.
url_fragment = (datastore_services.StringProperty(indexed=True,
required=True))
# Tags associated with the blog post.
tags = datastore_services.StringProperty(indexed=True, repeated=True)
# The thumbnail filename of the blog post.It's value will be none until
# a thumbnail is added to the blog post.It can be None only when blog
# post is a draft.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# Time when the blog post model was last published. Value will be None
# if the blog post has never been published.
published_on = (datastore_services.DateTimeProperty(indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BlogPostSummaryModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been associated as a part of the
BlogPostModel to the user and thus does not need a separate user
association.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user (author_id), but this
isn't exported because noteworthy details that belong to this
model have already been exported as a part of the BlogPostModel.
"""
return dict(
super(BlogPostSummaryModel, cls).get_export_policy(), **{
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'summary': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'published_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
})
class BlogPostModel(base_models.BaseModel):
"""Model to store blog post data.
The id of instances of this class is in the form of random hash of
12 chars.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY: Literal[True] = True
# The ID of the user the blog post is authored by.
author_id = datastore_services.StringProperty(indexed=True, required=True)
# Title of the blog post.
title = datastore_services.StringProperty(indexed=True, required=True)
# Content of the blog post.
content = datastore_services.TextProperty(indexed=False, required=True)
# The unique url fragment of the blog post. If the user directly enters the
# blog post's url in the editor or the homepage, the blogPostModel will be
# queried using the url fragment to retrieve data for populating the editor
# dashboard / blog post page.
url_fragment = (datastore_services.StringProperty(indexed=True,
required=True))
# Tags associated with the blog post.
tags = datastore_services.StringProperty(indexed=True, repeated=True)
# The thumbnail filename of the blog post. It's value will be None until
# a thumbnail is added to the blog post. It can be None only when blog
# post is a draft.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# Time when the blog post model was last published. Value will be None
# if the blog has never been published.
published_on = (datastore_services.DateTimeProperty(indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BlogPostModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there can
be multiple blog post models relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user to export."""
return dict(
super(BlogPostModel, cls).get_export_policy(),
**{
# We do not export the author_id because we should not
# export internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'title': base_models.EXPORT_POLICY.EXPORTED,
'content': base_models.EXPORT_POLICY.EXPORTED,
'url_fragment': base_models.EXPORT_POLICY.EXPORTED,
'tags': base_models.EXPORT_POLICY.EXPORTED,
'thumbnail_filename': base_models.EXPORT_POLICY.EXPORTED,
'published_on': base_models.EXPORT_POLICY.EXPORTED,
})
@classmethod
def generate_new_blog_post_id(cls) -> str:
"""Generates a new blog post ID which is unique and is in the form of
random hash of 12 chars.
Returns:
str. A blog post ID that is different from the IDs of all
the existing blog posts.
Raises:
Exception. There were too many collisions with existing blog post
IDs when attempting to generate a new blog post ID.
"""
for _ in range(base_models.MAX_RETRIES):
blog_post_id = utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
if not cls.get_by_id(blog_post_id):
return blog_post_id
raise Exception(
'New blog post id generator is producing too many collisions.')
@classmethod
def create(cls, blog_post_id: str, author_id: str) -> BlogPostModel:
"""Creates a new BlogPostModel entry.
Args:
blog_post_id: str. Blog Post ID of the newly-created blog post.
author_id: str. User ID of the author.
Returns:
BlogPostModel. The newly created BlogPostModel instance.
Raises:
Exception. A blog post with the given blog post ID exists already.
"""
if cls.get_by_id(blog_post_id):
raise Exception(
'A blog post with the given blog post ID exists already.')
entity = cls(id=blog_post_id,
author_id=author_id,
content='',
title='',
published_on=None,
url_fragment='',
tags=[],
thumbnail_filename=None)
entity.update_timestamps()
entity.put()
return entity
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional[BlogPostModel]:
"""Gets BlogPostModel by url_fragment. Returns None if the blog post
with the given url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the blog post.
Returns:
BlogPostModel | None. The blog post model of the Blog or None if not
found.
"""
return BlogPostModel.query(
datastore_services.all_of(cls.url_fragment == url_fragment,
cls.deleted == False) # pylint: disable=singleton-comparison
).get()
@classmethod
def export_data(cls, user_id: str) -> Dict[str, BlogPostModelDataDict]:
"""Exports the data from BlogPostModel into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from BlogPostModel.
"""
user_data: Dict[str, BlogPostModelDataDict] = {}
blog_post_models: Sequence[BlogPostModel] = cls.get_all().filter(
cls.author_id == user_id).fetch()
for blog_post_model in blog_post_models:
user_data[blog_post_model.id] = {
'title':
blog_post_model.title,
'content':
blog_post_model.content,
'url_fragment':
blog_post_model.url_fragment,
'tags':
blog_post_model.tags,
'thumbnail_filename':
blog_post_model.thumbnail_filename,
'published_on':
utils.get_time_in_millisecs(blog_post_model.published_on),
}
return user_data
class SentEmailModel(base_models.BaseModel):
"""Records the content and metadata of an email sent from Oppia.
This model is read-only; entries cannot be modified once created. The
id/key of instances of this class has the form '[intent].[random hash]'.
"""
# TODO(sll): Implement functionality to get all emails sent to a particular
# user with a given intent within a given time period.
# The user ID of the email recipient.
recipient_id = (datastore_services.StringProperty(required=True,
indexed=True))
# The email address of the recipient.
recipient_email = datastore_services.StringProperty(required=True)
# The user ID of the email sender. For site-generated emails this is equal
# to SYSTEM_COMMITTER_ID.
sender_id = datastore_services.StringProperty(required=True, indexed=True)
# The email address used to send the notification. This should be either
# the noreply address or the system address.
sender_email = datastore_services.StringProperty(required=True)
# The intent of the email.
intent = datastore_services.StringProperty(
required=True,
indexed=True,
choices=[
feconf.EMAIL_INTENT_SIGNUP, feconf.EMAIL_INTENT_MARKETING,
feconf.EMAIL_INTENT_DAILY_BATCH,
feconf.EMAIL_INTENT_EDITOR_ROLE_NOTIFICATION,
feconf.EMAIL_INTENT_FEEDBACK_MESSAGE_NOTIFICATION,
feconf.EMAIL_INTENT_SUBSCRIPTION_NOTIFICATION,
feconf.EMAIL_INTENT_SUGGESTION_NOTIFICATION,
feconf.EMAIL_INTENT_UNPUBLISH_EXPLORATION,
feconf.EMAIL_INTENT_DELETE_EXPLORATION,
feconf.EMAIL_INTENT_REPORT_BAD_CONTENT,
feconf.EMAIL_INTENT_QUERY_STATUS_NOTIFICATION,
feconf.EMAIL_INTENT_ONBOARD_REVIEWER,
feconf.EMAIL_INTENT_REMOVE_REVIEWER,
feconf.EMAIL_INTENT_ADDRESS_CONTRIBUTOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_REVIEW_CREATOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_REVIEW_CONTRIBUTOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_ADD_CONTRIBUTOR_DASHBOARD_REVIEWERS,
feconf.EMAIL_INTENT_VOICEOVER_APPLICATION_UPDATES,
feconf.EMAIL_INTENT_ACCOUNT_DELETED, feconf.BULK_EMAIL_INTENT_TEST
])
# The subject line of the email.
subject = datastore_services.TextProperty(required=True)
# The HTML content of the email body.
html_body = datastore_services.TextProperty(required=True)
# The datetime the email was sent, in UTC.
sent_datetime = (datastore_services.DateTimeProperty(required=True,
indexed=True))
# The hash of the recipient id, email subject and message body.
email_hash = datastore_services.StringProperty(indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data corresponding to a user: recipient_id,
recipient_email, sender_id, and sender_email, but this isn't deleted
because this model is needed for auditing purposes.
"""
return base_models.DELETION_POLICY.KEEP
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Users already have access to this data since emails were sent
to them.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because users already have access to noteworthy details of this data
(since emails were sent to them).
"""
return dict(
super(cls, cls).get_export_policy(), **{
'recipient_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'recipient_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'email_hash': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether SentEmailModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.recipient_id == user_id,
cls.sender_id == user_id,
)).get(keys_only=True) is not None
@classmethod
def _generate_id(cls, intent: str) -> str:
"""Generates an ID for a new SentEmailModel instance.
Args:
intent: str. The intent string, i.e. the purpose of the email.
Valid intent strings are defined in feconf.py.
Returns:
str. The newly-generated ID for the SentEmailModel instance.
Raises:
Exception. The id generator for SentEmailModel is producing
too many collisions.
"""
id_prefix = '%s.' % intent
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
new_id = '%s.%s' % (id_prefix,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(
base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for SentEmailModel is producing too many '
'collisions.')
@classmethod
def create(cls, recipient_id: str, recipient_email: str, sender_id: str,
sender_email: str, intent: str, subject: str, html_body: str,
sent_datetime: datetime.datetime) -> None:
"""Creates a new SentEmailModel entry.
Args:
recipient_id: str. The user ID of the email recipient.
recipient_email: str. The email address of the recipient.
sender_id: str. The user ID of the email sender.
sender_email: str. The email address used to send the notification.
intent: str. The intent string, i.e. the purpose of the email.
subject: str. The subject line of the email.
html_body: str. The HTML content of the email body.
sent_datetime: datetime.datetime. The datetime the email was sent,
in UTC.
"""
instance_id = cls._generate_id(intent)
email_model_instance = cls(id=instance_id,
recipient_id=recipient_id,
recipient_email=recipient_email,
sender_id=sender_id,
sender_email=sender_email,
intent=intent,
subject=subject,
html_body=html_body,
sent_datetime=sent_datetime)
email_model_instance.update_timestamps()
email_model_instance.put()
def _pre_put_hook(self) -> None:
"""Operations to perform just before the model is `put` into storage."""
super(SentEmailModel, self)._pre_put_hook()
self.email_hash = self._generate_hash(self.recipient_id, self.subject,
self.html_body)
@classmethod
def get_by_hash(
cls,
email_hash: str,
sent_datetime_lower_bound: Optional[datetime.datetime] = None
) -> List['SentEmailModel']:
"""Returns all messages with a given email_hash.
This also takes an optional sent_datetime_lower_bound argument,
which is a datetime instance. If this is given, only
SentEmailModel instances sent after sent_datetime_lower_bound
should be returned.
Args:
email_hash: str. The hash value of the email.
sent_datetime_lower_bound: datetime.datetime. The lower bound on
sent_datetime of the email to be searched.
Returns:
list(SentEmailModel). A list of emails which have the given hash
value and sent more recently than sent_datetime_lower_bound.
Raises:
Exception. The sent_datetime_lower_bound is not a valid
datetime.datetime.
"""
if sent_datetime_lower_bound is not None:
if not isinstance(sent_datetime_lower_bound, datetime.datetime):
raise Exception('Expected datetime, received %s of type %s' %
(sent_datetime_lower_bound,
type(sent_datetime_lower_bound)))
query = cls.query().filter(cls.email_hash == email_hash)
if sent_datetime_lower_bound is not None:
query = query.filter(cls.sent_datetime > sent_datetime_lower_bound)
messages = cast(List[SentEmailModel], query.fetch())
return messages
@classmethod
def _generate_hash(cls, recipient_id: str, email_subject: str,
email_body: str) -> str:
"""Generate hash for a given recipient_id, email_subject and cleaned
email_body.
Args:
recipient_id: str. The user ID of the email recipient.
email_subject: str. The subject line of the email.
email_body: str. The HTML content of the email body.
Returns:
str. The generated hash value of the given email.
"""
hash_value = utils.convert_to_hash(
recipient_id + email_subject + email_body, 100)
return hash_value
@classmethod
def check_duplicate_message(cls, recipient_id: str, email_subject: str,
email_body: str) -> bool:
"""Check for a given recipient_id, email_subject and cleaned
email_body, whether a similar message has been sent in the last
DUPLICATE_EMAIL_INTERVAL_MINS.
Args:
recipient_id: str. The user ID of the email recipient.
email_subject: str. The subject line of the email.
email_body: str. The HTML content of the email body.
Returns:
bool. Whether a similar message has been sent to the same recipient
in the last DUPLICATE_EMAIL_INTERVAL_MINS.
"""
email_hash = cls._generate_hash(recipient_id, email_subject,
email_body)
datetime_now = datetime.datetime.utcnow()
time_interval = datetime.timedelta(
minutes=feconf.DUPLICATE_EMAIL_INTERVAL_MINS)
sent_datetime_lower_bound = datetime_now - time_interval
messages = cls.get_by_hash(
email_hash, sent_datetime_lower_bound=sent_datetime_lower_bound)
for message in messages:
if (message.recipient_id == recipient_id
and message.subject == email_subject
and message.html_body == email_body):
return True
return False
class BulkEmailModel(base_models.BaseModel):
"""Records the content of an email sent from Oppia to multiple users.
This model is read-only; entries cannot be modified once created. The
id/key of instances of this model is randomly generated string of
length 12.
"""
# The user IDs of the email recipients.
recipient_ids = datastore_services.JsonProperty(default=[],
compressed=True)
# The user ID of the email sender. For site-generated emails this is equal
# to SYSTEM_COMMITTER_ID.
sender_id = datastore_services.StringProperty(required=True, indexed=True)
# The email address used to send the notification.
sender_email = datastore_services.StringProperty(required=True)
# The intent of the email.
intent = datastore_services.StringProperty(
required=True,
indexed=True,
choices=[
feconf.BULK_EMAIL_INTENT_MARKETING,
feconf.BULK_EMAIL_INTENT_IMPROVE_EXPLORATION,
feconf.BULK_EMAIL_INTENT_CREATE_EXPLORATION,
feconf.BULK_EMAIL_INTENT_CREATOR_REENGAGEMENT,
feconf.BULK_EMAIL_INTENT_LEARNER_REENGAGEMENT,
feconf.BULK_EMAIL_INTENT_ML_JOB_FAILURE
])
# The subject line of the email.
subject = datastore_services.TextProperty(required=True)
# The HTML content of the email body.
html_body = datastore_services.TextProperty(required=True)
# The datetime the email was sent, in UTC.
sent_datetime = (datastore_services.DateTimeProperty(required=True,
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data corresponding to a user: recipient_ids,
sender_id, and sender_email, but this isn't deleted because this model
is needed for auditing purposes.
"""
return base_models.DELETION_POLICY.KEEP
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Users already have access to this data since the emails were sent
to them.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because users already have access to noteworthy details of this data
(since emails were sent to them).
"""
return dict(
super(cls, cls).get_export_policy(), **{
'recipient_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BulkEmailModel exists for user. Since recipient_ids
can't be indexed it also can't be checked by this method, we can allow
this because the deletion policy for this model is keep , thus even the
deleted user's id will remain here.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return (cls.query(cls.sender_id == user_id).get(keys_only=True)
is not None)
@classmethod
def create(cls, instance_id: str, recipient_ids: List[str], sender_id: str,
sender_email: str, intent: str, subject: str, html_body: str,
sent_datetime: datetime.datetime) -> None:
"""Creates a new BulkEmailModel entry.
Args:
instance_id: str. The ID of the instance.
recipient_ids: list(str). The user IDs of the email recipients.
sender_id: str. The user ID of the email sender.
sender_email: str. The email address used to send the notification.
intent: str. The intent string, i.e. the purpose of the email.
subject: str. The subject line of the email.
html_body: str. The HTML content of the email body.
sent_datetime: datetime.datetime. The date and time the email
was sent, in UTC.
"""
email_model_instance = cls(id=instance_id,
recipient_ids=recipient_ids,
sender_id=sender_id,
sender_email=sender_email,
intent=intent,
subject=subject,
html_body=html_body,
sent_datetime=sent_datetime)
email_model_instance.update_timestamps()
email_model_instance.put()
class AppFeedbackReportModel(base_models.BaseModel):
"""Model for storing feedback reports sent from learners.
Instances of this model contain information about learner's device and Oppia
app settings, as well as information provided by the user in the feedback
report.
The id of each model instance is determined by concatenating the platform,
the timestamp of the report's submission date (in sec since epoch, in UTC),
and a hash of a string representation of a random int.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The platform (web or Android) that the report is sent from and that the
# feedback corresponds to.
platform = datastore_services.StringProperty(required=True,
indexed=True,
choices=PLATFORM_CHOICES)
# The ID of the user that scrubbed this report, if it has been scrubbed.
scrubbed_by = datastore_services.StringProperty(required=False,
indexed=True)
# Unique ID for the ticket this report is assigned to (see
# AppFeedbackReportTicketModel for how this is constructed). This defaults
# to None since initially, new reports received will not be assigned to a
# ticket.
ticket_id = datastore_services.StringProperty(required=False, indexed=True)
# The local datetime of when the report was submitted by the user on their
# device. This may be much earlier than the model entity's creation date if
# the report was locally cached for a long time on an Android device.
submitted_on = datastore_services.DateTimeProperty(required=True,
indexed=True)
# The nuber of hours offset from UTC of the user's local timezone.
local_timezone_offset_hrs = datastore_services.IntegerProperty(
required=False, indexed=True)
# The type of feedback for this report; this can be an arbitrary string
# since future iterations of the report structure may introduce new types
# and we cannot rely on the backend updates to fully sync with the frontend
# report updates.
report_type = datastore_services.StringProperty(required=True,
indexed=True)
# The category that this feedback is for. Possible categories include:
# suggestion_feature, suggestion_language, suggestion_other,
# issue_lesson_question, issue_general_language, issue_audio_language,
# issue_text_language, issue_topics, issue_profile, issue_other, crash.
category = datastore_services.StringProperty(required=True, indexed=True)
# The version of the app; on Android this is the package version name (e.g.
# 0.1-alpha-abcdef1234) and on web this is the release version (e.g. 3.0.8).
platform_version = datastore_services.StringProperty(required=True,
indexed=True)
# The entry point location that the user is accessing the feedback report
# from on both web & Android devices. Possible entry points include:
# navigation_drawer, lesson_player, revision_card, or crash.
entry_point = datastore_services.StringProperty(required=True,
indexed=True)
# Additional topic / story / exploration IDs that may be collected depending
# on the entry_point used to send the report; a lesson player entry point
# will have topic_id, story_id, and exploration_id, while revision cards
# will have topic_id and subtopic_id.
entry_point_topic_id = datastore_services.StringProperty(required=False,
indexed=True)
entry_point_story_id = datastore_services.StringProperty(required=False,
indexed=True)
entry_point_exploration_id = datastore_services.StringProperty(
required=False, indexed=True)
entry_point_subtopic_id = datastore_services.IntegerProperty(
required=False, indexed=True)
# The text language on Oppia set by the user in its ISO-639 language code;
# this is set by the user in Oppia's app preferences on all platforms.
text_language_code = datastore_services.StringProperty(required=True,
indexed=True)
# The audio language ISO-639 code on Oppia set by the user; this is set in
# Oppia's app preferences on all platforms.
audio_language_code = datastore_services.StringProperty(required=True,
indexed=True)
# The user's country locale represented as a ISO-3166 code; the locale is
# determined by the user's Android device settings.
android_device_country_locale_code = datastore_services.StringProperty(
required=False, indexed=True)
# The Android device model used to submit the report.
android_device_model = datastore_services.StringProperty(required=False,
indexed=True)
# The Android SDK version on the user's device.
android_sdk_version = datastore_services.IntegerProperty(required=False,
indexed=True)
# The feedback collected for Android reports; None if the platform is 'web'.
android_report_info = datastore_services.JsonProperty(required=False,
indexed=False)
# The schema version for the feedback report info; None if the platform is
# 'web'.
android_report_info_schema_version = datastore_services.IntegerProperty(
required=False, indexed=True)
# The feedback collected for Web reports; None if the platform is 'android'.
web_report_info = datastore_services.JsonProperty(required=False,
indexed=False)
# The schema version for the feedback report info; None if the platform is
# 'android'.
web_report_info_schema_version = datastore_services.IntegerProperty(
required=False, indexed=True)
# TODO(#13523): Change 'android_report_info' and 'web_report_info' to domain
# objects/TypedDict to remove Any from type-annotation below.
@classmethod
def create(cls, entity_id: str, platform: str,
submitted_on: datetime.datetime, local_timezone_offset_hrs: int,
report_type: str, category: str, platform_version: str,
android_device_country_locale_code: Optional[str],
android_sdk_version: Optional[int],
android_device_model: Optional[str], entry_point: str,
entry_point_topic_id: Optional[str],
entry_point_story_id: Optional[str],
entry_point_exploration_id: Optional[str],
entry_point_subtopic_id: Optional[str], text_language_code: str,
audio_language_code: str,
android_report_info: Optional[Dict[str, Any]],
web_report_info: Optional[Dict[str, Any]]) -> str:
"""Creates a new AppFeedbackReportModel instance and returns its ID.
Args:
entity_id: str. The ID used for this entity.
platform: str. The platform the report is submitted on.
submitted_on: datetime.datetime. The date and time the report was
submitted, in the user's local time zone.
local_timezone_offset_hrs: int. The hours offset from UTC of the
user's local time zone.
report_type: str. The type of report.
category: str. The category the report is providing feedback on.
platform_version: str. The version of Oppia that the report was
submitted on.
android_device_country_locale_code: str|None. The ISO-3166 code for
the user's country locale or None if it's a web report.
android_sdk_version: int|None. The SDK version running when on the
device or None if it's a web report.
android_device_model: str|None. The device model of the Android
device, or None if it's a web report.
entry_point: str. The entry point used to start the report.
entry_point_topic_id: str|None. The current topic ID depending on
the type of entry point used.
entry_point_story_id: str|None. The current story ID depending on
the type of entry point used.
entry_point_exploration_id: str|None. The current exploration ID
depending on the type of entry point used.
entry_point_subtopic_id: int|None. The current subtopic ID depending
on the type of entry point used.
text_language_code: str. The ISO-639 language code for the text
language set by the user on the Oppia app.
audio_language_code: str. The language code for the audio language
set by the user on the Oppia app, as defined by Oppia (not
necessarily an ISO-639 code).
android_report_info: dict|None. The information collected as part
of the Android-specific feedback report.
web_report_info: dict|None. The information collected as part of the
web-specific feedback report.
Returns:
AppFeedbackReportModel. The newly created AppFeedbackReportModel
instance.
"""
android_schema_version = None
web_schema_version = None
if platform == PLATFORM_CHOICE_ANDROID:
android_schema_version = (
feconf.CURRENT_ANDROID_REPORT_SCHEMA_VERSION)
else:
web_schema_version = (feconf.CURRENT_WEB_REPORT_SCHEMA_VERSION)
report_entity = cls(
id=entity_id,
platform=platform,
submitted_on=submitted_on,
local_timezone_offset_hrs=local_timezone_offset_hrs,
report_type=report_type,
category=category,
platform_version=platform_version,
android_device_country_locale_code=(
android_device_country_locale_code),
android_sdk_version=android_sdk_version,
android_device_model=android_device_model,
entry_point=entry_point,
entry_point_topic_id=entry_point_topic_id,
entry_point_exploration_id=entry_point_exploration_id,
entry_point_story_id=entry_point_story_id,
entry_point_subtopic_id=entry_point_subtopic_id,
text_language_code=text_language_code,
audio_language_code=audio_language_code,
android_report_info=android_report_info,
android_report_info_schema_version=android_schema_version,
web_report_info=web_report_info,
web_report_info_schema_version=web_schema_version)
report_entity.update_timestamps()
report_entity.put()
return entity_id
@classmethod
def generate_id(cls, platform: str,
submitted_on_datetime: datetime.datetime) -> str:
"""Generates key for the instance of AppFeedbackReportModel class in the
required format with the arguments provided.
Args:
platform: str. The platform the user is the report from.
submitted_on_datetime: datetime.datetime. The datetime that the
report was submitted on in UTC.
Returns:
str. The generated ID for this entity using platform,
submitted_on_sec, and a random string, of the form
'[platform].[submitted_on_msec].[random hash]'.
Raises:
Exception. If the id generator is producing too many collisions.
"""
submitted_datetime_in_msec = utils.get_time_in_millisecs(
submitted_on_datetime)
for _ in range(base_models.MAX_RETRIES):
random_hash = utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (platform, int(submitted_datetime_in_msec),
random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportModel is producing too '
'many collisions.')
@classmethod
def get_all_unscrubbed_expiring_report_models(
cls) -> Sequence[AppFeedbackReportModel]:
"""Fetches the reports that are past their 90-days in storage and must
be scrubbed.
Returns:
list(AppFeedbackReportModel). A list of AppFeedbackReportModel
entities that need to be scrubbed.
"""
datetime_now = datetime.datetime.utcnow()
datetime_before_which_to_scrub = datetime_now - (
feconf.APP_FEEDBACK_REPORT_MAXIMUM_LIFESPAN +
datetime.timedelta(days=1))
# The below return checks for '== None' rather than 'is None' since
# the latter throws "Cannot filter a non-Node argument; received False".
report_models: Sequence[AppFeedbackReportModel] = cls.query(
cls.created_on < datetime_before_which_to_scrub,
cls.scrubbed_by == None # pylint: disable=singleton-comparison
).fetch()
return report_models
@classmethod
def get_filter_options_for_field(
cls, filter_field: FILTER_FIELD_NAMES) -> List[str]:
"""Fetches values that can be used to filter reports by.
Args:
filter_field: FILTER_FIELD_NAME. The enum type of the field we want
to fetch all possible values for.
Returns:
list(str). The possible values that the field name can have.
"""
query = cls.query(projection=[filter_field.name], distinct=True)
filter_values = []
if filter_field == FILTER_FIELD_NAMES.report_type:
filter_values = [model.report_type for model in query]
elif filter_field == FILTER_FIELD_NAMES.platform:
filter_values = [model.platform for model in query]
elif filter_field == FILTER_FIELD_NAMES.entry_point:
filter_values = [model.entry_point for model in query]
elif filter_field == FILTER_FIELD_NAMES.submitted_on:
filter_values = [model.submitted_on.date() for model in query]
elif filter_field == FILTER_FIELD_NAMES.android_device_model:
filter_values = [model.android_device_model for model in query]
elif filter_field == FILTER_FIELD_NAMES.android_sdk_version:
filter_values = [model.android_sdk_version for model in query]
elif filter_field == FILTER_FIELD_NAMES.text_language_code:
filter_values = [model.text_language_code for model in query]
elif filter_field == FILTER_FIELD_NAMES.audio_language_code:
filter_values = [model.audio_language_code for model in query]
elif filter_field == FILTER_FIELD_NAMES.platform_version:
filter_values = [model.platform_version for model in query]
elif filter_field == (
FILTER_FIELD_NAMES.android_device_country_locale_code):
filter_values = [
model.android_device_country_locale_code for model in query
]
else:
raise utils.InvalidInputException(
'The field %s is not a valid field to filter reports on' %
(filter_field.name))
return filter_values
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model stores the user ID of who has scrubbed this report for auditing
purposes but otherwise does not contain data directly corresponding to
the user themselves.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data referencing user and will be exported."""
return dict(
super(cls, cls).get_export_policy(), **{
'platform':
base_models.EXPORT_POLICY.EXPORTED,
'scrubbed_by':
base_models.EXPORT_POLICY.EXPORTED,
'ticket_id':
base_models.EXPORT_POLICY.EXPORTED,
'submitted_on':
base_models.EXPORT_POLICY.EXPORTED,
'local_timezone_offset_hrs':
base_models.EXPORT_POLICY.EXPORTED,
'report_type':
base_models.EXPORT_POLICY.EXPORTED,
'category':
base_models.EXPORT_POLICY.EXPORTED,
'platform_version':
base_models.EXPORT_POLICY.EXPORTED,
'android_device_country_locale_code':
(base_models.EXPORT_POLICY.NOT_APPLICABLE),
'android_device_model':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_sdk_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_topic_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_story_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_exploration_id':
(base_models.EXPORT_POLICY.NOT_APPLICABLE),
'entry_point_subtopic_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'text_language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'audio_language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_report_info':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_report_info_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'web_report_info':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'web_report_info_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def export_data(cls, user_id: str) -> Dict[str, Dict[str, str]]:
"""Exports the data from AppFeedbackReportModel into dict format for
Takeout.
Args:
user_id: str. The ID of the user whose data should be exported;
this would be the ID of the user who has scrubbed the report.
Returns:
dict. Dictionary of the data from AppFeedbackReportModel.
"""
user_data = {}
report_models: Sequence[AppFeedbackReportModel] = (
cls.get_all().filter(cls.scrubbed_by == user_id).fetch())
for report_model in report_models:
submitted_on_msec = utils.get_time_in_millisecs(
report_model.submitted_on)
user_data[report_model.id] = {
'scrubbed_by':
report_model.scrubbed_by,
'platform':
report_model.platform,
'ticket_id':
report_model.ticket_id,
'submitted_on':
utils.get_human_readable_time_string(submitted_on_msec),
'local_timezone_offset_hrs':
(report_model.local_timezone_offset_hrs),
'report_type':
report_model.report_type,
'category':
report_model.category,
'platform_version':
report_model.platform_version
}
return user_data
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there
are multiple reports relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback reports will be moderator."""
return feconf.ROLE_ID_MODERATOR
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether AppFeedbackReportModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether a model is associated with the user.
"""
return cls.query(cls.scrubbed_by == user_id).get(
keys_only=True) is not None
class AppFeedbackReportTicketModel(base_models.BaseModel):
"""Model for storing tickets created to triage feedback reports.
Instances of this model contain information about ticket and associated
reports.
The id of each model instance is created by combining the entity's
ticket_name hash, creation timestamp, and a random 16-character string.
"""
# A name for the ticket given by the maintainer, limited to 100 characters.
ticket_name = datastore_services.StringProperty(required=True,
indexed=True)
# The platform that the reports in this ticket pertain to.
platform = datastore_services.StringProperty(required=True,
indexed=True,
choices=PLATFORM_CHOICES)
# The Github repository that has the associated issue for this ticket. The
# possible values correspond to GITHUB_REPO_CHOICES. If None then the
# ticket has not yet been assigned to a Github issue.
github_issue_repo_name = datastore_services.StringProperty(
required=False, indexed=True, choices=GITHUB_REPO_CHOICES)
# The Github issue number that applies to this ticket.
github_issue_number = datastore_services.IntegerProperty(required=False,
indexed=True)
# Whether this ticket has been archived.
archived = datastore_services.BooleanProperty(required=True, indexed=True)
# The datetime in UTC that the newest report in this ticket was created on,
# to help with sorting tickets. If all reports assigned to this ticket have
# been reassigned to a different ticket then this timestamp is None.
newest_report_timestamp = datastore_services.DateTimeProperty(
required=False, indexed=True)
# A list of report IDs associated with this ticket.
report_ids = datastore_services.StringProperty(indexed=True, repeated=True)
@classmethod
def create(cls, entity_id: str, ticket_name: str, platform: str,
github_issue_repo_name: Optional[str],
github_issue_number: Optional[int],
newest_report_timestamp: datetime.datetime,
report_ids: List[str]) -> str:
"""Creates a new AppFeedbackReportTicketModel instance and returns its
ID.
Args:
entity_id: str. The ID used for this entity.
ticket_name: str. The name assigned to the ticket by the moderator.
platform: str. The platform that this ticket fixes an issue on,
corresponding to one of PLATFORM_CHOICES.
github_issue_repo_name: str. The name of the Github repo with the
associated Github issue for this ticket.
github_issue_number: int|None. The Github issue number associated
with the ticket, if it has one.
newest_report_timestamp: datetime.datetime. The date and time of the
newest report that is a part of this ticket, by submission
datetime.
report_ids: list(str). The report_ids that are a part of this
ticket.
Returns:
AppFeedbackReportModel. The newly created AppFeedbackReportModel
instance.
"""
ticket_entity = cls(id=entity_id,
ticket_name=ticket_name,
platform=platform,
github_issue_repo_name=github_issue_repo_name,
github_issue_number=github_issue_number,
archived=False,
newest_report_timestamp=newest_report_timestamp,
report_ids=report_ids)
ticket_entity.update_timestamps()
ticket_entity.put()
return entity_id
@classmethod
def generate_id(cls, ticket_name: str) -> str:
"""Generates key for the instance of AppFeedbackReportTicketModel
class in the required format with the arguments provided.
Args:
ticket_name: str. The name assigned to the ticket on creation.
Returns:
str. The generated ID for this entity using the current datetime in
milliseconds (as the entity's creation timestamp), a SHA1 hash of
the ticket_name, and a random string, of the form
'[creation_datetime_msec]:[hash(ticket_name)]:[random hash]'.
Raises:
Exception. If the id generator is producing too many collisions.
"""
current_datetime_in_msec = utils.get_time_in_millisecs(
datetime.datetime.utcnow())
for _ in range(base_models.MAX_RETRIES):
name_hash = utils.convert_to_hash(ticket_name,
base_models.ID_LENGTH)
random_hash = utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (int(current_datetime_in_msec), name_hash,
random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportTicketModel is producing too'
'many collisions.')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any information directly corresponding to a
user.
"""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'ticket_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'platform': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_repo_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_number':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'archived': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'newest_report_timestamp':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'report_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback report tickets will be
moderator.
"""
return feconf.ROLE_ID_MODERATOR
class ClassifierTrainingJobModel(base_models.BaseModel):
"""Model for storing classifier training jobs.
The id of instances of this class has the form
'[exp_id].[random hash of 12 chars]'.
"""
# The ID of the algorithm used to create the model.
algorithm_id = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the interaction to which the algorithm belongs.
interaction_id = (datastore_services.StringProperty(required=True,
indexed=True))
# The exploration_id of the exploration to whose state the model belongs.
exp_id = datastore_services.StringProperty(required=True, indexed=True)
# The exploration version at the time this training job was created.
exp_version = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The name of the state to which the model belongs.
state_name = datastore_services.StringProperty(required=True, indexed=True)
# The status of the training job. It can be either NEW, COMPLETE or PENDING.
status = datastore_services.StringProperty(
required=True,
choices=feconf.ALLOWED_TRAINING_JOB_STATUSES,
default=feconf.TRAINING_JOB_STATUS_PENDING,
indexed=True)
# The training data which is to be populated when retrieving the job.
# The list contains dicts where each dict represents a single training
# data group. The training data are computed from answers that have been
# anonymized and that are not connected to any existing or deleted users.
training_data = datastore_services.JsonProperty(default=None)
# The time when the job's status should next be checked.
# It is incremented by TTL when a job with status NEW is picked up by VM.
next_scheduled_check_time = datastore_services.DateTimeProperty(
required=True, indexed=True)
# The algorithm version for the classifier. Algorithm version identifies
# the format of the classifier_data as well as the prediction API to be
# used.
algorithm_version = datastore_services.IntegerProperty(required=True,
indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'algorithm_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exp_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exp_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'state_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'training_data': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_scheduled_check_time':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'algorithm_version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def _generate_id(cls, exp_id: str) -> str:
"""Generates a unique id for the training job of the form
'[exp_id].[random hash of 16 chars]'.
Args:
exp_id: str. ID of the exploration.
Returns:
str. ID of the new ClassifierTrainingJobModel instance.
Raises:
Exception. The id generator for ClassifierTrainingJobModel is
producing too many collisions.
"""
for _ in range(base_models.MAX_RETRIES):
new_id = '%s.%s' % (exp_id,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(
base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for ClassifierTrainingJobModel is producing '
'too many collisions.')
@classmethod
def create(cls, algorithm_id: str, interaction_id: str, exp_id: str,
exp_version: int, next_scheduled_check_time: datetime.datetime,
training_data: TrainingDataUnionType, state_name: str,
status: str, algorithm_version: int) -> str:
"""Creates a new ClassifierTrainingJobModel entry.
Args:
algorithm_id: str. ID of the algorithm used to generate the model.
interaction_id: str. ID of the interaction to which the algorithm
belongs.
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
next_scheduled_check_time: datetime.datetime. The next scheduled
time to check the job.
training_data: dict. The data used in training phase.
state_name: str. The name of the state to which the classifier
belongs.
status: str. The status of the training job.
algorithm_version: int. The version of the classifier model to be
trained.
Returns:
str. ID of the new ClassifierModel entry.
Raises:
Exception. A model with the same ID already exists.
"""
instance_id = cls._generate_id(exp_id)
training_job_instance = cls(
id=instance_id,
algorithm_id=algorithm_id,
interaction_id=interaction_id,
exp_id=exp_id,
exp_version=exp_version,
next_scheduled_check_time=next_scheduled_check_time,
state_name=state_name,
status=status,
training_data=training_data,
algorithm_version=algorithm_version)
training_job_instance.update_timestamps()
training_job_instance.put()
return instance_id
@classmethod
def query_new_and_pending_training_jobs(
cls,
offset: int) -> Tuple[Sequence['ClassifierTrainingJobModel'], int]:
"""Gets the next 10 jobs which are either in status "new" or "pending",
ordered by their next_scheduled_check_time attribute.
Args:
offset: int. Number of query results to skip.
Returns:
tuple(list(ClassifierTrainingJobModel), int).
A tuple containing the list of the ClassifierTrainingJobModels
with status new or pending and the offset value.
"""
query = (cls.get_all().filter(
datastore_services.all_of(
cls.status.IN([
feconf.TRAINING_JOB_STATUS_NEW,
feconf.TRAINING_JOB_STATUS_PENDING
]), cls.next_scheduled_check_time <=
datetime.datetime.utcnow())).order(
cls.next_scheduled_check_time))
classifier_job_models: Sequence[ClassifierTrainingJobModel] = (
query.fetch(NEW_AND_PENDING_TRAINING_JOBS_FETCH_LIMIT,
offset=offset))
offset = offset + len(classifier_job_models)
return classifier_job_models, offset
# TODO(#13523): Change 'job_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@classmethod
def create_multi(cls, job_dicts_list: List[Dict[str, Any]]) -> List[str]:
"""Creates multiple new ClassifierTrainingJobModel entries.
Args:
job_dicts_list: list(dict). The list of dicts where each dict
represents the attributes of one ClassifierTrainingJobModel.
Returns:
list(str). List of job IDs.
"""
job_models = []
job_ids = []
for job_dict in job_dicts_list:
instance_id = cls._generate_id(job_dict['exp_id'])
training_job_instance = cls(
id=instance_id,
algorithm_id=job_dict['algorithm_id'],
interaction_id=job_dict['interaction_id'],
exp_id=job_dict['exp_id'],
exp_version=job_dict['exp_version'],
next_scheduled_check_time=job_dict[
'next_scheduled_check_time'],
state_name=job_dict['state_name'],
status=job_dict['status'],
training_data=job_dict['training_data'],
algorithm_version=job_dict['algorithm_version'])
job_models.append(training_job_instance)
job_ids.append(instance_id)
cls.update_timestamps_multi(job_models)
cls.put_multi(job_models)
return job_ids
