class GeneralSuggestionModel(base_models.BaseModel):
"""Model to store suggestions made by Oppia users.
The ID of the suggestions created is the same as the ID of the thread
linked to the suggestion.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of suggestion.
suggestion_type = datastore_services.StringProperty(
required=True, indexed=True, choices=feconf.SUGGESTION_TYPE_CHOICES)
# The type of the target entity which the suggestion is linked to.
target_type = datastore_services.StringProperty(
required=True,
indexed=True,
choices=feconf.SUGGESTION_TARGET_TYPE_CHOICES)
# The ID of the target entity being suggested to.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# The version number of the target entity at the time of creation of the
# suggestion.
target_version_at_submission = datastore_services.IntegerProperty(
required=True, indexed=True)
# Status of the suggestion.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=STATUS_CHOICES)
# The ID of the author of the suggestion.
author_id = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the reviewer who accepted/rejected the suggestion.
final_reviewer_id = datastore_services.StringProperty(indexed=True)
# The change command linked to the suggestion. Contains the details of the
# change.
change_cmd = datastore_services.JsonProperty(required=True)
# The category to score the suggestor in. This field will contain 2 values
# separated by a ., the first will be a value from SCORE_TYPE_CHOICES and
# the second will be the subcategory of the suggestion.
score_category = (datastore_services.StringProperty(required=True,
indexed=True))
# The ISO 639-1 code used to query suggestions by language, or None if the
# suggestion type is not queryable by language.
language_code = datastore_services.StringProperty(indexed=True)
# A flag that indicates whether the suggestion is edited by the reviewer.
edited_by_reviewer = datastore_services.BooleanProperty(default=False,
indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id, and final_reviewer_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple unshared instance since there
are multiple suggestions per user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'suggestion_type': base_models.EXPORT_POLICY.EXPORTED,
'target_type': base_models.EXPORT_POLICY.EXPORTED,
'target_id': base_models.EXPORT_POLICY.EXPORTED,
'target_version_at_submission':
base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
# The author_id and final_reviewer_id are not exported since
# we do not want to reveal internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'final_reviewer_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'change_cmd': base_models.EXPORT_POLICY.EXPORTED,
'score_category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.EXPORTED,
'edited_by_reviewer': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralSuggestionModel exists for the user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.author_id == user_id,
cls.final_reviewer_id == user_id)).get(
keys_only=True) is not None
# TODO(#13523): Change 'change_cmd' to TypedDict/Domain Object
# to remove Any used below.
@classmethod
def create(cls, suggestion_type: str, target_type: str, target_id: str,
target_version_at_submission: int, status: str, author_id: str,
final_reviewer_id: str, change_cmd: Dict[str, Any],
score_category: str, thread_id: str,
language_code: Optional[str]) -> None:
"""Creates a new SuggestionModel entry.
Args:
suggestion_type: str. The type of the suggestion.
target_type: str. The type of target entity being edited.
target_id: str. The ID of the target entity being edited.
target_version_at_submission: int. The version number of the target
entity at the time of creation of the suggestion.
status: str. The status of the suggestion.
author_id: str. The ID of the user who submitted the suggestion.
final_reviewer_id: str. The ID of the reviewer who has
accepted/rejected the suggestion.
change_cmd: dict. The actual content of the suggestion.
score_category: str. The scoring category for the suggestion.
thread_id: str. The ID of the feedback thread linked to the
suggestion.
language_code: str|None. The ISO 639-1 code used to query
suggestions by language, or None if the suggestion type is not
queryable by language.
Raises:
Exception. There is already a suggestion with the given id.
"""
instance_id = thread_id
if cls.get_by_id(instance_id):
raise Exception('There is already a suggestion with the given'
' id: %s' % instance_id)
cls(id=instance_id,
suggestion_type=suggestion_type,
target_type=target_type,
target_id=target_id,
target_version_at_submission=target_version_at_submission,
status=status,
author_id=author_id,
final_reviewer_id=final_reviewer_id,
change_cmd=change_cmd,
score_category=score_category,
language_code=language_code).put()
@classmethod
def query_suggestions(
cls, query_fields_and_values: List[Tuple[str, str]]
) -> List['GeneralSuggestionModel']:
"""Queries for suggestions.
Args:
query_fields_and_values: list(tuple(str, str)). A list of queries.
The first element in each tuple is the field to be queried, and
the second element is the corresponding value to query for.
Returns:
list(SuggestionModel). A list of suggestions that match the given
query values, up to a maximum of feconf.DEFAULT_QUERY_LIMIT
suggestions.
"""
query = cls.query()
for (field, value) in query_fields_and_values:
if field not in feconf.ALLOWED_SUGGESTION_QUERY_FIELDS:
raise Exception('Not allowed to query on field %s' % field)
query = query.filter(getattr(cls, field) == value)
return cast(List[GeneralSuggestionModel],
query.fetch(feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_translation_suggestions_in_review_with_exp_id(
cls, exp_id: str,
language_code: str) -> List['GeneralSuggestionModel']:
"""Returns translation suggestions which are in review with target_id
== exp_id.
Args:
exp_id: str. Exploration ID matching the target ID of the
translation suggestions.
language_code: str. Language code.
Returns:
list(SuggestionModel). A list of translation suggestions in review
with target_id of exp_id. The number of returned results is capped
by feconf.DEFAULT_QUERY_LIMIT.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.language_code == language_code).filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT).filter(
cls.target_id == exp_id).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_translation_suggestion_ids_with_exp_ids(
cls, exp_ids: List[str]) -> List[str]:
"""Gets the ids of translation suggestions corresponding to
explorations with the given exploration ids.
Args:
exp_ids: list(str). List of exploration ids to query for.
Returns:
list(str). A list of translation suggestion ids that
correspond to the given exploration ids. Note: it is not
guaranteed that the suggestion ids returned are ordered by the
exploration ids in exp_ids.
"""
query = (cls.get_all().filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT).filter(
cls.target_id.IN(exp_ids)))
suggestion_models = []
offset, more = (0, True)
while more:
results = cast(
List[GeneralSuggestionModel],
query.fetch(feconf.DEFAULT_QUERY_LIMIT, offset=offset))
if len(results):
offset = offset + len(results)
suggestion_models.extend(results)
else:
more = False
return [suggestion_model.id for suggestion_model in suggestion_models]
@classmethod
def get_all_stale_suggestion_ids(cls) -> List[str]:
"""Gets the ids of the suggestions which were last updated before the
threshold time.
Returns:
list(str). A list of the ids of the suggestions that are stale.
"""
threshold_time = (
datetime.datetime.utcnow() -
datetime.timedelta(0, 0, 0, THRESHOLD_TIME_BEFORE_ACCEPT_IN_MSECS))
suggestion_models = cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.last_updated < threshold_time).fetch())
return [suggestion_model.id for suggestion_model in suggestion_models]
@classmethod
def get_suggestions_waiting_too_long_for_review(
cls) -> List['GeneralSuggestionModel']:
"""Returns a list of suggestions that have been waiting for a review
longer than SUGGESTION_REVIEW_WAIT_TIME_THRESHOLD_IN_DAYS days on the
Contributor Dashboard. MAX_NUMBER_OF_SUGGESTIONS_TO_EMAIL_ADMIN
suggestions are returned, sorted in descending order by their review
wait time.
Returns:
list(GeneralSuggestionModel). A list of suggestions, sorted in
descending order by their review wait time.
Raises:
Exception. If there are no suggestion types offered on the
Contributor Dashboard.
"""
if not feconf.CONTRIBUTOR_DASHBOARD_SUGGESTION_TYPES:
raise Exception(
'Expected the suggestion types offered on the Contributor '
'Dashboard to be nonempty.')
threshold_time = (datetime.datetime.utcnow() - datetime.timedelta(
days=SUGGESTION_REVIEW_WAIT_TIME_THRESHOLD_IN_DAYS))
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.last_updated < threshold_time).filter(
cls.suggestion_type.IN(
feconf.CONTRIBUTOR_DASHBOARD_SUGGESTION_TYPES)).order(
cls.last_updated).fetch(
MAX_NUMBER_OF_SUGGESTIONS_TO_EMAIL_ADMIN))
@classmethod
def get_in_review_suggestions_in_score_categories(
cls, score_categories: List[str],
user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all suggestions which are in review in the given
score_categories.
Args:
score_categories: list(str). List of score categories to query for.
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
Returns:
list(SuggestionModel). A list of suggestions that are in the given
score categories, which are in review, but not created by the
given user.
"""
if len(score_categories) == 0:
raise Exception('Received empty list of score categories')
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.score_category.IN(score_categories)).filter(
cls.author_id != user_id).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_in_review_translation_suggestions(
cls, user_id: str,
language_codes: List[str]) -> List['GeneralSuggestionModel']:
"""Gets all translation suggestions which are in review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
language_codes: list(str). The list of language codes.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which are in review, but not created by the given user.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT
).filter(cls.author_id != user_id).filter(
cls.language_code.IN(language_codes)).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_in_review_question_suggestions(
cls, user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all question suggestions which are in review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which are in review, but not created by the given user.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_ADD_QUESTION).
filter(cls.author_id != user_id).fetch(feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_question_suggestions_waiting_longest_for_review(
cls) -> List['GeneralSuggestionModel']:
"""Returns MAX_QUESTION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS number
of question suggestions, sorted in descending order by review wait
time.
Returns:
list(GeneralSuggestionModel). A list of question suggestions,
sorted in descending order based on how long the suggestions have
been waiting for review.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_ADD_QUESTION).order(
cls.last_updated).fetch(
MAX_QUESTION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS))
@classmethod
def get_translation_suggestions_waiting_longest_for_review(
cls, language_code: str) -> List['GeneralSuggestionModel']:
"""Returns MAX_TRANSLATION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS
number of translation suggestions in the specified language code,
sorted in descending order by review wait time.
Args:
language_code: str. The ISO 639-1 language code of the translation
suggestions.
Returns:
list(GeneralSuggestionModel). A list of translation suggestions,
sorted in descending order based on how long the suggestions have
been waiting for review.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT
).filter(cls.language_code == language_code).order(
cls.last_updated).fetch(
MAX_TRANSLATION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS))
@classmethod
def get_user_created_suggestions_of_suggestion_type(
cls, suggestion_type: str,
user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all suggestions of suggestion_type which the user has created.
Args:
suggestion_type: str. The type of suggestion to query for.
user_id: str. The id of the user trying to make this query.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which the given user has created.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(
cls.suggestion_type == suggestion_type).filter(
cls.author_id == user_id).order(-cls.created_on).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_all_score_categories(cls) -> List[str]:
"""Gets all the score categories for which suggestions have been
created.
Returns:
list(str). A list of all the score categories.
"""
query_set = cast(
List[GeneralSuggestionModel],
cls.query(projection=['score_category'], distinct=True))
return [data.score_category for data in query_set]
# TODO(#13523): Change 'change_cmd' to TypedDict/Domain Object
# to remove Any used below.
@classmethod
def export_data(
cls, user_id: str
) -> Dict[str, Dict[str, Union[str, int, bool, Dict[str, Any], None]]]:
"""Exports the data from GeneralSuggestionModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralSuggestionModel.
"""
user_data = dict()
suggestion_models = cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.author_id == user_id).fetch())
for suggestion_model in suggestion_models:
user_data[suggestion_model.id] = {
'suggestion_type':
suggestion_model.suggestion_type,
'target_type':
suggestion_model.target_type,
'target_id':
suggestion_model.target_id,
'target_version_at_submission':
(suggestion_model.target_version_at_submission),
'status':
suggestion_model.status,
'change_cmd':
suggestion_model.change_cmd,
'language_code':
suggestion_model.language_code,
'edited_by_reviewer':
suggestion_model.edited_by_reviewer
}
return user_data
class TopicModel(base_models.VersionedModel):
"""Model for storing Topics.
This class should only be imported by the topic services file
and the topic model test file.
"""
SNAPSHOT_METADATA_CLASS = TopicSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = TopicSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = TopicCommitLogEntryModel
ALLOW_REVERT = False
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (datastore_services.StringProperty(required=True,
indexed=True))
# The abbreviated name of the topic.
abbreviated_name = (datastore_services.StringProperty(indexed=True,
default=''))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
# The thumbnail size in bytes of the topic.
thumbnail_size_in_bytes = (datastore_services.IntegerProperty(
indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# This consists of the list of objects referencing canonical stories that
# are part of this topic.
canonical_story_references = (datastore_services.JsonProperty(
repeated=True, indexed=False))
# This consists of the list of objects referencing additional stories that
# are part of this topic.
additional_story_references = (datastore_services.JsonProperty(
repeated=True, indexed=False))
# The schema version for the story reference object on each of the above 2
# lists.
story_reference_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# This consists of the list of uncategorized skill ids that are not part of
# any subtopic.
uncategorized_skill_ids = (datastore_services.StringProperty(repeated=True,
indexed=True))
# The list of subtopics that are part of the topic.
subtopics = datastore_services.JsonProperty(repeated=True, indexed=False)
# The schema version of the subtopic dict.
subtopic_schema_version = (datastore_services.IntegerProperty(
required=True, indexed=True))
# The id for the next subtopic.
next_subtopic_id = datastore_services.IntegerProperty(required=True)
# The ISO 639-1 code for the language this topic is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The url fragment of the topic.
url_fragment = (datastore_services.StringProperty(required=True,
indexed=True))
# Whether to show practice tab in the Topic viewer page.
practice_tab_is_displayed = datastore_services.BooleanProperty(
required=True, default=False)
# The content of the meta tag in the Topic viewer page.
meta_tag_content = datastore_services.StringProperty(indexed=True,
default='')
# The page title fragment used in the Topic viewer web page.
# For example, if the full Topic viewer web page title is
# 'Learn Fractions | Add, Subtract, Multiply and Divide | Oppia'
# the page title fragment field represents the middle value 'Add, Subtract,
# Multiply and Divide'.
page_title_fragment_for_web = datastore_services.StringProperty(
indexed=True, default='')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't be allowed.
def _prepare_additional_models(
self) -> Mapping[str, base_models.BaseModel]:
"""Prepares additional models needed for the commit process.
Returns:
dict(str, BaseModel). Additional models needed for
the commit process. Contains the TopicRightsModel.
"""
return {'rights_model': TopicRightsModel.get_by_id(self.id)}
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id, commit_type, commit_message, commit_cmds,
additional_models)
topic_rights = cast(TopicRightsModel,
additional_models['rights_model'])
if topic_rights.topic_is_published:
status = constants.ACTIVITY_STATUS_PUBLIC
else:
status = constants.ACTIVITY_STATUS_PRIVATE
topic_commit_log_entry = TopicCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, status, False)
topic_commit_log_entry.topic_id = self.id
return {
'snapshot_metadata_model':
models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': topic_commit_log_entry,
'versioned_model': models_to_put['versioned_model'],
}
@classmethod
def get_by_name(cls, topic_name: str) -> Optional[TopicModel]:
"""Gets TopicModel by topic_name. Returns None if the topic with
name topic_name doesn't exist.
Args:
topic_name: str. The name of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
return cls.get_all().filter(
cls.canonical_name == topic_name.lower()).get()
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional[TopicModel]:
"""Gets TopicModel by url_fragment. Returns None if the topic with
name url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
# TODO(#10210): Make fetching by URL fragment faster.
return cls.get_all().filter(cls.url_fragment == url_fragment).get()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'abbreviated_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_size_in_bytes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_reference_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_ids':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopics':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_subtopic_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'meta_tag_content':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'page_title_fragment_for_web':
(base_models.EXPORT_POLICY.NOT_APPLICABLE),
'practice_tab_is_displayed':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
})
class QuestionSkillLinkModel(base_models.BaseModel):
"""Model for storing Question-Skill Links.
The ID of instances of this class has the form '[question_id]:[skill_id]'.
"""
# The ID of the question.
question_id = (
datastore_services.StringProperty(required=True, indexed=True))
# The ID of the skill to which the question is linked.
skill_id = datastore_services.StringProperty(required=True, indexed=True)
# The difficulty of the skill.
skill_difficulty = (
datastore_services.FloatProperty(required=True, indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'question_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_difficulty': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_model_id(cls, question_id: str, skill_id: str) -> str:
"""Returns the model id by combining the questions and skill id.
Args:
question_id: str. The ID of the question.
skill_id: str. The ID of the skill to which the question is linked.
Returns:
str. The calculated model id.
"""
return '%s:%s' % (question_id, skill_id)
@classmethod
def create(
cls,
question_id: str,
skill_id: str,
skill_difficulty: float
) -> QuestionSkillLinkModel:
"""Creates a new QuestionSkillLinkModel entry.
Args:
question_id: str. The ID of the question.
skill_id: str. The ID of the skill to which the question is linked.
skill_difficulty: float. The difficulty between [0, 1] of the skill.
Raises:
Exception. The given question is already linked to the given skill.
Returns:
QuestionSkillLinkModel. Instance of the new QuestionSkillLinkModel
entry.
"""
question_skill_link_id = cls.get_model_id(question_id, skill_id)
if cls.get(question_skill_link_id, strict=False) is not None:
raise Exception(
'The given question is already linked to given skill')
question_skill_link_model_instance = cls(
id=question_skill_link_id,
question_id=question_id,
skill_id=skill_id,
skill_difficulty=skill_difficulty
)
return question_skill_link_model_instance
@classmethod
def get_total_question_count_for_skill_ids(
cls, skill_ids: List[str]
) -> int:
"""Returns the number of questions assigned to the given skill_ids.
Args:
skill_ids: list(str). Skill IDs for which the question count is
requested.
Returns:
int. The number of questions assigned to the given skill_ids.
"""
total_question_count = cls.query().filter(
cls.skill_id.IN(skill_ids)).count()
return total_question_count
@classmethod
def get_question_skill_links_by_skill_ids(
cls, question_count: int, skill_ids: List[str], offset: int
) -> Sequence[QuestionSkillLinkModel]:
"""Fetches the list of QuestionSkillLinkModels linked to the skill in
batches.
Args:
question_count: int. The number of questions to be returned.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
offset: int. Number of query results to skip.
Returns:
list(QuestionSkillLinkModel). The QuestionSkillLinkModels
corresponding to given skill_ids.
"""
question_skill_count = min(
len(skill_ids), constants.MAX_SKILLS_PER_QUESTION
) * question_count
return cls.query(
cls.skill_id.IN(skill_ids)
).order(-cls.last_updated).fetch(question_skill_count, offset=offset)
@classmethod
def get_question_skill_links_based_on_difficulty_equidistributed_by_skill(
cls,
total_question_count: int,
skill_ids: List[str],
difficulty_requested: float
) -> List[QuestionSkillLinkModel]:
"""Fetches the list of constant number of random QuestionSkillLinkModels
linked to the skills, sorted by the absolute value of the difference
between skill difficulty and the requested difficulty.
Args:
total_question_count: int. The number of questions expected.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
difficulty_requested: float. The skill difficulty of the questions
requested to be fetched.
Returns:
list(QuestionSkillLinkModel). A list of random
QuestionSkillLinkModels corresponding to given skill_ids, with
total_question_count/len(skill_ids) number of questions for
each skill. If not evenly divisible, it will be rounded up.
If not enough questions for a skill, just return all questions
it links to.
"""
if len(skill_ids) > feconf.MAX_NUMBER_OF_SKILL_IDS:
raise Exception('Please keep the number of skill IDs below 20.')
if (not skill_ids) or (total_question_count == 0):
return []
question_count_per_skill = int(
math.ceil(python_utils.divide( # type: ignore[no-untyped-call]
float(total_question_count), float(len(skill_ids)))))
question_skill_link_mapping = {}
# For fetching the questions randomly we have used a random offset.
# But this is a temporary solution since this method scales linearly.
# Other alternative methods were:
# 1) Using a random id in question id filter
# 2) Adding an additional column that can be filtered upon.
# But these methods are not viable because google datastore limits
# each query to have at most one inequality filter. So we can't filter
# on both question_id and difficulty. Please see
# https://github.com/oppia/oppia/pull/9061#issuecomment-629765809
# for more details.
def get_offset(query: datastore_services.Query) -> int:
"""Helper function to get the offset."""
question_count = query.count()
if question_count > 2 * question_count_per_skill:
return utils.get_random_int(
question_count - (question_count_per_skill * 2))
return 0
for skill_id in skill_ids:
query = cls.query(cls.skill_id == skill_id)
equal_questions_query = query.filter(
cls.skill_difficulty == difficulty_requested)
# We fetch more questions here in order to try and ensure that the
# eventual number of returned questions is sufficient to meet the
# number requested, even after deduplication.
new_question_skill_link_models: List[QuestionSkillLinkModel] = list(
equal_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(equal_questions_query)
)
)
for model in new_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
new_question_skill_link_models.remove(model)
if len(new_question_skill_link_models) >= question_count_per_skill:
new_question_skill_link_models = random.sample(
new_question_skill_link_models, question_count_per_skill)
else:
# Fetch QuestionSkillLinkModels with difficulty smaller than
# requested difficulty.
easier_questions_query = query.filter(
cls.skill_difficulty < difficulty_requested)
easier_question_skill_link_models: List[
QuestionSkillLinkModel
] = list(
easier_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(easier_questions_query)
)
)
for model in easier_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
easier_question_skill_link_models.remove(model)
question_extra_count = (
len(new_question_skill_link_models) +
len(easier_question_skill_link_models) -
question_count_per_skill)
if question_extra_count >= 0:
easier_question_skill_link_models = random.sample(
easier_question_skill_link_models,
question_count_per_skill -
len(new_question_skill_link_models)
)
new_question_skill_link_models.extend(
easier_question_skill_link_models)
else:
# Fetch QuestionSkillLinkModels with difficulty larger than
# requested difficulty.
new_question_skill_link_models.extend(
easier_question_skill_link_models)
harder_questions_query = query.filter(
cls.skill_difficulty > difficulty_requested)
harder_question_skill_link_models: List[
QuestionSkillLinkModel
] = list(
harder_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(harder_questions_query)
)
)
for model in harder_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
harder_question_skill_link_models.remove(model)
question_extra_count = (
len(new_question_skill_link_models) +
len(harder_question_skill_link_models) -
question_count_per_skill)
if question_extra_count >= 0:
harder_question_skill_link_models = (
random.sample(
harder_question_skill_link_models,
question_count_per_skill -
len(new_question_skill_link_models)
))
new_question_skill_link_models.extend(
harder_question_skill_link_models)
new_question_skill_link_models = (
new_question_skill_link_models[:question_count_per_skill])
for model in new_question_skill_link_models:
if model.question_id not in question_skill_link_mapping:
question_skill_link_mapping[model.question_id] = model
return list(question_skill_link_mapping.values())
@classmethod
def get_question_skill_links_equidistributed_by_skill(
cls, total_question_count: int, skill_ids: List[str]
) -> List[QuestionSkillLinkModel]:
"""Fetches the list of constant number of random
QuestionSkillLinkModels linked to the skills.
Args:
total_question_count: int. The number of questions expected.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
Returns:
list(QuestionSkillLinkModel). A list of random
QuestionSkillLinkModels corresponding to given skill_ids, with
total_question_count/len(skill_ids) number of questions for
each skill. If not evenly divisible, it will be rounded up.
If not enough questions for a skill, just return all questions
it links to.
"""
if len(skill_ids) > feconf.MAX_NUMBER_OF_SKILL_IDS:
raise Exception('Please keep the number of skill IDs below 20.')
if not skill_ids:
return []
question_count_per_skill = int(
math.ceil(
python_utils.divide( # type: ignore[no-untyped-call]
float(total_question_count), float(len(skill_ids)))))
question_skill_link_models = []
existing_question_ids = []
def get_offset(query: datastore_services.Query) -> int:
"""Helper function to get the offset."""
question_count = query.count()
if question_count > 2 * question_count_per_skill:
return utils.get_random_int(
question_count - (question_count_per_skill * 2))
return 0
for skill_id in skill_ids:
query = cls.query(cls.skill_id == skill_id)
# We fetch more questions here in order to try and ensure that the
# eventual number of returned questions is sufficient to meet the
# number requested, even after deduplication.
new_question_skill_link_models: List[QuestionSkillLinkModel] = list(
query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(query)
)
)
# Deduplicate if the same question is linked to multiple skills.
for model in new_question_skill_link_models:
if model.question_id in existing_question_ids:
new_question_skill_link_models.remove(model)
if len(new_question_skill_link_models) > question_count_per_skill:
sampled_question_skill_link_models = random.sample(
new_question_skill_link_models,
question_count_per_skill
)
else:
sampled_question_skill_link_models = (
new_question_skill_link_models)
question_skill_link_models.extend(
sampled_question_skill_link_models)
existing_question_ids.extend([
model.question_id for model in (
sampled_question_skill_link_models)
])
return question_skill_link_models
@classmethod
def get_all_question_ids_linked_to_skill_id(
cls, skill_id: str
) -> List[str]:
"""Returns a list of all question ids corresponding to the given skill
id.
Args:
skill_id: str. ID of the skill.
Returns:
list(str). The list of all question ids corresponding to the given
skill id.
"""
question_skill_link_models = cls.query().filter(
cls.skill_id == skill_id,
cls.deleted == False) # pylint: disable=singleton-comparison
question_ids = [
model.question_id for model in question_skill_link_models
]
return question_ids
@classmethod
def get_models_by_skill_id(
cls, skill_id: str
) -> Sequence[QuestionSkillLinkModel]:
"""Returns a list of QuestionSkillLink domains of a particular skill ID.
Args:
skill_id: str. ID of the skill.
Returns:
list(QuestionSkillLinkModel)|None. The list of question skill link
domains that are linked to the skill ID. None if the skill
ID doesn't exist.
"""
return cls.get_all().filter(cls.skill_id == skill_id).fetch()
@classmethod
def get_models_by_question_id(
cls, question_id: str
) -> Sequence[QuestionSkillLinkModel]:
"""Returns a list of QuestionSkillLinkModels of a particular
question ID.
Args:
question_id: str. ID of the question.
Returns:
list(QuestionSkillLinkModel)|None. The list of question skill link
models that are linked to the question ID, or None if there are no
question skill link models associated with the question ID.
"""
return cls.get_all().filter(cls.question_id == question_id).fetch()
@classmethod
def put_multi_question_skill_links(
cls, question_skill_links: List[QuestionSkillLinkModel]
) -> None:
"""Puts multiple question skill link models into the datastore.
Args:
question_skill_links: list(QuestionSkillLink). The list of
question skill link domain objects to put into the datastore.
"""
cls.update_timestamps_multi(question_skill_links)
cls.put_multi(question_skill_links)
@classmethod
def delete_multi_question_skill_links(
cls, question_skill_links: List[QuestionSkillLinkModel]
) -> None:
"""Deletes multiple question skill links from the datastore.
Args:
question_skill_links: list(QuestionSkillLinkModel). The list of
question skill link domain objects to delete from the datastore.
"""
cls.delete_multi(question_skill_links)
class SkillSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Skill.
This should be used whenever the content blob of the skill is not
needed (e.g. search results, etc).
A SkillSummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version.
The key of each instance is the skill id.
"""
# The description of the skill.
description = datastore_services.StringProperty(required=True,
indexed=True)
# The number of misconceptions associated with the skill.
misconception_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The number of worked examples in the skill.
worked_examples_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The ISO 639-1 code for the language this skill is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# Time when the skill model was last updated (not to be
# confused with last_updated, which is the time when the
# skill *summary* model was last updated).
skill_model_last_updated = (datastore_services.DateTimeProperty(
required=True, indexed=True))
# Time when the skill model was created (not to be confused
# with created_on, which is the time when the skill *summary*
# model was created).
skill_model_created_on = (datastore_services.DateTimeProperty(
required=True, indexed=True))
version = datastore_services.IntegerProperty(required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconception_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'worked_examples_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def fetch_page(
cls, page_size: int, urlsafe_start_cursor: Optional[str],
sort_by: Optional[str]
) -> Tuple[List['SkillSummaryModel'], Optional[str], bool]:
"""Returns the models according to values specified.
Args:
page_size: int. Number of skills to fetch.
urlsafe_start_cursor: str|None. The cursor to the next page or
None. If None, this means that the search should start from the
first page of results.
sort_by: str|None. A string indicating how to sort the result.
Returns:
3-tuple(query_models, urlsafe_start_cursor, more). where:
query_models: list(SkillSummary). The list of summaries
of skills starting at the given cursor.
urlsafe_start_cursor: str or None. A query cursor pointing to
the next batch of results. If there are no more results,
this might be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results
after this batch.
"""
cursor = (datastore_services.make_cursor(
urlsafe_cursor=urlsafe_start_cursor))
sort = -cls.skill_model_created_on
if sort_by == (
constants.
TOPIC_SKILL_DASHBOARD_SORT_OPTIONS['DecreasingCreatedOn']):
sort = cls.skill_model_created_on
elif sort_by == (
constants.
TOPIC_SKILL_DASHBOARD_SORT_OPTIONS['IncreasingUpdatedOn']):
sort = -cls.skill_model_last_updated
elif sort_by == (
constants.
TOPIC_SKILL_DASHBOARD_SORT_OPTIONS['DecreasingUpdatedOn']):
sort = cls.skill_model_last_updated
sort_query = cls.query().order(sort)
query_models, next_cursor, _ = (sort_query.fetch_page(
page_size, start_cursor=cursor))
# TODO(#13462): Refactor this so that we don't do the lookup.
# Do a forward lookup so that we can know if there are more values.
plus_one_query_models, _, _ = (sort_query.fetch_page(
page_size + 1, start_cursor=cursor))
# The urlsafe returns bytes and we need to decode them to string.
more_results = len(plus_one_query_models) == page_size + 1
new_urlsafe_start_cursor = (next_cursor.urlsafe().decode('utf-8') if
(next_cursor and more_results) else None)
return (cast(List[SkillSummaryModel],
query_models), new_urlsafe_start_cursor, more_results)
class TopicSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Topic.
This should be used whenever the content blob of the topic is not
needed (e.g. search results, etc).
A TopicSummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version,
url_fragment.
The key of each instance is the topic id.
"""
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (datastore_services.StringProperty(required=True,
indexed=True))
# The ISO 639-1 code for the language this topic is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# The url fragment of the topic.
url_fragment = (datastore_services.StringProperty(required=True,
indexed=True))
# Time when the topic model was last updated (not to be
# confused with last_updated, which is the time when the
# topic *summary* model was last updated).
topic_model_last_updated = (datastore_services.DateTimeProperty(
required=True, indexed=True))
# Time when the topic model was created (not to be confused
# with created_on, which is the time when the topic *summary*
# model was created).
topic_model_created_on = (datastore_services.DateTimeProperty(
required=True, indexed=True))
# The number of canonical stories that are part of this topic.
canonical_story_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The number of additional stories that are part of this topic.
additional_story_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The total number of skills in the topic (including those that are
# uncategorized).
total_skill_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The total number of published chapters in the topic.
total_published_node_count = (datastore_services.IntegerProperty(
required=True, indexed=True))
# The number of skills that are not part of any subtopic.
uncategorized_skill_count = (datastore_services.IntegerProperty(
required=True, indexed=True))
# The number of subtopics of the topic.
subtopic_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_skill_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_published_node_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class CollectionCommitLogEntryModel(base_models.BaseCommitLogEntryModel):
"""Log of commits to collections.
A new instance of this model is created and saved every time a commit to
CollectionModel or CollectionRightsModel occurs.
The id for this model is of the form 'collection-[collection_id]-[version]'.
"""
# The id of the collection being edited.
collection_id = (datastore_services.StringProperty(indexed=True,
required=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: user_id field.
"""
return (base_models.DELETION_POLICY.
PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""The history of commits is not relevant for the purposes of Takeout
since commits don't contain relevant data corresponding to users.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because the history of commits isn't deemed as useful for users since
commit logs don't contain relevant data corresponding to those users.
"""
return dict(
super(cls, cls).get_export_policy(),
**{'collection_id': base_models.EXPORT_POLICY.NOT_APPLICABLE})
@classmethod
def get_instance_id(cls, collection_id: str, version: int) -> str:
"""This function returns the generated id for the get_commit function
in the parent class.
Args:
collection_id: str. The id of the collection being edited.
version: int. The version number of the collection after the commit.
Returns:
str. The commit id with the collection id and version number.
"""
return 'collection-%s-%s' % (collection_id, version)
@classmethod
def get_all_non_private_commits(
cls,
page_size: int,
urlsafe_start_cursor: Optional[str],
max_age: Optional[datetime.timedelta] = None
) -> Tuple[Sequence[CollectionCommitLogEntryModel], Optional[str], bool]:
"""Fetches a list of all the non-private commits sorted by their last
updated attribute.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
max_age: datetime.timedelta. An instance of datetime.timedelta
representing the maximum age of the non-private commits to be
fetched.
Raises:
ValueError. The max_age is neither an instance of datetime.timedelta
nor None.
Returns:
3-tuple of (results, cursor, more). Where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if not isinstance(max_age, datetime.timedelta) and max_age is not None:
raise ValueError(
'max_age must be a datetime.timedelta instance or None.')
query = cls.query(cls.post_commit_is_private == False) # pylint: disable=singleton-comparison
if max_age:
query = query.filter(
cls.last_updated >= datetime.datetime.utcnow() - max_age)
return cls._fetch_page_sorted_by_last_updated(query, page_size,
urlsafe_start_cursor)
class SkillOpportunityModel(base_models.BaseModel):
"""Model for opportunities to add questions to skills.
The id of each instance is the id of the corresponding skill.
A new instance of this model is created each time a SkillModel is created.
When a SkillModel's skill description changes, the corresponding instance
of this model is also updated.
"""
# The description of the opportunity's skill.
skill_description = (datastore_services.StringProperty(required=True,
indexed=True))
# The number of questions associated with this opportunity's skill.
question_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'skill_description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_count': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_skill_opportunities(
cls, page_size: int, urlsafe_start_cursor: Optional[str]
) -> Tuple[Sequence['SkillOpportunityModel'], Optional[str], bool]:
"""Returns a list of skill opportunities available for adding questions.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
Returns:
3-tuple of (results, cursor, more). As described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: list(SkillOpportunityModel). A list
of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
start_cursor = datastore_services.make_cursor(
urlsafe_cursor=urlsafe_start_cursor)
created_on_query = cls.get_all().order(cls.created_on)
fetch_result: Tuple[Sequence[SkillOpportunityModel],
datastore_services.Cursor,
bool] = created_on_query.fetch_page(
page_size, start_cursor=start_cursor)
query_models, cursor, _ = fetch_result
# TODO(#13462): Refactor this so that we don't do the lookup.
# Do a forward lookup so that we can know if there are more values.
fetch_result = created_on_query.fetch_page(page_size + 1,
start_cursor=start_cursor)
plus_one_query_models, _, _ = fetch_result
more_results = len(plus_one_query_models) == page_size + 1
# The urlsafe returns bytes and we need to decode them to string.
return (query_models,
(cursor.urlsafe().decode('utf-8') if cursor else None),
more_results)
@classmethod
def delete_all(cls) -> None:
"""Deletes all entities of this class."""
keys = cls.query().fetch(keys_only=True)
datastore_services.delete_multi(keys)
class AppFeedbackReportTicketModel(base_models.BaseModel):
"""Model for storing tickets created to triage feedback reports.
Instances of this model contain information about ticket and associated
reports.
The id of each model instance is created by combining the entity's
ticket_name hash, creation timestamp, and a random 16-character string.
"""
# A name for the ticket given by the maintainer, limited to 100 characters.
ticket_name = datastore_services.StringProperty(required=True, indexed=True)
# The platform that the reports in this ticket pertain to.
platform = datastore_services.StringProperty(
required=True, indexed=True,
choices=PLATFORM_CHOICES)
# The Github repository that has the associated issue for this ticket. The
# possible values correspond to GITHUB_REPO_CHOICES. If None then the
# ticket has not yet been assigned to a Github issue.
github_issue_repo_name = datastore_services.StringProperty(
required=False, indexed=True,
choices=GITHUB_REPO_CHOICES)
# The Github issue number that applies to this ticket.
github_issue_number = datastore_services.IntegerProperty(
required=False, indexed=True)
# Whether this ticket has been archived.
archived = datastore_services.BooleanProperty(required=True, indexed=True)
# The datetime in UTC that the newest report in this ticket was created on,
# to help with sorting tickets. If all reports assigned to this ticket have
# been reassigned to a different ticket then this timestamp is None.
newest_report_timestamp = datastore_services.DateTimeProperty(
required=False, indexed=True)
# A list of report IDs associated with this ticket.
report_ids = datastore_services.StringProperty(indexed=True, repeated=True)
@classmethod
def create(
cls,
entity_id: str,
ticket_name: str,
platform: str,
github_issue_repo_name: Optional[str],
github_issue_number: Optional[int],
newest_report_timestamp: datetime.datetime,
report_ids: List[str]
) -> str:
"""Creates a new AppFeedbackReportTicketModel instance and returns its
ID.
Args:
entity_id: str. The ID used for this entity.
ticket_name: str. The name assigned to the ticket by the moderator.
platform: str. The platform that this ticket fixes an issue on,
corresponding to one of PLATFORM_CHOICES.
github_issue_repo_name: str. The name of the Github repo with the
associated Github issue for this ticket.
github_issue_number: int|None. The Github issue number associated
with the ticket, if it has one.
newest_report_timestamp: datetime.datetime. The date and time of the
newest report that is a part of this ticket, by submission
datetime.
report_ids: list(str). The report_ids that are a part of this
ticket.
Returns:
AppFeedbackReportModel. The newly created AppFeedbackReportModel
instance.
"""
ticket_entity = cls(
id=entity_id, ticket_name=ticket_name, platform=platform,
github_issue_repo_name=github_issue_repo_name,
github_issue_number=github_issue_number, archived=False,
newest_report_timestamp=newest_report_timestamp,
report_ids=report_ids)
ticket_entity.update_timestamps()
ticket_entity.put()
return entity_id
@classmethod
def generate_id(cls, ticket_name: str) -> str:
"""Generates key for the instance of AppFeedbackReportTicketModel
class in the required format with the arguments provided.
Args:
ticket_name: str. The name assigned to the ticket on creation.
Returns:
str. The generated ID for this entity using the current datetime in
milliseconds (as the entity's creation timestamp), a SHA1 hash of
the ticket_name, and a random string, of the form
'[creation_datetime_msec]:[hash(ticket_name)]:[random hash]'.
"""
current_datetime_in_msec = utils.get_time_in_millisecs(
datetime.datetime.utcnow())
for _ in range(base_models.MAX_RETRIES):
name_hash = utils.convert_to_hash(
ticket_name, base_models.ID_LENGTH)
random_hash = utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (
int(current_datetime_in_msec), name_hash, random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportTicketModel is producing too'
'many collisions.')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any information directly corresponding to a
user.
"""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'ticket_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'platform': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_repo_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_number': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'archived': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'newest_report_timestamp': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'report_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback report tickets will be
moderator.
"""
return feconf.ROLE_ID_MODERATOR
class AppFeedbackReportModel(base_models.BaseModel):
"""Model for storing feedback reports sent from learners.
Instances of this model contain information about learner's device and Oppia
app settings, as well as information provided by the user in the feedback
report.
The id of each model instance is determined by concatenating the platform,
the timestamp of the report's submission date (in sec since epoch, in UTC),
and a hash of a string representation of a random int.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The platform (web or Android) that the report is sent from and that the
# feedback corresponds to.
platform = datastore_services.StringProperty(
required=True, indexed=True,
choices=PLATFORM_CHOICES)
# The ID of the user that scrubbed this report, if it has been scrubbed.
scrubbed_by = datastore_services.StringProperty(
required=False, indexed=True)
# Unique ID for the ticket this report is assigned to (see
# AppFeedbackReportTicketModel for how this is constructed). This defaults
# to None since initially, new reports received will not be assigned to a
# ticket.
ticket_id = datastore_services.StringProperty(required=False, indexed=True)
# The local datetime of when the report was submitted by the user on their
# device. This may be much earlier than the model entity's creation date if
# the report was locally cached for a long time on an Android device.
submitted_on = datastore_services.DateTimeProperty(
required=True, indexed=True)
# The nuber of hours offset from UTC of the user's local timezone.
local_timezone_offset_hrs = datastore_services.IntegerProperty(
required=False, indexed=True)
# The type of feedback for this report; this can be an arbitrary string
# since future iterations of the report structure may introduce new types
# and we cannot rely on the backend updates to fully sync with the frontend
# report updates.
report_type = datastore_services.StringProperty(required=True, indexed=True)
# The category that this feedback is for. Possible categories include:
# suggestion_feature, suggestion_language, suggestion_other,
# issue_lesson_question, issue_general_language, issue_audio_language,
# issue_text_language, issue_topics, issue_profile, issue_other, crash.
category = datastore_services.StringProperty(required=True, indexed=True)
# The version of the app; on Android this is the package version name (e.g.
# 0.1-alpha-abcdef1234) and on web this is the release version (e.g. 3.0.8).
platform_version = datastore_services.StringProperty(
required=True, indexed=True)
# The entry point location that the user is accessing the feedback report
# from on both web & Android devices. Possible entry points include:
# navigation_drawer, lesson_player, revision_card, or crash.
entry_point = datastore_services.StringProperty(required=True, indexed=True)
# Additional topic / story / exploration IDs that may be collected depending
# on the entry_point used to send the report; a lesson player entry point
# will have topic_id, story_id, and exploration_id, while revision cards
# will have topic_id and subtopic_id.
entry_point_topic_id = datastore_services.StringProperty(
required=False, indexed=True)
entry_point_story_id = datastore_services.StringProperty(
required=False, indexed=True)
entry_point_exploration_id = datastore_services.StringProperty(
required=False, indexed=True)
entry_point_subtopic_id = datastore_services.IntegerProperty(
required=False, indexed=True)
# The text language on Oppia set by the user in its ISO-639 language code;
# this is set by the user in Oppia's app preferences on all platforms.
text_language_code = datastore_services.StringProperty(
required=True, indexed=True)
# The audio language ISO-639 code on Oppia set by the user; this is set in
# Oppia's app preferences on all platforms.
audio_language_code = datastore_services.StringProperty(
required=True, indexed=True)
# The user's country locale represented as a ISO-3166 code; the locale is
# determined by the user's Android device settings.
android_device_country_locale_code = datastore_services.StringProperty(
required=False, indexed=True)
# The Android device model used to submit the report.
android_device_model = datastore_services.StringProperty(
required=False, indexed=True)
# The Android SDK version on the user's device.
android_sdk_version = datastore_services.IntegerProperty(
required=False, indexed=True)
# The feedback collected for Android reports; None if the platform is 'web'.
android_report_info = datastore_services.JsonProperty(
required=False, indexed=False)
# The schema version for the feedback report info; None if the platform is
# 'web'.
android_report_info_schema_version = datastore_services.IntegerProperty(
required=False, indexed=True)
# The feedback collected for Web reports; None if the platform is 'android'.
web_report_info = datastore_services.JsonProperty(
required=False, indexed=False)
# The schema version for the feedback report info; None if the platform is
# 'android'.
web_report_info_schema_version = datastore_services.IntegerProperty(
required=False, indexed=True)
# TODO(#13523): Change 'android_report_info' and 'web_report_info' to domain
# objects/TypedDict to remove Any from type-annotation below.
@classmethod
def create(
cls,
entity_id: str,
platform: str,
submitted_on: datetime.datetime,
local_timezone_offset_hrs: int,
report_type: str,
category: str,
platform_version: str,
android_device_country_locale_code: Optional[str],
android_sdk_version: Optional[int],
android_device_model: Optional[str],
entry_point: str,
entry_point_topic_id: Optional[str],
entry_point_story_id: Optional[str],
entry_point_exploration_id: Optional[str],
entry_point_subtopic_id: Optional[str],
text_language_code: str,
audio_language_code: str,
android_report_info: Optional[Dict[str, Any]],
web_report_info: Optional[Dict[str, Any]]
) -> str:
"""Creates a new AppFeedbackReportModel instance and returns its ID.
Args:
entity_id: str. The ID used for this entity.
platform: str. The platform the report is submitted on.
submitted_on: datetime.datetime. The date and time the report was
submitted, in the user's local time zone.
local_timezone_offset_hrs: int. The hours offset from UTC of the
user's local time zone.
report_type: str. The type of report.
category: str. The category the report is providing feedback on.
platform_version: str. The version of Oppia that the report was
submitted on.
android_device_country_locale_code: str|None. The ISO-3166 code for
the user's country locale or None if it's a web report.
android_sdk_version: int|None. The SDK version running when on the
device or None if it's a web report.
android_device_model: str|None. The device model of the Android
device, or None if it's a web report.
entry_point: str. The entry point used to start the report.
entry_point_topic_id: str|None. The current topic ID depending on
the type of entry point used.
entry_point_story_id: str|None. The current story ID depending on
the type of entry point used.
entry_point_exploration_id: str|None. The current exploration ID
depending on the type of entry point used.
entry_point_subtopic_id: int|None. The current subtopic ID depending
on the type of entry point used.
text_language_code: str. The ISO-639 language code for the text
language set by the user on the Oppia app.
audio_language_code: str. The language code for the audio language
set by the user on the Oppia app, as defined by Oppia (not
necessarily an ISO-639 code).
android_report_info: dict|None. The information collected as part
of the Android-specific feedback report.
web_report_info: dict|None. The information collected as part of the
web-specific feedback report.
Returns:
AppFeedbackReportModel. The newly created AppFeedbackReportModel
instance.
"""
android_schema_version = None
web_schema_version = None
if platform == PLATFORM_CHOICE_ANDROID:
android_schema_version = (
feconf.CURRENT_ANDROID_REPORT_SCHEMA_VERSION)
else:
web_schema_version = (
feconf.CURRENT_WEB_REPORT_SCHEMA_VERSION)
report_entity = cls(
id=entity_id, platform=platform, submitted_on=submitted_on,
local_timezone_offset_hrs=local_timezone_offset_hrs,
report_type=report_type, category=category,
platform_version=platform_version,
android_device_country_locale_code=(
android_device_country_locale_code),
android_sdk_version=android_sdk_version,
android_device_model=android_device_model, entry_point=entry_point,
entry_point_topic_id=entry_point_topic_id,
entry_point_exploration_id=entry_point_exploration_id,
entry_point_story_id=entry_point_story_id,
entry_point_subtopic_id=entry_point_subtopic_id,
text_language_code=text_language_code,
audio_language_code=audio_language_code,
android_report_info=android_report_info,
android_report_info_schema_version=android_schema_version,
web_report_info=web_report_info,
web_report_info_schema_version=web_schema_version)
report_entity.update_timestamps()
report_entity.put()
return entity_id
@classmethod
def generate_id(
cls,
platform: str,
submitted_on_datetime: datetime.datetime
) -> str:
"""Generates key for the instance of AppFeedbackReportModel class in the
required format with the arguments provided.
Args:
platform: str. The platform the user is the report from.
submitted_on_datetime: datetime.datetime. The datetime that the
report was submitted on in UTC.
Returns:
str. The generated ID for this entity using platform,
submitted_on_sec, and a random string, of the form
'[platform].[submitted_on_msec].[random hash]'.
"""
submitted_datetime_in_msec = utils.get_time_in_millisecs(
submitted_on_datetime)
for _ in range(base_models.MAX_RETRIES):
random_hash = utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (
platform, int(submitted_datetime_in_msec), random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportModel is producing too '
'many collisions.')
@classmethod
def get_all_unscrubbed_expiring_report_models(
cls) -> Sequence['AppFeedbackReportModel']:
"""Fetches the reports that are past their 90-days in storage and must
be scrubbed.
Returns:
list(AppFeedbackReportModel). A list of AppFeedbackReportModel
entities that need to be scrubbed.
"""
datetime_now = datetime.datetime.utcnow()
datetime_before_which_to_scrub = datetime_now - (
feconf.APP_FEEDBACK_REPORT_MAXIMUM_LIFESPAN +
datetime.timedelta(days=1))
# The below return checks for '== None' rather than 'is None' since
# the latter throws "Cannot filter a non-Node argument; received False".
report_models: Sequence['AppFeedbackReportModel'] = cls.query(
cls.created_on < datetime_before_which_to_scrub,
cls.scrubbed_by == None).fetch() # pylint: disable=singleton-comparison
return report_models
@classmethod
def get_filter_options_for_field(cls, filter_field: str) -> List[str]:
"""Fetches values that can be used to filter reports by.
Args:
filter_field: FILTER_FIELD_NAME. The enum type of the field we want
to fetch all possible values for.
Returns:
list(str). The possible values that the field name can have.
"""
query = cls.query(projection=[filter_field.name], distinct=True) # type: ignore[attr-defined]
filter_values = []
if filter_field == FILTER_FIELD_NAMES.report_type:
filter_values = [model.report_type for model in query]
elif filter_field == FILTER_FIELD_NAMES.platform:
filter_values = [model.platform for model in query]
elif filter_field == FILTER_FIELD_NAMES.entry_point:
filter_values = [model.entry_point for model in query]
elif filter_field == FILTER_FIELD_NAMES.submitted_on:
filter_values = [model.submitted_on.date() for model in query]
elif filter_field == FILTER_FIELD_NAMES.android_device_model:
filter_values = [model.android_device_model for model in query]
elif filter_field == FILTER_FIELD_NAMES.android_sdk_version:
filter_values = [model.android_sdk_version for model in query]
elif filter_field == FILTER_FIELD_NAMES.text_language_code:
filter_values = [model.text_language_code for model in query]
elif filter_field == FILTER_FIELD_NAMES.audio_language_code:
filter_values = [model.audio_language_code for model in query]
elif filter_field == FILTER_FIELD_NAMES.platform_version:
filter_values = [model.platform_version for model in query]
elif filter_field == (
FILTER_FIELD_NAMES.android_device_country_locale_code):
filter_values = [
model.android_device_country_locale_code for model in query]
else:
raise utils.InvalidInputException(
'The field %s is not a valid field to filter reports on' % (
filter_field.name)) # type: ignore[attr-defined]
return filter_values
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model stores the user ID of who has scrubbed this report for auditing
purposes but otherwise does not contain data directly corresponding to
the user themselves.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data referencing user and will be exported."""
return dict(super(cls, cls).get_export_policy(), **{
'platform': base_models.EXPORT_POLICY.EXPORTED,
'scrubbed_by': base_models.EXPORT_POLICY.EXPORTED,
'ticket_id': base_models.EXPORT_POLICY.EXPORTED,
'submitted_on': base_models.EXPORT_POLICY.EXPORTED,
'local_timezone_offset_hrs': base_models.EXPORT_POLICY.EXPORTED,
'report_type': base_models.EXPORT_POLICY.EXPORTED,
'category': base_models.EXPORT_POLICY.EXPORTED,
'platform_version': base_models.EXPORT_POLICY.EXPORTED,
'android_device_country_locale_code': (
base_models.EXPORT_POLICY.NOT_APPLICABLE),
'android_device_model': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_sdk_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_topic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_story_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entry_point_exploration_id': (
base_models.EXPORT_POLICY.NOT_APPLICABLE),
'entry_point_subtopic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'text_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'audio_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_report_info': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'android_report_info_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'web_report_info': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'web_report_info_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def export_data(cls, user_id: str) -> Dict[str, Dict[str, str]]:
"""Exports the data from AppFeedbackReportModel into dict format for
Takeout.
Args:
user_id: str. The ID of the user whose data should be exported;
this would be the ID of the user who has scrubbed the report.
Returns:
dict. Dictionary of the data from AppFeedbackReportModel.
"""
user_data = {}
report_models: Sequence[AppFeedbackReportModel] = (
cls.get_all().filter(cls.scrubbed_by == user_id).fetch())
for report_model in report_models:
submitted_on_msec = utils.get_time_in_millisecs(
report_model.submitted_on)
user_data[report_model.id] = {
'scrubbed_by': report_model.scrubbed_by,
'platform': report_model.platform,
'ticket_id': report_model.ticket_id,
'submitted_on': utils.get_human_readable_time_string(
submitted_on_msec),
'local_timezone_offset_hrs': (
report_model.local_timezone_offset_hrs),
'report_type': report_model.report_type,
'category': report_model.category,
'platform_version': report_model.platform_version
}
return user_data
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there
are multiple reports relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback reports will be moderator."""
return feconf.ROLE_ID_MODERATOR
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether AppFeedbackReportModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether a model is associated with the user.
"""
return cls.query(
cls.scrubbed_by == user_id).get(keys_only=True) is not None
class ClassifierTrainingJobModel(base_models.BaseModel):
"""Model for storing classifier training jobs.
The id of instances of this class has the form
'[exp_id].[random hash of 12 chars]'.
"""
# The ID of the algorithm used to create the model.
algorithm_id = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the interaction to which the algorithm belongs.
interaction_id = (datastore_services.StringProperty(required=True,
indexed=True))
# The exploration_id of the exploration to whose state the model belongs.
exp_id = datastore_services.StringProperty(required=True, indexed=True)
# The exploration version at the time this training job was created.
exp_version = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The name of the state to which the model belongs.
state_name = datastore_services.StringProperty(required=True, indexed=True)
# The status of the training job. It can be either NEW, COMPLETE or PENDING.
status = datastore_services.StringProperty(
required=True,
choices=feconf.ALLOWED_TRAINING_JOB_STATUSES,
default=feconf.TRAINING_JOB_STATUS_PENDING,
indexed=True)
# The training data which is to be populated when retrieving the job.
# The list contains dicts where each dict represents a single training
# data group. The training data are computed from answers that have been
# anonymized and that are not connected to any existing or deleted users.
training_data = datastore_services.JsonProperty(default=None)
# The time when the job's status should next be checked.
# It is incremented by TTL when a job with status NEW is picked up by VM.
next_scheduled_check_time = datastore_services.DateTimeProperty(
required=True, indexed=True)
# The algorithm version for the classifier. Algorithm version identifies
# the format of the classifier_data as well as the prediction API to be
# used.
algorithm_version = datastore_services.IntegerProperty(required=True,
indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'algorithm_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exp_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exp_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'state_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'training_data': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_scheduled_check_time':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'algorithm_version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def _generate_id(cls, exp_id: str) -> str:
"""Generates a unique id for the training job of the form
'[exp_id].[random hash of 16 chars]'.
Args:
exp_id: str. ID of the exploration.
Returns:
str. ID of the new ClassifierTrainingJobModel instance.
Raises:
Exception. The id generator for ClassifierTrainingJobModel is
producing too many collisions.
"""
for _ in range(base_models.MAX_RETRIES):
new_id = '%s.%s' % (
exp_id,
utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for ClassifierTrainingJobModel is producing '
'too many collisions.')
@classmethod
def create(cls, algorithm_id: str, interaction_id: str, exp_id: str,
exp_version: int, next_scheduled_check_time: datetime.datetime,
training_data: TrainingDataUnionType, state_name: str,
status: str, algorithm_version: int) -> str:
"""Creates a new ClassifierTrainingJobModel entry.
Args:
algorithm_id: str. ID of the algorithm used to generate the model.
interaction_id: str. ID of the interaction to which the algorithm
belongs.
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
next_scheduled_check_time: datetime.datetime. The next scheduled
time to check the job.
training_data: dict. The data used in training phase.
state_name: str. The name of the state to which the classifier
belongs.
status: str. The status of the training job.
algorithm_version: int. The version of the classifier model to be
trained.
Returns:
str. ID of the new ClassifierModel entry.
Raises:
Exception. A model with the same ID already exists.
"""
instance_id = cls._generate_id(exp_id)
training_job_instance = cls(
id=instance_id,
algorithm_id=algorithm_id,
interaction_id=interaction_id,
exp_id=exp_id,
exp_version=exp_version,
next_scheduled_check_time=next_scheduled_check_time,
state_name=state_name,
status=status,
training_data=training_data,
algorithm_version=algorithm_version)
training_job_instance.update_timestamps()
training_job_instance.put()
return instance_id
@classmethod
def query_new_and_pending_training_jobs(
cls,
offset: int) -> Tuple[Sequence[ClassifierTrainingJobModel], int]:
"""Gets the next 10 jobs which are either in status "new" or "pending",
ordered by their next_scheduled_check_time attribute.
Args:
offset: int. Number of query results to skip.
Returns:
tuple(list(ClassifierTrainingJobModel), int).
A tuple containing the list of the ClassifierTrainingJobModels
with status new or pending and the offset value.
"""
query = (cls.get_all().filter(
datastore_services.all_of(
cls.status.IN([
feconf.TRAINING_JOB_STATUS_NEW,
feconf.TRAINING_JOB_STATUS_PENDING
]), cls.next_scheduled_check_time <=
datetime.datetime.utcnow())).order(
cls.next_scheduled_check_time))
classifier_job_models: Sequence[ClassifierTrainingJobModel] = (
query.fetch(NEW_AND_PENDING_TRAINING_JOBS_FETCH_LIMIT,
offset=offset))
offset = offset + len(classifier_job_models)
return classifier_job_models, offset
# TODO(#13523): Change 'job_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@classmethod
def create_multi(cls, job_dicts_list: List[Dict[str, Any]]) -> List[str]:
"""Creates multiple new ClassifierTrainingJobModel entries.
Args:
job_dicts_list: list(dict). The list of dicts where each dict
represents the attributes of one ClassifierTrainingJobModel.
Returns:
list(str). List of job IDs.
"""
job_models = []
job_ids = []
for job_dict in job_dicts_list:
instance_id = cls._generate_id(job_dict['exp_id'])
training_job_instance = cls(
id=instance_id,
algorithm_id=job_dict['algorithm_id'],
interaction_id=job_dict['interaction_id'],
exp_id=job_dict['exp_id'],
exp_version=job_dict['exp_version'],
next_scheduled_check_time=job_dict[
'next_scheduled_check_time'],
state_name=job_dict['state_name'],
status=job_dict['status'],
training_data=job_dict['training_data'],
algorithm_version=job_dict['algorithm_version'])
job_models.append(training_job_instance)
job_ids.append(instance_id)
cls.update_timestamps_multi(job_models)
cls.put_multi(job_models)
return job_ids
class MachineTranslationModel(base_models.BaseModel):
"""Model for storing machine generated translations for the purpose of
preventing duplicate generation. Machine translations are used for reference
purpose only and therefore are context agnostic. Model instances are mapped
by a deterministic key generated from the source and target language codes,
followed by a SHA-1 hash of the untranslated source text formated as
follows:
[source_language_code].[target_language_code].[hashed_source_text]
See MachineTranslationModel._generate_id() below for details.
The same origin text, source_language_code, and target_language_code always
maps to the same key and therefore always returns the same translated_text.
"""
# The untranslated source text.
source_text = datastore_services.TextProperty(required=True, indexed=False)
# A SHA-1 hash of the source text. This can be used to index the datastore
# by source text.
hashed_source_text = datastore_services.StringProperty(required=True,
indexed=True)
# The language code for the source text language. Must be different from
# target_language_code.
source_language_code = datastore_services.StringProperty(required=True,
indexed=True)
# The language code for the target translation language. Must be different
# from source_language_code.
target_language_code = datastore_services.StringProperty(required=True,
indexed=True)
# The machine generated translation of the source text into the target
# language.
translated_text = datastore_services.TextProperty(required=True,
indexed=False)
@classmethod
def create(cls, source_language_code: str, target_language_code: str,
source_text: str, translated_text: str) -> Optional[str]:
"""Creates a new MachineTranslationModel instance and returns its ID.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
source_text: str. The untranslated source text.
translated_text: str. The machine generated translation of the
source text into the target language.
Returns:
str|None. The id of the newly created
MachineTranslationModel instance, or None if the inputs are
invalid.
"""
if source_language_code is target_language_code:
return None
# SHA-1 always produces a 40 digit hash. 50 is chosen here to prevent
# convert_to_hash from truncating the hash.
hashed_source_text = utils.convert_to_hash(source_text, 50)
entity_id = cls._generate_id(source_language_code,
target_language_code, hashed_source_text)
translation_entity = cls(id=entity_id,
hashed_source_text=hashed_source_text,
source_language_code=source_language_code,
target_language_code=target_language_code,
source_text=source_text,
translated_text=translated_text)
translation_entity.put()
return entity_id
@staticmethod
def _generate_id(source_language_code: str, target_language_code: str,
hashed_source_text: str) -> str:
"""Generates a valid, deterministic key for a MachineTranslationModel
instance.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
hashed_source_text: str. An SHA-1 hash of the untranslated source
text.
Returns:
str. The deterministically generated identifier for this entity of
the form:
[source_language_code].[target_language_code].[hashed_source_text]
"""
return (
'%s.%s.%s' %
(source_language_code, target_language_code, hashed_source_text))
@classmethod
def get_machine_translation(
cls, source_language_code: str, target_language_code: str,
source_text: str) -> Optional['MachineTranslationModel']:
"""Gets MachineTranslationModel by language codes and source text.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
source_text: str. The untranslated source text.
Returns:
MachineTranslationModel|None. The MachineTranslationModel
instance corresponding to the given inputs, if such a translation
exists, or None if no translation is found.
"""
hashed_source_text = utils.convert_to_hash(source_text, 50)
instance_id = cls._generate_id(source_language_code,
target_language_code,
hashed_source_text)
return cls.get(instance_id, strict=False)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model is not associated with users."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is not associated with users."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model is not associated with users."""
return dict(
super(cls, cls).get_export_policy(), **{
'source_text': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'hashed_source_text': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'source_language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'translated_text': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class StateTrainingJobsMappingModel(base_models.BaseModel):
"""Model for mapping exploration attributes to a ClassifierTrainingJob.
The ID of instances of this class has the form
[exp_id].[exp_version].[state_name].
"""
# The exploration_id of the exploration to whose state the model belongs.
exp_id = datastore_services.StringProperty(required=True, indexed=True)
# The exploration version at the time the corresponding classifier's
# training job was created.
exp_version = (datastore_services.IntegerProperty(required=True,
indexed=True))
# The name of the state to which the model belongs.
state_name = datastore_services.StringProperty(required=True, indexed=True)
# The IDs of the training jobs corresponding to the exploration state. Each
# algorithm_id corresponding to the interaction of the exploration state is
# mapped to its unique job_id.
algorithm_ids_to_job_ids = datastore_services.JsonProperty(required=True,
indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'exp_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exp_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'state_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'algorithm_ids_to_job_ids':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def _generate_id(cls, exp_id: str, exp_version: int,
state_name: str) -> str:
"""Generates a unique ID for the Classifier Exploration Mapping of the
form [exp_id].[exp_version].[state_name].
Args:
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
state_name: unicode. The name of the state to which the classifier
belongs.
Returns:
str. ID of the new Classifier Exploration Mapping instance.
"""
return '%s.%s.%s' % (exp_id, exp_version, state_name)
@classmethod
def get_models(
cls, exp_id: str, exp_version: int, state_names: List[str]
) -> List[Optional[StateTrainingJobsMappingModel]]:
"""Retrieves the Classifier Exploration Mapping models given Exploration
attributes.
Args:
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
state_names: list(unicode). The state names for which we retrieve
the mapping models.
Returns:
list(ClassifierExplorationMappingModel|None). The model instances
for the classifier exploration mapping.
"""
mapping_ids = []
for state_name in state_names:
mapping_id = cls._generate_id(exp_id, exp_version, state_name)
mapping_ids.append(mapping_id)
mapping_instances = cls.get_multi(mapping_ids)
return mapping_instances
@classmethod
def get_model(cls, exp_id: str, exp_version: int,
state_name: str) -> Optional[StateTrainingJobsMappingModel]:
"""Retrieves the Classifier Exploration Mapping model for given
exploration.
Args:
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
state_name: unicode. The state name for which we retrieve
the mapping model.
Returns:
ClassifierExplorationMappingModel|None. The model instance
for the classifier exploration mapping. It returns None if the no
entry for given <exp_id, exp_version, state_name> is found.
"""
mapping_id = cls._generate_id(exp_id, exp_version, state_name)
model = cls.get_by_id(mapping_id)
return model
@classmethod
def create(cls, exp_id: str, exp_version: int, state_name: str,
algorithm_ids_to_job_ids: Dict[str, str]) -> str:
"""Creates a new ClassifierExplorationMappingModel entry.
Args:
exp_id: str. ID of the exploration.
exp_version: int. The exploration version at the time
this training job was created.
state_name: unicode. The name of the state to which the classifier
belongs.
algorithm_ids_to_job_ids: dict(str, str). The mapping from
algorithm IDs to the IDs of their corresponding classifier
training jobs.
Returns:
str. ID of the new ClassifierExplorationMappingModel entry.
Raises:
Exception. A model with the same ID already exists.
"""
instance_id = cls._generate_id(exp_id, exp_version, state_name)
if not cls.get_by_id(instance_id):
mapping_instance = cls(
id=instance_id,
exp_id=exp_id,
exp_version=exp_version,
state_name=state_name,
algorithm_ids_to_job_ids=algorithm_ids_to_job_ids)
mapping_instance.update_timestamps()
mapping_instance.put()
return instance_id
raise Exception('A model with the same ID already exists.')
@classmethod
def create_multi(
cls, state_training_jobs_mappings: List[
classifier_domain.StateTrainingJobsMapping]
) -> List[str]:
"""Creates multiple new StateTrainingJobsMappingModel entries.
Args:
state_training_jobs_mappings: list(StateTrainingJobsMapping). The
list of StateTrainingJobsMapping domain objects.
Returns:
list(int). The list of mapping IDs.
"""
mapping_models = []
mapping_ids = []
for state_training_job_mapping in state_training_jobs_mappings:
instance_id = cls._generate_id(
state_training_job_mapping.exp_id,
state_training_job_mapping.exp_version,
state_training_job_mapping.state_name)
mapping_instance = cls(
id=instance_id,
exp_id=state_training_job_mapping.exp_id,
exp_version=state_training_job_mapping.exp_version,
state_name=state_training_job_mapping.state_name,
algorithm_ids_to_job_ids=(
state_training_job_mapping.algorithm_ids_to_job_ids))
mapping_models.append(mapping_instance)
mapping_ids.append(instance_id)
cls.update_timestamps_multi(mapping_models)
cls.put_multi(mapping_models)
return mapping_ids
class TranslationContributionStatsModel(base_models.BaseModel):
"""Records the contributor dashboard translation contribution stats. There
is one instance of this model per (language_code, contributor_user_id,
topic_id) tuple. See related design doc for more details:
https://docs.google.com/document/d/1JEDiy-f1vnBLwibu8hsfuo3JObBWiaFvDTTU9L18zpY/edit#
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The ISO 639-1 language code for which the translation contributions were
# made.
language_code = datastore_services.StringProperty(required=True,
indexed=True)
# The user ID of the translation contributor.
contributor_user_id = datastore_services.StringProperty(required=True,
indexed=True)
# The topic ID of the translation contribution.
topic_id = datastore_services.StringProperty(required=True, indexed=True)
# The number of submitted translations.
submitted_translations_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The total word count of submitted translations. Excludes HTML tags and
# attributes.
submitted_translation_word_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The number of accepted translations.
accepted_translations_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The number of accepted translations without reviewer edits.
accepted_translations_without_reviewer_edits_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The total word count of accepted translations. Excludes HTML tags and
# attributes.
accepted_translation_word_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The number of rejected translations.
rejected_translations_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The total word count of rejected translations. Excludes HTML tags and
# attributes.
rejected_translation_word_count = datastore_services.IntegerProperty(
required=True, indexed=True)
# The unique last_updated dates of the translation suggestions.
contribution_dates = datastore_services.DateProperty(repeated=True,
indexed=True)
@classmethod
def create(cls, language_code: str, contributor_user_id: str,
topic_id: str, submitted_translations_count: int,
submitted_translation_word_count: int,
accepted_translations_count: int,
accepted_translations_without_reviewer_edits_count: int,
accepted_translation_word_count: int,
rejected_translations_count: int,
rejected_translation_word_count: int,
contribution_dates: List[datetime.date]) -> str:
"""Creates a new TranslationContributionStatsModel instance and returns
its ID.
"""
entity_id = cls.generate_id(language_code, contributor_user_id,
topic_id)
entity = cls(
id=entity_id,
language_code=language_code,
contributor_user_id=contributor_user_id,
topic_id=topic_id,
submitted_translations_count=submitted_translations_count,
submitted_translation_word_count=submitted_translation_word_count,
accepted_translations_count=accepted_translations_count,
accepted_translations_without_reviewer_edits_count=(
accepted_translations_without_reviewer_edits_count),
accepted_translation_word_count=accepted_translation_word_count,
rejected_translations_count=rejected_translations_count,
rejected_translation_word_count=rejected_translation_word_count,
contribution_dates=contribution_dates)
entity.update_timestamps()
entity.put()
return entity_id
@staticmethod
def generate_id(language_code: str, contributor_user_id: str,
topic_id: str) -> str:
"""Generates a unique ID for a TranslationContributionStatsModel
instance.
Args:
language_code: str. ISO 639-1 language code.
contributor_user_id: str. User ID.
topic_id: str. Topic ID.
Returns:
str. An ID of the form:
[language_code].[contributor_user_id].[topic_id]
"""
return ('%s.%s.%s' % (language_code, contributor_user_id, topic_id))
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get().
# https://mypy.readthedocs.io/en/stable/error_code_list.html#check-validity-of-overrides-override
@classmethod
def get( # type: ignore[override]
cls, language_code: str, contributor_user_id: str,
topic_id: str) -> Optional['TranslationContributionStatsModel']:
"""Gets the TranslationContributionStatsModel matching the supplied
language_code, contributor_user_id, topic_id.
Returns:
TranslationContributionStatsModel|None. The matching
TranslationContributionStatsModel, or None if no such model
instance exists.
"""
entity_id = cls.generate_id(language_code, contributor_user_id,
topic_id)
return cls.get_by_id(entity_id)
@classmethod
def get_all_by_user_id(
cls, user_id: str) -> List['TranslationContributionStatsModel']:
"""Gets all TranslationContributionStatsModels matching the supplied
user_id.
Returns:
list(TranslationContributionStatsModel). The matching
TranslationContributionStatsModels.
"""
return cast(
List[TranslationContributionStatsModel],
cls.get_all().filter(cls.contributor_user_id == user_id).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether TranslationContributionStatsModel references the
supplied user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.contributor_user_id == user_id).get(
keys_only=True) is not None
@classmethod
def get_deletion_policy(cls) -> base_models.DELETION_POLICY:
"""Model contains corresponding to a user: contributor_user_id."""
return base_models.DELETION_POLICY.DELETE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple languages and topics relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'language_code':
base_models.EXPORT_POLICY.EXPORTED,
# User ID is not exported in order to keep internal ids private.
'contributor_user_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_id':
base_models.EXPORT_POLICY.EXPORTED,
'submitted_translations_count':
base_models.EXPORT_POLICY.EXPORTED,
'submitted_translation_word_count':
base_models.EXPORT_POLICY.EXPORTED,
'accepted_translations_count':
base_models.EXPORT_POLICY.EXPORTED,
'accepted_translations_without_reviewer_edits_count':
base_models.EXPORT_POLICY.EXPORTED,
'accepted_translation_word_count':
base_models.EXPORT_POLICY.EXPORTED,
'rejected_translations_count':
base_models.EXPORT_POLICY.EXPORTED,
'rejected_translation_word_count':
base_models.EXPORT_POLICY.EXPORTED,
'contribution_dates':
base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def apply_deletion_policy(cls, user_id: str) -> None:
"""Delete instances of TranslationContributionStatsModel for the user.
Args:
user_id: str. The ID of the user whose data should be deleted.
"""
datastore_services.delete_multi(
cast(
List[datastore_services.Key],
cls.query(cls.contributor_user_id == user_id).fetch(
keys_only=True)))
@classmethod
def export_data(
cls,
user_id: str) -> Dict[str, Dict[str, Union[str, int, List[str]]]]:
"""Exports the data from TranslationContributionStatsModel into dict
format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from TranslationContributionStatsModel.
"""
user_data = dict()
stats_models = cast(
List[TranslationContributionStatsModel],
cls.get_all().filter(cls.contributor_user_id == user_id).fetch())
for model in stats_models:
user_data[model.id] = {
'language_code':
model.language_code,
'topic_id':
model.topic_id,
'submitted_translations_count':
(model.submitted_translations_count),
'submitted_translation_word_count':
(model.submitted_translation_word_count),
'accepted_translations_count':
(model.accepted_translations_count),
'accepted_translations_without_reviewer_edits_count':
(model.accepted_translations_without_reviewer_edits_count),
'accepted_translation_word_count':
(model.accepted_translation_word_count),
'rejected_translations_count':
(model.rejected_translations_count),
'rejected_translation_word_count':
(model.rejected_translation_word_count),
'contribution_dates':
[date.isoformat() for date in model.contribution_dates]
}
return user_data
class GeneralVoiceoverApplicationModel(base_models.BaseModel):
"""A general model for voiceover application of an entity.
The ID of the voiceover application will be a random hashed value.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of entity to which the user will be assigned as a voice artist
# once the application will get approved.
target_type = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the entity to which the application belongs.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# The language code for the voiceover audio.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The status of the application. One of: accepted, rejected, in-review.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=STATUS_CHOICES)
# The HTML content written in the given language_code.
# This will typically be a snapshot of the content of the initial card of
# the target.
content = datastore_services.TextProperty(required=True)
# The filename of the voiceover audio. The filename will have
# datetime-randomId(length 6)-language_code.mp3 pattern.
filename = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the author of the voiceover application.
author_id = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the reviewer who accepted/rejected the voiceover application.
final_reviewer_id = datastore_services.StringProperty(indexed=True)
# The plain text message submitted by the reviewer while rejecting the
# application.
rejection_message = datastore_services.TextProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id, and final_reviewer_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralVoiceoverApplicationModel exists for the user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.author_id == user_id,
cls.final_reviewer_id == user_id)).get(
keys_only=True) is not None
@classmethod
def get_user_voiceover_applications(
cls,
author_id: str,
status: Optional[str] = None
) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover application submitted by the given user.
Args:
author_id: str. The id of the user created the voiceover
application.
status: str|None. The status of the voiceover application.
If the status is None, the query will fetch all the
voiceover applications.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
applications submitted by the given user.
"""
if status in STATUS_CHOICES:
voiceover_application_query = cls.query(
datastore_services.all_of(cls.author_id == author_id,
cls.status == status))
else:
voiceover_application_query = cls.query(cls.author_id == author_id)
return cast(List[GeneralVoiceoverApplicationModel],
voiceover_application_query.fetch())
@classmethod
def get_reviewable_voiceover_applications(
cls, user_id: str) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover application which a given user can
review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own voiceover application, so the
voiceover application created by the user will be excluded.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
applications which the given user can review.
"""
return cast(
List[GeneralVoiceoverApplicationModel],
cls.query(
datastore_services.all_of(
cls.author_id != user_id,
cls.status == STATUS_IN_REVIEW)).fetch())
@classmethod
def get_voiceover_applications(
cls, target_type: str, target_id: str,
language_code: str) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover applications submitted for a give entity
in a given language.
Args:
target_type: str. The type of entity.
target_id: str. The ID of the targeted entity.
language_code: str. The code of the language in which the voiceover
application is submitted.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
application which is submitted to a give entity in a given language.
"""
return cast(
List[GeneralVoiceoverApplicationModel],
cls.query(
datastore_services.all_of(
cls.target_type == target_type, cls.target_id == target_id,
cls.language_code == language_code)).fetch())
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple voiceover applications relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'target_type': base_models.EXPORT_POLICY.EXPORTED,
'target_id': base_models.EXPORT_POLICY.EXPORTED,
'language_code': base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
'content': base_models.EXPORT_POLICY.EXPORTED,
'filename': base_models.EXPORT_POLICY.EXPORTED,
# The author_id and final_reviewer_id are not exported in order to
# keep internal ids private.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'final_reviewer_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rejection_message': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def export_data(cls, user_id: str) -> Dict[str, Dict[str, Optional[str]]]:
"""(Takeout) Exports the data from GeneralVoiceoverApplicationModel
into dict format.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralVoiceoverApplicationModel.
"""
user_data = dict()
voiceover_models = cast(List[GeneralVoiceoverApplicationModel],
cls.query(cls.author_id == user_id).fetch())
for voiceover_model in voiceover_models:
user_data[voiceover_model.id] = {
'target_type': voiceover_model.target_type,
'target_id': voiceover_model.target_id,
'language_code': voiceover_model.language_code,
'status': voiceover_model.status,
'content': voiceover_model.content,
'filename': voiceover_model.filename,
'rejection_message': voiceover_model.rejection_message
}
return user_data
class CollectionModel(base_models.VersionedModel):
"""Versioned storage model for an Oppia collection.
This class should only be imported by the collection services file
and the collection model test file.
"""
SNAPSHOT_METADATA_CLASS = CollectionSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = CollectionSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = CollectionCommitLogEntryModel
ALLOW_REVERT = True
# What this collection is called.
title = datastore_services.StringProperty(required=True)
# The category this collection belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this collection.
objective = datastore_services.TextProperty(default='', indexed=False)
# The language code of this collection.
language_code = datastore_services.StringProperty(
default=constants.DEFAULT_LANGUAGE_CODE, indexed=True)
# Tags associated with this collection.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# The version of all property blob schemas.
schema_version = datastore_services.IntegerProperty(required=True,
default=1,
indexed=True)
# A dict representing the contents of a collection. Currently, this
# contains the list of nodes. This dict should contain collection data
# whose structure might need to be changed in the future.
collection_contents = (datastore_services.JsonProperty(default={},
indexed=False))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_collection_count(cls) -> int:
"""Returns the total number of collections."""
return cls.get_all().count()
# TODO(#13523): Change 'model_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@staticmethod
def convert_to_valid_dict(model_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the CollectionModel dict.
Some old CollectionModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionModel from
them. We need to remove or replace these fields and values.
Args:
model_dict: dict. The content of the model. Some fields and field
values might no longer exist in the CollectionModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
# The nodes field is moved to collection_contents dict. We
# need to move the values from nodes field to collection_contents dict
# and delete nodes.
if 'nodes' in model_dict and model_dict['nodes']:
model_dict['collection_contents']['nodes'] = (copy.deepcopy(
model_dict['nodes']))
del model_dict['nodes']
return model_dict
# TODO(#13523): Change 'snapshot_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
def _reconstitute(self, snapshot_dict: Dict[str, Any]) -> CollectionModel:
"""Populates the model instance with the snapshot.
Some old CollectionModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the the snapshot.
"""
self.populate(**CollectionModel.convert_to_valid_dict(snapshot_dict))
return self
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def _trusted_commit(self, committer_id: str, commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(CollectionModel,
self)._trusted_commit(committer_id, commit_type, commit_message,
commit_cmds)
collection_rights = CollectionRightsModel.get_by_id(self.id)
collection_commit_log = CollectionCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, collection_rights.status,
collection_rights.community_owned)
collection_commit_log.collection_id = self.id
collection_commit_log.update_timestamps()
collection_commit_log.put()
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.delete_multi().
# https://mypy.readthedocs.io/en/stable/error_code_list.html#check-validity-of-overrides-override
@classmethod
def delete_multi( # type: ignore[override]
cls,
entity_ids: List[str],
committer_id: str,
commit_message: str,
force_deletion: bool = False) -> None:
"""Deletes the given cls instances with the given entity_ids.
Note that this extends the superclass method.
Args:
entity_ids: list(str). Ids of entities to delete.
committer_id: str. The user_id of the user who committed the change.
commit_message: str. The commit description message.
force_deletion: bool. If True these models are deleted completely
from storage, otherwise there are only marked as deleted.
Default is False.
"""
super(CollectionModel, cls).delete_multi(entity_ids,
committer_id,
commit_message,
force_deletion=force_deletion)
if not force_deletion:
commit_log_models = []
collection_rights_models = CollectionRightsModel.get_multi(
entity_ids, include_deleted=True)
versioned_models = cls.get_multi(entity_ids, include_deleted=True)
for model, rights_model in python_utils.ZIP(
versioned_models, collection_rights_models):
# Ruling out the possibility of None for mypy type checking.
assert model is not None
assert rights_model is not None
collection_commit_log = CollectionCommitLogEntryModel.create(
model.id, model.version, committer_id,
cls._COMMIT_TYPE_DELETE, commit_message,
[{
'cmd': cls.CMD_DELETE_COMMIT
}], rights_model.status, rights_model.community_owned)
collection_commit_log.collection_id = model.id
commit_log_models.append(collection_commit_log)
CollectionCommitLogEntryModel.update_timestamps_multi(
commit_log_models)
datastore_services.put_multi(commit_log_models)
class AppFeedbackReportStatsModel(base_models.BaseModel):
"""Model for storing aggregate report stats on the tickets created.
Instances of this model contain statistics for different report types based
on the ticket they are assigned to and the date of the aggregation is on.
The id of each model instance is calculated by concatenating the platform,
ticket ID, and the date (in isoformat) this entity is tracking stats for.
"""
# The unique ticket ID that this entity is aggregating for.
ticket_id = datastore_services.StringProperty(required=True, indexed=True)
# The platform that these statistics are for.
platform = datastore_services.StringProperty(
required=True, indexed=True,
choices=PLATFORM_CHOICES)
# The date in UTC that this entity is tracking on -- this should correspond
# to the creation date of the reports aggregated in this model.
stats_tracking_date = datastore_services.DateProperty(
required=True, indexed=True)
# The total number of reports submitted on this date.
total_reports_submitted = datastore_services.IntegerProperty(
required=True, indexed=True)
# JSON struct that maps the daily statistics for this ticket on the date
# specified in stats_tracking_date. The JSON will map each param_name
# (defined by a domain const ALLOWED_STATS_PARAM_NAMES) to a dictionary of
# all the possible param_values for that parameter and the number of reports
# submitted on that day that satisfy that param value, similar to e.g.:
#
# param_name1 : { param_value1 : report_count1,
# param_value2 : report_count2,
# param_value3 : report_count3 },
# param_name2 : { param_value1 : report_count1,
# param_value2 : report_count2,
# param_value3 : report_count3 } }.
daily_param_stats = datastore_services.JsonProperty(
required=True, indexed=False)
# The schema version for parameter statistics in this entity.
daily_param_stats_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
@classmethod
def create(
cls,
entity_id: str,
platform: str,
ticket_id: str,
stats_tracking_date: datetime.date,
total_reports_submitted: int,
daily_param_stats: Dict[str, Dict[str, int]]
) -> str:
"""Creates a new AppFeedbackReportStatsModel instance and returns its
ID.
Args:
entity_id: str. The ID used for this entity.
ticket_id: str. The ID for the ticket these stats aggregate on.
platform: str. The platform the stats are aggregating for.
stats_tracking_date: datetime.date. The date in UTC that this entity
is tracking stats for.
total_reports_submitted: int. The total number of reports submitted
on this date.
daily_param_stats: dict. The daily stats for this entity, keyed
by the parameter witch each value mapping a parameter value to
the number of reports that satisfy that parameter value.
Returns:
AppFeedbackReportStatsModel. The newly created
AppFeedbackReportStatsModel instance.
"""
stats_entity = cls(
id=entity_id, ticket_id=ticket_id, platform=platform,
stats_tracking_date=stats_tracking_date,
total_reports_submitted=total_reports_submitted,
daily_param_stats=daily_param_stats,
daily_param_stats_schema_version=(
feconf.CURRENT_FEEDBACK_REPORT_STATS_SCHEMA_VERSION))
stats_entity.update_timestamps()
stats_entity.put()
return entity_id
@classmethod
def calculate_id(
cls,
platform: str,
ticket_id: Optional[str],
stats_tracking_date: datetime.date
) -> str:
"""Generates key for the instance of AppFeedbackReportStatsModel
class in the required format with the arguments provided.
Args:
platform: str. The platform this entity is aggregating on.
ticket_id: str. The ID for the ticket these stats aggregate on.
stats_tracking_date: date. The date these stats are tracking on.
Returns:
str. The ID for this entity of the form
'[platform]:[ticket_id]:[stats_date in YYYY-MM-DD]'.
"""
if ticket_id is None:
ticket_id = UNTICKETED_ANDROID_REPORTS_STATS_TICKET_ID
return '%s:%s:%s' % (
platform, ticket_id, stats_tracking_date.isoformat())
@classmethod
def get_stats_for_ticket(
cls, ticket_id: str
) -> Sequence['AppFeedbackReportStatsModel']:
"""Fetches the stats for a single ticket.
Args:
ticket_id: str. The ID of the ticket to get stats for.
Returns:
list(str). A list of IDs corresponding to
AppFeedbackReportStatsModel entities that record stats on the
ticket.
"""
return cls.query(cls.ticket_id == ticket_id).fetch()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any information directly corresponding to a
user.
"""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'ticket_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'platform': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'stats_tracking_date': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_reports_submitted': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'daily_param_stats_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'daily_param_stats': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback reports stats will be
moderator.
"""
return feconf.ROLE_ID_MODERATOR
class CollectionRightsModel(base_models.VersionedModel):
"""Storage model for rights related to a collection.
The id of each instance is the id of the corresponding collection.
"""
SNAPSHOT_METADATA_CLASS = CollectionRightsSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = CollectionRightsSnapshotContentModel
ALLOW_REVERT = False
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to voiceover this collection.
voice_artist_ids = (datastore_services.StringProperty(indexed=True,
repeated=True))
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# Whether this collection is owned by the community.
community_owned = (datastore_services.BooleanProperty(indexed=True,
default=False))
# For private collections, whether this collection can be viewed
# by anyone who has the URL. If the collection is not private, this
# setting is ignored.
viewable_if_private = (datastore_services.BooleanProperty(indexed=True,
default=False))
# Time, in milliseconds, when the collection was first published.
first_published_msec = (datastore_services.FloatProperty(indexed=True,
default=None))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE,
indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE, constants.ACTIVITY_STATUS_PUBLIC
])
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, voice_artist_ids, editor_ids,
and owner_ids fields.
"""
return (base_models.DELETION_POLICY.
PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users contribute to collections and have varying rights.
"""
return (base_models.MODEL_ASSOCIATION_TO_USER.
ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'owner_ids': 'owned_collection_ids',
'editor_ids': 'editable_collection_ids',
'voice_artist_ids': 'voiced_collection_ids',
'viewer_ids': 'viewable_collection_ids'
}
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export/delete corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'owner_ids': base_models.EXPORT_POLICY.EXPORTED,
'editor_ids': base_models.EXPORT_POLICY.EXPORTED,
'voice_artist_ids': base_models.EXPORT_POLICY.EXPORTED,
'viewer_ids': base_models.EXPORT_POLICY.EXPORTED,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewable_if_private':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'first_published_msec':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionRightsModel references the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.owner_ids == user_id, cls.editor_ids == user_id,
cls.voice_artist_ids == user_id,
cls.viewer_ids == user_id)).get(keys_only=True) is not None
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def save(self, committer_id: str, commit_message: str,
commit_cmds: List[Dict[str, Any]]) -> None:
"""Updates the collection rights model by applying the given
commit_cmds, then saves it.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and additional arguments for that command.
"""
super(CollectionRightsModel, self).commit(committer_id, commit_message,
commit_cmds)
# TODO(#13523): Change 'model_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@staticmethod
def convert_to_valid_dict(model_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the CollectionRightsModel dict.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
model_dict: dict. The content of the model. Some fields and field
values might no longer exist in the CollectionRightsModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
# The status field could historically take the value 'publicized', this
# value is now equivalent to 'public'.
if model_dict['status'] == 'publicized':
model_dict['status'] = constants.ACTIVITY_STATUS_PUBLIC
# The voice_artist_ids field was previously named translator_ids. We
# need to move the values from translator_ids field to voice_artist_ids
# and delete translator_ids.
if 'translator_ids' in model_dict and model_dict['translator_ids']:
model_dict['voice_artist_ids'] = model_dict['translator_ids']
del model_dict['translator_ids']
# We need to remove pseudonymous IDs from all the fields that contain
# user IDs.
for field_name in ('owner_ids', 'editor_ids', 'voice_artist_ids',
'viewer_ids'):
model_dict[field_name] = [
user_id for user_id in model_dict[field_name]
if not utils.is_pseudonymous_id(user_id)
]
return model_dict
# TODO(#13523): Change 'snapshot_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
def _reconstitute(self, snapshot_dict: Dict[str,
Any]) -> CollectionRightsModel:
"""Populates the model instance with the snapshot.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the the snapshot.
"""
self.populate(
**CollectionRightsModel.convert_to_valid_dict(snapshot_dict))
return self
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def _trusted_commit(self, committer_id: str, commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]) -> None:
"""Record the event to the commit log after the model commit.
Note that this overrides the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(CollectionRightsModel,
self)._trusted_commit(committer_id, commit_type, commit_message,
commit_cmds)
# Create and delete events will already be recorded in the
# CollectionModel.
if commit_type not in ['create', 'delete']:
CollectionCommitLogEntryModel(
id=('rights-%s-%s' % (self.id, self.version)),
user_id=committer_id,
collection_id=self.id,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=None,
post_commit_status=self.status,
post_commit_community_owned=self.community_owned,
post_commit_is_private=(
self.status == constants.ACTIVITY_STATUS_PRIVATE)).put()
snapshot_metadata_model = self.SNAPSHOT_METADATA_CLASS.get(
self.get_snapshot_id(self.id, self.version))
# Ruling out the possibility of None for mypy type checking.
assert snapshot_metadata_model is not None
snapshot_metadata_model.content_user_ids = list(
sorted(
set(self.owner_ids) | set(self.editor_ids)
| set(self.voice_artist_ids) | set(self.viewer_ids)))
commit_cmds_user_ids = set()
for commit_cmd in commit_cmds:
user_id_attribute_names = next(
cmd['user_id_attribute_names']
for cmd in feconf.COLLECTION_RIGHTS_CHANGE_ALLOWED_COMMANDS
if cmd['name'] == commit_cmd['cmd'])
for user_id_attribute_name in user_id_attribute_names:
commit_cmds_user_ids.add(commit_cmd[user_id_attribute_name])
snapshot_metadata_model.commit_cmds_user_ids = list(
sorted(commit_cmds_user_ids))
snapshot_metadata_model.update_timestamps()
snapshot_metadata_model.put()
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of CollectionRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of CollectionRightsModel
in a python dict format. In this case, we are returning all the
ids of collections that the user is connected to, so they either
own, edit, voice, or have permission to view.
"""
owned_collections = cls.get_all().filter(cls.owner_ids == user_id)
editable_collections = cls.get_all().filter(cls.editor_ids == user_id)
voiced_collections = (cls.get_all().filter(
cls.voice_artist_ids == user_id))
viewable_collections = cls.get_all().filter(cls.viewer_ids == user_id)
owned_collection_ids = [col.key.id() for col in owned_collections]
editable_collection_ids = [
col.key.id() for col in editable_collections
]
voiced_collection_ids = [col.key.id() for col in voiced_collections]
viewable_collection_ids = [
col.key.id() for col in viewable_collections
]
return {
'owned_collection_ids': owned_collection_ids,
'editable_collection_ids': editable_collection_ids,
'voiced_collection_ids': voiced_collection_ids,
'viewable_collection_ids': viewable_collection_ids
}
class StorySummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Story.
This should be used whenever the content blob of the story is not
needed (e.g. search results, etc).
A StorySummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version.
The key of each instance is the story id.
"""
# The title of the story.
title = datastore_services.StringProperty(required=True, indexed=True)
# The ISO 639-1 code for the language this story is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# A high-level description of the story.
description = datastore_services.TextProperty(required=True, indexed=False)
# Time when the story model was last updated (not to be
# confused with last_updated, which is the time when the
# story *summary* model was last updated).
story_model_last_updated = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# Time when the story model was created (not to be confused
# with created_on, which is the time when the story *summary*
# model was created).
story_model_created_on = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# The titles of the nodes in the story, in the same order as present there.
node_titles = (
datastore_services.StringProperty(repeated=True, indexed=True))
# The thumbnail filename of the story.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the story.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
# The url fragment for the story.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_titles': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class CollectionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia collection.
This should be used whenever the content blob of the collection is not
needed (e.g. search results, etc).
A CollectionSummaryModel instance stores the following information:
id, title, category, objective, language_code, tags, ratings,
last_updated, created_on, status (private, public),
community_owned, owner_ids, editor_ids,
viewer_ids, version.
The key of each instance is the collection id.
"""
# What this collection is called.
title = datastore_services.StringProperty(required=True)
# The category this collection belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this collection.
objective = datastore_services.TextProperty(required=True, indexed=False)
# The ISO 639-1 code for the language this collection is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# Tags associated with this collection.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# Aggregate user-assigned ratings of the collection.
ratings = datastore_services.JsonProperty(default=None, indexed=False)
# Time when the collection model was last updated (not to be
# confused with last_updated, which is the time when the
# collection *summary* model was last updated).
collection_model_last_updated = (datastore_services.DateTimeProperty(
indexed=True))
# Time when the collection model was created (not to be confused
# with created_on, which is the time when the collection *summary*
# model was created).
collection_model_created_on = (datastore_services.DateTimeProperty(
indexed=True))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE,
indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE, constants.ACTIVITY_STATUS_PUBLIC
])
# Whether this collection is owned by the community.
community_owned = (datastore_services.BooleanProperty(required=True,
indexed=True))
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who have contributed (humans who have made a
# positive (not just a revert) change to the collection's content).
# NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be
# synchronized, meaning that the keys in contributors_summary need be
# equal to the contributor_ids list.
contributor_ids = (datastore_services.StringProperty(indexed=True,
repeated=True))
# A dict representing the contributors of non-trivial commits to this
# collection. Each key of this dict is a user_id, and the corresponding
# value is the number of non-trivial commits that the user has made.
contributors_summary = (datastore_services.JsonProperty(default={},
indexed=False))
# The version number of the collection after this commit. Only populated
# for commits to an collection (as opposed to its rights, etc.).
version = datastore_services.IntegerProperty()
# The number of nodes(explorations) that are within this collection.
node_count = datastore_services.IntegerProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, editor_ids, owner_ids, contributor_ids,
and contributors_summary fields.
"""
return (base_models.DELETION_POLICY.
PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the
CollectionRightsModel, and thus does not need an export_data
function.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because noteworthy details that belong to this model have already been
exported as a part of the CollectionRightsModel.
"""
return dict(
super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributors_summary':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_count': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionSummaryModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id,
cls.contributor_ids == user_id)).get(
keys_only=True) is not None
@classmethod
def get_non_private(cls) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with non-private collection summary models.
Returns:
iterable. An iterable with non-private collection summary models.
"""
return cls.get_all().filter(
cls.status != constants.ACTIVITY_STATUS_PRIVATE).fetch(
feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_private_at_least_viewable(
cls, user_id: str) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with private collection summary models that are
at least viewable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with private collection summary models that
are at least viewable by the given user.
"""
return cls.get_all().filter(
cls.status == constants.ACTIVITY_STATUS_PRIVATE).filter(
datastore_services.any_of(cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id)).fetch(
feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_at_least_editable(
cls, user_id: str) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with collection summary models that are at least
editable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with collection summary models that are at
least viewable by the given user.
"""
return CollectionSummaryModel.get_all().filter(
datastore_services.any_of(
CollectionSummaryModel.owner_ids == user_id,
CollectionSummaryModel.editor_ids == user_id)).fetch(
feconf.DEFAULT_QUERY_LIMIT)
class StoryModel(base_models.VersionedModel):
"""Model for storing stories.
This class should only be imported by the story services file
and the story model test file.
"""
SNAPSHOT_METADATA_CLASS = StorySnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = StorySnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = StoryCommitLogEntryModel
ALLOW_REVERT = False
# The title of the story.
title = datastore_services.StringProperty(required=True, indexed=True)
# The thumbnail filename of the story.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the story.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
# The thumbnail size of the story.
thumbnail_size_in_bytes = datastore_services.IntegerProperty(indexed=True)
# A high-level description of the story.
description = datastore_services.TextProperty(indexed=False)
# A set of notes, that describe the characters, main storyline, and setting.
notes = datastore_services.TextProperty(indexed=False)
# The ISO 639-1 code for the language this story is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The story contents dict specifying the list of story nodes and the
# connection between them. Modelled by class StoryContents
# (see story_domain.py for its current schema).
story_contents = datastore_services.JsonProperty(default={}, indexed=False)
# The schema version for the story_contents.
story_contents_schema_version = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The topic id to which the story belongs.
corresponding_topic_id = (
datastore_services.StringProperty(indexed=True, required=True))
# The url fragment for the story.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# The content of the meta tag in the Story viewer page.
meta_tag_content = datastore_services.StringProperty(
indexed=True, default='')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id,
commit_type,
commit_message,
commit_cmds,
additional_models
)
story_commit_log_entry = StoryCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False
)
story_commit_log_entry.story_id = self.id
return {
'snapshot_metadata_model': models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': story_commit_log_entry,
'versioned_model': models_to_put['versioned_model'],
}
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_size_in_bytes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'notes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_contents_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'corresponding_topic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'meta_tag_content': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional[StoryModel]:
"""Gets StoryModel by url_fragment. Returns None if the story with
name url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the story.
Returns:
StoryModel|None. The story model of the story or None if not
found.
"""
return cls.get_all().filter(cls.url_fragment == url_fragment).get()
class ExplorationOpportunitySummaryModel(base_models.BaseModel):
"""Summary of translation and voiceover opportunities in an exploration.
The id of each instance is the id of the corresponding exploration.
"""
topic_id = datastore_services.StringProperty(required=True, indexed=True)
topic_name = datastore_services.StringProperty(required=True, indexed=True)
story_id = datastore_services.StringProperty(required=True, indexed=True)
story_title = datastore_services.StringProperty(required=True,
indexed=True)
chapter_title = (datastore_services.StringProperty(required=True,
indexed=True))
content_count = (datastore_services.IntegerProperty(required=True,
indexed=True))
incomplete_translation_language_codes = datastore_services.StringProperty(
repeated=True, indexed=True)
translation_counts = (datastore_services.JsonProperty(default={},
indexed=False))
language_codes_with_assigned_voice_artists = (
datastore_services.StringProperty(repeated=True, indexed=True))
language_codes_needing_voice_artists = datastore_services.StringProperty(
repeated=True, indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'topic_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_title':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'chapter_title':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'content_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'incomplete_translation_language_codes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'translation_counts':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_codes_with_assigned_voice_artists':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_codes_needing_voice_artists':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_all_translation_opportunities(
cls, page_size: int, urlsafe_start_cursor: Optional[str],
language_code: str, topic_name: str
) -> Tuple[Sequence['ExplorationOpportunitySummaryModel'], Optional[str],
bool]:
"""Returns a list of opportunities available for translation in a
specific language.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
language_code: str. The language for which translation opportunities
are to be fetched.
topic_name: str or None. The topic for which translation
opportunities should be fetched. If topic_name is None or empty,
fetch translation opportunities from all topics.
Returns:
3-tuple of (results, cursor, more). As described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: list(ExplorationOpportunitySummaryModel). A list
of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
if urlsafe_start_cursor:
start_cursor = datastore_services.make_cursor(
urlsafe_cursor=urlsafe_start_cursor)
else:
start_cursor = datastore_services.make_cursor()
language_query = cls.query(
cls.incomplete_translation_language_codes == language_code).order(
cls.topic_name)
if topic_name:
language_query = language_query.filter(
cls.topic_name == topic_name)
fetch_result: Tuple[Sequence[ExplorationOpportunitySummaryModel],
datastore_services.Cursor,
bool] = language_query.fetch_page(
page_size, start_cursor=start_cursor)
results, cursor, _ = fetch_result
# TODO(#13462): Refactor this so that we don't do the lookup.
# Do a forward lookup so that we can know if there are more values.
fetch_result = (language_query.fetch_page(page_size + 1,
start_cursor=start_cursor))
plus_one_query_models, _, _ = fetch_result
more_results = len(plus_one_query_models) == page_size + 1
# The urlsafe returns bytes and we need to decode them to string.
return (results,
(cursor.urlsafe().decode('utf-8') if cursor else None),
more_results)
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_all_voiceover_opportunities(
cls, page_size: int, urlsafe_start_cursor: Optional[str],
language_code: str
) -> Tuple[Sequence['ExplorationOpportunitySummaryModel'], Optional[str],
bool]:
"""Returns a list of opportunities available for voiceover in a
specific language.
Args:
page_size: int. The maximum number of entities to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned entities starts from this datastore cursor.
Otherwise, the returned entities start from the beginning
of the full list of entities.
language_code: str. The language for which voiceover opportunities
to be fetched.
Returns:
3-tuple of (results, cursor, more). As described in fetch_page() at:
https://developers.google.com/appengine/docs/python/ndb/queryclass,
where:
results: list(ExplorationOpportunitySummaryModel). A list
of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
start_cursor = datastore_services.make_cursor(
urlsafe_cursor=urlsafe_start_cursor)
language_created_on_query = cls.query(
cls.language_codes_needing_voice_artists == language_code).order(
cls.created_on)
fetch_result: Tuple[Sequence[ExplorationOpportunitySummaryModel],
datastore_services.Cursor,
bool] = language_created_on_query.fetch_page(
page_size, start_cursor=start_cursor)
results, cursor, _ = fetch_result
# TODO(#13462): Refactor this so that we don't do the lookup.
# Do a forward lookup so that we can know if there are more values.
fetch_result = language_created_on_query.fetch_page(
page_size + 1, start_cursor=start_cursor)
plus_one_query_models, _, _ = fetch_result
more_results = len(plus_one_query_models) == page_size + 1
# The urlsafe returns bytes and we need to decode them to string.
return (results,
(cursor.urlsafe().decode('utf-8') if cursor else None),
more_results)
@classmethod
def get_by_topic(
cls,
topic_id: str) -> Sequence['ExplorationOpportunitySummaryModel']:
"""Returns all the models corresponding to the specific topic.
Returns:
list(ExplorationOpportunitySummaryModel). A list of
ExplorationOpportunitySummaryModel having given topic_id.
"""
return cls.query(cls.topic_id == topic_id).fetch()
@classmethod
def delete_all(cls) -> None:
"""Deletes all entities of this class."""
keys = cls.query().fetch(keys_only=True)
datastore_services.delete_multi(keys)
class SubtopicPageModel(base_models.VersionedModel):
"""Model for storing Subtopic pages.
This stores the HTML data for a subtopic page.
"""
SNAPSHOT_METADATA_CLASS = SubtopicPageSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = SubtopicPageSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = SubtopicPageCommitLogEntryModel
ALLOW_REVERT = False
# The topic id that this subtopic is a part of.
topic_id = datastore_services.StringProperty(required=True, indexed=True)
# The json data of the subtopic consisting of subtitled_html,
# recorded_voiceovers and written_translations fields.
page_contents = datastore_services.JsonProperty(required=True)
# The schema version for the page_contents field.
page_contents_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# The ISO 639-1 code for the language this subtopic page is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(SubtopicPageModel, self)._trusted_commit(
committer_id, commit_type, commit_message, commit_cmds)
subtopic_page_commit_log_entry = SubtopicPageCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False
)
subtopic_page_commit_log_entry.subtopic_page_id = self.id
subtopic_page_commit_log_entry.update_timestamps()
subtopic_page_commit_log_entry.put()
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'topic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'page_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'page_contents_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class SkillModel(base_models.VersionedModel):
"""Model for storing Skills.
This class should only be imported by the skill services file
and the skill model test file.
"""
SNAPSHOT_METADATA_CLASS = SkillSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = SkillSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = SkillCommitLogEntryModel
ALLOW_REVERT = False
# The description of the skill.
description = datastore_services.StringProperty(required=True,
indexed=True)
# The schema version for each of the misconception dicts.
misconceptions_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# The schema version for each of the rubric dicts.
rubric_schema_version = datastore_services.IntegerProperty(required=True,
indexed=True)
# A list of misconceptions associated with the skill, in which each
# element is a dict.
misconceptions = (datastore_services.JsonProperty(repeated=True,
indexed=False))
# The rubrics for the skill that explain each difficulty level.
rubrics = datastore_services.JsonProperty(repeated=True, indexed=False)
# The ISO 639-1 code for the language this skill is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The schema version for the skill_contents.
skill_contents_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# A dict representing the skill contents.
skill_contents = datastore_services.JsonProperty(indexed=False)
# The prerequisite skills for the skill.
prerequisite_skill_ids = (datastore_services.StringProperty(repeated=True,
indexed=True))
# The id to be used by the next misconception added.
next_misconception_id = (datastore_services.IntegerProperty(required=True,
indexed=False))
# The id that the skill is merged into, in case the skill has been
# marked as duplicate to another one and needs to be merged.
# This is an optional field.
superseding_skill_id = datastore_services.StringProperty(indexed=True)
# A flag indicating whether deduplication is complete for this skill.
# It will initially be False, and set to true only when there is a value
# for superseding_skill_id and the merge was completed.
all_questions_merged = (datastore_services.BooleanProperty(indexed=True,
required=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_merged_skills(cls) -> List['SkillModel']:
"""Returns the skill models which have been merged.
Returns:
list(SkillModel). List of skill models which have been merged.
"""
return [
skill for skill in cls.query()
if (skill.superseding_skill_id is not None and (
len(skill.superseding_skill_id) > 0))
]
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(self, committer_id: str, commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(SkillModel, self)._trusted_commit(committer_id, commit_type,
commit_message, commit_cmds)
skill_commit_log_entry = SkillCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False)
skill_commit_log_entry.skill_id = self.id
skill_commit_log_entry.update_timestamps()
skill_commit_log_entry.put()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconceptions_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rubric_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconceptions': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rubrics': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_contents_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'prerequisite_skill_ids':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_misconception_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'superseding_skill_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'all_questions_merged':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_by_description(cls, description: str) -> Optional['SkillModel']:
"""Gets SkillModel by description. Returns None if the skill with
description doesn't exist.
Args:
description: str. The description of the skill.
Returns:
SkillModel|None. The skill model of the skill or None if not
found.
"""
return cast(
Optional[SkillModel],
SkillModel.query().filter(cls.description == description).filter(
cls.deleted == False).get()) # pylint: disable=singleton-comparison
class GeneralFeedbackMessageModel(base_models.BaseModel):
"""Feedback messages. One or more of these messages make a thread.
The id of instances of this class has the form [thread_id].[message_id]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# ID corresponding to an entry of FeedbackThreadModel.
thread_id = datastore_services.StringProperty(required=True, indexed=True)
# 0-based sequential numerical ID. Sorting by this field will create the
# thread in chronological order.
message_id = datastore_services.IntegerProperty(required=True,
indexed=True)
# ID of the user who posted this message. This may be None if the feedback
# was given anonymously by a learner.
author_id = datastore_services.StringProperty(indexed=True)
# New thread status. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the status changes.
updated_status = (datastore_services.StringProperty(choices=STATUS_CHOICES,
indexed=True))
# New thread subject. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the subject changes.
updated_subject = datastore_services.StringProperty(indexed=True)
# Message text. Allowed not to exist (e.g. post only to update the status).
text = datastore_services.TextProperty(indexed=False)
# Whether the incoming message is received by email (as opposed to via
# the web).
received_via_email = datastore_services.BooleanProperty(default=False,
indexed=True,
required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple feedback messages relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'thread_id': base_models.EXPORT_POLICY.EXPORTED,
'message_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the author_id because we should not export
# internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'updated_status': base_models.EXPORT_POLICY.EXPORTED,
'updated_subject': base_models.EXPORT_POLICY.EXPORTED,
'text': base_models.EXPORT_POLICY.EXPORTED,
'received_via_email': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackMessageModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@classmethod
def export_data(
cls,
user_id: str) -> Dict[str, Dict[str, Union[str, int, bool, None]]]:
"""Exports the data from GeneralFeedbackMessageModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackMessageModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackMessageModel] = (
cls.get_all().filter(cls.author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'thread_id': feedback_model.thread_id,
'message_id': feedback_model.message_id,
'updated_status': feedback_model.updated_status,
'updated_subject': feedback_model.updated_subject,
'text': feedback_model.text,
'received_via_email': feedback_model.received_via_email
}
return user_data
@classmethod
def _generate_id(cls, thread_id: str, message_id: int) -> str:
"""Generates full message ID given the thread ID and message ID.
Args:
thread_id: str. Thread ID of the thread to which the message
belongs.
message_id: int. Message ID of the message.
Returns:
str. Full message ID.
"""
return '.'.join([thread_id, str(message_id)])
@property
def entity_id(self) -> str:
"""Returns the entity_id corresponding to this thread instance.
Returns:
str. The entity_id.
"""
return self.id.split('.')[1]
@property
def entity_type(self) -> str:
"""Returns the entity_type corresponding to this thread instance.
Returns:
str. The entity_type.
"""
return self.id.split('.')[0]
@classmethod
def create(
cls,
message_identifier: feedback_domain.FullyQualifiedMessageIdentifier
) -> GeneralFeedbackMessageModel:
"""Creates a new GeneralFeedbackMessageModel entry.
Args:
message_identifier: FullyQualifiedMessageIdentifier. The message
identifier consists of the thread_id and its corresponding
message_id.
Returns:
GeneralFeedbackMessageModel. Instance of the new
GeneralFeedbackMessageModel entry.
Raises:
Exception. A message with the same ID already exists
in the given thread.
"""
return cls.create_multi([message_identifier])[0]
@classmethod
def create_multi(
cls, message_identifiers: List[
feedback_domain.FullyQualifiedMessageIdentifier]
) -> List[GeneralFeedbackMessageModel]:
"""Creates a new GeneralFeedbackMessageModel entry for each
(thread_id, message_id) pair.
Args:
message_identifiers: list(FullyQualifiedMessageIdentifier). Each
message identifier consists of the thread_id and its
corresponding message_id.
Returns:
list(GeneralFeedbackMessageModel). Instances of the new
GeneralFeedbackMessageModel entries.
Raises:
Exception. The number of thread_ids must be equal to the number of
message_ids.
Exception. A message with the same ID already exists
in the given thread.
"""
thread_ids = [
message_identifier.thread_id
for message_identifier in message_identifiers
]
message_ids = [
message_identifier.message_id
for message_identifier in message_identifiers
]
# Generate the new ids.
instance_ids = [
cls._generate_id(thread_id, message_id)
for thread_id, message_id in zip(thread_ids, message_ids)
]
# Check if the new ids are valid.
current_instances = cls.get_multi(instance_ids)
conflict_ids = [
current_instance.id for current_instance in current_instances
if current_instance is not None
]
if len(conflict_ids) > 0:
raise Exception(
'The following feedback message ID(s) conflicted on '
'create: %s' % (' '.join(conflict_ids)))
return [cls(id=instance_id) for instance_id in instance_ids]
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get().
@classmethod
def get( # type: ignore[override]
cls,
thread_id: str,
message_id: int,
strict: bool = True) -> Optional[GeneralFeedbackMessageModel]:
"""Gets the GeneralFeedbackMessageModel entry for the given ID. Raises
an error if no undeleted message with the given ID is found and
strict == True.
Args:
thread_id: str. ID of the thread.
message_id: int. ID of the message.
strict: bool. Whether to raise an error if no FeedbackMessageModel
entry is found for the given IDs.
Returns:
GeneralFeedbackMessageModel or None. If strict == False and no
undeleted message with the given message_id exists in the
datastore, then returns None. Otherwise, returns the
GeneralFeedbackMessageModel instance that corresponds to the
given ID.
Raises:
EntityNotFoundError. The value of strict is True and either
(i) message ID is not valid
(ii) message is marked as deleted.
No error will be raised if strict == False.
"""
instance_id = cls._generate_id(thread_id, message_id)
return super(GeneralFeedbackMessageModel, cls).get(instance_id,
strict=strict)
@classmethod
def get_messages(cls,
thread_id: str) -> Sequence[GeneralFeedbackMessageModel]:
"""Returns a list of messages in the given thread. The number of
messages returned is capped by feconf.DEFAULT_QUERY_LIMIT.
Args:
thread_id: str. ID of the thread.
Returns:
list(GeneralFeedbackMessageModel). A list of messages in the
given thread, up to a maximum of feconf.DEFAULT_QUERY_LIMIT
messages.
"""
return cls.get_all().filter(cls.thread_id == thread_id).fetch(
feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_most_recent_message(cls,
thread_id: str) -> GeneralFeedbackMessageModel:
"""Returns the last message in the thread.
Args:
thread_id: str. ID of the thread.
Returns:
GeneralFeedbackMessageModel. Last message in the thread.
"""
thread = GeneralFeedbackThreadModel.get_by_id(thread_id)
message = cls.get(thread_id, thread.message_count - 1)
# Ruling out the possibility of None for mypy type checking.
assert message is not None
return message
@classmethod
def get_message_count(cls, thread_id: str) -> int:
"""Returns the number of messages in the thread. Includes the
deleted entries.
Args:
thread_id: str. ID of the thread.
Returns:
int. Number of messages in the thread.
"""
return cls.get_message_counts([thread_id])[0]
@classmethod
def get_message_counts(cls, thread_ids: List[str]) -> List[int]:
"""Returns a list containing the number of messages in the threads.
Includes the deleted entries.
Args:
thread_ids: list(str). ID of the threads.
Returns:
list(int). List of the message counts for the threads.
"""
thread_models = GeneralFeedbackThreadModel.get_multi(thread_ids)
assert None not in thread_models
return [
thread_model.message_count if thread_model else None
for thread_model in thread_models
]
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_all_messages(
cls, page_size: int, urlsafe_start_cursor: Optional[str]
) -> Tuple[Sequence[GeneralFeedbackMessageModel], Optional[str], bool]:
"""Fetches a list of all the messages sorted by their last updated
attribute.
Args:
page_size: int. The maximum number of messages to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned messages starts from this datastore cursor.
Otherwise, the returned messages start from the beginning
of the full list of messages.
Returns:
3-tuple of (results, cursor, more). Where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
return cls._fetch_page_sorted_by_last_updated(cls.query(), page_size,
urlsafe_start_cursor)
class TopicRightsModel(base_models.VersionedModel):
"""Storage model for rights related to a topic.
The id of each instance is the id of the corresponding topic.
"""
SNAPSHOT_METADATA_CLASS = TopicRightsSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = TopicRightsSnapshotContentModel
ALLOW_REVERT = False
# The user_ids of the managers of this topic.
manager_ids = datastore_services.StringProperty(indexed=True,
repeated=True)
# Whether this topic is published.
topic_is_published = datastore_services.BooleanProperty(indexed=True,
required=True,
default=False)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: manager_ids field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether TopicRightsModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.manager_ids == user_id).get(
keys_only=True) is not None
@classmethod
def get_by_user(cls, user_id: str) -> Sequence[TopicRightsModel]:
"""Retrieves the rights object for all topics assigned to given user
Args:
user_id: str. ID of user.
Returns:
list(TopicRightsModel). The list of TopicRightsModel objects in
which the given user is a manager.
"""
return cls.query(cls.manager_ids == user_id).fetch()
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id, commit_type, commit_message, commit_cmds,
additional_models)
if self.topic_is_published:
status = constants.ACTIVITY_STATUS_PUBLIC
else:
status = constants.ACTIVITY_STATUS_PRIVATE
topic_commit_log = TopicCommitLogEntryModel(
id=('rights-%s-%s' % (self.id, self.version)),
user_id=committer_id,
topic_id=self.id,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=None,
post_commit_status=status,
post_commit_community_owned=False,
post_commit_is_private=not self.topic_is_published)
snapshot_metadata_model = models_to_put['snapshot_metadata_model']
snapshot_metadata_model.content_user_ids = list(
sorted(set(self.manager_ids)))
commit_cmds_user_ids = set()
for commit_cmd in commit_cmds:
user_id_attribute_names = next(
cmd['user_id_attribute_names']
for cmd in feconf.TOPIC_RIGHTS_CHANGE_ALLOWED_COMMANDS
if cmd['name'] == commit_cmd['cmd'])
for user_id_attribute_name in user_id_attribute_names:
commit_cmds_user_ids.add(commit_cmd[user_id_attribute_name])
snapshot_metadata_model.commit_cmds_user_ids = list(
sorted(commit_cmds_user_ids))
return {
'snapshot_metadata_model':
models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': topic_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users contribute to topics and their rights.
"""
return (base_models.MODEL_ASSOCIATION_TO_USER.
ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'manager_ids': base_models.EXPORT_POLICY.EXPORTED,
'topic_is_published': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {'manager_ids': 'managed_topic_ids'}
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of TopicRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of TopicRightsModel in a dict
format. In this case, we are returning all the ids of the topics
this user manages.
"""
managed_topics = cls.get_all().filter(cls.manager_ids == user_id)
managed_topic_ids = [right.id for right in managed_topics]
return {'managed_topic_ids': managed_topic_ids}
class GeneralFeedbackThreadUserModel(base_models.BaseModel):
"""Model for storing the ids of the messages in the thread that are read by
the user.
Instances of this class have keys of the form [user_id].[thread_id]
"""
user_id = datastore_services.StringProperty(required=True, indexed=True)
thread_id = datastore_services.StringProperty(required=True, indexed=True)
message_ids_read_by_user = (datastore_services.IntegerProperty(
repeated=True, indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to delete corresponding to a user:
user_id field.
"""
return base_models.DELETION_POLICY.DELETE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple feedback threads relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'user_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thread_id':
base_models.EXPORT_POLICY.EXPORTED_AS_KEY_FOR_TAKEOUT_DICT,
'message_ids_read_by_user': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def apply_deletion_policy(cls, user_id: str) -> None:
"""Delete instance of GeneralFeedbackThreadUserModel for the user.
Args:
user_id: str. The ID of the user whose data should be deleted.
"""
keys = cls.query(cls.user_id == user_id).fetch(keys_only=True)
datastore_services.delete_multi(keys)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackThreadUserModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.user_id == user_id).get(
keys_only=True) is not None
@classmethod
def generate_full_id(cls, user_id: str, thread_id: str) -> str:
"""Generates the full message id of the format:
<user_id.thread_id>.
Args:
user_id: str. The user id.
thread_id: str. The thread id.
Returns:
str. The full message id.
"""
return '%s.%s' % (user_id, thread_id)
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get().
@classmethod
def get( # type: ignore[override]
cls, user_id: str,
thread_id: str) -> Optional[GeneralFeedbackThreadUserModel]:
"""Gets the FeedbackThreadUserModel corresponding to the given user and
the thread.
Args:
user_id: str. The id of the user.
thread_id: str. The id of the thread.
Returns:
FeedbackThreadUserModel. The FeedbackThreadUserModel instance which
matches with the given user_id, and thread id.
"""
instance_id = cls.generate_full_id(user_id, thread_id)
return super(GeneralFeedbackThreadUserModel, cls).get(instance_id,
strict=False)
@classmethod
def create(cls, user_id: str,
thread_id: str) -> GeneralFeedbackThreadUserModel:
"""Creates a new FeedbackThreadUserModel instance and returns it.
Args:
user_id: str. The id of the user.
thread_id: str. The id of the thread.
Returns:
FeedbackThreadUserModel. The newly created FeedbackThreadUserModel
instance.
"""
return cls.create_multi(user_id, [thread_id])[0]
@classmethod
def create_multi(
cls, user_id: str,
thread_ids: List[str]) -> List[GeneralFeedbackThreadUserModel]:
"""Creates new FeedbackThreadUserModel instances for user_id for each
of the thread_ids.
Args:
user_id: str. The id of the user.
thread_ids: list(str). The ids of the threads.
Returns:
list(FeedbackThreadUserModel). The newly created
FeedbackThreadUserModel instances.
"""
new_instances = []
for thread_id in thread_ids:
instance_id = cls.generate_full_id(user_id, thread_id)
new_instance = cls(id=instance_id,
user_id=user_id,
thread_id=thread_id)
new_instances.append(new_instance)
GeneralFeedbackThreadUserModel.update_timestamps_multi(new_instances)
GeneralFeedbackThreadUserModel.put_multi(new_instances)
return new_instances
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get_multi().
@classmethod
def get_multi( # type: ignore[override]
cls, user_id: str, thread_ids: List[str]) -> List[
Optional[GeneralFeedbackThreadUserModel]]:
"""Gets the ExplorationUserDataModel corresponding to the given user and
the thread ids.
Args:
user_id: str. The id of the user.
thread_ids: list(str). The ids of the threads.
Returns:
list(FeedbackThreadUserModel). The FeedbackThreadUserModels
corresponding to the given user ans thread ids.
"""
instance_ids = [
cls.generate_full_id(user_id, thread_id)
for thread_id in thread_ids
]
return super(GeneralFeedbackThreadUserModel,
cls).get_multi(instance_ids)
@classmethod
def export_data(cls, user_id: str) -> Dict[str, Dict[str, List[str]]]:
"""Takeout: Export GeneralFeedbackThreadUserModel user-based properties.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. A dict containing the user-relevant properties of
GeneralFeedbackThreadUserModel, i.e., which messages have been
read by the user (as a list of ids) in each thread.
"""
found_models = cls.get_all().filter(cls.user_id == user_id)
user_data = {}
for user_model in found_models:
user_data[user_model.thread_id] = {
'message_ids_read_by_user': user_model.message_ids_read_by_user
}
return user_data
class QuestionModel(base_models.VersionedModel):
"""Model for storing Questions.
The ID of instances of this class are in form of random hash of 12 chars.
"""
SNAPSHOT_METADATA_CLASS = QuestionSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = QuestionSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = QuestionCommitLogEntryModel
ALLOW_REVERT = True
# An object representing the question state data.
question_state_data = (
datastore_services.JsonProperty(indexed=False, required=True))
# The schema version for the question state data.
question_state_data_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# The ISO 639-1 code for the language this question is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The skill ids linked to this question.
linked_skill_ids = datastore_services.StringProperty(
indexed=True, repeated=True)
# The optional skill misconception ids marked as not relevant to the
# question.
# Note: Misconception ids are represented in two ways. In the Misconception
# domain object the id is a number. But in the context of a question
# (used here), the skill id needs to be included along with the
# misconception id, this is because questions can have multiple skills
# attached to it. Hence, the format for this field will be
# <skill-id>-<misconceptionid>.
inapplicable_skill_misconception_ids = datastore_services.StringProperty(
indexed=True, repeated=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'question_state_data': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_state_data_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'linked_skill_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'inapplicable_skill_misconception_ids':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def _get_new_id(cls) -> str:
"""Generates a unique ID for the question in the form of random hash
of 12 chars.
Returns:
new_id: str. ID of the new QuestionModel instance.
Raises:
Exception. The ID generator for QuestionModel is
producing too many collisions.
"""
for _ in range(base_models.MAX_RETRIES):
new_id = utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for QuestionModel is producing too many '
'collisions.')
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(QuestionModel, self)._trusted_commit(
committer_id, commit_type, commit_message, commit_cmds)
question_commit_log = QuestionCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False
)
question_commit_log.question_id = self.id
question_commit_log.update_timestamps()
question_commit_log.put()
# TODO(#13523): Change 'question_state_data' to TypedDict/Domain Object
# to remove Any used below.
@classmethod
def create(
cls,
question_state_data: Dict[str, Any],
language_code: str,
version: int,
linked_skill_ids: List[str],
inapplicable_skill_misconception_ids: List[str]
) -> QuestionModel:
"""Creates a new QuestionModel entry.
Args:
question_state_data: dict. An dict representing the question
state data.
language_code: str. The ISO 639-1 code for the language this
question is written in.
version: int. The version of the question.
linked_skill_ids: list(str). The skill ids linked to the question.
inapplicable_skill_misconception_ids: list(str). The optional
skill misconception ids marked as not applicable to the
question.
Returns:
QuestionModel. Instance of the new QuestionModel entry.
Raises:
Exception. A model with the same ID already exists.
"""
instance_id = cls._get_new_id()
question_model_instance = cls(
id=instance_id,
question_state_data=question_state_data,
language_code=language_code,
version=version,
linked_skill_ids=linked_skill_ids,
inapplicable_skill_misconception_ids=(
inapplicable_skill_misconception_ids))
return question_model_instance
@classmethod
def put_multi_questions(cls, questions: List[QuestionModel]) -> None:
"""Puts multiple question models into the datastore.
Args:
questions: list(Question). The list of question objects
to put into the datastore.
"""
cls.update_timestamps_multi(questions)
cls.put_multi(questions)
class GeneralFeedbackThreadModel(base_models.BaseModel):
"""Threads for each entity.
The id of instances of this class has the form
[entity_type].[entity_id].[generated_string]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of entity the thread is linked to.
entity_type = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the entity the thread is linked to.
entity_id = datastore_services.StringProperty(required=True, indexed=True)
# ID of the user who started the thread. This may be None if the feedback
# was given anonymously by a learner.
original_author_id = datastore_services.StringProperty(indexed=True)
# Latest status of the thread.
status = datastore_services.StringProperty(
default=STATUS_CHOICES_OPEN,
choices=STATUS_CHOICES,
required=True,
indexed=True,
)
# Latest subject of the thread.
subject = datastore_services.StringProperty(indexed=True, required=True)
# Summary text of the thread.
summary = datastore_services.TextProperty(indexed=False)
# Specifies whether this thread has a related suggestion.
has_suggestion = datastore_services.BooleanProperty(indexed=True,
default=False,
required=True)
# Cached value of the number of messages in the thread.
message_count = datastore_services.IntegerProperty(indexed=True, default=0)
# Cached text of the last message in the thread with non-empty content, or
# None if there is no such message.
last_nonempty_message_text = datastore_services.TextProperty(indexed=False)
# Cached ID for the user of the last message in the thread with non-empty
# content, or None if the message was made anonymously or if there is no
# such message.
last_nonempty_message_author_id = (datastore_services.StringProperty(
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
original_author_id and last_nonempty_message_author_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there
are multiple feedback threads relevant to a particular user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'entity_type': base_models.EXPORT_POLICY.EXPORTED,
'entity_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the original_author_id because we should not
# export internal user ids.
'original_author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.EXPORTED,
'subject': base_models.EXPORT_POLICY.EXPORTED,
'summary': base_models.EXPORT_POLICY.EXPORTED,
'has_suggestion': base_models.EXPORT_POLICY.EXPORTED,
'message_count': base_models.EXPORT_POLICY.EXPORTED,
'last_nonempty_message_text':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_nonempty_message_author_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_updated': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def get_field_names_for_takeout(cls) -> Dict[str, str]:
"""Indicates that the last_updated variable is exported under the
name "last_updated_msec" in Takeout.
"""
return dict(
super(cls, cls).get_field_names_for_takeout(),
**{'last_updated': 'last_updated_msec'})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackThreadModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.original_author_id == user_id,
cls.last_nonempty_message_author_id == user_id)).get(
keys_only=True) is not None
@classmethod
def export_data(
cls, user_id: str) -> Dict[str, Dict[str, Union[str, bool, None]]]:
"""Exports the data from GeneralFeedbackThreadModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackThreadModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackThreadModel] = (
cls.get_all().filter(cls.original_author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'entity_type':
feedback_model.entity_type,
'entity_id':
feedback_model.entity_id,
'status':
feedback_model.status,
'subject':
feedback_model.subject,
'has_suggestion':
feedback_model.has_suggestion,
'summary':
feedback_model.summary,
'message_count':
feedback_model.message_count,
'last_updated_msec':
utils.get_time_in_millisecs(feedback_model.last_updated)
}
return user_data
@classmethod
def generate_new_thread_id(cls, entity_type: str, entity_id: str) -> str:
"""Generates a new thread ID which is unique.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
Returns:
str. A thread ID that is different from the IDs of all
the existing threads within the given entity.
Raises:
Exception. There were too many collisions with existing thread IDs
when attempting to generate a new thread ID.
"""
for _ in range(_MAX_RETRIES):
thread_id = (
'%s.%s.%s%s' %
(entity_type, entity_id,
utils.base64_from_int(
int(utils.get_current_time_in_millisecs())),
utils.base64_from_int(utils.get_random_int(_RAND_RANGE))))
if not cls.get_by_id(thread_id):
return thread_id
raise Exception(
'New thread id generator is producing too many collisions.')
@classmethod
def create(cls, thread_id: str) -> GeneralFeedbackThreadModel:
"""Creates a new FeedbackThreadModel entry.
Args:
thread_id: str. Thread ID of the newly-created thread.
Returns:
GeneralFeedbackThreadModel. The newly created FeedbackThreadModel
instance.
Raises:
Exception. A thread with the given thread ID exists already.
"""
if cls.get_by_id(thread_id):
raise Exception('Feedback thread ID conflict on create.')
return cls(id=thread_id)
@classmethod
def get_threads(
cls,
entity_type: str,
entity_id: str,
limit: int = feconf.DEFAULT_QUERY_LIMIT
) -> Sequence[GeneralFeedbackThreadModel]:
"""Returns a list of threads associated with the entity, ordered
by their "last updated" field. The number of entities fetched is
limited by the `limit` argument to this method, whose default
value is equal to the default query limit.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
limit: int. The maximum possible number of items in the returned
list.
Returns:
list(GeneralFeedbackThreadModel). List of threads associated with
the entity. Doesn't include deleted entries.
"""
return cls.get_all().filter(cls.entity_type == entity_type).filter(
cls.entity_id == entity_id).order(-cls.last_updated).fetch(limit)
class QuestionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia question.
This should be used whenever the content blob of the question is not
needed (e.g. in search results, etc).
A QuestionSummaryModel instance stores the following information:
question_model_last_updated, question_model_created_on,
question_state_data.
The key of each instance is the question id.
"""
# Time when the question model was last updated (not to be
# confused with last_updated, which is the time when the
# question *summary* model was last updated).
question_model_last_updated = datastore_services.DateTimeProperty(
indexed=True, required=True)
# Time when the question model was created (not to be confused
# with created_on, which is the time when the question *summary*
# model was created).
question_model_created_on = datastore_services.DateTimeProperty(
indexed=True, required=True)
# The html content for the question.
question_content = (
datastore_services.TextProperty(indexed=False, required=True))
# The ID of the interaction.
interaction_id = (
datastore_services.StringProperty(indexed=True, required=True))
# The misconception ids addressed in the question. This includes
# tagged misconceptions ids as well as inapplicable misconception
# ids in the question.
misconception_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the QuestionModel
export_data function, and thus a new export_data function does not
need to be defined here.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because because noteworthy details that belong to this model have
already been exported as a part of the QuestionModel export_data
function.
"""
return dict(super(cls, cls).get_export_policy(), **{
'question_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_content': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconception_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class TaskEntryModel(base_models.BaseModel):
"""Model representation of an actionable task from the improvements tab.
The ID of a task has the form: "[entity_type].[entity_id].[entity_version].
[task_type].[target_type].[target_id]".
"""
# Utility field which results in a 20% speedup compared to querying by each
# of the invididual fields used to compose it.
# Value has the form: "[entity_type].[entity_id].[entity_version]".
composite_entity_id = datastore_services.StringProperty(required=True,
indexed=True)
# The type of entity a task entry refers to.
entity_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_ENTITY_TYPES)
# The ID of the entity a task entry refers to.
entity_id = datastore_services.StringProperty(required=True, indexed=True)
# The version of the entity a task entry refers to.
entity_version = datastore_services.IntegerProperty(required=True,
indexed=True)
# The type of task a task entry tracks.
task_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_TYPES)
# The type of sub-entity a task entry focuses on. Value is None when an
# entity does not have any meaningful sub-entities to target.
target_type = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_TARGET_TYPES)
# Uniquely identifies the sub-entity a task entry focuses on. Value is None
# when an entity does not have any meaningful sub-entities to target.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# A sentence generated by Oppia to describe why the task was created.
issue_description = datastore_services.StringProperty(default=None,
required=False,
indexed=True)
# Tracks the state/progress of a task entry.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=TASK_STATUS_CHOICES)
# ID of the user who closed the task, if any.
resolver_id = datastore_services.StringProperty(default=None,
required=False,
indexed=True)
# The date and time at which a task was closed or deprecated.
resolved_on = datastore_services.DateTimeProperty(default=None,
required=False,
indexed=True)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether any TaskEntryModel references the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.resolver_id == user_id).get() is not None
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to delete corresponding to a user:
resolver_id field.
It is okay to delete task entries since, after they are resolved, they
only act as a historical record. The removal just removes the historical
record.
"""
return base_models.DELETION_POLICY.DELETE
@classmethod
def apply_deletion_policy(cls, user_id: str) -> None:
"""Delete instances of TaskEntryModel for the user.
Args:
user_id: str. The ID of the user whose data should be deleted.
"""
task_entry_keys = (cls.query(cls.resolver_id == user_id).fetch(
keys_only=True))
datastore_services.delete_multi(task_entry_keys)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users resolve tasks.
"""
return (base_models.MODEL_ASSOCIATION_TO_USER.
ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user:
TaskEntryModel contains the ID of the user that acted on a task.
"""
return dict(
super(cls, cls).get_export_policy(), **{
'composite_entity_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'entity_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'task_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'issue_description': base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
'resolver_id': base_models.EXPORT_POLICY.EXPORTED,
'resolved_on': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'resolver_id': 'task_ids_resolved_by_user',
'issue_description': 'issue_descriptions',
'status': 'statuses',
'resolved_on': 'resolution_msecs'
}
@staticmethod
def export_data(user_id: str) -> Dict[str, List[str]]:
"""Returns the user-relevant properties of TaskEntryModels.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. The user-relevant properties of TaskEntryModel in a dict
format. In this case, we are returning all the ids of the tasks
which were closed by this user.
"""
task_ids_resolved_by_user = TaskEntryModel.query(
TaskEntryModel.resolver_id == user_id)
return {
'task_ids_resolved_by_user':
([t.id for t in task_ids_resolved_by_user]),
'issue_descriptions':
([t.issue_description for t in task_ids_resolved_by_user]),
'statuses': ([t.status for t in task_ids_resolved_by_user]),
'resolution_msecs':
([t.resolved_on for t in task_ids_resolved_by_user]),
}
@classmethod
def generate_task_id(cls, entity_type: str, entity_id: str,
entity_version: int, task_type: str, target_type: str,
target_id: str) -> str:
"""Generates a new task entry ID.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
task_type: str. The type of task a task entry tracks.
target_type: str. The type of sub-entity a task entry refers to.
target_id: str. The ID of the sub-entity a task entry refers to.
Returns:
str. The ID for the given task.
"""
return feconf.TASK_ENTRY_ID_TEMPLATE % (entity_type, entity_id,
entity_version, task_type,
target_type, target_id)
@classmethod
def generate_composite_entity_id(cls, entity_type: str, entity_id: str,
entity_version: int) -> str:
"""Generates a new composite_entity_id value.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
Returns:
str. The composite_entity_id for the given task.
"""
return feconf.COMPOSITE_ENTITY_ID_TEMPLATE % (entity_type, entity_id,
entity_version)
@classmethod
def create(cls,
entity_type: str,
entity_id: str,
entity_version: int,
task_type: str,
target_type: str,
target_id: str,
issue_description: Optional[str] = None,
status: str = constants.TASK_STATUS_OBSOLETE,
resolver_id: Optional[str] = None,
resolved_on: Optional[str] = None) -> str:
"""Creates a new task entry and puts it in storage.
Args:
entity_type: str. The type of entity a task entry refers to.
entity_id: str. The ID of the entity a task entry refers to.
entity_version: int. The version of the entity a task entry refers
to.
task_type: str. The type of task a task entry tracks.
target_type: str. The type of sub-entity a task entry refers to.
target_id: str. The ID of the sub-entity a task entry refers to.
issue_description: str. Sentence generated by Oppia to describe why
the task was created.
status: str. Tracks the state/progress of a task entry.
resolver_id: str. ID of the user who closed the task, if any.
resolved_on: str. The date and time at which a task was closed or
deprecated.
Returns:
str. The ID of the new task.
Raises:
Exception. A task corresponding to the provided identifier values
(entity_type, entity_id, entity_version, task_type, target_type,
target_id) already exists in storage.
"""
task_id = cls.generate_task_id(entity_type, entity_id, entity_version,
task_type, target_type, target_id)
if cls.get_by_id(task_id) is not None:
raise Exception('Task id %s already exists' % task_id)
composite_entity_id = cls.generate_composite_entity_id(
entity_type, entity_id, entity_version)
task_entry = cls(id=task_id,
composite_entity_id=composite_entity_id,
entity_type=entity_type,
entity_id=entity_id,
entity_version=entity_version,
task_type=task_type,
target_type=target_type,
target_id=target_id,
issue_description=issue_description,
status=status,
resolver_id=resolver_id,
resolved_on=resolved_on)
task_entry.update_timestamps()
task_entry.put()
return task_id
