class GeneralSuggestionModel(base_models.BaseModel):
"""Model to store suggestions made by Oppia users.
The ID of the suggestions created is the same as the ID of the thread
linked to the suggestion.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of suggestion.
suggestion_type = datastore_services.StringProperty(
required=True, indexed=True, choices=feconf.SUGGESTION_TYPE_CHOICES)
# The type of the target entity which the suggestion is linked to.
target_type = datastore_services.StringProperty(
required=True,
indexed=True,
choices=feconf.SUGGESTION_TARGET_TYPE_CHOICES)
# The ID of the target entity being suggested to.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# The version number of the target entity at the time of creation of the
# suggestion.
target_version_at_submission = datastore_services.IntegerProperty(
required=True, indexed=True)
# Status of the suggestion.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=STATUS_CHOICES)
# The ID of the author of the suggestion.
author_id = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the reviewer who accepted/rejected the suggestion.
final_reviewer_id = datastore_services.StringProperty(indexed=True)
# The change command linked to the suggestion. Contains the details of the
# change.
change_cmd = datastore_services.JsonProperty(required=True)
# The category to score the suggestor in. This field will contain 2 values
# separated by a ., the first will be a value from SCORE_TYPE_CHOICES and
# the second will be the subcategory of the suggestion.
score_category = (datastore_services.StringProperty(required=True,
indexed=True))
# The ISO 639-1 code used to query suggestions by language, or None if the
# suggestion type is not queryable by language.
language_code = datastore_services.StringProperty(indexed=True)
# A flag that indicates whether the suggestion is edited by the reviewer.
edited_by_reviewer = datastore_services.BooleanProperty(default=False,
indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id, and final_reviewer_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple unshared instance since there
are multiple suggestions per user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'suggestion_type': base_models.EXPORT_POLICY.EXPORTED,
'target_type': base_models.EXPORT_POLICY.EXPORTED,
'target_id': base_models.EXPORT_POLICY.EXPORTED,
'target_version_at_submission':
base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
# The author_id and final_reviewer_id are not exported since
# we do not want to reveal internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'final_reviewer_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'change_cmd': base_models.EXPORT_POLICY.EXPORTED,
'score_category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.EXPORTED,
'edited_by_reviewer': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralSuggestionModel exists for the user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.author_id == user_id,
cls.final_reviewer_id == user_id)).get(
keys_only=True) is not None
# TODO(#13523): Change 'change_cmd' to TypedDict/Domain Object
# to remove Any used below.
@classmethod
def create(cls, suggestion_type: str, target_type: str, target_id: str,
target_version_at_submission: int, status: str, author_id: str,
final_reviewer_id: str, change_cmd: Dict[str, Any],
score_category: str, thread_id: str,
language_code: Optional[str]) -> None:
"""Creates a new SuggestionModel entry.
Args:
suggestion_type: str. The type of the suggestion.
target_type: str. The type of target entity being edited.
target_id: str. The ID of the target entity being edited.
target_version_at_submission: int. The version number of the target
entity at the time of creation of the suggestion.
status: str. The status of the suggestion.
author_id: str. The ID of the user who submitted the suggestion.
final_reviewer_id: str. The ID of the reviewer who has
accepted/rejected the suggestion.
change_cmd: dict. The actual content of the suggestion.
score_category: str. The scoring category for the suggestion.
thread_id: str. The ID of the feedback thread linked to the
suggestion.
language_code: str|None. The ISO 639-1 code used to query
suggestions by language, or None if the suggestion type is not
queryable by language.
Raises:
Exception. There is already a suggestion with the given id.
"""
instance_id = thread_id
if cls.get_by_id(instance_id):
raise Exception('There is already a suggestion with the given'
' id: %s' % instance_id)
cls(id=instance_id,
suggestion_type=suggestion_type,
target_type=target_type,
target_id=target_id,
target_version_at_submission=target_version_at_submission,
status=status,
author_id=author_id,
final_reviewer_id=final_reviewer_id,
change_cmd=change_cmd,
score_category=score_category,
language_code=language_code).put()
@classmethod
def query_suggestions(
cls, query_fields_and_values: List[Tuple[str, str]]
) -> List['GeneralSuggestionModel']:
"""Queries for suggestions.
Args:
query_fields_and_values: list(tuple(str, str)). A list of queries.
The first element in each tuple is the field to be queried, and
the second element is the corresponding value to query for.
Returns:
list(SuggestionModel). A list of suggestions that match the given
query values, up to a maximum of feconf.DEFAULT_QUERY_LIMIT
suggestions.
"""
query = cls.query()
for (field, value) in query_fields_and_values:
if field not in feconf.ALLOWED_SUGGESTION_QUERY_FIELDS:
raise Exception('Not allowed to query on field %s' % field)
query = query.filter(getattr(cls, field) == value)
return cast(List[GeneralSuggestionModel],
query.fetch(feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_translation_suggestions_in_review_with_exp_id(
cls, exp_id: str,
language_code: str) -> List['GeneralSuggestionModel']:
"""Returns translation suggestions which are in review with target_id
== exp_id.
Args:
exp_id: str. Exploration ID matching the target ID of the
translation suggestions.
language_code: str. Language code.
Returns:
list(SuggestionModel). A list of translation suggestions in review
with target_id of exp_id. The number of returned results is capped
by feconf.DEFAULT_QUERY_LIMIT.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.language_code == language_code).filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT).filter(
cls.target_id == exp_id).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_translation_suggestion_ids_with_exp_ids(
cls, exp_ids: List[str]) -> List[str]:
"""Gets the ids of translation suggestions corresponding to
explorations with the given exploration ids.
Args:
exp_ids: list(str). List of exploration ids to query for.
Returns:
list(str). A list of translation suggestion ids that
correspond to the given exploration ids. Note: it is not
guaranteed that the suggestion ids returned are ordered by the
exploration ids in exp_ids.
"""
query = (cls.get_all().filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT).filter(
cls.target_id.IN(exp_ids)))
suggestion_models = []
offset, more = (0, True)
while more:
results = cast(
List[GeneralSuggestionModel],
query.fetch(feconf.DEFAULT_QUERY_LIMIT, offset=offset))
if len(results):
offset = offset + len(results)
suggestion_models.extend(results)
else:
more = False
return [suggestion_model.id for suggestion_model in suggestion_models]
@classmethod
def get_all_stale_suggestion_ids(cls) -> List[str]:
"""Gets the ids of the suggestions which were last updated before the
threshold time.
Returns:
list(str). A list of the ids of the suggestions that are stale.
"""
threshold_time = (
datetime.datetime.utcnow() -
datetime.timedelta(0, 0, 0, THRESHOLD_TIME_BEFORE_ACCEPT_IN_MSECS))
suggestion_models = cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.last_updated < threshold_time).fetch())
return [suggestion_model.id for suggestion_model in suggestion_models]
@classmethod
def get_suggestions_waiting_too_long_for_review(
cls) -> List['GeneralSuggestionModel']:
"""Returns a list of suggestions that have been waiting for a review
longer than SUGGESTION_REVIEW_WAIT_TIME_THRESHOLD_IN_DAYS days on the
Contributor Dashboard. MAX_NUMBER_OF_SUGGESTIONS_TO_EMAIL_ADMIN
suggestions are returned, sorted in descending order by their review
wait time.
Returns:
list(GeneralSuggestionModel). A list of suggestions, sorted in
descending order by their review wait time.
Raises:
Exception. If there are no suggestion types offered on the
Contributor Dashboard.
"""
if not feconf.CONTRIBUTOR_DASHBOARD_SUGGESTION_TYPES:
raise Exception(
'Expected the suggestion types offered on the Contributor '
'Dashboard to be nonempty.')
threshold_time = (datetime.datetime.utcnow() - datetime.timedelta(
days=SUGGESTION_REVIEW_WAIT_TIME_THRESHOLD_IN_DAYS))
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.last_updated < threshold_time).filter(
cls.suggestion_type.IN(
feconf.CONTRIBUTOR_DASHBOARD_SUGGESTION_TYPES)).order(
cls.last_updated).fetch(
MAX_NUMBER_OF_SUGGESTIONS_TO_EMAIL_ADMIN))
@classmethod
def get_in_review_suggestions_in_score_categories(
cls, score_categories: List[str],
user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all suggestions which are in review in the given
score_categories.
Args:
score_categories: list(str). List of score categories to query for.
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
Returns:
list(SuggestionModel). A list of suggestions that are in the given
score categories, which are in review, but not created by the
given user.
"""
if len(score_categories) == 0:
raise Exception('Received empty list of score categories')
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.score_category.IN(score_categories)).filter(
cls.author_id != user_id).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_in_review_translation_suggestions(
cls, user_id: str,
language_codes: List[str]) -> List['GeneralSuggestionModel']:
"""Gets all translation suggestions which are in review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
language_codes: list(str). The list of language codes.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which are in review, but not created by the given user.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT
).filter(cls.author_id != user_id).filter(
cls.language_code.IN(language_codes)).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_in_review_question_suggestions(
cls, user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all question suggestions which are in review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own suggestions, suggestions
authored by the user will be excluded.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which are in review, but not created by the given user.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_ADD_QUESTION).
filter(cls.author_id != user_id).fetch(feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_question_suggestions_waiting_longest_for_review(
cls) -> List['GeneralSuggestionModel']:
"""Returns MAX_QUESTION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS number
of question suggestions, sorted in descending order by review wait
time.
Returns:
list(GeneralSuggestionModel). A list of question suggestions,
sorted in descending order based on how long the suggestions have
been waiting for review.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type ==
feconf.SUGGESTION_TYPE_ADD_QUESTION).order(
cls.last_updated).fetch(
MAX_QUESTION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS))
@classmethod
def get_translation_suggestions_waiting_longest_for_review(
cls, language_code: str) -> List['GeneralSuggestionModel']:
"""Returns MAX_TRANSLATION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS
number of translation suggestions in the specified language code,
sorted in descending order by review wait time.
Args:
language_code: str. The ISO 639-1 language code of the translation
suggestions.
Returns:
list(GeneralSuggestionModel). A list of translation suggestions,
sorted in descending order based on how long the suggestions have
been waiting for review.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.status == STATUS_IN_REVIEW).filter(
cls.suggestion_type == feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT
).filter(cls.language_code == language_code).order(
cls.last_updated).fetch(
MAX_TRANSLATION_SUGGESTIONS_TO_FETCH_FOR_REVIEWER_EMAILS))
@classmethod
def get_user_created_suggestions_of_suggestion_type(
cls, suggestion_type: str,
user_id: str) -> List['GeneralSuggestionModel']:
"""Gets all suggestions of suggestion_type which the user has created.
Args:
suggestion_type: str. The type of suggestion to query for.
user_id: str. The id of the user trying to make this query.
Returns:
list(SuggestionModel). A list of suggestions that are of the given
type, which the given user has created.
"""
return cast(
List[GeneralSuggestionModel],
cls.get_all().filter(
cls.suggestion_type == suggestion_type).filter(
cls.author_id == user_id).order(-cls.created_on).fetch(
feconf.DEFAULT_QUERY_LIMIT))
@classmethod
def get_all_score_categories(cls) -> List[str]:
"""Gets all the score categories for which suggestions have been
created.
Returns:
list(str). A list of all the score categories.
"""
query_set = cast(
List[GeneralSuggestionModel],
cls.query(projection=['score_category'], distinct=True))
return [data.score_category for data in query_set]
# TODO(#13523): Change 'change_cmd' to TypedDict/Domain Object
# to remove Any used below.
@classmethod
def export_data(
cls, user_id: str
) -> Dict[str, Dict[str, Union[str, int, bool, Dict[str, Any], None]]]:
"""Exports the data from GeneralSuggestionModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralSuggestionModel.
"""
user_data = {}
suggestion_models = cast(
List[GeneralSuggestionModel],
cls.get_all().filter(cls.author_id == user_id).fetch())
for suggestion_model in suggestion_models:
user_data[suggestion_model.id] = {
'suggestion_type':
suggestion_model.suggestion_type,
'target_type':
suggestion_model.target_type,
'target_id':
suggestion_model.target_id,
'target_version_at_submission':
(suggestion_model.target_version_at_submission),
'status':
suggestion_model.status,
'change_cmd':
suggestion_model.change_cmd,
'language_code':
suggestion_model.language_code,
'edited_by_reviewer':
suggestion_model.edited_by_reviewer
}
return user_data
class GeneralFeedbackThreadModel(base_models.BaseModel):
"""Threads for each entity.
The id of instances of this class has the form
[entity_type].[entity_id].[generated_string]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of entity the thread is linked to.
entity_type = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the entity the thread is linked to.
entity_id = datastore_services.StringProperty(required=True, indexed=True)
# ID of the user who started the thread. This may be None if the feedback
# was given anonymously by a learner.
original_author_id = datastore_services.StringProperty(indexed=True)
# Latest status of the thread.
status = datastore_services.StringProperty(
default=STATUS_CHOICES_OPEN,
choices=STATUS_CHOICES,
required=True,
indexed=True,
)
# Latest subject of the thread.
subject = datastore_services.StringProperty(indexed=True, required=True)
# Summary text of the thread.
summary = datastore_services.TextProperty(indexed=False)
# Specifies whether this thread has a related suggestion.
has_suggestion = datastore_services.BooleanProperty(indexed=True,
default=False,
required=True)
# Cached value of the number of messages in the thread.
message_count = datastore_services.IntegerProperty(indexed=True, default=0)
# Cached text of the last message in the thread with non-empty content, or
# None if there is no such message.
last_nonempty_message_text = datastore_services.TextProperty(indexed=False)
# Cached ID for the user of the last message in the thread with non-empty
# content, or None if the message was made anonymously or if there is no
# such message.
last_nonempty_message_author_id = (datastore_services.StringProperty(
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
original_author_id and last_nonempty_message_author_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there
are multiple feedback threads relevant to a particular user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'entity_type': base_models.EXPORT_POLICY.EXPORTED,
'entity_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the original_author_id because we should not
# export internal user ids.
'original_author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.EXPORTED,
'subject': base_models.EXPORT_POLICY.EXPORTED,
'summary': base_models.EXPORT_POLICY.EXPORTED,
'has_suggestion': base_models.EXPORT_POLICY.EXPORTED,
'message_count': base_models.EXPORT_POLICY.EXPORTED,
'last_nonempty_message_text':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_nonempty_message_author_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_updated': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def get_field_names_for_takeout(cls) -> Dict[str, str]:
"""Indicates that the last_updated variable is exported under the
name "last_updated_msec" in Takeout.
"""
return dict(
super(cls, cls).get_field_names_for_takeout(),
**{'last_updated': 'last_updated_msec'})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackThreadModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.original_author_id == user_id,
cls.last_nonempty_message_author_id == user_id)).get(
keys_only=True) is not None
@classmethod
def export_data(
cls, user_id: str) -> Dict[str, Dict[str, Union[str, bool, None]]]:
"""Exports the data from GeneralFeedbackThreadModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackThreadModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackThreadModel] = (
cls.get_all().filter(cls.original_author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'entity_type':
feedback_model.entity_type,
'entity_id':
feedback_model.entity_id,
'status':
feedback_model.status,
'subject':
feedback_model.subject,
'has_suggestion':
feedback_model.has_suggestion,
'summary':
feedback_model.summary,
'message_count':
feedback_model.message_count,
'last_updated_msec':
utils.get_time_in_millisecs(feedback_model.last_updated)
}
return user_data
@classmethod
def generate_new_thread_id(cls, entity_type: str, entity_id: str) -> str:
"""Generates a new thread ID which is unique.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
Returns:
str. A thread ID that is different from the IDs of all
the existing threads within the given entity.
Raises:
Exception. There were too many collisions with existing thread IDs
when attempting to generate a new thread ID.
"""
for _ in python_utils.RANGE(_MAX_RETRIES):
thread_id = (
'%s.%s.%s%s' %
(entity_type, entity_id,
utils.base64_from_int(
int(utils.get_current_time_in_millisecs())),
utils.base64_from_int(utils.get_random_int(_RAND_RANGE))))
if not cls.get_by_id(thread_id):
return thread_id
raise Exception(
'New thread id generator is producing too many collisions.')
@classmethod
def create(cls, thread_id: str) -> 'GeneralFeedbackThreadModel':
"""Creates a new FeedbackThreadModel entry.
Args:
thread_id: str. Thread ID of the newly-created thread.
Returns:
GeneralFeedbackThreadModel. The newly created FeedbackThreadModel
instance.
Raises:
Exception. A thread with the given thread ID exists already.
"""
if cls.get_by_id(thread_id):
raise Exception('Feedback thread ID conflict on create.')
return cls(id=thread_id)
@classmethod
def get_threads(
cls,
entity_type: str,
entity_id: str,
limit: int = feconf.DEFAULT_QUERY_LIMIT
) -> Sequence['GeneralFeedbackThreadModel']:
"""Returns a list of threads associated with the entity, ordered
by their "last updated" field. The number of entities fetched is
limited by the `limit` argument to this method, whose default
value is equal to the default query limit.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
limit: int. The maximum possible number of items in the returned
list.
Returns:
list(GeneralFeedbackThreadModel). List of threads associated with
the entity. Doesn't include deleted entries.
"""
return cls.get_all().filter(cls.entity_type == entity_type).filter(
cls.entity_id == entity_id).order(-cls.last_updated).fetch(limit)
class AppFeedbackReportTicketModel(base_models.BaseModel):
"""Model for storing tickets created to triage feedback reports.
Instances of this model contain information about ticket and associated
reports.
The id of each model instance is created by combining the entity's
ticket_name hash, creation timestamp, and a random 16-character string.
"""
# A name for the ticket given by the maintainer, limited to 100 characters.
ticket_name = datastore_services.StringProperty(required=True,
indexed=True)
# The platform that the reports in this ticket pertain to.
platform = datastore_services.StringProperty(required=True,
indexed=True,
choices=PLATFORM_CHOICES)
# The Github repository that has the associated issue for this ticket. The
# possible values correspond to GITHUB_REPO_CHOICES. If None then the
# ticket has not yet been assigned to a Github issue.
github_issue_repo_name = datastore_services.StringProperty(
required=False, indexed=True, choices=GITHUB_REPO_CHOICES)
# The Github issue number that applies to this ticket.
github_issue_number = datastore_services.IntegerProperty(required=False,
indexed=True)
# Whether this ticket has been archived.
archived = datastore_services.BooleanProperty(required=True, indexed=True)
# The datetime in UTC that the newest report in this ticket was created on,
# to help with sorting tickets. If all reports assigned to this ticket have
# been reassigned to a different ticket then this timestamp is None.
newest_report_timestamp = datastore_services.DateTimeProperty(
required=False, indexed=True)
# A list of report IDs associated with this ticket.
report_ids = datastore_services.StringProperty(indexed=True, repeated=True)
@classmethod
def create(cls, entity_id: str, ticket_name: str, platform: str,
github_issue_repo_name: Optional[str],
github_issue_number: Optional[int],
newest_report_timestamp: datetime.datetime,
report_ids: List[str]) -> str:
"""Creates a new AppFeedbackReportTicketModel instance and returns its
ID.
Args:
entity_id: str. The ID used for this entity.
ticket_name: str. The name assigned to the ticket by the moderator.
platform: str. The platform that this ticket fixes an issue on,
corresponding to one of PLATFORM_CHOICES.
github_issue_repo_name: str. The name of the Github repo with the
associated Github issue for this ticket.
github_issue_number: int|None. The Github issue number associated
with the ticket, if it has one.
newest_report_timestamp: datetime.datetime. The date and time of the
newest report that is a part of this ticket, by submission
datetime.
report_ids: list(str). The report_ids that are a part of this
ticket.
Returns:
AppFeedbackReportModel. The newly created AppFeedbackReportModel
instance.
"""
ticket_entity = cls(id=entity_id,
ticket_name=ticket_name,
platform=platform,
github_issue_repo_name=github_issue_repo_name,
github_issue_number=github_issue_number,
archived=False,
newest_report_timestamp=newest_report_timestamp,
report_ids=report_ids)
ticket_entity.update_timestamps()
ticket_entity.put()
return entity_id
@classmethod
def generate_id(cls, ticket_name: str) -> str:
"""Generates key for the instance of AppFeedbackReportTicketModel
class in the required format with the arguments provided.
Args:
ticket_name: str. The name assigned to the ticket on creation.
Returns:
str. The generated ID for this entity using the current datetime in
milliseconds (as the entity's creation timestamp), a SHA1 hash of
the ticket_name, and a random string, of the form
'[creation_datetime_msec]:[hash(ticket_name)]:[random hash]'.
"""
current_datetime_in_msec = utils.get_time_in_millisecs(
datetime.datetime.utcnow())
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
name_hash = utils.convert_to_hash(ticket_name,
base_models.ID_LENGTH)
random_hash = utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
new_id = '%s.%s.%s' % (int(current_datetime_in_msec), name_hash,
random_hash)
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for AppFeedbackReportTicketModel is producing too'
'many collisions.')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any information directly corresponding to a
user.
"""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'ticket_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'platform': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_repo_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'github_issue_number':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'archived': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'newest_report_timestamp':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'report_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@staticmethod
def get_lowest_supported_role() -> str:
"""The lowest supported role for feedback report tickets will be
moderator.
"""
return feconf.ROLE_ID_MODERATOR
class JobModel(base_models.BaseModel):
"""Class representing a datastore entity for a long-running job."""
# The job type.
job_type = datastore_services.StringProperty(indexed=True)
# The time at which the job was queued, in milliseconds since the epoch.
time_queued_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was started, in milliseconds since the epoch.
# This is never set if the job was canceled before it was started.
time_started_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was completed, failed or canceled, in
# milliseconds since the epoch.
time_finished_msec = datastore_services.FloatProperty(indexed=True)
# The current status code for the job.
status_code = datastore_services.StringProperty(
indexed=True,
default=STATUS_CODE_NEW,
choices=[
STATUS_CODE_NEW, STATUS_CODE_QUEUED, STATUS_CODE_STARTED,
STATUS_CODE_COMPLETED, STATUS_CODE_FAILED, STATUS_CODE_CANCELED
])
# Any metadata for the job, such as the root pipeline id for mapreduce
# jobs.
metadata = datastore_services.JsonProperty(indexed=False)
# The output of the job. This is only populated if the job has status code
# STATUS_CODE_COMPLETED, and is None otherwise. If populated, this is
# expected to be a list of strings.
output = datastore_services.JsonProperty(indexed=False)
# The error message, if applicable. Only populated if the job has status
# code STATUS_CODE_FAILED or STATUS_CODE_CANCELED; None otherwise.
error = datastore_services.TextProperty(indexed=False)
# Whether the datastore models associated with this job have been cleaned
# up (i.e., deleted).
has_been_cleaned_up = (datastore_services.BooleanProperty(default=False,
indexed=True))
# Store additional params passed with job.
additional_job_params = datastore_services.JsonProperty(default=None)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'job_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_queued_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_started_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_finished_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'metadata': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'output': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'error': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'has_been_cleaned_up':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_job_params':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@property
def is_cancelable(self) -> bool:
"""Checks if the job is cancelable.
Returns:
bool. Whether the job's status_code is 'queued' or 'started'.
"""
# Whether the job is currently in 'queued' or 'started' status.
return self.status_code in [STATUS_CODE_QUEUED, STATUS_CODE_STARTED]
@classmethod
def get_all_unfinished_jobs(cls, limit: int) -> Sequence['JobModel']:
"""Gets at most `limit` unfinished jobs.
Args:
limit: int. A limit on the number of jobs to return.
Returns:
list(JobModel) or None. A list of at most `limit` number
of unfinished jobs.
"""
return cls.query().filter(
JobModel.status_code.IN([
STATUS_CODE_QUEUED, STATUS_CODE_STARTED
])).order(-cls.time_queued_msec).fetch(limit)
@classmethod
def get_unfinished_jobs(cls, job_type: str) -> datastore_services.Query:
"""Gets jobs that are unfinished.
Args:
job_type: str. The type of jobs that may be unfinished.
Returns:
list(JobModel) or None. A list of all jobs that belong
to the given job_type.
"""
return cls.query().filter(cls.job_type == job_type).filter(
JobModel.status_code.IN([STATUS_CODE_QUEUED, STATUS_CODE_STARTED]))
@classmethod
def do_unfinished_jobs_exist(cls, job_type: str) -> bool:
"""Checks if unfinished jobs exist.
Args:
job_type: str. Type of job for which to check.
Returns:
bool. True if unfinished jobs exist, otherwise false.
"""
return bool(cls.get_unfinished_jobs(job_type).count(limit=1))
class GeneralFeedbackMessageModel(base_models.BaseModel):
"""Feedback messages. One or more of these messages make a thread.
The id of instances of this class has the form [thread_id].[message_id]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# ID corresponding to an entry of FeedbackThreadModel.
thread_id = datastore_services.StringProperty(required=True, indexed=True)
# 0-based sequential numerical ID. Sorting by this field will create the
# thread in chronological order.
message_id = datastore_services.IntegerProperty(required=True,
indexed=True)
# ID of the user who posted this message. This may be None if the feedback
# was given anonymously by a learner.
author_id = datastore_services.StringProperty(indexed=True)
# New thread status. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the status changes.
updated_status = (datastore_services.StringProperty(choices=STATUS_CHOICES,
indexed=True))
# New thread subject. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the subject changes.
updated_subject = datastore_services.StringProperty(indexed=True)
# Message text. Allowed not to exist (e.g. post only to update the status).
text = datastore_services.TextProperty(indexed=False)
# Whether the incoming message is received by email (as opposed to via
# the web).
received_via_email = datastore_services.BooleanProperty(default=False,
indexed=True,
required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple feedback messages relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'thread_id': base_models.EXPORT_POLICY.EXPORTED,
'message_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the author_id because we should not export
# internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'updated_status': base_models.EXPORT_POLICY.EXPORTED,
'updated_subject': base_models.EXPORT_POLICY.EXPORTED,
'text': base_models.EXPORT_POLICY.EXPORTED,
'received_via_email': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackMessageModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@classmethod
def export_data(
cls,
user_id: str) -> Dict[str, Dict[str, Union[str, int, bool, None]]]:
"""Exports the data from GeneralFeedbackMessageModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackMessageModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackMessageModel] = (
cls.get_all().filter(cls.author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'thread_id': feedback_model.thread_id,
'message_id': feedback_model.message_id,
'updated_status': feedback_model.updated_status,
'updated_subject': feedback_model.updated_subject,
'text': feedback_model.text,
'received_via_email': feedback_model.received_via_email
}
return user_data
@classmethod
def _generate_id(cls, thread_id: str, message_id: int) -> str:
"""Generates full message ID given the thread ID and message ID.
Args:
thread_id: str. Thread ID of the thread to which the message
belongs.
message_id: int. Message ID of the message.
Returns:
str. Full message ID.
"""
return '.'.join([thread_id, python_utils.UNICODE(message_id)])
@property
def entity_id(self) -> str:
"""Returns the entity_id corresponding to this thread instance.
Returns:
str. The entity_id.
"""
return self.id.split('.')[1]
@property
def entity_type(self) -> str:
"""Returns the entity_type corresponding to this thread instance.
Returns:
str. The entity_type.
"""
return self.id.split('.')[0]
@classmethod
def create(
cls,
message_identifier: feedback_domain.FullyQualifiedMessageIdentifier
) -> 'GeneralFeedbackMessageModel':
"""Creates a new GeneralFeedbackMessageModel entry.
Args:
message_identifier: FullyQualifiedMessageIdentifier. The message
identifier consists of the thread_id and its corresponding
message_id.
Returns:
GeneralFeedbackMessageModel. Instance of the new
GeneralFeedbackMessageModel entry.
Raises:
Exception. A message with the same ID already exists
in the given thread.
"""
return cls.create_multi([message_identifier])[0]
@classmethod
def create_multi(
cls, message_identifiers: List[
feedback_domain.FullyQualifiedMessageIdentifier]
) -> List['GeneralFeedbackMessageModel']:
"""Creates a new GeneralFeedbackMessageModel entry for each
(thread_id, message_id) pair.
Args:
message_identifiers: list(FullyQualifiedMessageIdentifier). Each
message identifier consists of the thread_id and its
corresponding message_id.
Returns:
list(GeneralFeedbackMessageModel). Instances of the new
GeneralFeedbackMessageModel entries.
Raises:
Exception. The number of thread_ids must be equal to the number of
message_ids.
Exception. A message with the same ID already exists
in the given thread.
"""
thread_ids = [
message_identifier.thread_id
for message_identifier in message_identifiers
]
message_ids = [
message_identifier.message_id
for message_identifier in message_identifiers
]
# Generate the new ids.
instance_ids = [
cls._generate_id(thread_id, message_id)
for thread_id, message_id in python_utils.ZIP(
thread_ids, message_ids)
]
# Check if the new ids are valid.
current_instances = cls.get_multi(instance_ids)
conflict_ids = [
current_instance.id for current_instance in current_instances
if current_instance is not None
]
if len(conflict_ids) > 0:
raise Exception(
'The following feedback message ID(s) conflicted on '
'create: %s' % (' '.join(conflict_ids)))
return [cls(id=instance_id) for instance_id in instance_ids]
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get().
@classmethod
def get( # type: ignore[override]
cls,
thread_id: str,
message_id: int,
strict: bool = True) -> Optional['GeneralFeedbackMessageModel']:
"""Gets the GeneralFeedbackMessageModel entry for the given ID. Raises
an error if no undeleted message with the given ID is found and
strict == True.
Args:
thread_id: str. ID of the thread.
message_id: int. ID of the message.
strict: bool. Whether to raise an error if no FeedbackMessageModel
entry is found for the given IDs.
Returns:
GeneralFeedbackMessageModel or None. If strict == False and no
undeleted message with the given message_id exists in the
datastore, then returns None. Otherwise, returns the
GeneralFeedbackMessageModel instance that corresponds to the
given ID.
Raises:
EntityNotFoundError. The value of strict is True and either
(i) message ID is not valid
(ii) message is marked as deleted.
No error will be raised if strict == False.
"""
instance_id = cls._generate_id(thread_id, message_id)
return super(GeneralFeedbackMessageModel, cls).get(instance_id,
strict=strict)
@classmethod
def get_messages(
cls, thread_id: str) -> Sequence['GeneralFeedbackMessageModel']:
"""Returns a list of messages in the given thread. The number of
messages returned is capped by feconf.DEFAULT_QUERY_LIMIT.
Args:
thread_id: str. ID of the thread.
Returns:
list(GeneralFeedbackMessageModel). A list of messages in the
given thread, up to a maximum of feconf.DEFAULT_QUERY_LIMIT
messages.
"""
return cls.get_all().filter(cls.thread_id == thread_id).fetch(
feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_most_recent_message(
cls, thread_id: str) -> 'GeneralFeedbackMessageModel':
"""Returns the last message in the thread.
Args:
thread_id: str. ID of the thread.
Returns:
GeneralFeedbackMessageModel. Last message in the thread.
"""
thread = GeneralFeedbackThreadModel.get_by_id(thread_id)
message = cls.get(thread_id, thread.message_count - 1)
# Ruling out the possibility of None for mypy type checking.
assert message is not None
return message
@classmethod
def get_message_count(cls, thread_id: str) -> int:
"""Returns the number of messages in the thread. Includes the
deleted entries.
Args:
thread_id: str. ID of the thread.
Returns:
int. Number of messages in the thread.
"""
return cls.get_message_counts([thread_id])[0]
@classmethod
def get_message_counts(cls, thread_ids: List[str]) -> List[int]:
"""Returns a list containing the number of messages in the threads.
Includes the deleted entries.
Args:
thread_ids: list(str). ID of the threads.
Returns:
list(int). List of the message counts for the threads.
"""
thread_models = GeneralFeedbackThreadModel.get_multi(thread_ids)
assert None not in thread_models
return [
thread_model.message_count if thread_model else None
for thread_model in thread_models
]
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_all_messages(
cls, page_size: int, urlsafe_start_cursor: Optional[str]
) -> Tuple[Sequence['GeneralFeedbackMessageModel'], Optional[str], bool]:
"""Fetches a list of all the messages sorted by their last updated
attribute.
Args:
page_size: int. The maximum number of messages to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned messages starts from this datastore cursor.
Otherwise, the returned messages start from the beginning
of the full list of messages.
Returns:
3-tuple of (results, cursor, more). Where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
return cls._fetch_page_sorted_by_last_updated(cls.query(), page_size,
urlsafe_start_cursor)
class CollectionRightsModel(base_models.VersionedModel):
"""Storage model for rights related to a collection.
The id of each instance is the id of the corresponding collection.
"""
SNAPSHOT_METADATA_CLASS = CollectionRightsSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = CollectionRightsSnapshotContentModel
ALLOW_REVERT = False
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to voiceover this collection.
voice_artist_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# Whether this collection is owned by the community.
community_owned = (
datastore_services.BooleanProperty(indexed=True, default=False))
# For private collections, whether this collection can be viewed
# by anyone who has the URL. If the collection is not private, this
# setting is ignored.
viewable_if_private = (
datastore_services.BooleanProperty(indexed=True, default=False))
# Time, in milliseconds, when the collection was first published.
first_published_msec = (
datastore_services.FloatProperty(indexed=True, default=None))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE,
constants.ACTIVITY_STATUS_PUBLIC
]
)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, voice_artist_ids, editor_ids,
and owner_ids fields.
"""
return (
base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE
)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users contribute to collections and have varying rights.
"""
return (
base_models
.MODEL_ASSOCIATION_TO_USER
.ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'owner_ids': 'owned_collection_ids',
'editor_ids': 'editable_collection_ids',
'voice_artist_ids': 'voiced_collection_ids',
'viewer_ids': 'viewable_collection_ids'
}
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export/delete corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'owner_ids': base_models.EXPORT_POLICY.EXPORTED,
'editor_ids': base_models.EXPORT_POLICY.EXPORTED,
'voice_artist_ids': base_models.EXPORT_POLICY.EXPORTED,
'viewer_ids': base_models.EXPORT_POLICY.EXPORTED,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewable_if_private': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'first_published_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionRightsModel references the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.voice_artist_ids == user_id,
cls.viewer_ids == user_id
)).get(keys_only=True) is not None
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def save(
self,
committer_id: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Updates the collection rights model by applying the given
commit_cmds, then saves it.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and additional arguments for that command.
"""
super(CollectionRightsModel, self).commit(
committer_id, commit_message, commit_cmds)
# TODO(#13523): Change 'model_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@staticmethod
def convert_to_valid_dict(model_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the CollectionRightsModel dict.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
model_dict: dict. The content of the model. Some fields and field
values might no longer exist in the CollectionRightsModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
# The status field could historically take the value 'publicized', this
# value is now equivalent to 'public'.
if model_dict['status'] == 'publicized':
model_dict['status'] = constants.ACTIVITY_STATUS_PUBLIC
# The voice_artist_ids field was previously named translator_ids. We
# need to move the values from translator_ids field to voice_artist_ids
# and delete translator_ids.
if 'translator_ids' in model_dict and model_dict['translator_ids']:
model_dict['voice_artist_ids'] = model_dict['translator_ids']
del model_dict['translator_ids']
# We need to remove pseudonymous IDs from all the fields that contain
# user IDs.
for field_name in (
'owner_ids', 'editor_ids', 'voice_artist_ids', 'viewer_ids'):
model_dict[field_name] = [
user_id for user_id in model_dict[field_name]
if not utils.is_pseudonymous_id(user_id)
]
return model_dict
# TODO(#13523): Change 'snapshot_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
def _reconstitute(
self, snapshot_dict: Dict[str, Any]
) -> CollectionRightsModel:
"""Populates the model instance with the snapshot.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the the snapshot.
"""
self.populate(
**CollectionRightsModel.convert_to_valid_dict(snapshot_dict))
return self
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this overrides the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id,
commit_type,
commit_message,
commit_cmds,
additional_models
)
snapshot_metadata_model = models_to_put['snapshot_metadata_model']
snapshot_metadata_model.content_user_ids = list(sorted(
set(self.owner_ids) |
set(self.editor_ids) |
set(self.voice_artist_ids) |
set(self.viewer_ids)
))
commit_cmds_user_ids = set()
for commit_cmd in commit_cmds:
user_id_attribute_names = next(
cmd['user_id_attribute_names']
for cmd in feconf.COLLECTION_RIGHTS_CHANGE_ALLOWED_COMMANDS
if cmd['name'] == commit_cmd['cmd']
)
for user_id_attribute_name in user_id_attribute_names:
commit_cmds_user_ids.add(commit_cmd[user_id_attribute_name])
snapshot_metadata_model.commit_cmds_user_ids = list(
sorted(commit_cmds_user_ids))
# Create and delete events will already be recorded in the
# CollectionModel.
if commit_type not in ['create', 'delete']:
collection_commit_log = CollectionCommitLogEntryModel(
id=('rights-%s-%s' % (self.id, self.version)),
user_id=committer_id,
collection_id=self.id,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=None,
post_commit_status=self.status,
post_commit_community_owned=self.community_owned,
post_commit_is_private=(
self.status == constants.ACTIVITY_STATUS_PRIVATE)
)
return {
'snapshot_metadata_model': (
models_to_put['snapshot_metadata_model']),
'snapshot_content_model': (
models_to_put['snapshot_content_model']),
'commit_log_model': collection_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
return models_to_put
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of CollectionRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of CollectionRightsModel
in a python dict format. In this case, we are returning all the
ids of collections that the user is connected to, so they either
own, edit, voice, or have permission to view.
"""
owned_collections = cls.get_all().filter(cls.owner_ids == user_id)
editable_collections = cls.get_all().filter(cls.editor_ids == user_id)
voiced_collections = (
cls.get_all().filter(cls.voice_artist_ids == user_id))
viewable_collections = cls.get_all().filter(cls.viewer_ids == user_id)
owned_collection_ids = [col.key.id() for col in owned_collections]
editable_collection_ids = [col.key.id() for col in editable_collections]
voiced_collection_ids = [col.key.id() for col in voiced_collections]
viewable_collection_ids = [col.key.id() for col in viewable_collections]
return {
'owned_collection_ids': owned_collection_ids,
'editable_collection_ids': editable_collection_ids,
'voiced_collection_ids': voiced_collection_ids,
'viewable_collection_ids': viewable_collection_ids
}
class CollectionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia collection.
This should be used whenever the content blob of the collection is not
needed (e.g. search results, etc).
A CollectionSummaryModel instance stores the following information:
id, title, category, objective, language_code, tags, ratings,
last_updated, created_on, status (private, public),
community_owned, owner_ids, editor_ids,
viewer_ids, version.
The key of each instance is the collection id.
"""
# What this collection is called.
title = datastore_services.StringProperty(required=True)
# The category this collection belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this collection.
objective = datastore_services.TextProperty(required=True, indexed=False)
# The ISO 639-1 code for the language this collection is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# Tags associated with this collection.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# Aggregate user-assigned ratings of the collection.
ratings = datastore_services.JsonProperty(default=None, indexed=False)
# Time when the collection model was last updated (not to be
# confused with last_updated, which is the time when the
# collection *summary* model was last updated).
collection_model_last_updated = (
datastore_services.DateTimeProperty(indexed=True))
# Time when the collection model was created (not to be confused
# with created_on, which is the time when the collection *summary*
# model was created).
collection_model_created_on = (
datastore_services.DateTimeProperty(indexed=True))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE,
constants.ACTIVITY_STATUS_PUBLIC
]
)
# Whether this collection is owned by the community.
community_owned = (
datastore_services.BooleanProperty(required=True, indexed=True))
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who have contributed (humans who have made a
# positive (not just a revert) change to the collection's content).
# NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be
# synchronized, meaning that the keys in contributors_summary need be
# equal to the contributor_ids list.
contributor_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# A dict representing the contributors of non-trivial commits to this
# collection. Each key of this dict is a user_id, and the corresponding
# value is the number of non-trivial commits that the user has made.
contributors_summary = (
datastore_services.JsonProperty(default={}, indexed=False))
# The version number of the collection after this commit. Only populated
# for commits to an collection (as opposed to its rights, etc.).
version = datastore_services.IntegerProperty()
# The number of nodes(explorations) that are within this collection.
node_count = datastore_services.IntegerProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, editor_ids, owner_ids, contributor_ids,
and contributors_summary fields.
"""
return (
base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE
)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the
CollectionRightsModel, and thus does not need an export_data
function.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because noteworthy details that belong to this model have already been
exported as a part of the CollectionRightsModel.
"""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributors_summary': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_count': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionSummaryModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id,
cls.contributor_ids == user_id)).get(keys_only=True) is not None
@classmethod
def get_non_private(cls) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with non-private collection summary models.
Returns:
iterable. An iterable with non-private collection summary models.
"""
return cls.get_all().filter(
cls.status != constants.ACTIVITY_STATUS_PRIVATE
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_private_at_least_viewable(
cls, user_id: str
) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with private collection summary models that are
at least viewable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with private collection summary models that
are at least viewable by the given user.
"""
return cls.get_all().filter(
cls.status == constants.ACTIVITY_STATUS_PRIVATE
).filter(
datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id
)
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_at_least_editable(
cls, user_id: str
) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with collection summary models that are at least
editable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with collection summary models that are at
least viewable by the given user.
"""
return CollectionSummaryModel.get_all().filter(
datastore_services.any_of(
CollectionSummaryModel.owner_ids == user_id,
CollectionSummaryModel.editor_ids == user_id
)
).fetch(feconf.DEFAULT_QUERY_LIMIT)
class ExplorationModel(base_models.VersionedModel):
"""Versioned storage model for an Oppia exploration.
This class should only be imported by the exploration services file
and the exploration model test file.
"""
SNAPSHOT_METADATA_CLASS = ExplorationSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = ExplorationSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = ExplorationCommitLogEntryModel
ALLOW_REVERT = True
# What this exploration is called.
title = datastore_services.StringProperty(required=True)
# The category this exploration belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this exploration.
objective = datastore_services.TextProperty(default='', indexed=False)
# The ISO 639-1 code for the language this exploration is written in.
language_code = datastore_services.StringProperty(
default=constants.DEFAULT_LANGUAGE_CODE, indexed=True)
# Tags (topics, skills, concepts, etc.) associated with this
# exploration.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# A blurb for this exploration.
blurb = datastore_services.TextProperty(default='', indexed=False)
# 'Author notes' for this exploration.
author_notes = datastore_services.TextProperty(default='', indexed=False)
# The version of the states blob schema.
states_schema_version = datastore_services.IntegerProperty(required=True,
default=0,
indexed=True)
# The name of the initial state of this exploration.
init_state_name = (datastore_services.StringProperty(required=True,
indexed=True))
# A dict representing the states of this exploration. This dict should
# not be empty.
states = datastore_services.JsonProperty(default={}, indexed=False)
# The dict of parameter specifications associated with this exploration.
# Each specification is a dict whose keys are param names and whose values
# are each dicts with a single key, 'obj_type', whose value is a string.
param_specs = datastore_services.JsonProperty(default={}, indexed=False)
# The list of parameter changes to be performed once at the start of a
# reader's encounter with an exploration.
param_changes = (datastore_services.JsonProperty(repeated=True,
indexed=False))
# A boolean indicating whether automatic text-to-speech is enabled in
# this exploration.
auto_tts_enabled = (datastore_services.BooleanProperty(default=True,
indexed=True))
# A boolean indicating whether correctness feedback is enabled in this
# exploration.
correctness_feedback_enabled = datastore_services.BooleanProperty(
default=False, indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'title':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'blurb':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'author_notes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'states_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'init_state_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'states':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'param_specs':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'param_changes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'auto_tts_enabled':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'correctness_feedback_enabled':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_exploration_count(cls) -> int:
"""Returns the total number of explorations."""
return cls.get_all().count()
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't be allowed.
def _prepare_additional_models(
self) -> Mapping[str, base_models.BaseModel]:
"""Prepares additional models needed for the commit process.
Returns:
dict(str, BaseModel). Additional models needed for
the commit process. Contains the ExplorationRightsModel.
"""
return {'rights_model': ExplorationRightsModel.get_by_id(self.id)}
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id, commit_type, commit_message, commit_cmds,
additional_models)
# The cast is needed because the additional_models is list of BaseModels
# and we want to hint the mypy that this is ExplorationRightsModel.
exploration_rights_model = cast(ExplorationRightsModel,
additional_models['rights_model'])
exploration_commit_log = ExplorationCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, exploration_rights_model.status,
exploration_rights_model.community_owned)
exploration_commit_log.exploration_id = self.id
return {
'snapshot_metadata_model':
models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': exploration_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.delete_multi().
@classmethod
def delete_multi( # type: ignore[override]
cls,
entity_ids: List[str],
committer_id: str,
commit_message: str,
force_deletion: bool = False) -> None:
"""Deletes the given cls instances with the given entity_ids.
Note that this extends the superclass method.
Args:
entity_ids: list(str). Ids of entities to delete.
committer_id: str. The user_id of the user who committed the change.
commit_message: str. The commit description message.
force_deletion: bool. If True these models are deleted completely
from storage, otherwise there are only marked as deleted.
Default is False.
"""
super(ExplorationModel,
cls).delete_multi(entity_ids,
committer_id,
commit_message,
force_deletion=force_deletion)
if not force_deletion:
commit_log_models = []
exp_rights_models = ExplorationRightsModel.get_multi(
entity_ids, include_deleted=True)
versioned_models = cls.get_multi(entity_ids, include_deleted=True)
versioned_and_exp_rights_models = zip(versioned_models,
exp_rights_models)
for model, rights_model in versioned_and_exp_rights_models:
# Ruling out the possibility of None for mypy type checking.
assert model is not None
assert rights_model is not None
exploration_commit_log = ExplorationCommitLogEntryModel.create(
model.id, model.version, committer_id,
feconf.COMMIT_TYPE_DELETE, commit_message,
[{
'cmd': cls.CMD_DELETE_COMMIT
}], rights_model.status, rights_model.community_owned)
exploration_commit_log.exploration_id = model.id
commit_log_models.append(exploration_commit_log)
ExplorationCommitLogEntryModel.update_timestamps_multi(
commit_log_models)
datastore_services.put_multi(commit_log_models)
# TODO(#13523): Change snapshot of this model to TypedDict/Domain Object
# to remove Any used below.
@staticmethod
def convert_to_valid_dict(snapshot_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the ExplorationModel dict.
Some old ExplorationModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a ExplorationModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict. The content of the model. Some fields and field
values might no longer exist in the ExplorationModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
if 'skill_tags' in snapshot_dict:
del snapshot_dict['skill_tags']
if 'default_skin' in snapshot_dict:
del snapshot_dict['default_skin']
if 'skin_customizations' in snapshot_dict:
del snapshot_dict['skin_customizations']
return snapshot_dict
# TODO(#13523): Change 'snapshot_dict' to TypedDict/Domain Object
# to remove Any used below.
def _reconstitute(self, snapshot_dict: Dict[str, Any]) -> ExplorationModel:
"""Populates the model instance with the snapshot.
Some old ExplorationSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a ExplorationModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the snapshot.
"""
self.populate(**ExplorationModel.convert_to_valid_dict(snapshot_dict))
return self
class SkillModel(base_models.VersionedModel):
"""Model for storing Skills.
This class should only be imported by the skill services file
and the skill model test file.
"""
SNAPSHOT_METADATA_CLASS = SkillSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = SkillSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = SkillCommitLogEntryModel
ALLOW_REVERT = False
# The description of the skill.
description = datastore_services.StringProperty(required=True,
indexed=True)
# The schema version for each of the misconception dicts.
misconceptions_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# The schema version for each of the rubric dicts.
rubric_schema_version = datastore_services.IntegerProperty(required=True,
indexed=True)
# A list of misconceptions associated with the skill, in which each
# element is a dict.
misconceptions = (datastore_services.JsonProperty(repeated=True,
indexed=False))
# The rubrics for the skill that explain each difficulty level.
rubrics = datastore_services.JsonProperty(repeated=True, indexed=False)
# The ISO 639-1 code for the language this skill is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The schema version for the skill_contents.
skill_contents_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# A dict representing the skill contents.
skill_contents = datastore_services.JsonProperty(indexed=False)
# The prerequisite skills for the skill.
prerequisite_skill_ids = (datastore_services.StringProperty(repeated=True,
indexed=True))
# The id to be used by the next misconception added.
next_misconception_id = (datastore_services.IntegerProperty(required=True,
indexed=False))
# The id that the skill is merged into, in case the skill has been
# marked as duplicate to another one and needs to be merged.
# This is an optional field.
superseding_skill_id = datastore_services.StringProperty(indexed=True)
# A flag indicating whether deduplication is complete for this skill.
# It will initially be False, and set to true only when there is a value
# for superseding_skill_id and the merge was completed.
all_questions_merged = (datastore_services.BooleanProperty(indexed=True,
required=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@classmethod
def get_merged_skills(cls) -> List['SkillModel']:
"""Returns the skill models which have been merged.
Returns:
list(SkillModel). List of skill models which have been merged.
"""
return [
skill for skill in cls.query()
if (skill.superseding_skill_id is not None and (
len(skill.superseding_skill_id) > 0))
]
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(self, committer_id: str, commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(SkillModel, self)._trusted_commit(committer_id, commit_type,
commit_message, commit_cmds)
skill_commit_log_entry = SkillCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False)
skill_commit_log_entry.skill_id = self.id
skill_commit_log_entry.update_timestamps()
skill_commit_log_entry.put()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconceptions_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rubric_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconceptions': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rubrics': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_contents_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'prerequisite_skill_ids':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_misconception_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'superseding_skill_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'all_questions_merged':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_by_description(cls, description: str) -> Optional['SkillModel']:
"""Gets SkillModel by description. Returns None if the skill with
description doesn't exist.
Args:
description: str. The description of the skill.
Returns:
SkillModel|None. The skill model of the skill or None if not
found.
"""
return cls.get_all().filter(cls.description == description).get()
class TopicModel(base_models.VersionedModel):
"""Model for storing Topics.
This class should only be imported by the topic services file
and the topic model test file.
"""
SNAPSHOT_METADATA_CLASS = TopicSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = TopicSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = TopicCommitLogEntryModel
ALLOW_REVERT = False
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (
datastore_services.StringProperty(required=True, indexed=True))
# The abbreviated name of the topic.
abbreviated_name = (
datastore_services.StringProperty(indexed=True, default=''))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
# The thumbnail size in bytes of the topic.
thumbnail_size_in_bytes = (
datastore_services.IntegerProperty(indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# This consists of the list of objects referencing canonical stories that
# are part of this topic.
canonical_story_references = (
datastore_services.JsonProperty(repeated=True, indexed=False))
# This consists of the list of objects referencing additional stories that
# are part of this topic.
additional_story_references = (
datastore_services.JsonProperty(repeated=True, indexed=False))
# The schema version for the story reference object on each of the above 2
# lists.
story_reference_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# This consists of the list of uncategorized skill ids that are not part of
# any subtopic.
uncategorized_skill_ids = (
datastore_services.StringProperty(repeated=True, indexed=True))
# The list of subtopics that are part of the topic.
subtopics = datastore_services.JsonProperty(repeated=True, indexed=False)
# The schema version of the subtopic dict.
subtopic_schema_version = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The id for the next subtopic.
next_subtopic_id = datastore_services.IntegerProperty(required=True)
# The ISO 639-1 code for the language this topic is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The url fragment of the topic.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# Whether to show practice tab in the Topic viewer page.
practice_tab_is_displayed = datastore_services.BooleanProperty(
required=True, default=False)
# The content of the meta tag in the Topic viewer page.
meta_tag_content = datastore_services.StringProperty(
indexed=True, default='')
# The page title fragment used in the Topic viewer web page.
# For example, if the full Topic viewer web page title is
# 'Learn Fractions | Add, Subtract, Multiply and Divide | Oppia'
# the page title fragment field represents the middle value 'Add, Subtract,
# Multiply and Divide'.
page_title_fragment_for_web = datastore_services.StringProperty(
indexed=True, default='')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(TopicModel, self)._trusted_commit(
committer_id, commit_type, commit_message, commit_cmds)
topic_rights = TopicRightsModel.get_by_id(self.id)
if topic_rights.topic_is_published:
status = constants.ACTIVITY_STATUS_PUBLIC
else:
status = constants.ACTIVITY_STATUS_PRIVATE
topic_commit_log_entry = TopicCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type,
commit_message, commit_cmds, status, False
)
topic_commit_log_entry.topic_id = self.id
topic_commit_log_entry.update_timestamps()
topic_commit_log_entry.put()
@classmethod
def get_by_name(cls, topic_name: str) -> Optional['TopicModel']:
"""Gets TopicModel by topic_name. Returns None if the topic with
name topic_name doesn't exist.
Args:
topic_name: str. The name of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
return cls.get_all().filter(
cls.canonical_name == topic_name.lower()
).get()
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional['TopicModel']:
"""Gets TopicModel by url_fragment. Returns None if the topic with
name url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
# TODO(#10210): Make fetching by URL fragment faster.
return cls.get_all().filter(cls.url_fragment == url_fragment).get()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'abbreviated_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_size_in_bytes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_reference_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopics': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_subtopic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'meta_tag_content': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'page_title_fragment_for_web': (
base_models.EXPORT_POLICY.NOT_APPLICABLE),
'practice_tab_is_displayed':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE,
})
class TopicRightsModel(base_models.VersionedModel):
"""Storage model for rights related to a topic.
The id of each instance is the id of the corresponding topic.
"""
SNAPSHOT_METADATA_CLASS = TopicRightsSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = TopicRightsSnapshotContentModel
ALLOW_REVERT = False
# The user_ids of the managers of this topic.
manager_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# Whether this topic is published.
topic_is_published = datastore_services.BooleanProperty(
indexed=True, required=True, default=False)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: manager_ids field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether TopicRightsModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
cls.manager_ids == user_id
).get(keys_only=True) is not None
@classmethod
def get_by_user(cls, user_id: str) -> Sequence['TopicRightsModel']:
"""Retrieves the rights object for all topics assigned to given user
Args:
user_id: str. ID of user.
Returns:
list(TopicRightsModel). The list of TopicRightsModel objects in
which the given user is a manager.
"""
return cls.query(cls.manager_ids == user_id).fetch()
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(TopicRightsModel, self)._trusted_commit(
committer_id, commit_type, commit_message, commit_cmds)
topic_rights = TopicRightsModel.get_by_id(self.id)
if topic_rights.topic_is_published:
status = constants.ACTIVITY_STATUS_PUBLIC
else:
status = constants.ACTIVITY_STATUS_PRIVATE
TopicCommitLogEntryModel(
id=('rights-%s-%s' % (self.id, self.version)),
user_id=committer_id,
topic_id=self.id,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=None,
post_commit_status=status,
post_commit_community_owned=False,
post_commit_is_private=not topic_rights.topic_is_published
).put()
snapshot_metadata_model = self.SNAPSHOT_METADATA_CLASS.get(
self.get_snapshot_id(self.id, self.version))
# Ruling out the possibility of None for mypy type checking.
assert snapshot_metadata_model is not None
snapshot_metadata_model.content_user_ids = list(sorted(set(
self.manager_ids)))
commit_cmds_user_ids = set()
for commit_cmd in commit_cmds:
user_id_attribute_names = next(
cmd['user_id_attribute_names']
for cmd in feconf.TOPIC_RIGHTS_CHANGE_ALLOWED_COMMANDS
if cmd['name'] == commit_cmd['cmd']
)
for user_id_attribute_name in user_id_attribute_names:
commit_cmds_user_ids.add(commit_cmd[user_id_attribute_name])
snapshot_metadata_model.commit_cmds_user_ids = list(
sorted(commit_cmds_user_ids))
snapshot_metadata_model.update_timestamps()
snapshot_metadata_model.put()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users contribute to topics and their rights.
"""
return (
base_models
.MODEL_ASSOCIATION_TO_USER
.ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'manager_ids': base_models.EXPORT_POLICY.EXPORTED,
'topic_is_published': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'manager_ids': 'managed_topic_ids'
}
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of TopicRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of TopicRightsModel in a dict
format. In this case, we are returning all the ids of the topics
this user manages.
"""
managed_topics = cls.get_all().filter(cls.manager_ids == user_id)
managed_topic_ids = [right.id for right in managed_topics]
return {
'managed_topic_ids': managed_topic_ids
}
class BlogPostRightsModel(base_models.BaseModel):
"""Storage model for rights related to a blog post.
The id of each instance is the blog_post_id.
"""
# The user_ids of the blog editors of this blog post.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# Whether this blog post is published or not.
# False if blog post is a draft, True if published.
blog_post_is_published = datastore_services.BooleanProperty(indexed=True,
required=True,
default=False)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to be deleted corresponding to a user: editor_ids
field. It does not delete the model but removes the user id from the
list of editor IDs corresponding to a blog post rights model.
"""
return base_models.DELETION_POLICY.DELETE
@classmethod
def deassign_user_from_all_blog_posts(cls, user_id: str) -> None:
"""Removes user_id from the list of editor_ids from all the blog
post rights models.
Args:
user_id: str. The ID of the user to be removed from editor ids.
"""
blog_post_rights_models = cls.get_all_by_user(user_id)
if blog_post_rights_models:
for rights_model in blog_post_rights_models:
rights_model.editor_ids.remove(user_id)
cls.update_timestamps_multi(blog_post_rights_models)
cls.put_multi(blog_post_rights_models)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BlogPostRightsModel references to the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
# NOTE: Even though `editor_ids` is repeated, we can compare it to a
# single value and it will return models where any of the editor IDs
# are equal to user_id.
cls.editor_ids == user_id).get(keys_only=True) is not None
@classmethod
def get_published_models_by_user(
cls,
user_id: str,
limit: Optional[int] = None) -> List[BlogPostRightsModel]:
"""Retrieves the blog post rights objects for published blog posts for
which the given user is an editor.
Args:
user_id: str. ID of the author of the blog post.
limit: int|None. The maximum number of BlogPostRightsModels to be
fetched. If None, all existing published models by user will be
fetched.
Returns:
list(BlogPostRightsModel). The list of BlogPostRightsModel objects
in which the given user is an editor. The list will be ordered
according to the time when the model was last updated.
"""
query = cls.query(
cls.editor_ids == user_id,
cls.blog_post_is_published == True # pylint: disable=singleton-comparison
).order(-cls.last_updated)
return list(query.fetch(limit) if limit is not None else query.fetch())
@classmethod
def get_draft_models_by_user(
cls,
user_id: str,
limit: Optional[int] = None) -> List[BlogPostRightsModel]:
"""Retrieves the blog post rights objects for draft blog posts for which
the given user is an editor.
Args:
user_id: str. ID of the author of the blog post.
limit: int|None. The maximum number of BlogPostRightsModels to be
fetched. If None, all existing draft models by user will be
fetched.
Returns:
list(BlogPostRightsModel). The list of BlogPostRightsModel objects
in which the given user is an editor. The list will be ordered
according to the time when the model was last updated.
"""
query = cls.query(
cls.editor_ids == user_id,
cls.blog_post_is_published == False # pylint: disable=singleton-comparison
).order(-cls.last_updated)
return list(query.fetch(limit) if limit is not None else query.fetch())
@classmethod
def get_all_by_user(cls, user_id: str) -> List[BlogPostRightsModel]:
"""Retrieves the blog post rights objects for all blog posts for which
the given user is an editor.
Args:
user_id: str. ID of the author of the blog post.
Returns:
list(BlogPostRightsModel). The list of BlogPostRightsModel objects
in which the given user is an editor.
"""
return list(cls.query(cls.editor_ids == user_id).fetch())
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users can edit the blog post.
"""
return (base_models.MODEL_ASSOCIATION_TO_USER.
ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(BlogPostRightsModel, cls).get_export_policy(), **{
'editor_ids': base_models.EXPORT_POLICY.EXPORTED,
'blog_post_is_published':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of BlogPostsRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of BlogPostsRightsModel.
in a python dict format. In this case, we are returning all the
ids of blog posts for which the user is an editor.
"""
editable_blog_posts: Sequence[BlogPostRightsModel] = (cls.query(
cls.editor_ids == user_id).fetch())
editable_blog_post_ids = [blog.id for blog in editable_blog_posts]
return {
'editable_blog_post_ids': editable_blog_post_ids,
}
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'editor_ids': 'editable_blog_post_ids',
}
@classmethod
def create(cls, blog_post_id: str, author_id: str) -> BlogPostRightsModel:
"""Creates a new BlogPostRightsModel entry.
Args:
blog_post_id: str. Blog Post ID of the newly-created blog post.
author_id: str. User ID of the author.
Returns:
BlogPostRightsModel. The newly created BlogPostRightsModel
instance.
Raises:
Exception. A blog post rights model with the given blog post ID
exists already.
"""
if cls.get_by_id(blog_post_id):
raise Exception(
'Blog Post ID conflict on creating new blog post rights model.'
)
entity = cls(id=blog_post_id, editor_ids=[author_id])
entity.update_timestamps()
entity.put()
return entity
