class BeamJobRunResultModel(base_models.BaseModel):
"""Represents the result of an Apache Beam job.
IMPORTANT: Datastore entities have a size limit of ~1MiB (1,048,572 bytes,
to be precise). To support jobs that may need to produce outputs that exceed
this this limit, BeamJobRunResultModels are split into _unordered_ batches.
Thus, to return the full results you must fetch all models with matching
job_model_id values and concatenate their results.
The IDs of BeamJobRunResultModel are inconsequential, and generated by using
the urlsafe base64 encoding of a uuid() value: ^[A-Za-z0-9-_]{22}$.
"""
# The ID of the BeamJobRunResultModel corresponding to the result.
job_id = datastore_services.StringProperty(required=True, indexed=True)
# The standard text output generated by the corresponding Apache Beam job.
stdout = datastore_services.TextProperty(required=False, indexed=False)
# The error output generated by the corresponding Apache Beam job.
stderr = datastore_services.TextProperty(required=False, indexed=False)
# We have ignored [override] here because the signature of this method
# doesn't match with signature of super class's get_new_id() method.
@classmethod
def get_new_id(cls) -> str: # type: ignore[override]
"""Generates an ID for a new BeamJobRunResultModel.
Returns:
str. The new ID.
"""
return _get_new_model_id(cls)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model doesn't contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(BeamJobRunResultModel, cls).get_export_policy(), **{
'job_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'stdout': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'stderr': base_models.EXPORT_POLICY.NOT_APPLICABLE,
})
class GeneralFeedbackThreadModel(base_models.BaseModel):
"""Threads for each entity.
The id of instances of this class has the form
[entity_type].[entity_id].[generated_string]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of entity the thread is linked to.
entity_type = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the entity the thread is linked to.
entity_id = datastore_services.StringProperty(required=True, indexed=True)
# ID of the user who started the thread. This may be None if the feedback
# was given anonymously by a learner.
original_author_id = datastore_services.StringProperty(indexed=True)
# Latest status of the thread.
status = datastore_services.StringProperty(
default=STATUS_CHOICES_OPEN,
choices=STATUS_CHOICES,
required=True,
indexed=True,
)
# Latest subject of the thread.
subject = datastore_services.StringProperty(indexed=True, required=True)
# Summary text of the thread.
summary = datastore_services.TextProperty(indexed=False)
# Specifies whether this thread has a related suggestion.
has_suggestion = datastore_services.BooleanProperty(indexed=True,
default=False,
required=True)
# Cached value of the number of messages in the thread.
message_count = datastore_services.IntegerProperty(indexed=True, default=0)
# Cached text of the last message in the thread with non-empty content, or
# None if there is no such message.
last_nonempty_message_text = datastore_services.TextProperty(indexed=False)
# Cached ID for the user of the last message in the thread with non-empty
# content, or None if the message was made anonymously or if there is no
# such message.
last_nonempty_message_author_id = (datastore_services.StringProperty(
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
original_author_id and last_nonempty_message_author_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there
are multiple feedback threads relevant to a particular user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'entity_type': base_models.EXPORT_POLICY.EXPORTED,
'entity_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the original_author_id because we should not
# export internal user ids.
'original_author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.EXPORTED,
'subject': base_models.EXPORT_POLICY.EXPORTED,
'summary': base_models.EXPORT_POLICY.EXPORTED,
'has_suggestion': base_models.EXPORT_POLICY.EXPORTED,
'message_count': base_models.EXPORT_POLICY.EXPORTED,
'last_nonempty_message_text':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_nonempty_message_author_id':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'last_updated': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def get_field_names_for_takeout(cls) -> Dict[str, str]:
"""Indicates that the last_updated variable is exported under the
name "last_updated_msec" in Takeout.
"""
return dict(
super(cls, cls).get_field_names_for_takeout(),
**{'last_updated': 'last_updated_msec'})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackThreadModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.original_author_id == user_id,
cls.last_nonempty_message_author_id == user_id)).get(
keys_only=True) is not None
@classmethod
def export_data(
cls, user_id: str) -> Dict[str, Dict[str, Union[str, bool, None]]]:
"""Exports the data from GeneralFeedbackThreadModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackThreadModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackThreadModel] = (
cls.get_all().filter(cls.original_author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'entity_type':
feedback_model.entity_type,
'entity_id':
feedback_model.entity_id,
'status':
feedback_model.status,
'subject':
feedback_model.subject,
'has_suggestion':
feedback_model.has_suggestion,
'summary':
feedback_model.summary,
'message_count':
feedback_model.message_count,
'last_updated_msec':
utils.get_time_in_millisecs(feedback_model.last_updated)
}
return user_data
@classmethod
def generate_new_thread_id(cls, entity_type: str, entity_id: str) -> str:
"""Generates a new thread ID which is unique.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
Returns:
str. A thread ID that is different from the IDs of all
the existing threads within the given entity.
Raises:
Exception. There were too many collisions with existing thread IDs
when attempting to generate a new thread ID.
"""
for _ in python_utils.RANGE(_MAX_RETRIES):
thread_id = (
'%s.%s.%s%s' %
(entity_type, entity_id,
utils.base64_from_int(
int(utils.get_current_time_in_millisecs())),
utils.base64_from_int(utils.get_random_int(_RAND_RANGE))))
if not cls.get_by_id(thread_id):
return thread_id
raise Exception(
'New thread id generator is producing too many collisions.')
@classmethod
def create(cls, thread_id: str) -> 'GeneralFeedbackThreadModel':
"""Creates a new FeedbackThreadModel entry.
Args:
thread_id: str. Thread ID of the newly-created thread.
Returns:
GeneralFeedbackThreadModel. The newly created FeedbackThreadModel
instance.
Raises:
Exception. A thread with the given thread ID exists already.
"""
if cls.get_by_id(thread_id):
raise Exception('Feedback thread ID conflict on create.')
return cls(id=thread_id)
@classmethod
def get_threads(
cls,
entity_type: str,
entity_id: str,
limit: int = feconf.DEFAULT_QUERY_LIMIT
) -> Sequence['GeneralFeedbackThreadModel']:
"""Returns a list of threads associated with the entity, ordered
by their "last updated" field. The number of entities fetched is
limited by the `limit` argument to this method, whose default
value is equal to the default query limit.
Args:
entity_type: str. The type of the entity.
entity_id: str. The ID of the entity.
limit: int. The maximum possible number of items in the returned
list.
Returns:
list(GeneralFeedbackThreadModel). List of threads associated with
the entity. Doesn't include deleted entries.
"""
return cls.get_all().filter(cls.entity_type == entity_type).filter(
cls.entity_id == entity_id).order(-cls.last_updated).fetch(limit)
class GeneralFeedbackMessageModel(base_models.BaseModel):
"""Feedback messages. One or more of these messages make a thread.
The id of instances of this class has the form [thread_id].[message_id]
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# ID corresponding to an entry of FeedbackThreadModel.
thread_id = datastore_services.StringProperty(required=True, indexed=True)
# 0-based sequential numerical ID. Sorting by this field will create the
# thread in chronological order.
message_id = datastore_services.IntegerProperty(required=True,
indexed=True)
# ID of the user who posted this message. This may be None if the feedback
# was given anonymously by a learner.
author_id = datastore_services.StringProperty(indexed=True)
# New thread status. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the status changes.
updated_status = (datastore_services.StringProperty(choices=STATUS_CHOICES,
indexed=True))
# New thread subject. Must exist in the first message of a thread. For the
# rest of the thread, should exist only when the subject changes.
updated_subject = datastore_services.StringProperty(indexed=True)
# Message text. Allowed not to exist (e.g. post only to update the status).
text = datastore_services.TextProperty(indexed=False)
# Whether the incoming message is received by email (as opposed to via
# the web).
received_via_email = datastore_services.BooleanProperty(default=False,
indexed=True,
required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple feedback messages relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'thread_id': base_models.EXPORT_POLICY.EXPORTED,
'message_id': base_models.EXPORT_POLICY.EXPORTED,
# We do not export the author_id because we should not export
# internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'updated_status': base_models.EXPORT_POLICY.EXPORTED,
'updated_subject': base_models.EXPORT_POLICY.EXPORTED,
'text': base_models.EXPORT_POLICY.EXPORTED,
'received_via_email': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralFeedbackMessageModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@classmethod
def export_data(
cls,
user_id: str) -> Dict[str, Dict[str, Union[str, int, bool, None]]]:
"""Exports the data from GeneralFeedbackMessageModel
into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralFeedbackMessageModel.
"""
user_data = {}
feedback_models: Sequence[GeneralFeedbackMessageModel] = (
cls.get_all().filter(cls.author_id == user_id).fetch())
for feedback_model in feedback_models:
user_data[feedback_model.id] = {
'thread_id': feedback_model.thread_id,
'message_id': feedback_model.message_id,
'updated_status': feedback_model.updated_status,
'updated_subject': feedback_model.updated_subject,
'text': feedback_model.text,
'received_via_email': feedback_model.received_via_email
}
return user_data
@classmethod
def _generate_id(cls, thread_id: str, message_id: int) -> str:
"""Generates full message ID given the thread ID and message ID.
Args:
thread_id: str. Thread ID of the thread to which the message
belongs.
message_id: int. Message ID of the message.
Returns:
str. Full message ID.
"""
return '.'.join([thread_id, python_utils.UNICODE(message_id)])
@property
def entity_id(self) -> str:
"""Returns the entity_id corresponding to this thread instance.
Returns:
str. The entity_id.
"""
return self.id.split('.')[1]
@property
def entity_type(self) -> str:
"""Returns the entity_type corresponding to this thread instance.
Returns:
str. The entity_type.
"""
return self.id.split('.')[0]
@classmethod
def create(
cls,
message_identifier: feedback_domain.FullyQualifiedMessageIdentifier
) -> 'GeneralFeedbackMessageModel':
"""Creates a new GeneralFeedbackMessageModel entry.
Args:
message_identifier: FullyQualifiedMessageIdentifier. The message
identifier consists of the thread_id and its corresponding
message_id.
Returns:
GeneralFeedbackMessageModel. Instance of the new
GeneralFeedbackMessageModel entry.
Raises:
Exception. A message with the same ID already exists
in the given thread.
"""
return cls.create_multi([message_identifier])[0]
@classmethod
def create_multi(
cls, message_identifiers: List[
feedback_domain.FullyQualifiedMessageIdentifier]
) -> List['GeneralFeedbackMessageModel']:
"""Creates a new GeneralFeedbackMessageModel entry for each
(thread_id, message_id) pair.
Args:
message_identifiers: list(FullyQualifiedMessageIdentifier). Each
message identifier consists of the thread_id and its
corresponding message_id.
Returns:
list(GeneralFeedbackMessageModel). Instances of the new
GeneralFeedbackMessageModel entries.
Raises:
Exception. The number of thread_ids must be equal to the number of
message_ids.
Exception. A message with the same ID already exists
in the given thread.
"""
thread_ids = [
message_identifier.thread_id
for message_identifier in message_identifiers
]
message_ids = [
message_identifier.message_id
for message_identifier in message_identifiers
]
# Generate the new ids.
instance_ids = [
cls._generate_id(thread_id, message_id)
for thread_id, message_id in python_utils.ZIP(
thread_ids, message_ids)
]
# Check if the new ids are valid.
current_instances = cls.get_multi(instance_ids)
conflict_ids = [
current_instance.id for current_instance in current_instances
if current_instance is not None
]
if len(conflict_ids) > 0:
raise Exception(
'The following feedback message ID(s) conflicted on '
'create: %s' % (' '.join(conflict_ids)))
return [cls(id=instance_id) for instance_id in instance_ids]
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.get().
@classmethod
def get( # type: ignore[override]
cls,
thread_id: str,
message_id: int,
strict: bool = True) -> Optional['GeneralFeedbackMessageModel']:
"""Gets the GeneralFeedbackMessageModel entry for the given ID. Raises
an error if no undeleted message with the given ID is found and
strict == True.
Args:
thread_id: str. ID of the thread.
message_id: int. ID of the message.
strict: bool. Whether to raise an error if no FeedbackMessageModel
entry is found for the given IDs.
Returns:
GeneralFeedbackMessageModel or None. If strict == False and no
undeleted message with the given message_id exists in the
datastore, then returns None. Otherwise, returns the
GeneralFeedbackMessageModel instance that corresponds to the
given ID.
Raises:
EntityNotFoundError. The value of strict is True and either
(i) message ID is not valid
(ii) message is marked as deleted.
No error will be raised if strict == False.
"""
instance_id = cls._generate_id(thread_id, message_id)
return super(GeneralFeedbackMessageModel, cls).get(instance_id,
strict=strict)
@classmethod
def get_messages(
cls, thread_id: str) -> Sequence['GeneralFeedbackMessageModel']:
"""Returns a list of messages in the given thread. The number of
messages returned is capped by feconf.DEFAULT_QUERY_LIMIT.
Args:
thread_id: str. ID of the thread.
Returns:
list(GeneralFeedbackMessageModel). A list of messages in the
given thread, up to a maximum of feconf.DEFAULT_QUERY_LIMIT
messages.
"""
return cls.get_all().filter(cls.thread_id == thread_id).fetch(
feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_most_recent_message(
cls, thread_id: str) -> 'GeneralFeedbackMessageModel':
"""Returns the last message in the thread.
Args:
thread_id: str. ID of the thread.
Returns:
GeneralFeedbackMessageModel. Last message in the thread.
"""
thread = GeneralFeedbackThreadModel.get_by_id(thread_id)
message = cls.get(thread_id, thread.message_count - 1)
# Ruling out the possibility of None for mypy type checking.
assert message is not None
return message
@classmethod
def get_message_count(cls, thread_id: str) -> int:
"""Returns the number of messages in the thread. Includes the
deleted entries.
Args:
thread_id: str. ID of the thread.
Returns:
int. Number of messages in the thread.
"""
return cls.get_message_counts([thread_id])[0]
@classmethod
def get_message_counts(cls, thread_ids: List[str]) -> List[int]:
"""Returns a list containing the number of messages in the threads.
Includes the deleted entries.
Args:
thread_ids: list(str). ID of the threads.
Returns:
list(int). List of the message counts for the threads.
"""
thread_models = GeneralFeedbackThreadModel.get_multi(thread_ids)
assert None not in thread_models
return [
thread_model.message_count if thread_model else None
for thread_model in thread_models
]
# TODO(#13523): Change the return value of the function below from
# tuple(list, str|None, bool) to a domain object.
@classmethod
def get_all_messages(
cls, page_size: int, urlsafe_start_cursor: Optional[str]
) -> Tuple[Sequence['GeneralFeedbackMessageModel'], Optional[str], bool]:
"""Fetches a list of all the messages sorted by their last updated
attribute.
Args:
page_size: int. The maximum number of messages to be returned.
urlsafe_start_cursor: str or None. If provided, the list of
returned messages starts from this datastore cursor.
Otherwise, the returned messages start from the beginning
of the full list of messages.
Returns:
3-tuple of (results, cursor, more). Where:
results: List of query results.
cursor: str or None. A query cursor pointing to the next
batch of results. If there are no more results, this might
be None.
more: bool. If True, there are (probably) more results after
this batch. If False, there are no further results after
this batch.
"""
return cls._fetch_page_sorted_by_last_updated(cls.query(), page_size,
urlsafe_start_cursor)
class JobModel(base_models.BaseModel):
"""Class representing a datastore entity for a long-running job."""
# The job type.
job_type = datastore_services.StringProperty(indexed=True)
# The time at which the job was queued, in milliseconds since the epoch.
time_queued_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was started, in milliseconds since the epoch.
# This is never set if the job was canceled before it was started.
time_started_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was completed, failed or canceled, in
# milliseconds since the epoch.
time_finished_msec = datastore_services.FloatProperty(indexed=True)
# The current status code for the job.
status_code = datastore_services.StringProperty(
indexed=True,
default=STATUS_CODE_NEW,
choices=[
STATUS_CODE_NEW, STATUS_CODE_QUEUED, STATUS_CODE_STARTED,
STATUS_CODE_COMPLETED, STATUS_CODE_FAILED, STATUS_CODE_CANCELED
])
# Any metadata for the job, such as the root pipeline id for mapreduce
# jobs.
metadata = datastore_services.JsonProperty(indexed=False)
# The output of the job. This is only populated if the job has status code
# STATUS_CODE_COMPLETED, and is None otherwise. If populated, this is
# expected to be a list of strings.
output = datastore_services.JsonProperty(indexed=False)
# The error message, if applicable. Only populated if the job has status
# code STATUS_CODE_FAILED or STATUS_CODE_CANCELED; None otherwise.
error = datastore_services.TextProperty(indexed=False)
# Whether the datastore models associated with this job have been cleaned
# up (i.e., deleted).
has_been_cleaned_up = (datastore_services.BooleanProperty(default=False,
indexed=True))
# Store additional params passed with job.
additional_job_params = datastore_services.JsonProperty(default=None)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'job_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_queued_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_started_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_finished_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'metadata': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'output': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'error': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'has_been_cleaned_up':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_job_params':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@property
def is_cancelable(self) -> bool:
"""Checks if the job is cancelable.
Returns:
bool. Whether the job's status_code is 'queued' or 'started'.
"""
# Whether the job is currently in 'queued' or 'started' status.
return self.status_code in [STATUS_CODE_QUEUED, STATUS_CODE_STARTED]
@classmethod
def get_all_unfinished_jobs(cls, limit: int) -> Sequence['JobModel']:
"""Gets at most `limit` unfinished jobs.
Args:
limit: int. A limit on the number of jobs to return.
Returns:
list(JobModel) or None. A list of at most `limit` number
of unfinished jobs.
"""
return cls.query().filter(
JobModel.status_code.IN([
STATUS_CODE_QUEUED, STATUS_CODE_STARTED
])).order(-cls.time_queued_msec).fetch(limit)
@classmethod
def get_unfinished_jobs(cls, job_type: str) -> datastore_services.Query:
"""Gets jobs that are unfinished.
Args:
job_type: str. The type of jobs that may be unfinished.
Returns:
list(JobModel) or None. A list of all jobs that belong
to the given job_type.
"""
return cls.query().filter(cls.job_type == job_type).filter(
JobModel.status_code.IN([STATUS_CODE_QUEUED, STATUS_CODE_STARTED]))
@classmethod
def do_unfinished_jobs_exist(cls, job_type: str) -> bool:
"""Checks if unfinished jobs exist.
Args:
job_type: str. Type of job for which to check.
Returns:
bool. True if unfinished jobs exist, otherwise false.
"""
return bool(cls.get_unfinished_jobs(job_type).count(limit=1))
class SentEmailModel(base_models.BaseModel):
"""Records the content and metadata of an email sent from Oppia.
This model is read-only; entries cannot be modified once created. The
id/key of instances of this class has the form '[intent].[random hash]'.
"""
# TODO(sll): Implement functionality to get all emails sent to a particular
# user with a given intent within a given time period.
# The user ID of the email recipient.
recipient_id = (datastore_services.StringProperty(required=True,
indexed=True))
# The email address of the recipient.
recipient_email = datastore_services.StringProperty(required=True)
# The user ID of the email sender. For site-generated emails this is equal
# to SYSTEM_COMMITTER_ID.
sender_id = datastore_services.StringProperty(required=True, indexed=True)
# The email address used to send the notification. This should be either
# the noreply address or the system address.
sender_email = datastore_services.StringProperty(required=True)
# The intent of the email.
intent = datastore_services.StringProperty(
required=True,
indexed=True,
choices=[
feconf.EMAIL_INTENT_SIGNUP, feconf.EMAIL_INTENT_MARKETING,
feconf.EMAIL_INTENT_DAILY_BATCH,
feconf.EMAIL_INTENT_EDITOR_ROLE_NOTIFICATION,
feconf.EMAIL_INTENT_FEEDBACK_MESSAGE_NOTIFICATION,
feconf.EMAIL_INTENT_SUBSCRIPTION_NOTIFICATION,
feconf.EMAIL_INTENT_SUGGESTION_NOTIFICATION,
feconf.EMAIL_INTENT_UNPUBLISH_EXPLORATION,
feconf.EMAIL_INTENT_DELETE_EXPLORATION,
feconf.EMAIL_INTENT_REPORT_BAD_CONTENT,
feconf.EMAIL_INTENT_QUERY_STATUS_NOTIFICATION,
feconf.EMAIL_INTENT_ONBOARD_REVIEWER,
feconf.EMAIL_INTENT_REMOVE_REVIEWER,
feconf.EMAIL_INTENT_ADDRESS_CONTRIBUTOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_REVIEW_CREATOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_REVIEW_CONTRIBUTOR_DASHBOARD_SUGGESTIONS,
feconf.EMAIL_INTENT_ADD_CONTRIBUTOR_DASHBOARD_REVIEWERS,
feconf.EMAIL_INTENT_VOICEOVER_APPLICATION_UPDATES,
feconf.EMAIL_INTENT_ACCOUNT_DELETED, feconf.BULK_EMAIL_INTENT_TEST
])
# The subject line of the email.
subject = datastore_services.TextProperty(required=True)
# The HTML content of the email body.
html_body = datastore_services.TextProperty(required=True)
# The datetime the email was sent, in UTC.
sent_datetime = (datastore_services.DateTimeProperty(required=True,
indexed=True))
# The hash of the recipient id, email subject and message body.
email_hash = datastore_services.StringProperty(indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data corresponding to a user: recipient_id,
recipient_email, sender_id, and sender_email, but this isn't deleted
because this model is needed for auditing purposes.
"""
return base_models.DELETION_POLICY.KEEP
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Users already have access to this data since emails were sent
to them.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because users already have access to noteworthy details of this data
(since emails were sent to them).
"""
return dict(
super(cls, cls).get_export_policy(), **{
'recipient_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'recipient_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'email_hash': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether SentEmailModel exists for user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(
cls.recipient_id == user_id,
cls.sender_id == user_id,
)).get(keys_only=True) is not None
@classmethod
def _generate_id(cls, intent: str) -> str:
"""Generates an ID for a new SentEmailModel instance.
Args:
intent: str. The intent string, i.e. the purpose of the email.
Valid intent strings are defined in feconf.py.
Returns:
str. The newly-generated ID for the SentEmailModel instance.
Raises:
Exception. The id generator for SentEmailModel is producing
too many collisions.
"""
id_prefix = '%s.' % intent
for _ in python_utils.RANGE(base_models.MAX_RETRIES):
new_id = '%s.%s' % (id_prefix,
utils.convert_to_hash(
python_utils.UNICODE(
utils.get_random_int(
base_models.RAND_RANGE)),
base_models.ID_LENGTH))
if not cls.get_by_id(new_id):
return new_id
raise Exception(
'The id generator for SentEmailModel is producing too many '
'collisions.')
@classmethod
def create(cls, recipient_id: str, recipient_email: str, sender_id: str,
sender_email: str, intent: str, subject: str, html_body: str,
sent_datetime: datetime.datetime) -> None:
"""Creates a new SentEmailModel entry.
Args:
recipient_id: str. The user ID of the email recipient.
recipient_email: str. The email address of the recipient.
sender_id: str. The user ID of the email sender.
sender_email: str. The email address used to send the notification.
intent: str. The intent string, i.e. the purpose of the email.
subject: str. The subject line of the email.
html_body: str. The HTML content of the email body.
sent_datetime: datetime.datetime. The datetime the email was sent,
in UTC.
"""
instance_id = cls._generate_id(intent)
email_model_instance = cls(id=instance_id,
recipient_id=recipient_id,
recipient_email=recipient_email,
sender_id=sender_id,
sender_email=sender_email,
intent=intent,
subject=subject,
html_body=html_body,
sent_datetime=sent_datetime)
email_model_instance.update_timestamps()
email_model_instance.put()
def _pre_put_hook(self) -> None:
"""Operations to perform just before the model is `put` into storage."""
super(SentEmailModel, self)._pre_put_hook()
self.email_hash = self._generate_hash(self.recipient_id, self.subject,
self.html_body)
@classmethod
def get_by_hash(
cls,
email_hash: str,
sent_datetime_lower_bound: Optional[datetime.datetime] = None
) -> List['SentEmailModel']:
"""Returns all messages with a given email_hash.
This also takes an optional sent_datetime_lower_bound argument,
which is a datetime instance. If this is given, only
SentEmailModel instances sent after sent_datetime_lower_bound
should be returned.
Args:
email_hash: str. The hash value of the email.
sent_datetime_lower_bound: datetime.datetime. The lower bound on
sent_datetime of the email to be searched.
Returns:
list(SentEmailModel). A list of emails which have the given hash
value and sent more recently than sent_datetime_lower_bound.
Raises:
Exception. The sent_datetime_lower_bound is not a valid
datetime.datetime.
"""
if sent_datetime_lower_bound is not None:
if not isinstance(sent_datetime_lower_bound, datetime.datetime):
raise Exception('Expected datetime, received %s of type %s' %
(sent_datetime_lower_bound,
type(sent_datetime_lower_bound)))
query = cls.query().filter(cls.email_hash == email_hash)
if sent_datetime_lower_bound is not None:
query = query.filter(cls.sent_datetime > sent_datetime_lower_bound)
messages = cast(List[SentEmailModel], query.fetch())
return messages
@classmethod
def _generate_hash(cls, recipient_id: str, email_subject: str,
email_body: str) -> str:
"""Generate hash for a given recipient_id, email_subject and cleaned
email_body.
Args:
recipient_id: str. The user ID of the email recipient.
email_subject: str. The subject line of the email.
email_body: str. The HTML content of the email body.
Returns:
str. The generated hash value of the given email.
"""
hash_value = utils.convert_to_hash(
recipient_id + email_subject + email_body, 100)
return hash_value
@classmethod
def check_duplicate_message(cls, recipient_id: str, email_subject: str,
email_body: str) -> bool:
"""Check for a given recipient_id, email_subject and cleaned
email_body, whether a similar message has been sent in the last
DUPLICATE_EMAIL_INTERVAL_MINS.
Args:
recipient_id: str. The user ID of the email recipient.
email_subject: str. The subject line of the email.
email_body: str. The HTML content of the email body.
Returns:
bool. Whether a similar message has been sent to the same recipient
in the last DUPLICATE_EMAIL_INTERVAL_MINS.
"""
email_hash = cls._generate_hash(recipient_id, email_subject,
email_body)
datetime_now = datetime.datetime.utcnow()
time_interval = datetime.timedelta(
minutes=feconf.DUPLICATE_EMAIL_INTERVAL_MINS)
sent_datetime_lower_bound = datetime_now - time_interval
messages = cls.get_by_hash(
email_hash, sent_datetime_lower_bound=sent_datetime_lower_bound)
for message in messages:
if (message.recipient_id == recipient_id
and message.subject == email_subject
and message.html_body == email_body):
return True
return False
class BulkEmailModel(base_models.BaseModel):
"""Records the content of an email sent from Oppia to multiple users.
This model is read-only; entries cannot be modified once created. The
id/key of instances of this model is randomly generated string of
length 12.
"""
# The user IDs of the email recipients.
recipient_ids = datastore_services.JsonProperty(default=[],
compressed=True)
# The user ID of the email sender. For site-generated emails this is equal
# to SYSTEM_COMMITTER_ID.
sender_id = datastore_services.StringProperty(required=True, indexed=True)
# The email address used to send the notification.
sender_email = datastore_services.StringProperty(required=True)
# The intent of the email.
intent = datastore_services.StringProperty(
required=True,
indexed=True,
choices=[
feconf.BULK_EMAIL_INTENT_MARKETING,
feconf.BULK_EMAIL_INTENT_IMPROVE_EXPLORATION,
feconf.BULK_EMAIL_INTENT_CREATE_EXPLORATION,
feconf.BULK_EMAIL_INTENT_CREATOR_REENGAGEMENT,
feconf.BULK_EMAIL_INTENT_LEARNER_REENGAGEMENT,
feconf.BULK_EMAIL_INTENT_ML_JOB_FAILURE
])
# The subject line of the email.
subject = datastore_services.TextProperty(required=True)
# The HTML content of the email body.
html_body = datastore_services.TextProperty(required=True)
# The datetime the email was sent, in UTC.
sent_datetime = (datastore_services.DateTimeProperty(required=True,
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data corresponding to a user: recipient_ids,
sender_id, and sender_email, but this isn't deleted because this model
is needed for auditing purposes.
"""
return base_models.DELETION_POLICY.KEEP
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Users already have access to this data since the emails were sent
to them.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because users already have access to noteworthy details of this data
(since emails were sent to them).
"""
return dict(
super(cls, cls).get_export_policy(), **{
'recipient_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sender_email': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'intent': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subject': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'html_body': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'sent_datetime': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BulkEmailModel exists for user. Since recipient_ids
can't be indexed it also can't be checked by this method, we can allow
this because the deletion policy for this model is keep , thus even the
deleted user's id will remain here.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return (cls.query(cls.sender_id == user_id).get(keys_only=True)
is not None)
@classmethod
def create(cls, instance_id: str, recipient_ids: List[str], sender_id: str,
sender_email: str, intent: str, subject: str, html_body: str,
sent_datetime: datetime.datetime) -> None:
"""Creates a new BulkEmailModel entry.
Args:
instance_id: str. The ID of the instance.
recipient_ids: list(str). The user IDs of the email recipients.
sender_id: str. The user ID of the email sender.
sender_email: str. The email address used to send the notification.
intent: str. The intent string, i.e. the purpose of the email.
subject: str. The subject line of the email.
html_body: str. The HTML content of the email body.
sent_datetime: datetime.datetime. The date and time the email
was sent, in UTC.
"""
email_model_instance = cls(id=instance_id,
recipient_ids=recipient_ids,
sender_id=sender_id,
sender_email=sender_email,
intent=intent,
subject=subject,
html_body=html_body,
sent_datetime=sent_datetime)
email_model_instance.update_timestamps()
email_model_instance.put()
class CollectionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia collection.
This should be used whenever the content blob of the collection is not
needed (e.g. search results, etc).
A CollectionSummaryModel instance stores the following information:
id, title, category, objective, language_code, tags, ratings,
last_updated, created_on, status (private, public),
community_owned, owner_ids, editor_ids,
viewer_ids, version.
The key of each instance is the collection id.
"""
# What this collection is called.
title = datastore_services.StringProperty(required=True)
# The category this collection belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this collection.
objective = datastore_services.TextProperty(required=True, indexed=False)
# The ISO 639-1 code for the language this collection is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# Tags associated with this collection.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# Aggregate user-assigned ratings of the collection.
ratings = datastore_services.JsonProperty(default=None, indexed=False)
# Time when the collection model was last updated (not to be
# confused with last_updated, which is the time when the
# collection *summary* model was last updated).
collection_model_last_updated = (
datastore_services.DateTimeProperty(indexed=True))
# Time when the collection model was created (not to be confused
# with created_on, which is the time when the collection *summary*
# model was created).
collection_model_created_on = (
datastore_services.DateTimeProperty(indexed=True))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE,
constants.ACTIVITY_STATUS_PUBLIC
]
)
# Whether this collection is owned by the community.
community_owned = (
datastore_services.BooleanProperty(required=True, indexed=True))
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who have contributed (humans who have made a
# positive (not just a revert) change to the collection's content).
# NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be
# synchronized, meaning that the keys in contributors_summary need be
# equal to the contributor_ids list.
contributor_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# A dict representing the contributors of non-trivial commits to this
# collection. Each key of this dict is a user_id, and the corresponding
# value is the number of non-trivial commits that the user has made.
contributors_summary = (
datastore_services.JsonProperty(default={}, indexed=False))
# The version number of the collection after this commit. Only populated
# for commits to an collection (as opposed to its rights, etc.).
version = datastore_services.IntegerProperty()
# The number of nodes(explorations) that are within this collection.
node_count = datastore_services.IntegerProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, editor_ids, owner_ids, contributor_ids,
and contributors_summary fields.
"""
return (
base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE
)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the
CollectionRightsModel, and thus does not need an export_data
function.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because noteworthy details that belong to this model have already been
exported as a part of the CollectionRightsModel.
"""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributors_summary': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_count': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionSummaryModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id,
cls.contributor_ids == user_id)).get(keys_only=True) is not None
@classmethod
def get_non_private(cls) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with non-private collection summary models.
Returns:
iterable. An iterable with non-private collection summary models.
"""
return cls.get_all().filter(
cls.status != constants.ACTIVITY_STATUS_PRIVATE
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_private_at_least_viewable(
cls, user_id: str
) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with private collection summary models that are
at least viewable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with private collection summary models that
are at least viewable by the given user.
"""
return cls.get_all().filter(
cls.status == constants.ACTIVITY_STATUS_PRIVATE
).filter(
datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.viewer_ids == user_id
)
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_at_least_editable(
cls, user_id: str
) -> Sequence[CollectionSummaryModel]:
"""Returns an iterable with collection summary models that are at least
editable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with collection summary models that are at
least viewable by the given user.
"""
return CollectionSummaryModel.get_all().filter(
datastore_services.any_of(
CollectionSummaryModel.owner_ids == user_id,
CollectionSummaryModel.editor_ids == user_id
)
).fetch(feconf.DEFAULT_QUERY_LIMIT)
class BlogPostModel(base_models.BaseModel):
"""Model to store blog post data.
The id of instances of this class is in the form of random hash of
12 chars.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY: Literal[True] = True
# The ID of the user the blog post is authored by.
author_id = datastore_services.StringProperty(indexed=True, required=True)
# Title of the blog post.
title = datastore_services.StringProperty(indexed=True, required=True)
# Content of the blog post.
content = datastore_services.TextProperty(indexed=False, required=True)
# The unique url fragment of the blog post. If the user directly enters the
# blog post's url in the editor or the homepage, the blogPostModel will be
# queried using the url fragment to retrieve data for populating the editor
# dashboard / blog post page.
url_fragment = (datastore_services.StringProperty(indexed=True,
required=True))
# Tags associated with the blog post.
tags = datastore_services.StringProperty(indexed=True, repeated=True)
# The thumbnail filename of the blog post. It's value will be None until
# a thumbnail is added to the blog post. It can be None only when blog
# post is a draft.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# Time when the blog post model was last published. Value will be None
# if the blog has never been published.
published_on = (datastore_services.DateTimeProperty(indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id field.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether BlogPostModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(cls.author_id == user_id).get(
keys_only=True) is not None
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there can
be multiple blog post models relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user to export."""
return dict(
super(BlogPostModel, cls).get_export_policy(),
**{
# We do not export the author_id because we should not
# export internal user ids.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'title': base_models.EXPORT_POLICY.EXPORTED,
'content': base_models.EXPORT_POLICY.EXPORTED,
'url_fragment': base_models.EXPORT_POLICY.EXPORTED,
'tags': base_models.EXPORT_POLICY.EXPORTED,
'thumbnail_filename': base_models.EXPORT_POLICY.EXPORTED,
'published_on': base_models.EXPORT_POLICY.EXPORTED,
})
@classmethod
def generate_new_blog_post_id(cls) -> str:
"""Generates a new blog post ID which is unique and is in the form of
random hash of 12 chars.
Returns:
str. A blog post ID that is different from the IDs of all
the existing blog posts.
Raises:
Exception. There were too many collisions with existing blog post
IDs when attempting to generate a new blog post ID.
"""
for _ in range(base_models.MAX_RETRIES):
blog_post_id = utils.convert_to_hash(
str(utils.get_random_int(base_models.RAND_RANGE)),
base_models.ID_LENGTH)
if not cls.get_by_id(blog_post_id):
return blog_post_id
raise Exception(
'New blog post id generator is producing too many collisions.')
@classmethod
def create(cls, blog_post_id: str, author_id: str) -> BlogPostModel:
"""Creates a new BlogPostModel entry.
Args:
blog_post_id: str. Blog Post ID of the newly-created blog post.
author_id: str. User ID of the author.
Returns:
BlogPostModel. The newly created BlogPostModel instance.
Raises:
Exception. A blog post with the given blog post ID exists already.
"""
if cls.get_by_id(blog_post_id):
raise Exception(
'A blog post with the given blog post ID exists already.')
entity = cls(id=blog_post_id,
author_id=author_id,
content='',
title='',
published_on=None,
url_fragment='',
tags=[],
thumbnail_filename=None)
entity.update_timestamps()
entity.put()
return entity
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional[BlogPostModel]:
"""Gets BlogPostModel by url_fragment. Returns None if the blog post
with the given url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the blog post.
Returns:
BlogPostModel | None. The blog post model of the Blog or None if not
found.
"""
return BlogPostModel.query(
datastore_services.all_of(cls.url_fragment == url_fragment,
cls.deleted == False) # pylint: disable=singleton-comparison
).get()
@classmethod
def export_data(cls, user_id: str) -> Dict[str, BlogPostModelDataDict]:
"""Exports the data from BlogPostModel into dict format for Takeout.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from BlogPostModel.
"""
user_data: Dict[str, BlogPostModelDataDict] = {}
blog_post_models: Sequence[BlogPostModel] = cls.get_all().filter(
cls.author_id == user_id).fetch()
for blog_post_model in blog_post_models:
user_data[blog_post_model.id] = {
'title':
blog_post_model.title,
'content':
blog_post_model.content,
'url_fragment':
blog_post_model.url_fragment,
'tags':
blog_post_model.tags,
'thumbnail_filename':
blog_post_model.thumbnail_filename,
'published_on':
utils.get_time_in_millisecs(blog_post_model.published_on),
}
return user_data
class MachineTranslationModel(base_models.BaseModel):
"""Model for storing machine generated translations for the purpose of
preventing duplicate generation. Machine translations are used for reference
purpose only and therefore are context agnostic. Model instances are mapped
by a deterministic key generated from the source and target language codes,
followed by a SHA-1 hash of the untranslated source text formated as
follows:
[source_language_code].[target_language_code].[hashed_source_text]
See MachineTranslationModel._generate_id() below for details.
The same origin text, source_language_code, and target_language_code always
maps to the same key and therefore always returns the same translated_text.
"""
# The untranslated source text.
source_text = datastore_services.TextProperty(required=True, indexed=False)
# A SHA-1 hash of the source text. This can be used to index the datastore
# by source text.
hashed_source_text = datastore_services.StringProperty(
required=True, indexed=True)
# The language code for the source text language. Must be different from
# target_language_code.
source_language_code = datastore_services.StringProperty(
required=True, indexed=True)
# The language code for the target translation language. Must be different
# from source_language_code.
target_language_code = datastore_services.StringProperty(
required=True, indexed=True)
# The machine generated translation of the source text into the target
# language.
translated_text = datastore_services.TextProperty(
required=True, indexed=False)
@classmethod
def create(
cls,
source_language_code: str,
target_language_code: str,
source_text: str,
translated_text: str
) -> Optional[str]:
"""Creates a new MachineTranslationModel instance and returns its ID.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
source_text: str. The untranslated source text.
translated_text: str. The machine generated translation of the
source text into the target language.
Returns:
str|None. The id of the newly created
MachineTranslationModel instance, or None if the inputs are
invalid.
"""
if source_language_code is target_language_code:
return None
# SHA-1 always produces a 40 digit hash. 50 is chosen here to prevent
# convert_to_hash from truncating the hash.
hashed_source_text = utils.convert_to_hash(source_text, 50)
entity_id = cls._generate_id(
source_language_code, target_language_code, hashed_source_text)
translation_entity = cls(
id=entity_id,
hashed_source_text=hashed_source_text,
source_language_code=source_language_code,
target_language_code=target_language_code,
source_text=source_text,
translated_text=translated_text)
translation_entity.put()
return entity_id
@staticmethod
def _generate_id(
source_language_code: str,
target_language_code: str,
hashed_source_text: str
) -> str:
"""Generates a valid, deterministic key for a MachineTranslationModel
instance.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
hashed_source_text: str. An SHA-1 hash of the untranslated source
text.
Returns:
str. The deterministically generated identifier for this entity of
the form:
[source_language_code].[target_language_code].[hashed_source_text]
"""
return (
'%s.%s.%s' % (
source_language_code, target_language_code, hashed_source_text)
)
@classmethod
def get_machine_translation(
cls,
source_language_code: str,
target_language_code: str,
source_text: str
) -> Optional[MachineTranslationModel]:
"""Gets MachineTranslationModel by language codes and source text.
Args:
source_language_code: str. The language code for the source text
language. Must be different from target_language_code.
target_language_code: str. The language code for the target
translation language. Must be different from
source_language_code.
source_text: str. The untranslated source text.
Returns:
MachineTranslationModel|None. The MachineTranslationModel
instance corresponding to the given inputs, if such a translation
exists, or None if no translation is found.
"""
hashed_source_text = utils.convert_to_hash(source_text, 50)
instance_id = cls._generate_id(
source_language_code, target_language_code, hashed_source_text)
return cls.get(instance_id, strict=False)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model is not associated with users."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is not associated with users."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model is not associated with users."""
return dict(super(cls, cls).get_export_policy(), **{
'source_text': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'hashed_source_text': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'source_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'target_language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'translated_text': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class ExplorationModel(base_models.VersionedModel):
"""Versioned storage model for an Oppia exploration.
This class should only be imported by the exploration services file
and the exploration model test file.
"""
SNAPSHOT_METADATA_CLASS = ExplorationSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = ExplorationSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = ExplorationCommitLogEntryModel
ALLOW_REVERT = True
# What this exploration is called.
title = datastore_services.StringProperty(required=True)
# The category this exploration belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this exploration.
objective = datastore_services.TextProperty(default='', indexed=False)
# The ISO 639-1 code for the language this exploration is written in.
language_code = datastore_services.StringProperty(
default=constants.DEFAULT_LANGUAGE_CODE, indexed=True)
# Tags (topics, skills, concepts, etc.) associated with this
# exploration.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# A blurb for this exploration.
blurb = datastore_services.TextProperty(default='', indexed=False)
# 'Author notes' for this exploration.
author_notes = datastore_services.TextProperty(default='', indexed=False)
# The version of the states blob schema.
states_schema_version = datastore_services.IntegerProperty(required=True,
default=0,
indexed=True)
# The name of the initial state of this exploration.
init_state_name = (datastore_services.StringProperty(required=True,
indexed=True))
# A dict representing the states of this exploration. This dict should
# not be empty.
states = datastore_services.JsonProperty(default={}, indexed=False)
# The dict of parameter specifications associated with this exploration.
# Each specification is a dict whose keys are param names and whose values
# are each dicts with a single key, 'obj_type', whose value is a string.
param_specs = datastore_services.JsonProperty(default={}, indexed=False)
# The list of parameter changes to be performed once at the start of a
# reader's encounter with an exploration.
param_changes = (datastore_services.JsonProperty(repeated=True,
indexed=False))
# A boolean indicating whether automatic text-to-speech is enabled in
# this exploration.
auto_tts_enabled = (datastore_services.BooleanProperty(default=True,
indexed=True))
# A boolean indicating whether correctness feedback is enabled in this
# exploration.
correctness_feedback_enabled = datastore_services.BooleanProperty(
default=False, indexed=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'title':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'blurb':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'author_notes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'states_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'init_state_name':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'states':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'param_specs':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'param_changes':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'auto_tts_enabled':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'correctness_feedback_enabled':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_exploration_count(cls) -> int:
"""Returns the total number of explorations."""
return cls.get_all().count()
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't be allowed.
def _prepare_additional_models(
self) -> Mapping[str, base_models.BaseModel]:
"""Prepares additional models needed for the commit process.
Returns:
dict(str, BaseModel). Additional models needed for
the commit process. Contains the ExplorationRightsModel.
"""
return {'rights_model': ExplorationRightsModel.get_by_id(self.id)}
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id, commit_type, commit_message, commit_cmds,
additional_models)
# The cast is needed because the additional_models is list of BaseModels
# and we want to hint the mypy that this is ExplorationRightsModel.
exploration_rights_model = cast(ExplorationRightsModel,
additional_models['rights_model'])
exploration_commit_log = ExplorationCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, exploration_rights_model.status,
exploration_rights_model.community_owned)
exploration_commit_log.exploration_id = self.id
return {
'snapshot_metadata_model':
models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': exploration_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.delete_multi().
@classmethod
def delete_multi( # type: ignore[override]
cls,
entity_ids: List[str],
committer_id: str,
commit_message: str,
force_deletion: bool = False) -> None:
"""Deletes the given cls instances with the given entity_ids.
Note that this extends the superclass method.
Args:
entity_ids: list(str). Ids of entities to delete.
committer_id: str. The user_id of the user who committed the change.
commit_message: str. The commit description message.
force_deletion: bool. If True these models are deleted completely
from storage, otherwise there are only marked as deleted.
Default is False.
"""
super(ExplorationModel,
cls).delete_multi(entity_ids,
committer_id,
commit_message,
force_deletion=force_deletion)
if not force_deletion:
commit_log_models = []
exp_rights_models = ExplorationRightsModel.get_multi(
entity_ids, include_deleted=True)
versioned_models = cls.get_multi(entity_ids, include_deleted=True)
versioned_and_exp_rights_models = zip(versioned_models,
exp_rights_models)
for model, rights_model in versioned_and_exp_rights_models:
# Ruling out the possibility of None for mypy type checking.
assert model is not None
assert rights_model is not None
exploration_commit_log = ExplorationCommitLogEntryModel.create(
model.id, model.version, committer_id,
feconf.COMMIT_TYPE_DELETE, commit_message,
[{
'cmd': cls.CMD_DELETE_COMMIT
}], rights_model.status, rights_model.community_owned)
exploration_commit_log.exploration_id = model.id
commit_log_models.append(exploration_commit_log)
ExplorationCommitLogEntryModel.update_timestamps_multi(
commit_log_models)
datastore_services.put_multi(commit_log_models)
# TODO(#13523): Change snapshot of this model to TypedDict/Domain Object
# to remove Any used below.
@staticmethod
def convert_to_valid_dict(snapshot_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the ExplorationModel dict.
Some old ExplorationModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a ExplorationModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict. The content of the model. Some fields and field
values might no longer exist in the ExplorationModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
if 'skill_tags' in snapshot_dict:
del snapshot_dict['skill_tags']
if 'default_skin' in snapshot_dict:
del snapshot_dict['default_skin']
if 'skin_customizations' in snapshot_dict:
del snapshot_dict['skin_customizations']
return snapshot_dict
# TODO(#13523): Change 'snapshot_dict' to TypedDict/Domain Object
# to remove Any used below.
def _reconstitute(self, snapshot_dict: Dict[str, Any]) -> ExplorationModel:
"""Populates the model instance with the snapshot.
Some old ExplorationSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a ExplorationModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the snapshot.
"""
self.populate(**ExplorationModel.convert_to_valid_dict(snapshot_dict))
return self
class TopicModel(base_models.VersionedModel):
"""Model for storing Topics.
This class should only be imported by the topic services file
and the topic model test file.
"""
SNAPSHOT_METADATA_CLASS = TopicSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = TopicSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = TopicCommitLogEntryModel
ALLOW_REVERT = False
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (
datastore_services.StringProperty(required=True, indexed=True))
# The abbreviated name of the topic.
abbreviated_name = (
datastore_services.StringProperty(indexed=True, default=''))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
# The thumbnail size in bytes of the topic.
thumbnail_size_in_bytes = (
datastore_services.IntegerProperty(indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# This consists of the list of objects referencing canonical stories that
# are part of this topic.
canonical_story_references = (
datastore_services.JsonProperty(repeated=True, indexed=False))
# This consists of the list of objects referencing additional stories that
# are part of this topic.
additional_story_references = (
datastore_services.JsonProperty(repeated=True, indexed=False))
# The schema version for the story reference object on each of the above 2
# lists.
story_reference_schema_version = datastore_services.IntegerProperty(
required=True, indexed=True)
# This consists of the list of uncategorized skill ids that are not part of
# any subtopic.
uncategorized_skill_ids = (
datastore_services.StringProperty(repeated=True, indexed=True))
# The list of subtopics that are part of the topic.
subtopics = datastore_services.JsonProperty(repeated=True, indexed=False)
# The schema version of the subtopic dict.
subtopic_schema_version = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The id for the next subtopic.
next_subtopic_id = datastore_services.IntegerProperty(required=True)
# The ISO 639-1 code for the language this topic is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The url fragment of the topic.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# Whether to show practice tab in the Topic viewer page.
practice_tab_is_displayed = datastore_services.BooleanProperty(
required=True, default=False)
# The content of the meta tag in the Topic viewer page.
meta_tag_content = datastore_services.StringProperty(
indexed=True, default='')
# The page title fragment used in the Topic viewer web page.
# For example, if the full Topic viewer web page title is
# 'Learn Fractions | Add, Subtract, Multiply and Divide | Oppia'
# the page title fragment field represents the middle value 'Add, Subtract,
# Multiply and Divide'.
page_title_fragment_for_web = datastore_services.StringProperty(
indexed=True, default='')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def _trusted_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
"""
super(TopicModel, self)._trusted_commit(
committer_id, commit_type, commit_message, commit_cmds)
topic_rights = TopicRightsModel.get_by_id(self.id)
if topic_rights.topic_is_published:
status = constants.ACTIVITY_STATUS_PUBLIC
else:
status = constants.ACTIVITY_STATUS_PRIVATE
topic_commit_log_entry = TopicCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type,
commit_message, commit_cmds, status, False
)
topic_commit_log_entry.topic_id = self.id
topic_commit_log_entry.update_timestamps()
topic_commit_log_entry.put()
@classmethod
def get_by_name(cls, topic_name: str) -> Optional['TopicModel']:
"""Gets TopicModel by topic_name. Returns None if the topic with
name topic_name doesn't exist.
Args:
topic_name: str. The name of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
return cls.get_all().filter(
cls.canonical_name == topic_name.lower()
).get()
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional['TopicModel']:
"""Gets TopicModel by url_fragment. Returns None if the topic with
name url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the topic.
Returns:
TopicModel|None. The topic model of the topic or None if not
found.
"""
# TODO(#10210): Make fetching by URL fragment faster.
return cls.get_all().filter(cls.url_fragment == url_fragment).get()
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'abbreviated_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_size_in_bytes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_references':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_reference_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopics': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'next_subtopic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'meta_tag_content': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'page_title_fragment_for_web': (
base_models.EXPORT_POLICY.NOT_APPLICABLE),
'practice_tab_is_displayed':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE,
})
class TopicSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Topic.
This should be used whenever the content blob of the topic is not
needed (e.g. search results, etc).
A TopicSummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version,
url_fragment.
The key of each instance is the topic id.
"""
# The name of the topic.
name = datastore_services.StringProperty(required=True, indexed=True)
# The canonical name of the topic, created by making `name` lowercase.
canonical_name = (
datastore_services.StringProperty(required=True, indexed=True))
# The ISO 639-1 code for the language this topic is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The description of the topic.
description = datastore_services.TextProperty(indexed=False)
# The url fragment of the topic.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# Time when the topic model was last updated (not to be
# confused with last_updated, which is the time when the
# topic *summary* model was last updated).
topic_model_last_updated = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# Time when the topic model was created (not to be confused
# with created_on, which is the time when the topic *summary*
# model was created).
topic_model_created_on = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# The number of canonical stories that are part of this topic.
canonical_story_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of additional stories that are part of this topic.
additional_story_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The total number of skills in the topic (including those that are
# uncategorized).
total_skill_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The total number of published chapters in the topic.
total_published_node_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of skills that are not part of any subtopic.
uncategorized_skill_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The number of subtopics of the topic.
subtopic_count = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The thumbnail filename of the topic.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the topic.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_name': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'topic_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'canonical_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_story_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_skill_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'total_published_node_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'uncategorized_skill_count':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'subtopic_count': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class StoryModel(base_models.VersionedModel):
"""Model for storing stories.
This class should only be imported by the story services file
and the story model test file.
"""
SNAPSHOT_METADATA_CLASS = StorySnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = StorySnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = StoryCommitLogEntryModel
ALLOW_REVERT = False
# The title of the story.
title = datastore_services.StringProperty(required=True, indexed=True)
# The thumbnail filename of the story.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the story.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
# The thumbnail size of the story.
thumbnail_size_in_bytes = datastore_services.IntegerProperty(indexed=True)
# A high-level description of the story.
description = datastore_services.TextProperty(indexed=False)
# A set of notes, that describe the characters, main storyline, and setting.
notes = datastore_services.TextProperty(indexed=False)
# The ISO 639-1 code for the language this story is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# The story contents dict specifying the list of story nodes and the
# connection between them. Modelled by class StoryContents
# (see story_domain.py for its current schema).
story_contents = datastore_services.JsonProperty(default={}, indexed=False)
# The schema version for the story_contents.
story_contents_schema_version = (
datastore_services.IntegerProperty(required=True, indexed=True))
# The topic id to which the story belongs.
corresponding_topic_id = (
datastore_services.StringProperty(indexed=True, required=True))
# The url fragment for the story.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
# The content of the meta tag in the Story viewer page.
meta_tag_content = datastore_services.StringProperty(
indexed=True, default='')
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
# TODO(#13523): Change 'commit_cmds' to TypedDict/Domain Object
# to remove Any used below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id,
commit_type,
commit_message,
commit_cmds,
additional_models
)
story_commit_log_entry = StoryCommitLogEntryModel.create(
self.id, self.version, committer_id, commit_type, commit_message,
commit_cmds, constants.ACTIVITY_STATUS_PUBLIC, False
)
story_commit_log_entry.story_id = self.id
return {
'snapshot_metadata_model': models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': story_commit_log_entry,
'versioned_model': models_to_put['versioned_model'],
}
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_size_in_bytes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'notes': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_contents_schema_version':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'corresponding_topic_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'meta_tag_content': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_by_url_fragment(cls, url_fragment: str) -> Optional[StoryModel]:
"""Gets StoryModel by url_fragment. Returns None if the story with
name url_fragment doesn't exist.
Args:
url_fragment: str. The url fragment of the story.
Returns:
StoryModel|None. The story model of the story or None if not
found.
"""
return cls.get_all().filter(cls.url_fragment == url_fragment).get()
class StorySummaryModel(base_models.BaseModel):
"""Summary model for an Oppia Story.
This should be used whenever the content blob of the story is not
needed (e.g. search results, etc).
A StorySummaryModel instance stores the following information:
id, description, language_code, last_updated, created_on, version.
The key of each instance is the story id.
"""
# The title of the story.
title = datastore_services.StringProperty(required=True, indexed=True)
# The ISO 639-1 code for the language this story is written in.
language_code = (
datastore_services.StringProperty(required=True, indexed=True))
# A high-level description of the story.
description = datastore_services.TextProperty(required=True, indexed=False)
# Time when the story model was last updated (not to be
# confused with last_updated, which is the time when the
# story *summary* model was last updated).
story_model_last_updated = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# Time when the story model was created (not to be confused
# with created_on, which is the time when the story *summary*
# model was created).
story_model_created_on = (
datastore_services.DateTimeProperty(required=True, indexed=True))
# The titles of the nodes in the story, in the same order as present there.
node_titles = (
datastore_services.StringProperty(repeated=True, indexed=True))
# The thumbnail filename of the story.
thumbnail_filename = datastore_services.StringProperty(indexed=True)
# The thumbnail background color of the story.
thumbnail_bg_color = datastore_services.StringProperty(indexed=True)
version = datastore_services.IntegerProperty(required=True)
# The url fragment for the story.
url_fragment = (
datastore_services.StringProperty(required=True, indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'description': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'story_model_created_on': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'node_titles': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_filename': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'thumbnail_bg_color': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'url_fragment': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class QuestionSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia question.
This should be used whenever the content blob of the question is not
needed (e.g. in search results, etc).
A QuestionSummaryModel instance stores the following information:
question_model_last_updated, question_model_created_on,
question_state_data.
The key of each instance is the question id.
"""
# Time when the question model was last updated (not to be
# confused with last_updated, which is the time when the
# question *summary* model was last updated).
question_model_last_updated = datastore_services.DateTimeProperty(
indexed=True, required=True)
# Time when the question model was created (not to be confused
# with created_on, which is the time when the question *summary*
# model was created).
question_model_created_on = datastore_services.DateTimeProperty(
indexed=True, required=True)
# The html content for the question.
question_content = (datastore_services.TextProperty(indexed=False,
required=True))
# The ID of the interaction.
interaction_id = (datastore_services.StringProperty(indexed=True,
required=True))
# The misconception ids addressed in the question. This includes
# tagged misconceptions ids as well as inapplicable misconception
# ids in the question.
misconception_ids = (datastore_services.StringProperty(indexed=True,
repeated=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the QuestionModel
export_data function, and thus a new export_data function does not
need to be defined here.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because because noteworthy details that belong to this model have
already been exported as a part of the QuestionModel export_data
function.
"""
return dict(
super(cls, cls).get_export_policy(), **{
'question_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'question_content': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'interaction_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'misconception_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
class GeneralVoiceoverApplicationModel(base_models.BaseModel):
"""A general model for voiceover application of an entity.
The ID of the voiceover application will be a random hashed value.
"""
# We use the model id as a key in the Takeout dict.
ID_IS_USED_AS_TAKEOUT_KEY = True
# The type of entity to which the user will be assigned as a voice artist
# once the application will get approved.
target_type = datastore_services.StringProperty(required=True,
indexed=True)
# The ID of the entity to which the application belongs.
target_id = datastore_services.StringProperty(required=True, indexed=True)
# The language code for the voiceover audio.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# The status of the application. One of: accepted, rejected, in-review.
status = datastore_services.StringProperty(required=True,
indexed=True,
choices=STATUS_CHOICES)
# The HTML content written in the given language_code.
# This will typically be a snapshot of the content of the initial card of
# the target.
content = datastore_services.TextProperty(required=True)
# The filename of the voiceover audio. The filename will have
# datetime-randomId(length 6)-language_code.mp3 pattern.
filename = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the author of the voiceover application.
author_id = datastore_services.StringProperty(required=True, indexed=True)
# The ID of the reviewer who accepted/rejected the voiceover application.
final_reviewer_id = datastore_services.StringProperty(indexed=True)
# The plain text message submitted by the reviewer while rejecting the
# application.
rejection_message = datastore_services.TextProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize corresponding to a user:
author_id, and final_reviewer_id fields.
"""
return base_models.DELETION_POLICY.LOCALLY_PSEUDONYMIZE
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether GeneralVoiceoverApplicationModel exists for the user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.author_id == user_id,
cls.final_reviewer_id == user_id)).get(
keys_only=True) is not None
@classmethod
def get_user_voiceover_applications(
cls,
author_id: str,
status: Optional[str] = None
) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover application submitted by the given user.
Args:
author_id: str. The id of the user created the voiceover
application.
status: str|None. The status of the voiceover application.
If the status is None, the query will fetch all the
voiceover applications.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
applications submitted by the given user.
"""
if status in STATUS_CHOICES:
voiceover_application_query = cls.query(
datastore_services.all_of(cls.author_id == author_id,
cls.status == status))
else:
voiceover_application_query = cls.query(cls.author_id == author_id)
return cast(List[GeneralVoiceoverApplicationModel],
voiceover_application_query.fetch())
@classmethod
def get_reviewable_voiceover_applications(
cls, user_id: str) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover application which a given user can
review.
Args:
user_id: str. The id of the user trying to make this query.
As a user cannot review their own voiceover application, so the
voiceover application created by the user will be excluded.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
applications which the given user can review.
"""
return cast(
List[GeneralVoiceoverApplicationModel],
cls.query(
datastore_services.all_of(
cls.author_id != user_id,
cls.status == STATUS_IN_REVIEW)).fetch())
@classmethod
def get_voiceover_applications(
cls, target_type: str, target_id: str,
language_code: str) -> List['GeneralVoiceoverApplicationModel']:
"""Returns a list of voiceover applications submitted for a give entity
in a given language.
Args:
target_type: str. The type of entity.
target_id: str. The ID of the targeted entity.
language_code: str. The code of the language in which the voiceover
application is submitted.
Returns:
list(GeneralVoiceoverApplicationModel). The list of voiceover
application which is submitted to a give entity in a given language.
"""
return cast(
List[GeneralVoiceoverApplicationModel],
cls.query(
datastore_services.all_of(
cls.target_type == target_type, cls.target_id == target_id,
cls.language_code == language_code)).fetch())
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as multiple instances per user since there are
multiple voiceover applications relevant to a user.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.MULTIPLE_INSTANCES_PER_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(),
**{
'target_type': base_models.EXPORT_POLICY.EXPORTED,
'target_id': base_models.EXPORT_POLICY.EXPORTED,
'language_code': base_models.EXPORT_POLICY.EXPORTED,
'status': base_models.EXPORT_POLICY.EXPORTED,
'content': base_models.EXPORT_POLICY.EXPORTED,
'filename': base_models.EXPORT_POLICY.EXPORTED,
# The author_id and final_reviewer_id are not exported in order to
# keep internal ids private.
'author_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'final_reviewer_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'rejection_message': base_models.EXPORT_POLICY.EXPORTED
})
@classmethod
def export_data(cls, user_id: str) -> Dict[str, Dict[str, Optional[str]]]:
"""(Takeout) Exports the data from GeneralVoiceoverApplicationModel
into dict format.
Args:
user_id: str. The ID of the user whose data should be exported.
Returns:
dict. Dictionary of the data from GeneralVoiceoverApplicationModel.
"""
user_data = {}
voiceover_models = cast(List[GeneralVoiceoverApplicationModel],
cls.query(cls.author_id == user_id).fetch())
for voiceover_model in voiceover_models:
user_data[voiceover_model.id] = {
'target_type': voiceover_model.target_type,
'target_id': voiceover_model.target_id,
'language_code': voiceover_model.language_code,
'status': voiceover_model.status,
'content': voiceover_model.content,
'filename': voiceover_model.filename,
'rejection_message': voiceover_model.rejection_message
}
return user_data
class CollectionModel(base_models.VersionedModel):
"""Versioned storage model for an Oppia collection.
This class should only be imported by the collection services file
and the collection model test file.
"""
SNAPSHOT_METADATA_CLASS = CollectionSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = CollectionSnapshotContentModel
COMMIT_LOG_ENTRY_CLASS = CollectionCommitLogEntryModel
ALLOW_REVERT = True
# What this collection is called.
title = datastore_services.StringProperty(required=True)
# The category this collection belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this collection.
objective = datastore_services.TextProperty(default='', indexed=False)
# The language code of this collection.
language_code = datastore_services.StringProperty(
default=constants.DEFAULT_LANGUAGE_CODE, indexed=True)
# Tags associated with this collection.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# The version of all property blob schemas.
schema_version = datastore_services.IntegerProperty(
required=True, default=1, indexed=True)
# A dict representing the contents of a collection. Currently, this
# contains the list of nodes. This dict should contain collection data
# whose structure might need to be changed in the future.
collection_contents = (
datastore_services.JsonProperty(default={}, indexed=False))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'schema_version': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'collection_contents': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_collection_count(cls) -> int:
"""Returns the total number of collections."""
return cls.get_all().count()
# TODO(#13523): Change 'model_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@staticmethod
def convert_to_valid_dict(model_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the CollectionModel dict.
Some old CollectionModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionModel from
them. We need to remove or replace these fields and values.
Args:
model_dict: dict. The content of the model. Some fields and field
values might no longer exist in the CollectionModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
# The nodes field is moved to collection_contents dict. We
# need to move the values from nodes field to collection_contents dict
# and delete nodes.
if 'nodes' in model_dict and model_dict['nodes']:
model_dict['collection_contents']['nodes'] = (
copy.deepcopy(model_dict['nodes']))
del model_dict['nodes']
return model_dict
# TODO(#13523): Change 'snapshot_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
def _reconstitute(self, snapshot_dict: Dict[str, Any]) -> CollectionModel:
"""Populates the model instance with the snapshot.
Some old CollectionModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the the snapshot.
"""
self.populate(
**CollectionModel.convert_to_valid_dict(snapshot_dict))
return self
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't be allowed.
def _prepare_additional_models(self) -> Mapping[str, base_models.BaseModel]:
"""Prepares additional models needed for the commit process.
Returns:
dict(str, BaseModel). Additional models needed for
the commit process. Contains the CollectionRightsModel.
"""
return {
'rights_model': CollectionRightsModel.get_by_id(self.id)
}
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this extends the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id,
commit_type,
commit_message,
commit_cmds,
additional_models
)
# The cast is needed because the additional_models is list of BaseModels
# and we want to hint the mypy that this is CollectionRightsModel.
collection_rights_model = cast(
CollectionRightsModel, additional_models['rights_model']
)
collection_commit_log = CollectionCommitLogEntryModel.create(
self.id,
self.version,
committer_id,
commit_type,
commit_message,
commit_cmds,
collection_rights_model.status,
collection_rights_model.community_owned
)
collection_commit_log.collection_id = self.id
return {
'snapshot_metadata_model': models_to_put['snapshot_metadata_model'],
'snapshot_content_model': models_to_put['snapshot_content_model'],
'commit_log_model': collection_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
# We have ignored [override] here because the signature of this method
# doesn't match with BaseModel.delete_multi().
# https://mypy.readthedocs.io/en/stable/error_code_list.html#check-validity-of-overrides-override
@classmethod
def delete_multi( # type: ignore[override]
cls,
entity_ids: List[str],
committer_id: str,
commit_message: str,
force_deletion: bool = False
) -> None:
"""Deletes the given cls instances with the given entity_ids.
Note that this extends the superclass method.
Args:
entity_ids: list(str). Ids of entities to delete.
committer_id: str. The user_id of the user who committed the change.
commit_message: str. The commit description message.
force_deletion: bool. If True these models are deleted completely
from storage, otherwise there are only marked as deleted.
Default is False.
"""
super(CollectionModel, cls).delete_multi(
entity_ids, committer_id,
commit_message, force_deletion=force_deletion)
if not force_deletion:
commit_log_models = []
collection_rights_models = CollectionRightsModel.get_multi(
entity_ids, include_deleted=True)
versioned_models = cls.get_multi(entity_ids, include_deleted=True)
for model, rights_model in zip(
versioned_models, collection_rights_models):
# Ruling out the possibility of None for mypy type checking.
assert model is not None
assert rights_model is not None
collection_commit_log = CollectionCommitLogEntryModel.create(
model.id, model.version, committer_id,
feconf.COMMIT_TYPE_DELETE,
commit_message, [{'cmd': cls.CMD_DELETE_COMMIT}],
rights_model.status, rights_model.community_owned
)
collection_commit_log.collection_id = model.id
commit_log_models.append(collection_commit_log)
CollectionCommitLogEntryModel.update_timestamps_multi(
commit_log_models)
datastore_services.put_multi(commit_log_models)
