class QuestionSkillLinkModel(base_models.BaseModel):
"""Model for storing Question-Skill Links.
The ID of instances of this class has the form '[question_id]:[skill_id]'.
"""
# The ID of the question.
question_id = (datastore_services.StringProperty(required=True,
indexed=True))
# The ID of the skill to which the question is linked.
skill_id = datastore_services.StringProperty(required=True, indexed=True)
# The difficulty of the skill.
skill_difficulty = (datastore_services.FloatProperty(required=True,
indexed=True))
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'question_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_id': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'skill_difficulty': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def get_model_id(cls, question_id: str, skill_id: str) -> str:
"""Returns the model id by combining the questions and skill id.
Args:
question_id: str. The ID of the question.
skill_id: str. The ID of the skill to which the question is linked.
Returns:
str. The calculated model id.
"""
return '%s:%s' % (question_id, skill_id)
@classmethod
def create(cls, question_id: str, skill_id: str,
skill_difficulty: float) -> 'QuestionSkillLinkModel':
"""Creates a new QuestionSkillLinkModel entry.
Args:
question_id: str. The ID of the question.
skill_id: str. The ID of the skill to which the question is linked.
skill_difficulty: float. The difficulty between [0, 1] of the skill.
Raises:
Exception. The given question is already linked to the given skill.
Returns:
QuestionSkillLinkModel. Instance of the new QuestionSkillLinkModel
entry.
"""
question_skill_link_id = cls.get_model_id(question_id, skill_id)
if cls.get(question_skill_link_id, strict=False) is not None:
raise Exception(
'The given question is already linked to given skill')
question_skill_link_model_instance = cls(
id=question_skill_link_id,
question_id=question_id,
skill_id=skill_id,
skill_difficulty=skill_difficulty)
return question_skill_link_model_instance
@classmethod
def get_total_question_count_for_skill_ids(cls,
skill_ids: List[str]) -> int:
"""Returns the number of questions assigned to the given skill_ids.
Args:
skill_ids: list(str). Skill IDs for which the question count is
requested.
Returns:
int. The number of questions assigned to the given skill_ids.
"""
total_question_count = cls.query().filter(
cls.skill_id.IN(skill_ids)).count()
return total_question_count
@classmethod
def get_question_skill_links_by_skill_ids(
cls, question_count: int, skill_ids: List[str],
offset: int) -> List['QuestionSkillLinkModel']:
"""Fetches the list of QuestionSkillLinkModels linked to the skill in
batches.
Args:
question_count: int. The number of questions to be returned.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
offset: int. Number of query results to skip.
Returns:
list(QuestionSkillLinkModel). The QuestionSkillLinkModels
corresponding to given skill_ids.
"""
question_skill_count = min(
len(skill_ids), constants.MAX_SKILLS_PER_QUESTION) * question_count
return cast(
List[QuestionSkillLinkModel],
cls.query(
cls.skill_id.IN(skill_ids)).order(-cls.last_updated).fetch(
question_skill_count, offset=offset))
@classmethod
def get_question_skill_links_based_on_difficulty_equidistributed_by_skill(
cls, total_question_count: int, skill_ids: List[str],
difficulty_requested: float) -> List['QuestionSkillLinkModel']:
"""Fetches the list of constant number of random QuestionSkillLinkModels
linked to the skills, sorted by the absolute value of the difference
between skill difficulty and the requested difficulty.
Args:
total_question_count: int. The number of questions expected.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
difficulty_requested: float. The skill difficulty of the questions
requested to be fetched.
Returns:
list(QuestionSkillLinkModel). A list of random
QuestionSkillLinkModels corresponding to given skill_ids, with
total_question_count/len(skill_ids) number of questions for
each skill. If not evenly divisible, it will be rounded up.
If not enough questions for a skill, just return all questions
it links to.
"""
if len(skill_ids) > feconf.MAX_NUMBER_OF_SKILL_IDS:
raise Exception('Please keep the number of skill IDs below 20.')
if (not skill_ids) or (total_question_count == 0):
return []
question_count_per_skill = int(
math.ceil(
python_utils.divide( # type: ignore[no-untyped-call]
float(total_question_count), float(len(skill_ids)))))
question_skill_link_mapping = {}
# For fetching the questions randomly we have used a random offset.
# But this is a temporary solution since this method scales linearly.
# Other alternative methods were:
# 1) Using a random id in question id filter
# 2) Adding an additional column that can be filtered upon.
# But these methods are not viable because google datastore limits
# each query to have at most one inequality filter. So we can't filter
# on both question_id and difficulty. Please see
# https://github.com/oppia/oppia/pull/9061#issuecomment-629765809
# for more details.
def get_offset(query: datastore_services.Query) -> int:
"""Helper function to get the offset."""
question_count = query.count()
if question_count > 2 * question_count_per_skill:
return utils.get_random_int(question_count -
(question_count_per_skill * 2))
return 0
for skill_id in skill_ids:
query = cls.query(cls.skill_id == skill_id)
equal_questions_query = query.filter(
cls.skill_difficulty == difficulty_requested)
# We fetch more questions here in order to try and ensure that the
# eventual number of returned questions is sufficient to meet the
# number requested, even after deduplication.
new_question_skill_link_models = cast(
List[QuestionSkillLinkModel],
equal_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(equal_questions_query)))
for model in new_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
new_question_skill_link_models.remove(model)
if len(new_question_skill_link_models) >= question_count_per_skill:
new_question_skill_link_models = random.sample(
new_question_skill_link_models, question_count_per_skill)
else:
# Fetch QuestionSkillLinkModels with difficulty smaller than
# requested difficulty.
easier_questions_query = query.filter(
cls.skill_difficulty < difficulty_requested)
easier_question_skill_link_models = cast(
List[QuestionSkillLinkModel],
easier_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(easier_questions_query)))
for model in easier_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
easier_question_skill_link_models.remove(model)
question_extra_count = (
len(new_question_skill_link_models) +
len(easier_question_skill_link_models) -
question_count_per_skill)
if question_extra_count >= 0:
easier_question_skill_link_models = random.sample(
easier_question_skill_link_models,
question_count_per_skill -
len(new_question_skill_link_models))
new_question_skill_link_models.extend(
easier_question_skill_link_models)
else:
# Fetch QuestionSkillLinkModels with difficulty larger than
# requested difficulty.
new_question_skill_link_models.extend(
easier_question_skill_link_models)
harder_questions_query = query.filter(
cls.skill_difficulty > difficulty_requested)
harder_question_skill_link_models = (
harder_questions_query.fetch(
limit=question_count_per_skill * 2,
offset=get_offset(harder_questions_query)))
harder_question_skill_link_models = cast(
List[QuestionSkillLinkModel],
harder_questions_query.fetch())
for model in harder_question_skill_link_models:
if model.question_id in question_skill_link_mapping:
harder_question_skill_link_models.remove(model)
question_extra_count = (
len(new_question_skill_link_models) +
len(harder_question_skill_link_models) -
question_count_per_skill)
if question_extra_count >= 0:
harder_question_skill_link_models = (random.sample(
harder_question_skill_link_models,
question_count_per_skill -
len(new_question_skill_link_models)))
new_question_skill_link_models.extend(
cast(List[QuestionSkillLinkModel],
harder_question_skill_link_models))
new_question_skill_link_models = (
new_question_skill_link_models[:question_count_per_skill])
for model in new_question_skill_link_models:
if model.question_id not in question_skill_link_mapping:
question_skill_link_mapping[model.question_id] = model
return list(question_skill_link_mapping.values())
@classmethod
def get_question_skill_links_equidistributed_by_skill(
cls, total_question_count: int,
skill_ids: List[str]) -> List['QuestionSkillLinkModel']:
"""Fetches the list of constant number of random
QuestionSkillLinkModels linked to the skills.
Args:
total_question_count: int. The number of questions expected.
skill_ids: list(str). The ids of skills for which the linked
question ids are to be retrieved.
Returns:
list(QuestionSkillLinkModel). A list of random
QuestionSkillLinkModels corresponding to given skill_ids, with
total_question_count/len(skill_ids) number of questions for
each skill. If not evenly divisible, it will be rounded up.
If not enough questions for a skill, just return all questions
it links to.
"""
if len(skill_ids) > feconf.MAX_NUMBER_OF_SKILL_IDS:
raise Exception('Please keep the number of skill IDs below 20.')
if not skill_ids:
return []
question_count_per_skill = int(
math.ceil(
python_utils.divide( # type: ignore[no-untyped-call]
float(total_question_count), float(len(skill_ids)))))
question_skill_link_models = []
existing_question_ids = []
def get_offset(query: datastore_services.Query) -> int:
"""Helper function to get the offset."""
question_count = query.count()
if question_count > 2 * question_count_per_skill:
return utils.get_random_int(question_count -
(question_count_per_skill * 2))
return 0
for skill_id in skill_ids:
query = cls.query(cls.skill_id == skill_id)
# We fetch more questions here in order to try and ensure that the
# eventual number of returned questions is sufficient to meet the
# number requested, even after deduplication.
new_question_skill_link_models = cast(
List[QuestionSkillLinkModel],
query.fetch(limit=question_count_per_skill * 2,
offset=get_offset(query)))
# Deduplicate if the same question is linked to multiple skills.
for model in new_question_skill_link_models:
if model.question_id in existing_question_ids:
new_question_skill_link_models.remove(model)
if len(new_question_skill_link_models) > question_count_per_skill:
sampled_question_skill_link_models = random.sample(
new_question_skill_link_models, question_count_per_skill)
else:
sampled_question_skill_link_models = (
new_question_skill_link_models)
question_skill_link_models.extend(
sampled_question_skill_link_models)
existing_question_ids.extend([
model.question_id
for model in (sampled_question_skill_link_models)
])
return question_skill_link_models
@classmethod
def get_all_question_ids_linked_to_skill_id(cls,
skill_id: str) -> List[str]:
"""Returns a list of all question ids corresponding to the given skill
id.
Args:
skill_id: str. ID of the skill.
Returns:
list(str). The list of all question ids corresponding to the given
skill id.
"""
question_skill_link_models = cls.query().filter(
cls.skill_id == skill_id, cls.deleted == False) #pylint: disable=singleton-comparison
question_ids = [
model.question_id for model in question_skill_link_models
]
return question_ids
@classmethod
def get_models_by_skill_id(
cls, skill_id: str) -> List['QuestionSkillLinkModel']:
"""Returns a list of QuestionSkillLink domains of a particular skill ID.
Args:
skill_id: str. ID of the skill.
Returns:
list(QuestionSkillLinkModel)|None. The list of question skill link
domains that are linked to the skill ID. None if the skill
ID doesn't exist.
"""
return cast(
List[QuestionSkillLinkModel],
QuestionSkillLinkModel.query().filter(
cls.skill_id == skill_id).fetch())
@classmethod
def get_models_by_question_id(
cls, question_id: str) -> List['QuestionSkillLinkModel']:
"""Returns a list of QuestionSkillLinkModels of a particular
question ID.
Args:
question_id: str. ID of the question.
Returns:
list(QuestionSkillLinkModel)|None. The list of question skill link
models that are linked to the question ID, or None if there are no
question skill link models associated with the question ID.
"""
return cast(List[QuestionSkillLinkModel],
QuestionSkillLinkModel.query().filter(
cls.question_id == question_id,
cls.deleted == False).fetch()) #pylint: disable=singleton-comparison
@classmethod
def put_multi_question_skill_links(
cls, question_skill_links: List['QuestionSkillLinkModel']) -> None:
"""Puts multiple question skill link models into the datastore.
Args:
question_skill_links: list(QuestionSkillLink). The list of
question skill link domain objects to put into the datastore.
"""
cls.update_timestamps_multi(question_skill_links)
cls.put_multi(question_skill_links)
@classmethod
def delete_multi_question_skill_links(
cls, question_skill_links: List['QuestionSkillLinkModel']) -> None:
"""Deletes multiple question skill links from the datastore.
Args:
question_skill_links: list(QuestionSkillLinkModel). The list of
question skill link domain objects to delete from the datastore.
"""
cls.delete_multi(question_skill_links)
class JobModel(base_models.BaseModel):
"""Class representing a datastore entity for a long-running job."""
# The job type.
job_type = datastore_services.StringProperty(indexed=True)
# The time at which the job was queued, in milliseconds since the epoch.
time_queued_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was started, in milliseconds since the epoch.
# This is never set if the job was canceled before it was started.
time_started_msec = datastore_services.FloatProperty(indexed=True)
# The time at which the job was completed, failed or canceled, in
# milliseconds since the epoch.
time_finished_msec = datastore_services.FloatProperty(indexed=True)
# The current status code for the job.
status_code = datastore_services.StringProperty(
indexed=True,
default=STATUS_CODE_NEW,
choices=[
STATUS_CODE_NEW, STATUS_CODE_QUEUED, STATUS_CODE_STARTED,
STATUS_CODE_COMPLETED, STATUS_CODE_FAILED, STATUS_CODE_CANCELED
])
# Any metadata for the job, such as the root pipeline id for mapreduce
# jobs.
metadata = datastore_services.JsonProperty(indexed=False)
# The output of the job. This is only populated if the job has status code
# STATUS_CODE_COMPLETED, and is None otherwise. If populated, this is
# expected to be a list of strings.
output = datastore_services.JsonProperty(indexed=False)
# The error message, if applicable. Only populated if the job has status
# code STATUS_CODE_FAILED or STATUS_CODE_CANCELED; None otherwise.
error = datastore_services.TextProperty(indexed=False)
# Whether the datastore models associated with this job have been cleaned
# up (i.e., deleted).
has_been_cleaned_up = (datastore_services.BooleanProperty(default=False,
indexed=True))
# Store additional params passed with job.
additional_job_params = datastore_services.JsonProperty(default=None)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model doesn't contain any data directly corresponding to a user."""
return base_models.DELETION_POLICY.NOT_APPLICABLE
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model does not contain user data."""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model doesn't contain any data directly corresponding to a user."""
return dict(
super(cls, cls).get_export_policy(), **{
'job_type': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_queued_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_started_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'time_finished_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'metadata': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'output': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'error': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'has_been_cleaned_up':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'additional_job_params':
base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@property
def is_cancelable(self) -> bool:
"""Checks if the job is cancelable.
Returns:
bool. Whether the job's status_code is 'queued' or 'started'.
"""
# Whether the job is currently in 'queued' or 'started' status.
return self.status_code in [STATUS_CODE_QUEUED, STATUS_CODE_STARTED]
@classmethod
def get_all_unfinished_jobs(cls, limit: int) -> Sequence['JobModel']:
"""Gets at most `limit` unfinished jobs.
Args:
limit: int. A limit on the number of jobs to return.
Returns:
list(JobModel) or None. A list of at most `limit` number
of unfinished jobs.
"""
return cls.query().filter(
JobModel.status_code.IN([
STATUS_CODE_QUEUED, STATUS_CODE_STARTED
])).order(-cls.time_queued_msec).fetch(limit)
@classmethod
def get_unfinished_jobs(cls, job_type: str) -> datastore_services.Query:
"""Gets jobs that are unfinished.
Args:
job_type: str. The type of jobs that may be unfinished.
Returns:
list(JobModel) or None. A list of all jobs that belong
to the given job_type.
"""
return cls.query().filter(cls.job_type == job_type).filter(
JobModel.status_code.IN([STATUS_CODE_QUEUED, STATUS_CODE_STARTED]))
@classmethod
def do_unfinished_jobs_exist(cls, job_type: str) -> bool:
"""Checks if unfinished jobs exist.
Args:
job_type: str. Type of job for which to check.
Returns:
bool. True if unfinished jobs exist, otherwise false.
"""
return bool(cls.get_unfinished_jobs(job_type).count(limit=1))
class CoreModel(base_models.BaseModel):
"""Simple BaseModel subclass with a 'prop' float property."""
prop = datastore_services.FloatProperty()
class CollectionRightsModel(base_models.VersionedModel):
"""Storage model for rights related to a collection.
The id of each instance is the id of the corresponding collection.
"""
SNAPSHOT_METADATA_CLASS = CollectionRightsSnapshotMetadataModel
SNAPSHOT_CONTENT_CLASS = CollectionRightsSnapshotContentModel
ALLOW_REVERT = False
# The user_ids of owners of this collection.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this collection.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to voiceover this collection.
voice_artist_ids = (
datastore_services.StringProperty(indexed=True, repeated=True))
# The user_ids of users who are allowed to view this collection.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# Whether this collection is owned by the community.
community_owned = (
datastore_services.BooleanProperty(indexed=True, default=False))
# For private collections, whether this collection can be viewed
# by anyone who has the URL. If the collection is not private, this
# setting is ignored.
viewable_if_private = (
datastore_services.BooleanProperty(indexed=True, default=False))
# Time, in milliseconds, when the collection was first published.
first_published_msec = (
datastore_services.FloatProperty(indexed=True, default=None))
# The publication status of this collection.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE, indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE,
constants.ACTIVITY_STATUS_PUBLIC
]
)
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, voice_artist_ids, editor_ids,
and owner_ids fields.
"""
return (
base_models.DELETION_POLICY.PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE
)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model is exported as one instance shared across users since multiple
users contribute to collections and have varying rights.
"""
return (
base_models
.MODEL_ASSOCIATION_TO_USER
.ONE_INSTANCE_SHARED_ACROSS_USERS)
@classmethod
def get_field_name_mapping_to_takeout_keys(cls) -> Dict[str, str]:
"""Defines the mapping of field names to takeout keys since this model
is exported as one instance shared across users.
"""
return {
'owner_ids': 'owned_collection_ids',
'editor_ids': 'editable_collection_ids',
'voice_artist_ids': 'voiced_collection_ids',
'viewer_ids': 'viewable_collection_ids'
}
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data to export/delete corresponding to a user."""
return dict(super(cls, cls).get_export_policy(), **{
'owner_ids': base_models.EXPORT_POLICY.EXPORTED,
'editor_ids': base_models.EXPORT_POLICY.EXPORTED,
'voice_artist_ids': base_models.EXPORT_POLICY.EXPORTED,
'viewer_ids': base_models.EXPORT_POLICY.EXPORTED,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewable_if_private': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'first_published_msec': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether CollectionRightsModel references the given user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(datastore_services.any_of(
cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.voice_artist_ids == user_id,
cls.viewer_ids == user_id
)).get(keys_only=True) is not None
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def save(
self,
committer_id: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]]
) -> None:
"""Updates the collection rights model by applying the given
commit_cmds, then saves it.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, which should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and additional arguments for that command.
"""
super(CollectionRightsModel, self).commit(
committer_id, commit_message, commit_cmds)
# TODO(#13523): Change 'model_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
@staticmethod
def convert_to_valid_dict(model_dict: Dict[str, Any]) -> Dict[str, Any]:
"""Replace invalid fields and values in the CollectionRightsModel dict.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
model_dict: dict. The content of the model. Some fields and field
values might no longer exist in the CollectionRightsModel
schema.
Returns:
dict. The content of the model. Only valid fields and values are
present.
"""
# The status field could historically take the value 'publicized', this
# value is now equivalent to 'public'.
if model_dict['status'] == 'publicized':
model_dict['status'] = constants.ACTIVITY_STATUS_PUBLIC
# The voice_artist_ids field was previously named translator_ids. We
# need to move the values from translator_ids field to voice_artist_ids
# and delete translator_ids.
if 'translator_ids' in model_dict and model_dict['translator_ids']:
model_dict['voice_artist_ids'] = model_dict['translator_ids']
del model_dict['translator_ids']
# We need to remove pseudonymous IDs from all the fields that contain
# user IDs.
for field_name in (
'owner_ids', 'editor_ids', 'voice_artist_ids', 'viewer_ids'):
model_dict[field_name] = [
user_id for user_id in model_dict[field_name]
if not utils.is_pseudonymous_id(user_id)
]
return model_dict
# TODO(#13523): Change 'snapshot_dict' to domain object/TypedDict to
# remove Any from type-annotation below.
def _reconstitute(
self, snapshot_dict: Dict[str, Any]
) -> CollectionRightsModel:
"""Populates the model instance with the snapshot.
Some old CollectionRightsSnapshotContentModels can contain fields
and field values that are no longer supported and would cause
an exception when we try to reconstitute a CollectionRightsModel from
them. We need to remove or replace these fields and values.
Args:
snapshot_dict: dict(str, *). The snapshot with the model
property values.
Returns:
VersionedModel. The instance of the VersionedModel class populated
with the the snapshot.
"""
self.populate(
**CollectionRightsModel.convert_to_valid_dict(snapshot_dict))
return self
# TODO(#13523): Change 'commit_cmds' to domain object/TypedDict to
# remove Any from type-annotation below.
def compute_models_to_commit(
self,
committer_id: str,
commit_type: str,
commit_message: str,
commit_cmds: List[Dict[str, Any]],
# We expect Mapping because we want to allow models that inherit
# from BaseModel as the values, if we used Dict this wouldn't
# be allowed.
additional_models: Mapping[str, base_models.BaseModel]
) -> base_models.ModelsToPutDict:
"""Record the event to the commit log after the model commit.
Note that this overrides the superclass method.
Args:
committer_id: str. The user_id of the user who committed the
change.
commit_type: str. The type of commit. Possible values are in
core.storage.base_models.COMMIT_TYPE_CHOICES.
commit_message: str. The commit description message.
commit_cmds: list(dict). A list of commands, describing changes
made in this model, should give sufficient information to
reconstruct the commit. Each dict always contains:
cmd: str. Unique command.
and then additional arguments for that command.
additional_models: dict(str, BaseModel). Additional models that are
needed for the commit process.
Returns:
ModelsToPutDict. A dict of models that should be put into
the datastore.
"""
models_to_put = super().compute_models_to_commit(
committer_id,
commit_type,
commit_message,
commit_cmds,
additional_models
)
snapshot_metadata_model = models_to_put['snapshot_metadata_model']
snapshot_metadata_model.content_user_ids = list(sorted(
set(self.owner_ids) |
set(self.editor_ids) |
set(self.voice_artist_ids) |
set(self.viewer_ids)
))
commit_cmds_user_ids = set()
for commit_cmd in commit_cmds:
user_id_attribute_names = next(
cmd['user_id_attribute_names']
for cmd in feconf.COLLECTION_RIGHTS_CHANGE_ALLOWED_COMMANDS
if cmd['name'] == commit_cmd['cmd']
)
for user_id_attribute_name in user_id_attribute_names:
commit_cmds_user_ids.add(commit_cmd[user_id_attribute_name])
snapshot_metadata_model.commit_cmds_user_ids = list(
sorted(commit_cmds_user_ids))
# Create and delete events will already be recorded in the
# CollectionModel.
if commit_type not in ['create', 'delete']:
collection_commit_log = CollectionCommitLogEntryModel(
id=('rights-%s-%s' % (self.id, self.version)),
user_id=committer_id,
collection_id=self.id,
commit_type=commit_type,
commit_message=commit_message,
commit_cmds=commit_cmds,
version=None,
post_commit_status=self.status,
post_commit_community_owned=self.community_owned,
post_commit_is_private=(
self.status == constants.ACTIVITY_STATUS_PRIVATE)
)
return {
'snapshot_metadata_model': (
models_to_put['snapshot_metadata_model']),
'snapshot_content_model': (
models_to_put['snapshot_content_model']),
'commit_log_model': collection_commit_log,
'versioned_model': models_to_put['versioned_model'],
}
return models_to_put
@classmethod
def export_data(cls, user_id: str) -> Dict[str, List[str]]:
"""(Takeout) Export user-relevant properties of CollectionRightsModel.
Args:
user_id: str. The user_id denotes which user's data to extract.
Returns:
dict. The user-relevant properties of CollectionRightsModel
in a python dict format. In this case, we are returning all the
ids of collections that the user is connected to, so they either
own, edit, voice, or have permission to view.
"""
owned_collections = cls.get_all().filter(cls.owner_ids == user_id)
editable_collections = cls.get_all().filter(cls.editor_ids == user_id)
voiced_collections = (
cls.get_all().filter(cls.voice_artist_ids == user_id))
viewable_collections = cls.get_all().filter(cls.viewer_ids == user_id)
owned_collection_ids = [col.key.id() for col in owned_collections]
editable_collection_ids = [col.key.id() for col in editable_collections]
voiced_collection_ids = [col.key.id() for col in voiced_collections]
viewable_collection_ids = [col.key.id() for col in viewable_collections]
return {
'owned_collection_ids': owned_collection_ids,
'editable_collection_ids': editable_collection_ids,
'voiced_collection_ids': voiced_collection_ids,
'viewable_collection_ids': viewable_collection_ids
}
class ExpSummaryModel(base_models.BaseModel):
"""Summary model for an Oppia exploration.
This should be used whenever the content blob of the exploration is not
needed (e.g. in search results, etc).
A ExpSummaryModel instance stores the following information:
id, title, category, objective, language_code, tags,
last_updated, created_on, status (private, public),
community_owned, owner_ids, editor_ids,
viewer_ids, version.
The key of each instance is the exploration id.
"""
# What this exploration is called.
title = datastore_services.StringProperty(required=True)
# The category this exploration belongs to.
category = datastore_services.StringProperty(required=True, indexed=True)
# The objective of this exploration.
objective = datastore_services.TextProperty(required=True, indexed=False)
# The ISO 639-1 code for the language this exploration is written in.
language_code = (datastore_services.StringProperty(required=True,
indexed=True))
# Tags associated with this exploration.
tags = datastore_services.StringProperty(repeated=True, indexed=True)
# Aggregate user-assigned ratings of the exploration.
ratings = datastore_services.JsonProperty(default=None, indexed=False)
# Scaled average rating for the exploration.
scaled_average_rating = datastore_services.FloatProperty(indexed=True)
# Time when the exploration model was last updated (not to be
# confused with last_updated, which is the time when the
# exploration *summary* model was last updated).
exploration_model_last_updated = (datastore_services.DateTimeProperty(
indexed=True))
# Time when the exploration model was created (not to be confused
# with created_on, which is the time when the exploration *summary*
# model was created).
exploration_model_created_on = (datastore_services.DateTimeProperty(
indexed=True))
# Time when the exploration was first published.
first_published_msec = datastore_services.FloatProperty(indexed=True)
# The publication status of this exploration.
status = datastore_services.StringProperty(
default=constants.ACTIVITY_STATUS_PRIVATE,
indexed=True,
choices=[
constants.ACTIVITY_STATUS_PRIVATE, constants.ACTIVITY_STATUS_PUBLIC
])
# Whether this exploration is owned by the community.
community_owned = (datastore_services.BooleanProperty(required=True,
indexed=True))
# The user_ids of owners of this exploration.
owner_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to edit this exploration.
editor_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who are allowed to voiceover this exploration.
voice_artist_ids = (datastore_services.StringProperty(indexed=True,
repeated=True))
# The user_ids of users who are allowed to view this exploration.
viewer_ids = datastore_services.StringProperty(indexed=True, repeated=True)
# The user_ids of users who have contributed (humans who have made a
# positive (not just a revert) change to the exploration's content).
# NOTE TO DEVELOPERS: contributor_ids and contributors_summary need to be
# synchronized, meaning that the keys in contributors_summary need be
# equal to the contributor_ids list.
contributor_ids = (datastore_services.StringProperty(indexed=True,
repeated=True))
# A dict representing the contributors of non-trivial commits to this
# exploration. Each key of this dict is a user_id, and the corresponding
# value is the number of non-trivial commits that the user has made.
contributors_summary = (datastore_services.JsonProperty(default={},
indexed=False))
# The version number of the exploration after this commit. Only populated
# for commits to an exploration (as opposed to its rights, etc.).
version = datastore_services.IntegerProperty()
@staticmethod
def get_deletion_policy() -> base_models.DELETION_POLICY:
"""Model contains data to pseudonymize or delete corresponding
to a user: viewer_ids, voice_artist_ids, editor_ids, owner_ids,
contributor_ids, and contributors_summary fields.
"""
return (base_models.DELETION_POLICY.
PSEUDONYMIZE_IF_PUBLIC_DELETE_IF_PRIVATE)
@classmethod
def has_reference_to_user_id(cls, user_id: str) -> bool:
"""Check whether ExpSummaryModel references user.
Args:
user_id: str. The ID of the user whose data should be checked.
Returns:
bool. Whether any models refer to the given user ID.
"""
return cls.query(
datastore_services.any_of(cls.owner_ids == user_id,
cls.editor_ids == user_id,
cls.voice_artist_ids == user_id,
cls.viewer_ids == user_id,
cls.contributor_ids == user_id)).get(
keys_only=True) is not None
@classmethod
def get_non_private(cls) -> Sequence[ExpSummaryModel]:
"""Returns an iterable with non-private ExpSummary models.
Returns:
iterable. An iterable with non-private ExpSummary models.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status != constants.ACTIVITY_STATUS_PRIVATE
).filter(ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_top_rated(cls, limit: int) -> Sequence[ExpSummaryModel]:
"""Fetches the top-rated exp summaries that are public in descending
order of scaled_average_rating.
Args:
limit: int. The maximum number of results to return.
Returns:
iterable. An iterable with the top rated exp summaries that are
public in descending order of scaled_average_rating.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).order(-ExpSummaryModel.scaled_average_rating).fetch(limit)
@classmethod
def get_private_at_least_viewable(
cls, user_id: str) -> Sequence[ExpSummaryModel]:
"""Fetches private exp summaries that are at least viewable by the
given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with private exp summaries that are at least
viewable by the given user.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status ==
constants.ACTIVITY_STATUS_PRIVATE).filter(
datastore_services.any_of(
ExpSummaryModel.owner_ids == user_id,
ExpSummaryModel.editor_ids == user_id,
ExpSummaryModel.voice_artist_ids == user_id,
ExpSummaryModel.viewer_ids == user_id)).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_at_least_editable(cls, user_id: str) -> Sequence[ExpSummaryModel]:
"""Fetches exp summaries that are at least editable by the given user.
Args:
user_id: str. The id of the given user.
Returns:
iterable. An iterable with exp summaries that are at least
editable by the given user.
"""
return ExpSummaryModel.query().filter(
datastore_services.any_of(
ExpSummaryModel.owner_ids == user_id,
ExpSummaryModel.editor_ids == user_id)).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).fetch(feconf.DEFAULT_QUERY_LIMIT)
@classmethod
def get_recently_published(cls, limit: int) -> Sequence[ExpSummaryModel]:
"""Fetches exp summaries that are recently published.
Args:
limit: int. The maximum number of results to return.
Returns:
iterable. An iterable with exp summaries that are
recently published. The returned list is sorted by the time of
publication with latest being first in the list.
"""
return ExpSummaryModel.query().filter(
ExpSummaryModel.status == constants.ACTIVITY_STATUS_PUBLIC).filter(
ExpSummaryModel.deleted == False # pylint: disable=singleton-comparison
).order(-ExpSummaryModel.first_published_msec).fetch(limit)
@staticmethod
def get_model_association_to_user(
) -> base_models.MODEL_ASSOCIATION_TO_USER:
"""Model data has already been exported as a part of the
ExplorationModel and thus does not need a separate export.
"""
return base_models.MODEL_ASSOCIATION_TO_USER.NOT_CORRESPONDING_TO_USER
@classmethod
def get_export_policy(cls) -> Dict[str, base_models.EXPORT_POLICY]:
"""Model contains data corresponding to a user, but this isn't exported
because because noteworthy details that belong to this model have
already been exported as a part of the ExplorationModel.
"""
return dict(
super(cls, cls).get_export_policy(), **{
'title': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'category': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'objective': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'language_code': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'tags': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'ratings': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'scaled_average_rating':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exploration_model_last_updated':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'exploration_model_created_on':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'first_published_msec':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'status': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'community_owned': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'owner_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'editor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'voice_artist_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'viewer_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributor_ids': base_models.EXPORT_POLICY.NOT_APPLICABLE,
'contributors_summary':
base_models.EXPORT_POLICY.NOT_APPLICABLE,
'version': base_models.EXPORT_POLICY.NOT_APPLICABLE
})
