def map(item):
if item.deleted or item.suggestion_type != (
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
exploration = exp_fetchers.get_exploration_by_id(suggestion.target_id)
if suggestion.change.state_name not in exploration.states:
return
state = exploration.states[suggestion.change.state_name]
subtitled_unicode_content_ids = []
customisation_args = state.interaction.customization_args
for ca_name in customisation_args:
subtitled_unicode_content_ids.extend(
state_domain.InteractionCustomizationArg.
traverse_by_schema_and_get(
customisation_args[ca_name].schema,
customisation_args[ca_name].value,
[schema_utils.SCHEMA_OBJ_TYPE_SUBTITLED_UNICODE],
lambda subtitled_unicode: subtitled_unicode.content_id))
content_id = suggestion.change.content_id
if content_id in subtitled_unicode_content_ids:
yield ('FOUND', '%s | %s' % (item.id, content_id))
yield ('PROCESSED', item.id)
def map(item):
if item.deleted or item.suggestion_type != (
feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
exploration = exp_fetchers.get_exploration_by_id(suggestion.target_id)
if suggestion.change.state_name not in exploration.states:
return
state = exploration.states[suggestion.change.state_name]
subtitled_unicode_content_ids = []
customisation_args = state.interaction.customization_args
for ca_name in customisation_args:
subtitled_unicode_content_ids.extend(
state_domain.InteractionCustomizationArg.
traverse_by_schema_and_get(
customisation_args[ca_name].schema,
customisation_args[ca_name].value,
[schema_utils.SCHEMA_OBJ_TYPE_SUBTITLED_UNICODE],
lambda subtitled_unicode: subtitled_unicode.content_id))
if suggestion.change.content_id in subtitled_unicode_content_ids:
if suggestion.change.cmd == exp_domain.CMD_ADD_WRITTEN_TRANSLATION:
suggestion.change.data_format = (
schema_utils.SCHEMA_TYPE_UNICODE)
suggestion.change.translation_html = html_cleaner.strip_html_tags(
suggestion.change.translation_html)
item.change_cmd = suggestion.change.to_dict()
item.update_timestamps(update_last_updated_time=False)
item.put()
yield ('UPDATED',
'%s | %s' % (item.id, suggestion.change.content_id))
yield ('PROCESSED', item.id)
def map(item):
if item.target_type != suggestion_models.TARGET_TYPE_EXPLORATION:
return
if item.suggestion_type != (
suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
html_string_list = suggestion.get_all_html_content_strings()
html_string = ''.join(html_string_list)
invalid_math_tags = (
html_validation_service.
validate_math_tags_in_html_with_attribute_math_content(html_string))
if len(invalid_math_tags) > 0:
yield (
SuggestionSvgFilenameValidationOneOffJob._ERROR_KEY,
item.id)
return
math_tags_with_invalid_svg_filename = (
html_validation_service.validate_svg_filenames_in_math_rich_text(
feconf.ENTITY_TYPE_EXPLORATION, item.target_id, html_string))
if len(math_tags_with_invalid_svg_filename) > 0:
yield (
SuggestionSvgFilenameValidationOneOffJob.
_INVALID_SVG_FILENAME_KEY, (
item.id, math_tags_with_invalid_svg_filename))
def map(item):
suggestion = suggestion_services.get_suggestion_from_model(item)
html_string_list = suggestion.get_all_html_content_strings()
html_string = ''.join(html_string_list)
if (html_validation_service.check_for_math_component_in_html(
html_string)):
yield ('Suggestion with Math', item.id)
def map(item):
# Exit early if the suggestion has been marked deleted, or if the
# suggestion has already set the language code property, or if the
# suggestion type is not queryable by language, since ndb automatically
# sets properties that aren't intialized to None.
if item.deleted or item.language_code or item.suggestion_type == (
suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
if suggestion.suggestion_type == (
suggestion_models.SUGGESTION_TYPE_ADD_QUESTION):
# Set the language code to be the language of the question.
suggestion.language_code = suggestion.change.question_dict[
'language_code']
elif suggestion.suggestion_type == (
suggestion_models.SUGGESTION_TYPE_TRANSLATE_CONTENT):
# Set the language code to be the language of the translation.
suggestion.language_code = suggestion.change.language_code
# Validate the suggestion before updating the storage model.
try:
suggestion.validate()
except Exception as e:
logging.error('Suggestion %s failed validation: %s' % (item.id, e))
yield (PopulateSuggestionLanguageCodeMigrationOneOffJob.
_VALIDATION_ERROR_KEY,
'Suggestion %s failed validation: %s' % (item.id, e))
return
item.language_code = suggestion.language_code
item.put()
yield ('%s_suggestion_migrated' % item.suggestion_type, item.id)
def _get_model_domain_object_instance(cls, item):
if item.target_type in TARGET_TYPE_TO_TARGET_MODEL:
return suggestion_services.get_suggestion_from_model(item)
else:
cls._add_error(
'target %s' % base_model_validators.ERROR_CATEGORY_TYPE_CHECK,
'Entity id %s: Target type %s is not allowed' %
(item.id, item.target_type))
return None
def map(item):
"""Implements the map function (generator). Computes word counts of
translations suggestions and outputs suggestion metadata.
Args:
item: GeneralSuggestionModel. An instance of GeneralSuggestionModel.
Yields:
tuple(key, recent_activity_commits). Where:
key: str. The entity ID of the corresponding
TranslationContributionStatsModel.
dict. Has the keys:
suggestion_status: str. The translation suggestion status.
edited_by_reviewer: bool. Whether the translation suggestion
was edited by a reviewer.
content_word_count: int. The word count of the translation
suggestion content HTML.
last_updated_date: date. The last updated date of the
translation suggestion.
"""
if item.suggestion_type != feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT:
return
suggestion = suggestion_services.get_suggestion_from_model(item)
# Try to extract the topic ID from the corresponding exploration
# opportunity.
topic_id = ''
exp_id = suggestion.target_id
exp_opportunity_dict = (
opportunity_services.get_exploration_opportunity_summaries_by_ids(
[exp_id]))
exp_opportunity = exp_opportunity_dict[exp_id]
if exp_opportunity is not None:
topic_id = exp_opportunity.topic_id
# Count the number of words in the original content, ignoring any HTML
# tags and attributes.
content_plain_text = html_cleaner.strip_html_tags(
suggestion.change.content_html)
content_word_count = len(content_plain_text.split())
key = suggestion_models.TranslationContributionStatsModel.generate_id(
suggestion.language_code, suggestion.author_id, topic_id)
translation_contribution_stats_dict = {
'suggestion_status': suggestion.status,
'edited_by_reviewer': suggestion.edited_by_reviewer,
'content_word_count': content_word_count,
'last_updated_date': suggestion.last_updated.date().isoformat()
}
yield (key, translation_contribution_stats_dict)
def map(item):
if item.deleted:
return
if isinstance(item, suggestion_models.GeneralSuggestionModel):
# Exit early if the suggestion type is not a part of the
# Contributor Dashboard or if the suggestion is not currently in
# review.
if item.suggestion_type == (
suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT) or (
item.status != suggestion_models.STATUS_IN_REVIEW):
return
suggestion = suggestion_services.get_suggestion_from_model(item)
yield ('%s%s%s%s%s' % (
PopulateContributionStatsOneOffJob.ITEM_TYPE_SUGGESTION,
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
suggestion.suggestion_type,
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
suggestion.language_code), 1)
else:
if item.can_review_translation_for_language_codes:
for language_code in (
item.can_review_translation_for_language_codes):
yield ('%s%s%s%s%s' % (
PopulateContributionStatsOneOffJob.ITEM_TYPE_REVIEWER,
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
(
PopulateContributionStatsOneOffJob
.ITEM_CATEGORY_TRANSLATION
),
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
language_code), 1)
if item.can_review_questions:
yield ('%s%s%s%s%s' % (
PopulateContributionStatsOneOffJob.ITEM_TYPE_REVIEWER,
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
PopulateContributionStatsOneOffJob.ITEM_CATEGORY_QUESTION,
PopulateContributionStatsOneOffJob.KEY_DELIMITER,
constants.DEFAULT_LANGUAGE_CODE), 1)
def map(item):
if item.deleted or item.suggestion_type != (
feconf.SUGGESTION_TYPE_ADD_QUESTION):
return
try:
# Suggestion class itself updates the question state dict of the
# suggestion while initializing the object.
suggestion = suggestion_services.get_suggestion_from_model(item)
except Exception as e:
yield ('MIGRATION_FAILURE', (item.id, e))
return
try:
suggestion.validate()
except Exception as e:
yield ('POST_MIGRATION_VALIDATION_FALIURE', (item.id, e))
return
item.change_cmd = suggestion.change.to_dict()
item.update_timestamps(update_last_updated_time=False)
item.put()
yield ('SUCCESS', item.id)
