def map(item): if item.deleted or item.suggestion_type != ( feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT): return suggestion = suggestion_services.get_suggestion_from_model(item) exploration = exp_fetchers.get_exploration_by_id(suggestion.target_id) if suggestion.change.state_name not in exploration.states: return state = exploration.states[suggestion.change.state_name] subtitled_unicode_content_ids = [] customisation_args = state.interaction.customization_args for ca_name in customisation_args: subtitled_unicode_content_ids.extend( state_domain.InteractionCustomizationArg. traverse_by_schema_and_get( customisation_args[ca_name].schema, customisation_args[ca_name].value, [schema_utils.SCHEMA_OBJ_TYPE_SUBTITLED_UNICODE], lambda subtitled_unicode: subtitled_unicode.content_id)) content_id = suggestion.change.content_id if content_id in subtitled_unicode_content_ids: yield ('FOUND', '%s | %s' % (item.id, content_id)) yield ('PROCESSED', item.id)
def map(item): if item.deleted or item.suggestion_type != ( feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT): return suggestion = suggestion_services.get_suggestion_from_model(item) exploration = exp_fetchers.get_exploration_by_id(suggestion.target_id) if suggestion.change.state_name not in exploration.states: return state = exploration.states[suggestion.change.state_name] subtitled_unicode_content_ids = [] customisation_args = state.interaction.customization_args for ca_name in customisation_args: subtitled_unicode_content_ids.extend( state_domain.InteractionCustomizationArg. traverse_by_schema_and_get( customisation_args[ca_name].schema, customisation_args[ca_name].value, [schema_utils.SCHEMA_OBJ_TYPE_SUBTITLED_UNICODE], lambda subtitled_unicode: subtitled_unicode.content_id)) if suggestion.change.content_id in subtitled_unicode_content_ids: if suggestion.change.cmd == exp_domain.CMD_ADD_WRITTEN_TRANSLATION: suggestion.change.data_format = ( schema_utils.SCHEMA_TYPE_UNICODE) suggestion.change.translation_html = html_cleaner.strip_html_tags( suggestion.change.translation_html) item.change_cmd = suggestion.change.to_dict() item.update_timestamps(update_last_updated_time=False) item.put() yield ('UPDATED', '%s | %s' % (item.id, suggestion.change.content_id)) yield ('PROCESSED', item.id)
def map(item): if item.target_type != suggestion_models.TARGET_TYPE_EXPLORATION: return if item.suggestion_type != ( suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT): return suggestion = suggestion_services.get_suggestion_from_model(item) html_string_list = suggestion.get_all_html_content_strings() html_string = ''.join(html_string_list) invalid_math_tags = ( html_validation_service. validate_math_tags_in_html_with_attribute_math_content(html_string)) if len(invalid_math_tags) > 0: yield ( SuggestionSvgFilenameValidationOneOffJob._ERROR_KEY, item.id) return math_tags_with_invalid_svg_filename = ( html_validation_service.validate_svg_filenames_in_math_rich_text( feconf.ENTITY_TYPE_EXPLORATION, item.target_id, html_string)) if len(math_tags_with_invalid_svg_filename) > 0: yield ( SuggestionSvgFilenameValidationOneOffJob. _INVALID_SVG_FILENAME_KEY, ( item.id, math_tags_with_invalid_svg_filename))
def map(item): suggestion = suggestion_services.get_suggestion_from_model(item) html_string_list = suggestion.get_all_html_content_strings() html_string = ''.join(html_string_list) if (html_validation_service.check_for_math_component_in_html( html_string)): yield ('Suggestion with Math', item.id)
def map(item): # Exit early if the suggestion has been marked deleted, or if the # suggestion has already set the language code property, or if the # suggestion type is not queryable by language, since ndb automatically # sets properties that aren't intialized to None. if item.deleted or item.language_code or item.suggestion_type == ( suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT): return suggestion = suggestion_services.get_suggestion_from_model(item) if suggestion.suggestion_type == ( suggestion_models.SUGGESTION_TYPE_ADD_QUESTION): # Set the language code to be the language of the question. suggestion.language_code = suggestion.change.question_dict[ 'language_code'] elif suggestion.suggestion_type == ( suggestion_models.SUGGESTION_TYPE_TRANSLATE_CONTENT): # Set the language code to be the language of the translation. suggestion.language_code = suggestion.change.language_code # Validate the suggestion before updating the storage model. try: suggestion.validate() except Exception as e: logging.error('Suggestion %s failed validation: %s' % (item.id, e)) yield (PopulateSuggestionLanguageCodeMigrationOneOffJob. _VALIDATION_ERROR_KEY, 'Suggestion %s failed validation: %s' % (item.id, e)) return item.language_code = suggestion.language_code item.put() yield ('%s_suggestion_migrated' % item.suggestion_type, item.id)
def _get_model_domain_object_instance(cls, item): if item.target_type in TARGET_TYPE_TO_TARGET_MODEL: return suggestion_services.get_suggestion_from_model(item) else: cls._add_error( 'target %s' % base_model_validators.ERROR_CATEGORY_TYPE_CHECK, 'Entity id %s: Target type %s is not allowed' % (item.id, item.target_type)) return None
def map(item): """Implements the map function (generator). Computes word counts of translations suggestions and outputs suggestion metadata. Args: item: GeneralSuggestionModel. An instance of GeneralSuggestionModel. Yields: tuple(key, recent_activity_commits). Where: key: str. The entity ID of the corresponding TranslationContributionStatsModel. dict. Has the keys: suggestion_status: str. The translation suggestion status. edited_by_reviewer: bool. Whether the translation suggestion was edited by a reviewer. content_word_count: int. The word count of the translation suggestion content HTML. last_updated_date: date. The last updated date of the translation suggestion. """ if item.suggestion_type != feconf.SUGGESTION_TYPE_TRANSLATE_CONTENT: return suggestion = suggestion_services.get_suggestion_from_model(item) # Try to extract the topic ID from the corresponding exploration # opportunity. topic_id = '' exp_id = suggestion.target_id exp_opportunity_dict = ( opportunity_services.get_exploration_opportunity_summaries_by_ids( [exp_id])) exp_opportunity = exp_opportunity_dict[exp_id] if exp_opportunity is not None: topic_id = exp_opportunity.topic_id # Count the number of words in the original content, ignoring any HTML # tags and attributes. content_plain_text = html_cleaner.strip_html_tags( suggestion.change.content_html) content_word_count = len(content_plain_text.split()) key = suggestion_models.TranslationContributionStatsModel.generate_id( suggestion.language_code, suggestion.author_id, topic_id) translation_contribution_stats_dict = { 'suggestion_status': suggestion.status, 'edited_by_reviewer': suggestion.edited_by_reviewer, 'content_word_count': content_word_count, 'last_updated_date': suggestion.last_updated.date().isoformat() } yield (key, translation_contribution_stats_dict)
def map(item): if item.deleted: return if isinstance(item, suggestion_models.GeneralSuggestionModel): # Exit early if the suggestion type is not a part of the # Contributor Dashboard or if the suggestion is not currently in # review. if item.suggestion_type == ( suggestion_models.SUGGESTION_TYPE_EDIT_STATE_CONTENT) or ( item.status != suggestion_models.STATUS_IN_REVIEW): return suggestion = suggestion_services.get_suggestion_from_model(item) yield ('%s%s%s%s%s' % ( PopulateContributionStatsOneOffJob.ITEM_TYPE_SUGGESTION, PopulateContributionStatsOneOffJob.KEY_DELIMITER, suggestion.suggestion_type, PopulateContributionStatsOneOffJob.KEY_DELIMITER, suggestion.language_code), 1) else: if item.can_review_translation_for_language_codes: for language_code in ( item.can_review_translation_for_language_codes): yield ('%s%s%s%s%s' % ( PopulateContributionStatsOneOffJob.ITEM_TYPE_REVIEWER, PopulateContributionStatsOneOffJob.KEY_DELIMITER, ( PopulateContributionStatsOneOffJob .ITEM_CATEGORY_TRANSLATION ), PopulateContributionStatsOneOffJob.KEY_DELIMITER, language_code), 1) if item.can_review_questions: yield ('%s%s%s%s%s' % ( PopulateContributionStatsOneOffJob.ITEM_TYPE_REVIEWER, PopulateContributionStatsOneOffJob.KEY_DELIMITER, PopulateContributionStatsOneOffJob.ITEM_CATEGORY_QUESTION, PopulateContributionStatsOneOffJob.KEY_DELIMITER, constants.DEFAULT_LANGUAGE_CODE), 1)
def map(item): if item.deleted or item.suggestion_type != ( feconf.SUGGESTION_TYPE_ADD_QUESTION): return try: # Suggestion class itself updates the question state dict of the # suggestion while initializing the object. suggestion = suggestion_services.get_suggestion_from_model(item) except Exception as e: yield ('MIGRATION_FAILURE', (item.id, e)) return try: suggestion.validate() except Exception as e: yield ('POST_MIGRATION_VALIDATION_FALIURE', (item.id, e)) return item.change_cmd = suggestion.change.to_dict() item.update_timestamps(update_last_updated_time=False) item.put() yield ('SUCCESS', item.id)