def map(item): from core.domain import exp_services exploration = exp_services.get_exploration_from_model(item) try: exploration.validate(strict=False) except utils.ValidationError as e: yield (item.id, unicode(e).encode('utf-8'))
def map(item): if item.deleted: return err_dict = {} try: exploration = exp_services.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % str(e), [item.id]) return html_list = exploration.get_all_html_content_strings() try: err_dict = html_validation_service.validate_customization_args( html_list) except Exception as e: yield ( 'Error in validating customization args for exploration %s' % ( item.id), [traceback.format_exc()]) return for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) for state_name, state in exploration.states.iteritems(): exp_and_state_key = '%s %s' % (item.id, state_name) yield (state.interaction.id, exp_and_state_key)
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) for state_name, state in exploration.states.iteritems(): hints_length = len(state.interaction.hints) if hints_length > 0: exp_and_state_key = '%s %s' % ( item.id, state_name.encode('utf-8')) yield (str(hints_length), exp_and_state_key)
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) explorations = [] # Fetch all versions of the exploration, if they exist. if exploration.version > 1: # Ignore latest version since we already have corresponding # exploration. versions = range(1, exploration.version) # Get all exploration versions for current exploration id. try: explorations = ( exp_services.get_multiple_explorations_by_version( exploration.id, versions)) except Exception as e: yield ('ERROR with exp_id %s' % item.id, str(e)) return # Append latest exploration to the list of explorations. explorations.append(exploration) # Retrieve list of snapshot models representing each version of the # exploration. versions = range(1, exploration.version + 1) snapshots_by_version = ( exp_models.ExplorationModel.get_snapshots_metadata( exploration.id, versions)) # Create and save state id mapping model for all exploration versions. for exploration, snapshot in zip(explorations, snapshots_by_version): if snapshot is None: yield ( 'ERROR with exp_id %s' % item.id, 'Error: No exploration snapshot metadata model instance ' 'found for exploration %s, version %d' % (exploration.id, exploration.version)) return change_list = snapshot['commit_cmds'] # Check if commit is to revert the exploration. if change_list and change_list[0]['cmd'].endswith( 'revert_version_number'): reverted_version = change_list[0]['version_number'] exp_services.create_and_save_state_id_mapping_model_for_reverted_exploration( # pylint: disable=line-too-long exploration.id, exploration.version - 1, reverted_version) else: exp_services.create_and_save_state_id_mapping_model( exploration, change_list) yield (exploration.id, exploration.version)
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) html_list = exploration.get_all_html_content_strings() err_dict = html_cleaner.validate_textangular_format(html_list, True) for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) for state_name, state in exploration.states.iteritems(): for fallback in state.interaction.fallbacks: num_submits = fallback.trigger.customization_args[ 'num_submits'] feedback = fallback.outcome.feedback[0] yield ('%s: %s' % (item.id, state_name.encode('utf-8')), '%s: %s' % (num_submits['value'], feedback.encode('utf-8')))
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) html_list = exploration.get_all_html_content_strings() err_dict = html_cleaner.validate_rte_format( html_list, feconf.RTE_FORMAT_TEXTANGULAR, run_migration=True) for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) exp_rights = rights_manager.get_exploration_rights(item.id) try: if exp_rights.status == rights_manager.ACTIVITY_STATUS_PRIVATE: exploration.validate() else: exploration.validate(strict=True) except utils.ValidationError as e: yield (item.id, unicode(e).encode('utf-8'))
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) # err_dict is a dictionary to store the invalid tags and the # invalid parent-child relations that we find. err_dict = {} # All the invalid html strings will be stored in this. err_dict['strings'] = [] allowed_parent_list = (feconf.RTE_CONTENT_SPEC['RTE_TYPE_TEXTANGULAR'] ['ALLOWED_PARENT_LIST']) allowed_tag_list = (feconf.RTE_CONTENT_SPEC['RTE_TYPE_TEXTANGULAR'] ['ALLOWED_TAG_LIST']) html_list = exploration.get_all_html_content_strings() for html_data in html_list: soup = bs4.BeautifulSoup(html_data, 'html.parser') # Text with no parent tag is also invalid. for content in soup.contents: if not content.name: err_dict['strings'].append(html_data) break for tag in soup.findAll(): # Checking for tags not allowed in RTE. if tag.name not in allowed_tag_list: if 'invalidTags' in err_dict: err_dict['invalidTags'].append(tag.name) else: err_dict['invalidTags'] = [tag.name] err_dict['strings'].append(html_data) # Checking for parent-child relation that are not # allowed in RTE. parent = tag.parent.name if (tag.name in allowed_tag_list) and ( parent not in allowed_parent_list[tag.name]): if tag.name in err_dict: err_dict[tag.name].append(parent) else: err_dict[tag.name] = [parent] err_dict['strings'].append(html_data) for key in err_dict: if err_dict[key]: yield (key, list(set(err_dict[key])))
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_rte_format( html_list, feconf.RTE_FORMAT_CKEDITOR) for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) # err_dict is a dictionary to store the invalid tags and the # invalid parent-child relations that we find. err_dict = {} htmlValidation_filepath = os.path.join(feconf.CONTENT_VALIDATION_DIR, 'htmlValidation.yaml') htmlValidation_yaml = utils.get_file_contents(htmlValidation_filepath) # Convert the yaml content to dictionary. htmlValidation_dict = utils.dict_from_yaml(htmlValidation_yaml) allowed_parent_list = htmlValidation_dict['allowed_parent_list'] allowed_tag_list = htmlValidation_dict['allowed_tag_list'] for state in exploration.states.itervalues(): # result is the list of all the html present in the state. result = exp_services.find_all_values_for_key( 'html', state.to_dict()) for html_data in result: html_data = html_data.encode('utf-8') soup = BeautifulSoup(html_data, 'html.parser') for tag in soup.findAll(): # Checking for tags not allowed in RTE. if tag.name not in allowed_tag_list: if 'invalidTags' in err_dict: err_dict['invalidTags'] += [tag.name] else: err_dict['invalidTags'] = [tag.name] # Checking for parent-child relation that are not # allowed in RTE. parent = tag.parent.name if (tag.name in allowed_tag_list and parent not in allowed_parent_list[tag.name]): if tag.name in err_dict: err_dict[tag.name] += [parent] else: err_dict[tag.name] = [parent] for key in err_dict.iterkeys(): err_dict[key] = list(set(err_dict[key])) yield ('Errors', err_dict)
def map(item): """Yields the id of each newly-created exploration copy, together with its YAML representation. Args: item: ExplorationModel. An exploration storage model. Yields: str. The string containing exploration id of each newly-created exploration. It is of the format: <exp_id>copy<copy_number> """ if ExpCopiesMRJobManager._entity_created_before_job_queued(item): for count in range(10): yield ('%scopy%d' % (item.id, count), exp_services.get_exploration_from_model(item).to_yaml())
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) for state_name, state in exploration.states.iteritems(): if state.interaction.id == 'ItemSelectionInput': choices = ( state.interaction.customization_args['choices']['value']) for group in state.interaction.answer_groups: for rule_spec in group.rule_specs: for rule_item in rule_spec.inputs['x']: if rule_item not in choices: yield (item.id, '%s: %s' % (state_name.encode('utf-8'), rule_item.encode('utf-8')))
def map(item): from core.domain import exp_services from core.domain import rights_manager if item.deleted: return exploration = exp_services.get_exploration_from_model(item) exp_summary = rights_manager.get_exploration_rights(item.id) try: if exp_summary.status == rights_manager.EXPLORATION_STATUS_PRIVATE: exploration.validate() else: exploration.validate(strict=True) except utils.ValidationError as e: yield (item.id, unicode(e).encode('utf-8'))
def map(item): if item.deleted: return exploration = exp_services.get_exploration_from_model(item) for state_name, state in exploration.states.iteritems(): if state.interaction.id == 'ItemSelectionInput': choices = ( state.interaction.customization_args['choices']['value']) for group in state.interaction.answer_groups: for rule_spec in group.rule_specs: for rule_item in rule_spec.inputs['x']: if rule_item not in choices: yield ( item.id, '%s: %s' % ( state_name.encode('utf-8'), rule_item.encode('utf-8')))
def map(item): from core.domain import exp_domain from core.domain import exp_services if item.deleted: return exp = exp_services.get_exploration_from_model(item) change_list = [] for state_name, state in exp.states.iteritems(): if state.interaction.id == 'InteractiveMap': old_value = {} for handler in state.interaction.handlers: handler_dict = handler.to_dict() old_value[handler_dict['name']] = ( handler_dict['rule_specs']) new_value = copy.deepcopy(old_value) for handler in new_value['submit']: if handler['definition']['rule_type'] != 'default': handler['definition']['inputs']['d'] *= 110 change_list.append({ 'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY, 'property_name': 'widget_handlers', 'state_name': state_name, 'old_value': old_value, 'new_value': new_value, }) if change_list: yield (item.id, 'started') try: exp_services.update_exploration( feconf.MIGRATION_BOT_USERNAME, item.id, change_list, 'Convert distances in rules for world map to kilometers.') yield (item.id, 'completed') except utils.ValidationError as e: if 'An objective must be specified' in str(e): yield ('ERROR-OBJECTIVE', item.id) else: raise
def map(item): from core.domain import exp_services if item.deleted: return exp = exp_services.get_exploration_from_model(item) change_list = [] for state_name, state in exp.states.iteritems(): if state.interaction.id == 'InteractiveMap': old_value = {} for handler in state.interaction.handlers: handler_dict = handler.to_dict() old_value[handler_dict['name']] = ( handler_dict['rule_specs']) new_value = copy.deepcopy(old_value) for handler in new_value['submit']: if handler['definition']['rule_type'] != 'default': handler['definition']['inputs']['d'] *= 110 change_list.append({ 'cmd': 'edit_state_property', 'property_name': 'widget_handlers', 'state_name': state_name, 'old_value': old_value, 'new_value': new_value, }) if change_list: yield (item.id, 'started') try: exp_services.update_exploration( feconf.MIGRATION_BOT_USERNAME, item.id, change_list, 'Convert distances in rules for world map to kilometers.') yield (item.id, 'completed') except utils.ValidationError as e: if 'An objective must be specified' in str(e): yield ('ERROR-OBJECTIVE', item.id) else: raise
def map(exp_model): current_version = exp_model.version version_numbers = range(1, current_version + 1) list_of_exploration_models = (exp_model.get_multi_versions( exp_model.id, version_numbers)) filenames = [] for exploration_model in list_of_exploration_models: exploration = exp_services.get_exploration_from_model( exploration_model) filenames_in_version = ( exp_services.get_image_filenames_from_exploration(exploration)) filenames = list(set().union(filenames, filenames_in_version)) for filename in filenames: file_system_class = (fs_domain.ExplorationFileSystem if feconf.DEV_MODE else fs_domain.GcsFileSystem) fs = fs_domain.AbstractFileSystem(file_system_class(exp_model.id)) filepath = (filename if feconf.DEV_MODE else 'image/%s' % filename) raw_data = fs.get(filepath) exp_services.save_original_and_compressed_versions_of_image( 'ADMIN', filename, exp_model.id, raw_data) yield (ADDED_COMPRESSED_VERSIONS_OF_IMAGES, exp_model.id)
def map(item): if item.deleted: return err_dict = {} try: exploration = exp_services.get_exploration_from_model(item) except Exception as e: yield ('Error %s in exploration' % str(e), [item.id]) return html_list = exploration.get_all_html_content_strings() try: err_dict = html_cleaner.validate_rte_format( html_list, feconf.RTE_FORMAT_CKEDITOR, run_migration=True) except Exception as e: yield ('Error %s in exploration %s' % (str(e), item.id), html_list) return for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): if item.deleted: return err_dict = {} try: exploration = exp_services.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % str(e), [item.id]) return html_list = exploration.get_all_html_content_strings() try: err_dict = html_validation_service.validate_rte_format( html_list, feconf.RTE_FORMAT_CKEDITOR, run_migration=True) except Exception as e: yield ('Error in validating rte format for exploration %s' % item.id, [traceback.format_exc()]) return for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def map(item): from core.domain import exp_services if ExpCopiesMRJobManager._entity_created_before_job_queued(item): for count in range(10): yield ("%scopy%d" % (item.id, count), exp_services.get_exploration_from_model(item).to_yaml())
def save_new_exp_with_states_schema_v0(self, exp_id, user_id, title): """Saves a new default exploration with a default version 0 states dictionary. This function should only be used for creating explorations in tests involving migration of datastore explorations that use an old states schema version. Note that it makes an explicit commit to the datastore instead of using the usual functions for updating and creating explorations. This is because the latter approach would result in an exploration with the *current* states schema version. Args: exp_id: str. The exploration ID. user_id: str. The user_id of the creator. title: str. The title of the exploration. """ exp_model = exp_models.ExplorationModel( id=exp_id, category='category', title=title, objective='Old objective', language_code='en', tags=[], blurb='', author_notes='', states_schema_version=0, init_state_name=feconf.DEFAULT_INIT_STATE_NAME, states=self.VERSION_0_STATES_DICT, param_specs={}, param_changes=[]) rights_manager.create_new_exploration_rights(exp_id, user_id) commit_message = 'New exploration created with title \'%s\'.' % title exp_model.commit(user_id, commit_message, [{ 'cmd': 'create_new', 'title': 'title', 'category': 'category', }]) exp_rights = exp_models.ExplorationRightsModel.get_by_id(exp_id) exp_summary_model = exp_models.ExpSummaryModel( id=exp_id, title=title, category='category', objective='Old objective', language_code='en', tags=[], ratings=feconf.get_empty_ratings(), scaled_average_rating=feconf.EMPTY_SCALED_AVERAGE_RATING, status=exp_rights.status, community_owned=exp_rights.community_owned, owner_ids=exp_rights.owner_ids, contributor_ids=[], contributors_summary={}, ) exp_summary_model.put() # Note: Also save state id mappping model for new exploration. If not # saved, it may cause errors in test cases. exploration = exp_services.get_exploration_from_model(exp_model) exp_services.create_and_save_state_id_mapping_model(exploration, [])
def map(item): if item.deleted: return try: exploration = exp_services.get_exploration_from_model(item) except Exception as e: yield ('ERROR get_exp_from_model: exp_id %s' % item.id, str(e)) return # Commit commands which are required to generate state id mapping for # given version of exploration from previous version of exploration. RELEVANT_COMMIT_CMDS = [ exp_domain.CMD_ADD_STATE, exp_domain.CMD_RENAME_STATE, exp_domain.CMD_DELETE_STATE, exp_models.ExplorationModel.CMD_REVERT_COMMIT ] explorations = [] # Fetch all versions of the exploration, if they exist. if exploration.version > 1: # Ignore latest version since we already have corresponding # exploration. versions = range(1, exploration.version) # Get all exploration versions for current exploration id. try: explorations = ( exp_services.get_multiple_explorations_by_version( exploration.id, versions)) except Exception as e: yield ( 'ERROR get_multiple_exp_by_version exp_id %s' % item.id, str(e)) return # Append latest exploration to the list of explorations. explorations.append(exploration) # Retrieve list of snapshot models representing each version of the # exploration. versions = range(1, exploration.version + 1) snapshots_by_version = ( exp_models.ExplorationModel.get_snapshots_metadata( exploration.id, versions)) # Create and save state id mapping model for all exploration versions. for exploration, snapshot in zip(explorations, snapshots_by_version): if snapshot is None: yield ( 'ERROR with exp_id %s' % item.id, 'Error: No exploration snapshot metadata model instance ' 'found for exploration %s, version %d' % ( exploration.id, exploration.version)) return change_list = [ exp_domain.ExplorationChange(change_cmd) for change_cmd in snapshot['commit_cmds'] if change_cmd['cmd'] in RELEVANT_COMMIT_CMDS ] try: # Check if commit is to revert the exploration. if change_list and change_list[0].cmd == ( exp_models.ExplorationModel.CMD_REVERT_COMMIT): reverted_version = change_list[0].version_number # pylint: disable=line-too-long state_id_mapping = exp_services.generate_state_id_mapping_model_for_reverted_exploration( exploration.id, exploration.version - 1, reverted_version) # pylint: enable=line-too-long else: state_id_mapping = ( exp_services.generate_state_id_mapping_model( exploration, change_list)) state_id_mapping_model = exp_models.StateIdMappingModel.create( state_id_mapping.exploration_id, state_id_mapping.exploration_version, state_id_mapping.state_names_to_ids, state_id_mapping.largest_state_id_used, overwrite=True) state_id_mapping_model.put() except Exception as e: yield ( 'ERROR with exp_id %s version %s' % ( item.id, exploration.version), traceback.format_exc()) return yield (exploration.id, exploration.version)
def map(item): from core.domain import exp_services if ExpCopiesMRJobManager._entity_created_before_job_queued(item): for count in range(10): yield ('%scopy%d' % (item.id, count), exp_services.get_exploration_from_model(item).to_yaml())