def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): hints_length = len(state.interaction.hints) if hints_length > 0: exp_and_state_key = '%s %s' % (item.id, state_name.encode('utf-8')) yield (python_utils.UNICODE(hints_length), exp_and_state_key)
def map(item): if not item.deleted: exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): if state.interaction.id == 'MathExpressionInput': for group in state.interaction.answer_groups: for rule_spec in group.rule_specs: rule = rule_spec.inputs['x'] if any(sep in rule for sep in SEPARATORS): yield (item.id, '%s: %s' % (state_name.encode('utf-8'), rule.encode('utf-8')))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) exp_rights = rights_manager.get_exploration_rights(item.id) try: if exp_rights.status == rights_manager.ACTIVITY_STATUS_PRIVATE: exploration.validate() else: exploration.validate(strict=True) except utils.ValidationError as e: yield (item.id, unicode(e).encode(encoding='utf-8'))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) exp_rights = rights_manager.get_exploration_rights(item.id) try: if exp_rights.status == rights_domain.ACTIVITY_STATUS_PRIVATE: exploration.validate() else: exploration.validate(strict=True) except utils.ValidationError as e: yield (item.id, python_utils.convert_to_bytes(e))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_rte_format( html_list, feconf.RTE_FORMAT_CKEDITOR) for key in err_dict: if err_dict[key]: yield (key, err_dict[key])
def _validate_translation_counts( cls, item, field_name_to_external_model_references): """Validate that translation_counts match the translations available in the exploration. Args: item: datastore_services.Model. ExplorationOpportunitySummaryModel to validate. field_name_to_external_model_references: dict(str, (list(base_model_validators.ExternalModelReference))). A dict keyed by field name. The field name represents a unique identifier provided by the storage model to which the external model is associated. Each value contains a list of ExternalModelReference objects corresponding to the field_name. For examples, all the external Exploration Models corresponding to a storage model can be associated with the field name 'exp_ids'. This dict is used for validation of External Model properties linked to the storage model. """ exploration_model_references = ( field_name_to_external_model_references['exploration_ids']) for exploration_model_reference in exploration_model_references: exploration_model = exploration_model_reference.model_instance if exploration_model is None or exploration_model.deleted: model_class = exploration_model_reference.model_class model_id = exploration_model_reference.model_id cls._add_error( 'exploration_ids %s' % ( base_model_validators.ERROR_CATEGORY_FIELD_CHECK), 'Entity id %s: based on field exploration_ids having' ' value %s, expected model %s with id %s but it doesn\'t' ' exist' % ( item.id, model_id, model_class.__name__, model_id)) continue exploration = exp_fetchers.get_exploration_from_model( exploration_model) exploration_translation_counts = ( exploration.get_translation_counts()) if exploration_translation_counts != item.translation_counts: cls._add_error( 'translation %s' % ( base_model_validators.ERROR_CATEGORY_COUNT_CHECK), 'Entity id %s: Translation counts: %s does not match the ' 'translation counts of external exploration model: %s' % ( item.id, item.translation_counts, exploration_translation_counts))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): interaction = state.interaction exp_and_state_key = '%s %s' % (item.id, state_name) if interaction.id == 'RatioExpressionInput': number_of_terms = ( interaction.customization_args['numberOfTerms'].value) if number_of_terms > 10: yield (python_utils.UNICODE(number_of_terms), exp_and_state_key) yield ('SUCCESS', 1)
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): if state.interaction.id == 'ItemSelectionInput': choices = ( state.interaction.customization_args['choices']['value']) for group in state.interaction.answer_groups: for rule_spec in group.rule_specs: for rule_item in rule_spec.inputs['x']: if rule_item not in choices: yield (item.id, '%s: %s' % (state_name.encode('utf-8'), rule_item.encode('utf-8')))
def map(item): if item.deleted: return try: exploration = exp_fetchers.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % str(e), [item.id]) return html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_rte_format( html_list, feconf.RTE_FORMAT_CKEDITOR) for key in err_dict: if err_dict[key]: yield ('%s Exp Id: %s' % (key, item.id), err_dict[key])
def map(item): if item.deleted: return err_dict = {} try: exploration = exp_fetchers.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % str(e), [item.id]) return html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_customization_args( html_list) for key in err_dict: if err_dict[key]: yield ('%s Exp Id: %s' % (key, item.id), err_dict[key])
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): if state.interaction.id == 'MultipleChoiceInput': choices_length = len( state.interaction.customization_args['choices']['value']) for anwer_group_index, answer_group in enumerate( state.interaction.answer_groups): for rule_index, rule_spec in enumerate( answer_group.rule_specs): if rule_spec.inputs['x'] >= choices_length: yield ( item.id, 'State name: %s, AnswerGroup: %s,' % (state_name.encode('utf-8'), anwer_group_index) + ' Rule: %s is invalid.' % (rule_index) + '(Indices here are 0-indexed.)')
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) invalid_tags_info_in_exp = [] for state_name, state in exploration.states.items(): html_string = ''.join(state.get_all_html_content_strings()) error_list = (html_validation_service. validate_math_content_attribute_in_html(html_string)) if len(error_list) > 0: invalid_tags_info_in_state = { 'state_name': state_name, 'error_list': error_list, 'no_of_invalid_tags': len(error_list) } invalid_tags_info_in_exp.append(invalid_tags_info_in_state) if len(invalid_tags_info_in_exp) > 0: yield ('Found invalid tags', (item.id, invalid_tags_info_in_exp))
def map(item): if item.deleted: return err_dict = {} try: exploration = exp_fetchers.get_exploration_from_model(item) except Exception as e: yield ('Error %s when loading exploration' % python_utils.UNICODE(e), [item.id]) return html_list = exploration.get_all_html_content_strings() err_dict = html_validation_service.validate_customization_args( html_list) for key in err_dict: err_value_with_exp_id = err_dict[key] err_value_with_exp_id.append('Exp ID: %s' % item.id) yield (key, err_value_with_exp_id)
def _is_used_logic_proof_interaction_query_satisfied( user_settings_model, _): """Determines whether a user has used logic proof interaction in any of the explorations created by the user. """ user_id = user_settings_model.id user_contributions = user_models.UserContributionsModel.get( user_id, strict=False) if user_contributions is None: return False exploration_ids = user_contributions.created_exploration_ids exploration_instances = ( datastore_services.fetch_multiple_entities_by_ids_and_models( [('ExplorationModel', exploration_ids)]))[0] for item in exploration_instances: exploration = exp_fetchers.get_exploration_from_model(item) for _, state in exploration.states.items(): if state.interaction.id == 'LogicProof': return True return False
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) exploration_status = (rights_manager.get_exploration_rights( item.id).status) for state_name, state in exploration.states.items(): html_string = ''.join(state.get_all_html_content_strings()) error_list = (html_validation_service.validate_math_tags_in_html( html_string)) if len(error_list) > 0: key = ('exp_id: %s, exp_status: %s failed validation.' % (item.id, exploration_status)) value_dict = { 'state_name': state_name, 'error_list': error_list, 'no_of_invalid_tags': len(error_list) } yield (key, value_dict)
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) validation_errors = [] for state_name, state in exploration.states.items(): if state.interaction.id == 'DragAndDropSortInput': for answer_group_index, answer_group in enumerate( state.interaction.answer_groups): for rule_index, rule_spec in enumerate( answer_group.rule_specs): for rule_input in rule_spec.inputs: value = rule_spec.inputs[rule_input] if value == '' or value == []: validation_errors.append( 'State name: %s, AnswerGroup: %s,' % (state_name, answer_group_index) + ' Rule input %s in rule with index %s' ' is empty. ' % (rule_input, rule_index)) if validation_errors: yield (item.id, validation_errors)
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) error_messages = [] for _, state in exploration.states.items(): if state.interaction.id is None: continue try: ca_specs = ( interaction_registry.Registry.get_interaction_by_id( state.interaction.id).customization_arg_specs ) customization_args_dict = {} for ca_name in state.interaction.customization_args: customization_args_dict[ca_name] = ( state.interaction.customization_args[ ca_name].to_customization_arg_dict() ) customization_args_util.validate_customization_args_and_values( 'interaction', state.interaction.id, customization_args_dict, ca_specs, fail_on_validation_errors=True ) except Exception as e: error_messages.append( '%s: %s' % (state.interaction.id, python_utils.UNICODE(e))) if error_messages: yield ( 'Failed customization args validation for exp ' 'id %s' % item.id, ', '.join(error_messages))
def map(item): if item.deleted: return exploration = exp_fetchers.get_exploration_from_model(item) try: exploration.validate() except Exception as e: logging.error('Exploration %s failed non-strict validation: %s' % (item.id, e)) yield ('validation_error', 'Exploration %s failed non-strict validation: %s' % (item.id, e)) return html_strings_in_exploration = '' for state in exploration.states.values(): html_strings_in_exploration += (''.join( state.get_all_html_content_strings())) list_of_latex_strings_without_svg = ( html_validation_service.get_latex_strings_without_svg_from_html( html_strings_in_exploration)) if len(list_of_latex_strings_without_svg) > 0: yield (ExplorationMathRichTextInfoModelGenerationOneOffJob. _SUCCESS_KEY, (item.id, list_of_latex_strings_without_svg))
def _get_model_domain_object_instance(self, item): return exp_fetchers.get_exploration_from_model(item)
def map(item): is_valid_math_expression = schema_utils.get_validator( 'is_valid_math_expression') is_valid_math_equation = schema_utils.get_validator( 'is_valid_math_equation') ltt = latex2text.LatexNodes2Text() unicode_to_text_mapping = ( MathExpressionValidationOneOffJob.UNICODE_TO_TEXT) inverse_trig_fns_mapping = ( MathExpressionValidationOneOffJob.INVERSE_TRIG_FNS_MAPPING) trig_fns = MathExpressionValidationOneOffJob.TRIG_FNS if not item.deleted: exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): if state.interaction.id == 'MathExpressionInput': for group in state.interaction.answer_groups: for rule_spec in group.rule_specs: rule_input = ltt.latex_to_text( rule_spec.inputs['x']) # Shifting powers in trig functions to the end. # For eg. 'sin^2(x)' -> 'sin(x)^2'. for trig_fn in trig_fns: rule_input = re.sub( r'%s(\^\d)\((.)\)' % trig_fn, r'%s(\2)\1' % trig_fn, rule_input) # Adding parens to trig functions that don't have # any. For eg. 'cosA' -> 'cos(A)'. for trig_fn in trig_fns: rule_input = re.sub(r'%s(?!\()(.)' % trig_fn, r'%s(\1)' % trig_fn, rule_input) # The pylatexenc lib outputs the unicode values of # special characters like sqrt and pi, which is why # they need to be replaced with their corresponding # text values before performing validation. for unicode_char, text in ( unicode_to_text_mapping.items()): rule_input = rule_input.replace( unicode_char, text) # Replacing trig functions that have format which is # incompatible with the validations. for invalid_trig_fn, valid_trig_fn in ( inverse_trig_fns_mapping.items()): rule_input = rule_input.replace( invalid_trig_fn, valid_trig_fn) validity = 'Invalid' if is_valid_math_expression(rule_input): validity = 'Valid Expression' elif is_valid_math_equation(rule_input): validity = 'Valid Equation' output_values = '%s %s: %s' % (item.id, state_name, rule_input) yield (validity, output_values.encode('utf-8'))
def map(item): if item.deleted: return exp_status = rights_manager.get_exploration_rights(item.id).status if exp_status == rights_domain.ACTIVITY_STATUS_PRIVATE: return def get_invalid_values(value_type, value, choices): """Checks that the html in SetOfHtmlString, ListOfSetsOfHtmlStrings, and DragAndDropHtmlString rule inputs have associated content ids in choices. Args: value_type: str. The type of the value. value: *. The value to migrate. choices: list(dict). The list of subtitled html dicts to find content ids from. Returns: *. The migrated rule input. """ invalid_values = [] if value_type == 'DragAndDropHtmlString': if value not in choices: invalid_values.append(value) if value_type == 'SetOfHtmlString': for html in value: invalid_values.extend( get_invalid_values('DragAndDropHtmlString', html, choices)) if value_type == 'ListOfSetsOfHtmlStrings': for html_set in value: invalid_values.extend( get_invalid_values('SetOfHtmlString', html_set, choices)) return invalid_values exploration = exp_fetchers.get_exploration_from_model(item) for state_name, state in exploration.states.items(): if state.interaction.id not in [ 'DragAndDropSortInput', 'ItemSelectionInput' ]: continue choices = [ choice.html for choice in state.interaction.customization_args['choices'].value ] solution = state.interaction.solution if solution is not None: if state.interaction.id == 'ItemSelectionInput': invalid_values = get_invalid_values( 'SetOfHtmlString', solution.correct_answer, choices) if invalid_values: yield (exploration.id, ('<ItemSelectionInput Answer> ' 'State: %s, Invalid Values: %s' % (state_name, invalid_values)).encode('utf-8')) if state.interaction.id == 'DragAndDropSortInput': invalid_values = get_invalid_values( 'ListOfSetsOfHtmlStrings', solution.correct_answer, choices) if invalid_values: yield (exploration.id, ('<DragAndDropSortInput Answer> ' 'State: %s, Invalid Values: %s' % (state_name, invalid_values)).encode('utf-8')) for group_i, group in enumerate(state.interaction.answer_groups): for rule_spec in group.rule_specs: rule_inputs = rule_spec.inputs rule_type = rule_spec.rule_type if state.interaction.id == 'ItemSelectionInput': # For all rule inputs for ItemSelectionInput, the x # input is of type SetOfHtmlString. invalid_values = get_invalid_values( 'SetOfHtmlString', rule_inputs['x'], choices) if invalid_values: yield (exploration.id, ('<ItemSelectionInput Rule> State: %s, ' 'Answer Group Index: %i, ' 'Invalid Values: %s' % (state_name, group_i, invalid_values)).encode('utf-8')) if state.interaction.id == 'DragAndDropSortInput': if rule_type in [ 'IsEqualToOrdering', 'IsEqualToOrderingWithOneItemAtIncorrectPosition' # pylint: disable=line-too-long ]: # For rule type IsEqualToOrdering and # IsEqualToOrderingWithOneItemAtIncorrectPosition, # the x is of type ListOfSetsOfHtmlStrings. invalid_values = get_invalid_values( 'ListOfSetsOfHtmlStrings', rule_inputs['x'], choices) if invalid_values: yield (exploration.id, ('<DragAndDropSortInput Rule> ' 'State: %s, ' 'Answer Group Index: %i, ' 'Invalid Values: %s' % (state_name, group_i, invalid_values)).encode('utf-8')) elif rule_type == 'HasElementXAtPositionY': # For rule type HasElementXAtPositionY, # the x input is of type DragAndDropHtmlString. The # y input is of type DragAndDropPositiveInt (no # validation required). invalid_values = get_invalid_values( 'DragAndDropHtmlString', rule_inputs['x'], choices) if invalid_values: yield (exploration.id, ('<DragAndDropSortInput Rule> ' 'State: %s, ' 'Answer Group Index: %i, ' 'Invalid Values: %s' % (state_name, group_i, invalid_values)).encode('utf-8')) elif rule_type == 'HasElementXBeforeElementY': # For rule type HasElementXBeforeElementY, # the x and y inputs are of type # DragAndDropHtmlString. for rule_input_name in ['x', 'y']: invalid_values = get_invalid_values( 'DragAndDropHtmlString', rule_inputs[rule_input_name], choices) if invalid_values: yield (exploration.id, ('<DragAndDropSortInput Rule> ' 'State: %s, ' 'Answer Group Index: %i, ' 'Invalid Values: %s' % (state_name, group_i, invalid_values)).encode('utf-8'))
def test_migration_then_reversion_maintains_valid_exploration(self): """This integration test simulates the behavior of the domain layer prior to the introduction of a states schema. In particular, it deals with an exploration that was created before any states schema migrations occur. The exploration is constructed using multiple change lists, then a migration is run. The test thereafter tests if reverting to a version prior to the migration still maintains a valid exploration. It tests both the exploration domain object and the exploration model stored in the datastore for validity. Note: It is important to distinguish between when the test is testing the exploration domain versus its model. It is operating at the domain layer when using exp_fetchers.get_exploration_by_id. Otherwise, it loads the model explicitly using exp_models.ExplorationModel.get and then converts it to an exploration domain object for validation using exp_fetchers.get_exploration_from_model. This is NOT the same process as exp_fetchers.get_exploration_by_id as it skips many steps which include the conversion pipeline (which is crucial to this test). """ exp_id = 'exp_id2' end_state_name = 'End' # Create an exploration with an old states schema version. swap_states_schema_41 = self.swap(feconf, 'CURRENT_STATE_SCHEMA_VERSION', 41) swap_exp_schema_46 = self.swap(exp_domain.Exploration, 'CURRENT_EXP_SCHEMA_VERSION', 46) with swap_states_schema_41, swap_exp_schema_46: self.save_new_valid_exploration(exp_id, self.albert_id, title='Old Title', end_state_name=end_state_name) caching_services.delete_multi( caching_services.CACHE_NAMESPACE_EXPLORATION, None, [exp_id]) # Load the exploration without using the conversion pipeline. All of # these changes are to happen on an exploration with states schema # version 41. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # In version 1, the title was 'Old title'. # In version 2, the title becomes 'New title'. exploration_model.title = 'New title' exploration_model.commit(self.albert_id, 'Changed title.', []) # Version 2 of exploration. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # Store state id mapping model for new exploration. exp_fetchers.get_exploration_from_model(exploration_model) # In version 3, a new state is added. exploration_model.states['New state'] = { 'solicit_answer_details': False, 'written_translations': { 'translations_mapping': { 'content': {}, 'default_outcome': {}, 'ca_placeholder_0': {}, } }, 'recorded_voiceovers': { 'voiceovers_mapping': { 'content': {}, 'default_outcome': {}, 'ca_placeholder_0': {}, } }, 'param_changes': [], 'classifier_model_id': None, 'content': { 'content_id': 'content', 'html': '<p>Unicode Characters ����</p>' }, 'next_content_id_index': 5, 'interaction': { 'answer_groups': [], 'confirmed_unclassified_answers': [], 'customization_args': { 'buttonText': { 'value': { 'content_id': 'ca_placeholder_0', 'unicode_str': 'Click me!', }, }, }, 'default_outcome': { 'dest': end_state_name, 'feedback': { 'content_id': 'default_outcome', 'html': '', }, 'labelled_as_correct': False, 'missing_prerequisite_skill_id': None, 'param_changes': [], 'refresher_exploration_id': None, }, 'hints': [], 'id': 'Continue', 'solution': None, }, } # Properly link in the new state to avoid an invalid exploration. init_state = exploration_model.states[feconf.DEFAULT_INIT_STATE_NAME] init_state['interaction']['default_outcome']['dest'] = 'New state' exploration_model.commit('committer_id_v3', 'Added new state', []) # Version 3 of exploration. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # Version 4 is an upgrade based on the migration job. commit_cmds = [ exp_domain.ExplorationChange({ 'cmd': exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION, 'from_version': str(exploration_model.states_schema_version), 'to_version': str(feconf.CURRENT_STATE_SCHEMA_VERSION) }) ] exp_services.update_exploration( feconf.MIGRATION_BOT_USERNAME, exploration_model.id, commit_cmds, 'Update exploration states from schema version %d to %d.' % (exploration_model.states_schema_version, feconf.CURRENT_STATE_SCHEMA_VERSION)) # Verify the latest version of the exploration has the most up-to-date # states schema version. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) exploration = exp_fetchers.get_exploration_from_model( exploration_model, run_conversion=False) self.assertEqual(exploration.states_schema_version, feconf.CURRENT_STATE_SCHEMA_VERSION) # The exploration should be valid after conversion. exploration.validate(strict=True) # Version 5 is a reversion to version 1. exp_services.revert_exploration('committer_id_v4', exp_id, 4, 1) # The exploration model itself should now be the old version # (pre-migration). exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) self.assertEqual(exploration_model.states_schema_version, 41) # The exploration domain object should be updated since it ran through # the conversion pipeline. exploration = exp_fetchers.get_exploration_by_id(exp_id) # The reversion after migration should still be an up-to-date # exploration. exp_fetchers.get_exploration_by_id will automatically # keep it up-to-date. self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML) # The exploration should be valid after reversion. exploration.validate(strict=True) snapshots_metadata = exp_services.get_exploration_snapshots_metadata( exp_id) # These are used to verify the correct history has been recorded after # both migration and reversion. commit_dict_5 = { 'committer_id': 'committer_id_v4', 'commit_message': 'Reverted exploration to version 1', 'version_number': 5, } commit_dict_4 = { 'committer_id': feconf.MIGRATION_BOT_USERNAME, 'commit_message': 'Update exploration states from schema version 41 to %d.' % feconf.CURRENT_STATE_SCHEMA_VERSION, 'commit_cmds': [{ 'cmd': exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION, 'from_version': '41', 'to_version': str(feconf.CURRENT_STATE_SCHEMA_VERSION) }], 'version_number': 4, } # Ensure there have been 5 commits. self.assertEqual(len(snapshots_metadata), 5) # Ensure the correct commit logs were entered during both migration and # reversion. Also, ensure the correct commit command was written during # migration. # These asserts check whether one dict is subset of the other. # The format is assertDictEqual(a, {**a, **b}) where a is the superset # and b is the subset. self.assertDictEqual(snapshots_metadata[3], { **snapshots_metadata[3], **commit_dict_4 }) self.assertDictEqual(snapshots_metadata[4], { **snapshots_metadata[4], **commit_dict_5 }) self.assertLess(snapshots_metadata[3]['created_on_ms'], snapshots_metadata[4]['created_on_ms']) # Ensure that if a converted, then reverted, then converted exploration # is saved, it will be the up-to-date version within the datastore. exp_services.update_exploration(self.albert_id, exp_id, [], 'Resave after reversion') exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) exploration = exp_fetchers.get_exploration_from_model( exploration_model, run_conversion=False) # This exploration should be both up-to-date and valid. self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML) exploration.validate()
def test_migration_then_reversion_maintains_valid_exploration(self): """This integration test simulates the behavior of the domain layer prior to the introduction of a states schema. In particular, it deals with an exploration that was created before any states schema migrations occur. The exploration is constructed using multiple change lists, then a migration job is run. The test thereafter tests if reverting to a version prior to the migration still maintains a valid exploration. It tests both the exploration domain object and the exploration model stored in the datastore for validity. Note: It is important to distinguish between when the test is testing the exploration domain versus its model. It is operating at the domain layer when using exp_fetchers.get_exploration_by_id. Otherwise, it loads the model explicitly using exp_models.ExplorationModel.get and then converts it to an exploration domain object for validation using exp_fetchers.get_exploration_from_model. This is NOT the same process as exp_fetchers.get_exploration_by_id as it skips many steps which include the conversion pipeline (which is crucial to this test). """ exp_id = 'exp_id2' # Create a exploration with states schema version 0. self.save_new_exp_with_states_schema_v0(exp_id, self.albert_id, 'Old Title') # Load the exploration without using the conversion pipeline. All of # these changes are to happen on an exploration with states schema # version 0. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # In version 1, the title was 'Old title'. # In version 2, the title becomes 'New title'. exploration_model.title = 'New title' exploration_model.commit(self.albert_id, 'Changed title.', []) # Version 2 of exploration. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # Store state id mapping model for new exploration. exploration = exp_fetchers.get_exploration_from_model( exploration_model) # In version 3, a new state is added. new_state = copy.deepcopy( self.VERSION_0_STATES_DICT[feconf.DEFAULT_INIT_STATE_NAME]) new_state['interaction']['id'] = 'TextInput' exploration_model.states['New state'] = new_state # Properly link in the new state to avoid an invalid exploration. init_state = exploration_model.states[feconf.DEFAULT_INIT_STATE_NAME] init_handler = init_state['interaction']['handlers'][0] init_handler['rule_specs'][0]['dest'] = 'New state' exploration_model.commit('committer_id_v3', 'Added new state', []) # Version 3 of exploration. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) # Store state id mapping model for new exploration. exploration = exp_fetchers.get_exploration_from_model( exploration_model) # Version 4 is an upgrade based on the migration job. # Start migration job on sample exploration. job_id = exp_jobs_one_off.ExplorationMigrationJobManager.create_new() exp_jobs_one_off.ExplorationMigrationJobManager.enqueue(job_id) self.process_and_flush_pending_tasks() # Verify the latest version of the exploration has the most up-to-date # states schema version. exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) exploration = exp_fetchers.get_exploration_from_model( exploration_model, run_conversion=False) self.assertEqual(exploration.states_schema_version, feconf.CURRENT_STATE_SCHEMA_VERSION) # The exploration should be valid after conversion. exploration.validate(strict=True) # Version 5 is a reversion to version 1. exp_services.revert_exploration('committer_id_v4', exp_id, 4, 1) # The exploration model itself should now be the old version # (pre-migration). exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) self.assertEqual(exploration_model.states_schema_version, 0) # The exploration domain object should be updated since it ran through # the conversion pipeline. exploration = exp_fetchers.get_exploration_by_id(exp_id) # The reversion after migration should still be an up-to-date # exploration. exp_fetchers.get_exploration_by_id will automatically # keep it up-to-date. self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML) # The exploration should be valid after reversion. exploration.validate(strict=True) snapshots_metadata = exp_services.get_exploration_snapshots_metadata( exp_id) # These are used to verify the correct history has been recorded after # both migration and reversion. commit_dict_5 = { 'committer_id': 'committer_id_v4', 'commit_message': 'Reverted exploration to version 1', 'version_number': 5, } commit_dict_4 = { 'committer_id': feconf.MIGRATION_BOT_USERNAME, 'commit_message': 'Update exploration states from schema version 0 to %d.' % feconf.CURRENT_STATE_SCHEMA_VERSION, 'commit_cmds': [{ 'cmd': exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION, 'from_version': '0', 'to_version': python_utils.UNICODE(feconf.CURRENT_STATE_SCHEMA_VERSION) }], 'version_number': 4, } # Ensure there have been 5 commits. self.assertEqual(len(snapshots_metadata), 5) # Ensure the correct commit logs were entered during both migration and # reversion. Also, ensure the correct commit command was written during # migration. self.assertDictContainsSubset(commit_dict_4, snapshots_metadata[3]) self.assertDictContainsSubset(commit_dict_5, snapshots_metadata[4]) self.assertLess(snapshots_metadata[3]['created_on_ms'], snapshots_metadata[4]['created_on_ms']) # Ensure that if a converted, then reverted, then converted exploration # is saved, it will be the up-to-date version within the datastore. exp_services.update_exploration(self.albert_id, exp_id, [], 'Resave after reversion') exploration_model = exp_models.ExplorationModel.get(exp_id, strict=True, version=None) exploration = exp_fetchers.get_exploration_from_model( exploration_model, run_conversion=False) # This exploration should be both up-to-date and valid. self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML) exploration.validate(strict=True)