def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
hints_length = len(state.interaction.hints)
if hints_length > 0:
exp_and_state_key = '%s %s' % (item.id,
state_name.encode('utf-8'))
yield (python_utils.UNICODE(hints_length), exp_and_state_key)
def map(item):
if not item.deleted:
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
if state.interaction.id == 'MathExpressionInput':
for group in state.interaction.answer_groups:
for rule_spec in group.rule_specs:
rule = rule_spec.inputs['x']
if any(sep in rule for sep in SEPARATORS):
yield (item.id,
'%s: %s' % (state_name.encode('utf-8'),
rule.encode('utf-8')))
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
exp_rights = rights_manager.get_exploration_rights(item.id)
try:
if exp_rights.status == rights_manager.ACTIVITY_STATUS_PRIVATE:
exploration.validate()
else:
exploration.validate(strict=True)
except utils.ValidationError as e:
yield (item.id, unicode(e).encode(encoding='utf-8'))
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
exp_rights = rights_manager.get_exploration_rights(item.id)
try:
if exp_rights.status == rights_domain.ACTIVITY_STATUS_PRIVATE:
exploration.validate()
else:
exploration.validate(strict=True)
except utils.ValidationError as e:
yield (item.id, python_utils.convert_to_bytes(e))
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
html_list = exploration.get_all_html_content_strings()
err_dict = html_validation_service.validate_rte_format(
html_list, feconf.RTE_FORMAT_CKEDITOR)
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def _validate_translation_counts(
cls, item, field_name_to_external_model_references):
"""Validate that translation_counts match the translations available in
the exploration.
Args:
item: datastore_services.Model. ExplorationOpportunitySummaryModel
to validate.
field_name_to_external_model_references:
dict(str, (list(base_model_validators.ExternalModelReference))).
A dict keyed by field name. The field name represents
a unique identifier provided by the storage
model to which the external model is associated. Each value
contains a list of ExternalModelReference objects corresponding
to the field_name. For examples, all the external Exploration
Models corresponding to a storage model can be associated
with the field name 'exp_ids'. This dict is used for
validation of External Model properties linked to the
storage model.
"""
exploration_model_references = (
field_name_to_external_model_references['exploration_ids'])
for exploration_model_reference in exploration_model_references:
exploration_model = exploration_model_reference.model_instance
if exploration_model is None or exploration_model.deleted:
model_class = exploration_model_reference.model_class
model_id = exploration_model_reference.model_id
cls._add_error(
'exploration_ids %s' % (
base_model_validators.ERROR_CATEGORY_FIELD_CHECK),
'Entity id %s: based on field exploration_ids having'
' value %s, expected model %s with id %s but it doesn\'t'
' exist' % (
item.id, model_id, model_class.__name__, model_id))
continue
exploration = exp_fetchers.get_exploration_from_model(
exploration_model)
exploration_translation_counts = (
exploration.get_translation_counts())
if exploration_translation_counts != item.translation_counts:
cls._add_error(
'translation %s' % (
base_model_validators.ERROR_CATEGORY_COUNT_CHECK),
'Entity id %s: Translation counts: %s does not match the '
'translation counts of external exploration model: %s' % (
item.id, item.translation_counts,
exploration_translation_counts))
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
interaction = state.interaction
exp_and_state_key = '%s %s' % (item.id, state_name)
if interaction.id == 'RatioExpressionInput':
number_of_terms = (
interaction.customization_args['numberOfTerms'].value)
if number_of_terms > 10:
yield (python_utils.UNICODE(number_of_terms),
exp_and_state_key)
yield ('SUCCESS', 1)
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
if state.interaction.id == 'ItemSelectionInput':
choices = (
state.interaction.customization_args['choices']['value'])
for group in state.interaction.answer_groups:
for rule_spec in group.rule_specs:
for rule_item in rule_spec.inputs['x']:
if rule_item not in choices:
yield (item.id,
'%s: %s' % (state_name.encode('utf-8'),
rule_item.encode('utf-8')))
def map(item):
if item.deleted:
return
try:
exploration = exp_fetchers.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s when loading exploration' % str(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
err_dict = html_validation_service.validate_rte_format(
html_list, feconf.RTE_FORMAT_CKEDITOR)
for key in err_dict:
if err_dict[key]:
yield ('%s Exp Id: %s' % (key, item.id), err_dict[key])
def map(item):
if item.deleted:
return
err_dict = {}
try:
exploration = exp_fetchers.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s when loading exploration' % str(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
err_dict = html_validation_service.validate_customization_args(
html_list)
for key in err_dict:
if err_dict[key]:
yield ('%s Exp Id: %s' % (key, item.id), err_dict[key])
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
if state.interaction.id == 'MultipleChoiceInput':
choices_length = len(
state.interaction.customization_args['choices']['value'])
for anwer_group_index, answer_group in enumerate(
state.interaction.answer_groups):
for rule_index, rule_spec in enumerate(
answer_group.rule_specs):
if rule_spec.inputs['x'] >= choices_length:
yield (
item.id, 'State name: %s, AnswerGroup: %s,' %
(state_name.encode('utf-8'), anwer_group_index)
+ ' Rule: %s is invalid.' % (rule_index) +
'(Indices here are 0-indexed.)')
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
invalid_tags_info_in_exp = []
for state_name, state in exploration.states.items():
html_string = ''.join(state.get_all_html_content_strings())
error_list = (html_validation_service.
validate_math_content_attribute_in_html(html_string))
if len(error_list) > 0:
invalid_tags_info_in_state = {
'state_name': state_name,
'error_list': error_list,
'no_of_invalid_tags': len(error_list)
}
invalid_tags_info_in_exp.append(invalid_tags_info_in_state)
if len(invalid_tags_info_in_exp) > 0:
yield ('Found invalid tags', (item.id, invalid_tags_info_in_exp))
def map(item):
if item.deleted:
return
err_dict = {}
try:
exploration = exp_fetchers.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s when loading exploration' %
python_utils.UNICODE(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
err_dict = html_validation_service.validate_customization_args(
html_list)
for key in err_dict:
err_value_with_exp_id = err_dict[key]
err_value_with_exp_id.append('Exp ID: %s' % item.id)
yield (key, err_value_with_exp_id)
def _is_used_logic_proof_interaction_query_satisfied(
user_settings_model, _):
"""Determines whether a user has used logic proof
interaction in any of the explorations created by the user.
"""
user_id = user_settings_model.id
user_contributions = user_models.UserContributionsModel.get(
user_id, strict=False)
if user_contributions is None:
return False
exploration_ids = user_contributions.created_exploration_ids
exploration_instances = (
datastore_services.fetch_multiple_entities_by_ids_and_models(
[('ExplorationModel', exploration_ids)]))[0]
for item in exploration_instances:
exploration = exp_fetchers.get_exploration_from_model(item)
for _, state in exploration.states.items():
if state.interaction.id == 'LogicProof':
return True
return False
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
exploration_status = (rights_manager.get_exploration_rights(
item.id).status)
for state_name, state in exploration.states.items():
html_string = ''.join(state.get_all_html_content_strings())
error_list = (html_validation_service.validate_math_tags_in_html(
html_string))
if len(error_list) > 0:
key = ('exp_id: %s, exp_status: %s failed validation.' %
(item.id, exploration_status))
value_dict = {
'state_name': state_name,
'error_list': error_list,
'no_of_invalid_tags': len(error_list)
}
yield (key, value_dict)
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
validation_errors = []
for state_name, state in exploration.states.items():
if state.interaction.id == 'DragAndDropSortInput':
for answer_group_index, answer_group in enumerate(
state.interaction.answer_groups):
for rule_index, rule_spec in enumerate(
answer_group.rule_specs):
for rule_input in rule_spec.inputs:
value = rule_spec.inputs[rule_input]
if value == '' or value == []:
validation_errors.append(
'State name: %s, AnswerGroup: %s,' %
(state_name, answer_group_index) +
' Rule input %s in rule with index %s'
' is empty. ' % (rule_input, rule_index))
if validation_errors:
yield (item.id, validation_errors)
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
error_messages = []
for _, state in exploration.states.items():
if state.interaction.id is None:
continue
try:
ca_specs = (
interaction_registry.Registry.get_interaction_by_id(
state.interaction.id).customization_arg_specs
)
customization_args_dict = {}
for ca_name in state.interaction.customization_args:
customization_args_dict[ca_name] = (
state.interaction.customization_args[
ca_name].to_customization_arg_dict()
)
customization_args_util.validate_customization_args_and_values(
'interaction',
state.interaction.id,
customization_args_dict,
ca_specs,
fail_on_validation_errors=True
)
except Exception as e:
error_messages.append(
'%s: %s' % (state.interaction.id, python_utils.UNICODE(e)))
if error_messages:
yield (
'Failed customization args validation for exp '
'id %s' % item.id, ', '.join(error_messages))
def map(item):
if item.deleted:
return
exploration = exp_fetchers.get_exploration_from_model(item)
try:
exploration.validate()
except Exception as e:
logging.error('Exploration %s failed non-strict validation: %s' %
(item.id, e))
yield ('validation_error',
'Exploration %s failed non-strict validation: %s' %
(item.id, e))
return
html_strings_in_exploration = ''
for state in exploration.states.values():
html_strings_in_exploration += (''.join(
state.get_all_html_content_strings()))
list_of_latex_strings_without_svg = (
html_validation_service.get_latex_strings_without_svg_from_html(
html_strings_in_exploration))
if len(list_of_latex_strings_without_svg) > 0:
yield (ExplorationMathRichTextInfoModelGenerationOneOffJob.
_SUCCESS_KEY, (item.id, list_of_latex_strings_without_svg))
def _get_model_domain_object_instance(self, item):
return exp_fetchers.get_exploration_from_model(item)
def map(item):
is_valid_math_expression = schema_utils.get_validator(
'is_valid_math_expression')
is_valid_math_equation = schema_utils.get_validator(
'is_valid_math_equation')
ltt = latex2text.LatexNodes2Text()
unicode_to_text_mapping = (
MathExpressionValidationOneOffJob.UNICODE_TO_TEXT)
inverse_trig_fns_mapping = (
MathExpressionValidationOneOffJob.INVERSE_TRIG_FNS_MAPPING)
trig_fns = MathExpressionValidationOneOffJob.TRIG_FNS
if not item.deleted:
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
if state.interaction.id == 'MathExpressionInput':
for group in state.interaction.answer_groups:
for rule_spec in group.rule_specs:
rule_input = ltt.latex_to_text(
rule_spec.inputs['x'])
# Shifting powers in trig functions to the end.
# For eg. 'sin^2(x)' -> 'sin(x)^2'.
for trig_fn in trig_fns:
rule_input = re.sub(
r'%s(\^\d)\((.)\)' % trig_fn,
r'%s(\2)\1' % trig_fn, rule_input)
# Adding parens to trig functions that don't have
# any. For eg. 'cosA' -> 'cos(A)'.
for trig_fn in trig_fns:
rule_input = re.sub(r'%s(?!\()(.)' % trig_fn,
r'%s(\1)' % trig_fn,
rule_input)
# The pylatexenc lib outputs the unicode values of
# special characters like sqrt and pi, which is why
# they need to be replaced with their corresponding
# text values before performing validation.
for unicode_char, text in (
unicode_to_text_mapping.items()):
rule_input = rule_input.replace(
unicode_char, text)
# Replacing trig functions that have format which is
# incompatible with the validations.
for invalid_trig_fn, valid_trig_fn in (
inverse_trig_fns_mapping.items()):
rule_input = rule_input.replace(
invalid_trig_fn, valid_trig_fn)
validity = 'Invalid'
if is_valid_math_expression(rule_input):
validity = 'Valid Expression'
elif is_valid_math_equation(rule_input):
validity = 'Valid Equation'
output_values = '%s %s: %s' % (item.id, state_name,
rule_input)
yield (validity, output_values.encode('utf-8'))
def map(item):
if item.deleted:
return
exp_status = rights_manager.get_exploration_rights(item.id).status
if exp_status == rights_domain.ACTIVITY_STATUS_PRIVATE:
return
def get_invalid_values(value_type, value, choices):
"""Checks that the html in SetOfHtmlString, ListOfSetsOfHtmlStrings,
and DragAndDropHtmlString rule inputs have associated content ids
in choices.
Args:
value_type: str. The type of the value.
value: *. The value to migrate.
choices: list(dict). The list of subtitled html dicts to find
content ids from.
Returns:
*. The migrated rule input.
"""
invalid_values = []
if value_type == 'DragAndDropHtmlString':
if value not in choices:
invalid_values.append(value)
if value_type == 'SetOfHtmlString':
for html in value:
invalid_values.extend(
get_invalid_values('DragAndDropHtmlString', html,
choices))
if value_type == 'ListOfSetsOfHtmlStrings':
for html_set in value:
invalid_values.extend(
get_invalid_values('SetOfHtmlString', html_set,
choices))
return invalid_values
exploration = exp_fetchers.get_exploration_from_model(item)
for state_name, state in exploration.states.items():
if state.interaction.id not in [
'DragAndDropSortInput', 'ItemSelectionInput'
]:
continue
choices = [
choice.html for choice in
state.interaction.customization_args['choices'].value
]
solution = state.interaction.solution
if solution is not None:
if state.interaction.id == 'ItemSelectionInput':
invalid_values = get_invalid_values(
'SetOfHtmlString', solution.correct_answer, choices)
if invalid_values:
yield (exploration.id,
('<ItemSelectionInput Answer> '
'State: %s, Invalid Values: %s' %
(state_name, invalid_values)).encode('utf-8'))
if state.interaction.id == 'DragAndDropSortInput':
invalid_values = get_invalid_values(
'ListOfSetsOfHtmlStrings', solution.correct_answer,
choices)
if invalid_values:
yield (exploration.id,
('<DragAndDropSortInput Answer> '
'State: %s, Invalid Values: %s' %
(state_name, invalid_values)).encode('utf-8'))
for group_i, group in enumerate(state.interaction.answer_groups):
for rule_spec in group.rule_specs:
rule_inputs = rule_spec.inputs
rule_type = rule_spec.rule_type
if state.interaction.id == 'ItemSelectionInput':
# For all rule inputs for ItemSelectionInput, the x
# input is of type SetOfHtmlString.
invalid_values = get_invalid_values(
'SetOfHtmlString', rule_inputs['x'], choices)
if invalid_values:
yield (exploration.id,
('<ItemSelectionInput Rule> State: %s, '
'Answer Group Index: %i, '
'Invalid Values: %s' %
(state_name, group_i,
invalid_values)).encode('utf-8'))
if state.interaction.id == 'DragAndDropSortInput':
if rule_type in [
'IsEqualToOrdering',
'IsEqualToOrderingWithOneItemAtIncorrectPosition' # pylint: disable=line-too-long
]:
# For rule type IsEqualToOrdering and
# IsEqualToOrderingWithOneItemAtIncorrectPosition,
# the x is of type ListOfSetsOfHtmlStrings.
invalid_values = get_invalid_values(
'ListOfSetsOfHtmlStrings', rule_inputs['x'],
choices)
if invalid_values:
yield (exploration.id,
('<DragAndDropSortInput Rule> '
'State: %s, '
'Answer Group Index: %i, '
'Invalid Values: %s' %
(state_name, group_i,
invalid_values)).encode('utf-8'))
elif rule_type == 'HasElementXAtPositionY':
# For rule type HasElementXAtPositionY,
# the x input is of type DragAndDropHtmlString. The
# y input is of type DragAndDropPositiveInt (no
# validation required).
invalid_values = get_invalid_values(
'DragAndDropHtmlString', rule_inputs['x'],
choices)
if invalid_values:
yield (exploration.id,
('<DragAndDropSortInput Rule> '
'State: %s, '
'Answer Group Index: %i, '
'Invalid Values: %s' %
(state_name, group_i,
invalid_values)).encode('utf-8'))
elif rule_type == 'HasElementXBeforeElementY':
# For rule type HasElementXBeforeElementY,
# the x and y inputs are of type
# DragAndDropHtmlString.
for rule_input_name in ['x', 'y']:
invalid_values = get_invalid_values(
'DragAndDropHtmlString',
rule_inputs[rule_input_name], choices)
if invalid_values:
yield (exploration.id,
('<DragAndDropSortInput Rule> '
'State: %s, '
'Answer Group Index: %i, '
'Invalid Values: %s' %
(state_name, group_i,
invalid_values)).encode('utf-8'))
def test_migration_then_reversion_maintains_valid_exploration(self):
"""This integration test simulates the behavior of the domain layer
prior to the introduction of a states schema. In particular, it deals
with an exploration that was created before any states schema
migrations occur. The exploration is constructed using multiple change
lists, then a migration is run. The test thereafter tests if
reverting to a version prior to the migration still maintains a valid
exploration. It tests both the exploration domain object and the
exploration model stored in the datastore for validity.
Note: It is important to distinguish between when the test is testing
the exploration domain versus its model. It is operating at the domain
layer when using exp_fetchers.get_exploration_by_id. Otherwise, it
loads the model explicitly using exp_models.ExplorationModel.get and
then converts it to an exploration domain object for validation using
exp_fetchers.get_exploration_from_model. This is NOT the same process
as exp_fetchers.get_exploration_by_id as it skips many steps which
include the conversion pipeline (which is crucial to this test).
"""
exp_id = 'exp_id2'
end_state_name = 'End'
# Create an exploration with an old states schema version.
swap_states_schema_41 = self.swap(feconf,
'CURRENT_STATE_SCHEMA_VERSION', 41)
swap_exp_schema_46 = self.swap(exp_domain.Exploration,
'CURRENT_EXP_SCHEMA_VERSION', 46)
with swap_states_schema_41, swap_exp_schema_46:
self.save_new_valid_exploration(exp_id,
self.albert_id,
title='Old Title',
end_state_name=end_state_name)
caching_services.delete_multi(
caching_services.CACHE_NAMESPACE_EXPLORATION, None, [exp_id])
# Load the exploration without using the conversion pipeline. All of
# these changes are to happen on an exploration with states schema
# version 41.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# In version 1, the title was 'Old title'.
# In version 2, the title becomes 'New title'.
exploration_model.title = 'New title'
exploration_model.commit(self.albert_id, 'Changed title.', [])
# Version 2 of exploration.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# Store state id mapping model for new exploration.
exp_fetchers.get_exploration_from_model(exploration_model)
# In version 3, a new state is added.
exploration_model.states['New state'] = {
'solicit_answer_details': False,
'written_translations': {
'translations_mapping': {
'content': {},
'default_outcome': {},
'ca_placeholder_0': {},
}
},
'recorded_voiceovers': {
'voiceovers_mapping': {
'content': {},
'default_outcome': {},
'ca_placeholder_0': {},
}
},
'param_changes': [],
'classifier_model_id': None,
'content': {
'content_id': 'content',
'html': '<p>Unicode Characters ����</p>'
},
'next_content_id_index': 5,
'interaction': {
'answer_groups': [],
'confirmed_unclassified_answers': [],
'customization_args': {
'buttonText': {
'value': {
'content_id': 'ca_placeholder_0',
'unicode_str': 'Click me!',
},
},
},
'default_outcome': {
'dest': end_state_name,
'feedback': {
'content_id': 'default_outcome',
'html': '',
},
'labelled_as_correct': False,
'missing_prerequisite_skill_id': None,
'param_changes': [],
'refresher_exploration_id': None,
},
'hints': [],
'id': 'Continue',
'solution': None,
},
}
# Properly link in the new state to avoid an invalid exploration.
init_state = exploration_model.states[feconf.DEFAULT_INIT_STATE_NAME]
init_state['interaction']['default_outcome']['dest'] = 'New state'
exploration_model.commit('committer_id_v3', 'Added new state', [])
# Version 3 of exploration.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# Version 4 is an upgrade based on the migration job.
commit_cmds = [
exp_domain.ExplorationChange({
'cmd':
exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION,
'from_version':
str(exploration_model.states_schema_version),
'to_version':
str(feconf.CURRENT_STATE_SCHEMA_VERSION)
})
]
exp_services.update_exploration(
feconf.MIGRATION_BOT_USERNAME, exploration_model.id, commit_cmds,
'Update exploration states from schema version %d to %d.' %
(exploration_model.states_schema_version,
feconf.CURRENT_STATE_SCHEMA_VERSION))
# Verify the latest version of the exploration has the most up-to-date
# states schema version.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
exploration = exp_fetchers.get_exploration_from_model(
exploration_model, run_conversion=False)
self.assertEqual(exploration.states_schema_version,
feconf.CURRENT_STATE_SCHEMA_VERSION)
# The exploration should be valid after conversion.
exploration.validate(strict=True)
# Version 5 is a reversion to version 1.
exp_services.revert_exploration('committer_id_v4', exp_id, 4, 1)
# The exploration model itself should now be the old version
# (pre-migration).
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
self.assertEqual(exploration_model.states_schema_version, 41)
# The exploration domain object should be updated since it ran through
# the conversion pipeline.
exploration = exp_fetchers.get_exploration_by_id(exp_id)
# The reversion after migration should still be an up-to-date
# exploration. exp_fetchers.get_exploration_by_id will automatically
# keep it up-to-date.
self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML)
# The exploration should be valid after reversion.
exploration.validate(strict=True)
snapshots_metadata = exp_services.get_exploration_snapshots_metadata(
exp_id)
# These are used to verify the correct history has been recorded after
# both migration and reversion.
commit_dict_5 = {
'committer_id': 'committer_id_v4',
'commit_message': 'Reverted exploration to version 1',
'version_number': 5,
}
commit_dict_4 = {
'committer_id':
feconf.MIGRATION_BOT_USERNAME,
'commit_message':
'Update exploration states from schema version 41 to %d.' %
feconf.CURRENT_STATE_SCHEMA_VERSION,
'commit_cmds': [{
'cmd':
exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION,
'from_version':
'41',
'to_version':
str(feconf.CURRENT_STATE_SCHEMA_VERSION)
}],
'version_number':
4,
}
# Ensure there have been 5 commits.
self.assertEqual(len(snapshots_metadata), 5)
# Ensure the correct commit logs were entered during both migration and
# reversion. Also, ensure the correct commit command was written during
# migration.
# These asserts check whether one dict is subset of the other.
# The format is assertDictEqual(a, {**a, **b}) where a is the superset
# and b is the subset.
self.assertDictEqual(snapshots_metadata[3], {
**snapshots_metadata[3],
**commit_dict_4
})
self.assertDictEqual(snapshots_metadata[4], {
**snapshots_metadata[4],
**commit_dict_5
})
self.assertLess(snapshots_metadata[3]['created_on_ms'],
snapshots_metadata[4]['created_on_ms'])
# Ensure that if a converted, then reverted, then converted exploration
# is saved, it will be the up-to-date version within the datastore.
exp_services.update_exploration(self.albert_id, exp_id, [],
'Resave after reversion')
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
exploration = exp_fetchers.get_exploration_from_model(
exploration_model, run_conversion=False)
# This exploration should be both up-to-date and valid.
self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML)
exploration.validate()
def test_migration_then_reversion_maintains_valid_exploration(self):
"""This integration test simulates the behavior of the domain layer
prior to the introduction of a states schema. In particular, it deals
with an exploration that was created before any states schema
migrations occur. The exploration is constructed using multiple change
lists, then a migration job is run. The test thereafter tests if
reverting to a version prior to the migration still maintains a valid
exploration. It tests both the exploration domain object and the
exploration model stored in the datastore for validity.
Note: It is important to distinguish between when the test is testing
the exploration domain versus its model. It is operating at the domain
layer when using exp_fetchers.get_exploration_by_id. Otherwise, it
loads the model explicitly using exp_models.ExplorationModel.get and
then converts it to an exploration domain object for validation using
exp_fetchers.get_exploration_from_model. This is NOT the same process
as exp_fetchers.get_exploration_by_id as it skips many steps which
include the conversion pipeline (which is crucial to this test).
"""
exp_id = 'exp_id2'
# Create a exploration with states schema version 0.
self.save_new_exp_with_states_schema_v0(exp_id, self.albert_id,
'Old Title')
# Load the exploration without using the conversion pipeline. All of
# these changes are to happen on an exploration with states schema
# version 0.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# In version 1, the title was 'Old title'.
# In version 2, the title becomes 'New title'.
exploration_model.title = 'New title'
exploration_model.commit(self.albert_id, 'Changed title.', [])
# Version 2 of exploration.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# Store state id mapping model for new exploration.
exploration = exp_fetchers.get_exploration_from_model(
exploration_model)
# In version 3, a new state is added.
new_state = copy.deepcopy(
self.VERSION_0_STATES_DICT[feconf.DEFAULT_INIT_STATE_NAME])
new_state['interaction']['id'] = 'TextInput'
exploration_model.states['New state'] = new_state
# Properly link in the new state to avoid an invalid exploration.
init_state = exploration_model.states[feconf.DEFAULT_INIT_STATE_NAME]
init_handler = init_state['interaction']['handlers'][0]
init_handler['rule_specs'][0]['dest'] = 'New state'
exploration_model.commit('committer_id_v3', 'Added new state', [])
# Version 3 of exploration.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
# Store state id mapping model for new exploration.
exploration = exp_fetchers.get_exploration_from_model(
exploration_model)
# Version 4 is an upgrade based on the migration job.
# Start migration job on sample exploration.
job_id = exp_jobs_one_off.ExplorationMigrationJobManager.create_new()
exp_jobs_one_off.ExplorationMigrationJobManager.enqueue(job_id)
self.process_and_flush_pending_tasks()
# Verify the latest version of the exploration has the most up-to-date
# states schema version.
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
exploration = exp_fetchers.get_exploration_from_model(
exploration_model, run_conversion=False)
self.assertEqual(exploration.states_schema_version,
feconf.CURRENT_STATE_SCHEMA_VERSION)
# The exploration should be valid after conversion.
exploration.validate(strict=True)
# Version 5 is a reversion to version 1.
exp_services.revert_exploration('committer_id_v4', exp_id, 4, 1)
# The exploration model itself should now be the old version
# (pre-migration).
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
self.assertEqual(exploration_model.states_schema_version, 0)
# The exploration domain object should be updated since it ran through
# the conversion pipeline.
exploration = exp_fetchers.get_exploration_by_id(exp_id)
# The reversion after migration should still be an up-to-date
# exploration. exp_fetchers.get_exploration_by_id will automatically
# keep it up-to-date.
self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML)
# The exploration should be valid after reversion.
exploration.validate(strict=True)
snapshots_metadata = exp_services.get_exploration_snapshots_metadata(
exp_id)
# These are used to verify the correct history has been recorded after
# both migration and reversion.
commit_dict_5 = {
'committer_id': 'committer_id_v4',
'commit_message': 'Reverted exploration to version 1',
'version_number': 5,
}
commit_dict_4 = {
'committer_id':
feconf.MIGRATION_BOT_USERNAME,
'commit_message':
'Update exploration states from schema version 0 to %d.' %
feconf.CURRENT_STATE_SCHEMA_VERSION,
'commit_cmds': [{
'cmd':
exp_domain.CMD_MIGRATE_STATES_SCHEMA_TO_LATEST_VERSION,
'from_version':
'0',
'to_version':
python_utils.UNICODE(feconf.CURRENT_STATE_SCHEMA_VERSION)
}],
'version_number':
4,
}
# Ensure there have been 5 commits.
self.assertEqual(len(snapshots_metadata), 5)
# Ensure the correct commit logs were entered during both migration and
# reversion. Also, ensure the correct commit command was written during
# migration.
self.assertDictContainsSubset(commit_dict_4, snapshots_metadata[3])
self.assertDictContainsSubset(commit_dict_5, snapshots_metadata[4])
self.assertLess(snapshots_metadata[3]['created_on_ms'],
snapshots_metadata[4]['created_on_ms'])
# Ensure that if a converted, then reverted, then converted exploration
# is saved, it will be the up-to-date version within the datastore.
exp_services.update_exploration(self.albert_id, exp_id, [],
'Resave after reversion')
exploration_model = exp_models.ExplorationModel.get(exp_id,
strict=True,
version=None)
exploration = exp_fetchers.get_exploration_from_model(
exploration_model, run_conversion=False)
# This exploration should be both up-to-date and valid.
self.assertEqual(exploration.to_yaml(), self.UPGRADED_EXP_YAML)
exploration.validate(strict=True)
