def map(item):
from core.domain import exp_services
exploration = exp_services.get_exploration_from_model(item)
try:
exploration.validate(strict=False)
except utils.ValidationError as e:
yield (item.id, unicode(e).encode('utf-8'))
def map(item):
if item.deleted:
return
err_dict = {}
try:
exploration = exp_services.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s when loading exploration' % str(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
try:
err_dict = html_validation_service.validate_customization_args(
html_list)
except Exception as e:
yield (
'Error in validating customization args for exploration %s' % (
item.id),
[traceback.format_exc()])
return
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
exp_and_state_key = '%s %s' % (item.id, state_name)
yield (state.interaction.id, exp_and_state_key)
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
exp_and_state_key = '%s %s' % (item.id, state_name)
yield (state.interaction.id, exp_and_state_key)
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
hints_length = len(state.interaction.hints)
if hints_length > 0:
exp_and_state_key = '%s %s' % (
item.id, state_name.encode('utf-8'))
yield (str(hints_length), exp_and_state_key)
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
explorations = []
# Fetch all versions of the exploration, if they exist.
if exploration.version > 1:
# Ignore latest version since we already have corresponding
# exploration.
versions = range(1, exploration.version)
# Get all exploration versions for current exploration id.
try:
explorations = (
exp_services.get_multiple_explorations_by_version(
exploration.id, versions))
except Exception as e:
yield ('ERROR with exp_id %s' % item.id, str(e))
return
# Append latest exploration to the list of explorations.
explorations.append(exploration)
# Retrieve list of snapshot models representing each version of the
# exploration.
versions = range(1, exploration.version + 1)
snapshots_by_version = (
exp_models.ExplorationModel.get_snapshots_metadata(
exploration.id, versions))
# Create and save state id mapping model for all exploration versions.
for exploration, snapshot in zip(explorations, snapshots_by_version):
if snapshot is None:
yield (
'ERROR with exp_id %s' % item.id,
'Error: No exploration snapshot metadata model instance '
'found for exploration %s, version %d' %
(exploration.id, exploration.version))
return
change_list = snapshot['commit_cmds']
# Check if commit is to revert the exploration.
if change_list and change_list[0]['cmd'].endswith(
'revert_version_number'):
reverted_version = change_list[0]['version_number']
exp_services.create_and_save_state_id_mapping_model_for_reverted_exploration( # pylint: disable=line-too-long
exploration.id, exploration.version - 1, reverted_version)
else:
exp_services.create_and_save_state_id_mapping_model(
exploration, change_list)
yield (exploration.id, exploration.version)
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
html_list = exploration.get_all_html_content_strings()
err_dict = html_cleaner.validate_textangular_format(html_list, True)
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
for fallback in state.interaction.fallbacks:
num_submits = fallback.trigger.customization_args[
'num_submits']
feedback = fallback.outcome.feedback[0]
yield ('%s: %s' % (item.id, state_name.encode('utf-8')),
'%s: %s' %
(num_submits['value'], feedback.encode('utf-8')))
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
html_list = exploration.get_all_html_content_strings()
err_dict = html_cleaner.validate_rte_format(
html_list, feconf.RTE_FORMAT_TEXTANGULAR, run_migration=True)
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
exp_rights = rights_manager.get_exploration_rights(item.id)
try:
if exp_rights.status == rights_manager.ACTIVITY_STATUS_PRIVATE:
exploration.validate()
else:
exploration.validate(strict=True)
except utils.ValidationError as e:
yield (item.id, unicode(e).encode('utf-8'))
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
# err_dict is a dictionary to store the invalid tags and the
# invalid parent-child relations that we find.
err_dict = {}
# All the invalid html strings will be stored in this.
err_dict['strings'] = []
allowed_parent_list = (feconf.RTE_CONTENT_SPEC['RTE_TYPE_TEXTANGULAR']
['ALLOWED_PARENT_LIST'])
allowed_tag_list = (feconf.RTE_CONTENT_SPEC['RTE_TYPE_TEXTANGULAR']
['ALLOWED_TAG_LIST'])
html_list = exploration.get_all_html_content_strings()
for html_data in html_list:
soup = bs4.BeautifulSoup(html_data, 'html.parser')
# Text with no parent tag is also invalid.
for content in soup.contents:
if not content.name:
err_dict['strings'].append(html_data)
break
for tag in soup.findAll():
# Checking for tags not allowed in RTE.
if tag.name not in allowed_tag_list:
if 'invalidTags' in err_dict:
err_dict['invalidTags'].append(tag.name)
else:
err_dict['invalidTags'] = [tag.name]
err_dict['strings'].append(html_data)
# Checking for parent-child relation that are not
# allowed in RTE.
parent = tag.parent.name
if (tag.name in allowed_tag_list) and (
parent not in allowed_parent_list[tag.name]):
if tag.name in err_dict:
err_dict[tag.name].append(parent)
else:
err_dict[tag.name] = [parent]
err_dict['strings'].append(html_data)
for key in err_dict:
if err_dict[key]:
yield (key, list(set(err_dict[key])))
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
html_list = exploration.get_all_html_content_strings()
err_dict = html_validation_service.validate_rte_format(
html_list, feconf.RTE_FORMAT_CKEDITOR)
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
exp_rights = rights_manager.get_exploration_rights(item.id)
try:
if exp_rights.status == rights_manager.ACTIVITY_STATUS_PRIVATE:
exploration.validate()
else:
exploration.validate(strict=True)
except utils.ValidationError as e:
yield (item.id, unicode(e).encode('utf-8'))
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
# err_dict is a dictionary to store the invalid tags and the
# invalid parent-child relations that we find.
err_dict = {}
htmlValidation_filepath = os.path.join(feconf.CONTENT_VALIDATION_DIR,
'htmlValidation.yaml')
htmlValidation_yaml = utils.get_file_contents(htmlValidation_filepath)
# Convert the yaml content to dictionary.
htmlValidation_dict = utils.dict_from_yaml(htmlValidation_yaml)
allowed_parent_list = htmlValidation_dict['allowed_parent_list']
allowed_tag_list = htmlValidation_dict['allowed_tag_list']
for state in exploration.states.itervalues():
# result is the list of all the html present in the state.
result = exp_services.find_all_values_for_key(
'html', state.to_dict())
for html_data in result:
html_data = html_data.encode('utf-8')
soup = BeautifulSoup(html_data, 'html.parser')
for tag in soup.findAll():
# Checking for tags not allowed in RTE.
if tag.name not in allowed_tag_list:
if 'invalidTags' in err_dict:
err_dict['invalidTags'] += [tag.name]
else:
err_dict['invalidTags'] = [tag.name]
# Checking for parent-child relation that are not
# allowed in RTE.
parent = tag.parent.name
if (tag.name in allowed_tag_list
and parent not in allowed_parent_list[tag.name]):
if tag.name in err_dict:
err_dict[tag.name] += [parent]
else:
err_dict[tag.name] = [parent]
for key in err_dict.iterkeys():
err_dict[key] = list(set(err_dict[key]))
yield ('Errors', err_dict)
def map(item):
"""Yields the id of each newly-created exploration copy, together with
its YAML representation.
Args:
item: ExplorationModel. An exploration storage model.
Yields:
str. The string containing exploration id of each newly-created
exploration. It is of the format:
<exp_id>copy<copy_number>
"""
if ExpCopiesMRJobManager._entity_created_before_job_queued(item):
for count in range(10):
yield ('%scopy%d' % (item.id, count),
exp_services.get_exploration_from_model(item).to_yaml())
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
if state.interaction.id == 'ItemSelectionInput':
choices = (
state.interaction.customization_args['choices']['value'])
for group in state.interaction.answer_groups:
for rule_spec in group.rule_specs:
for rule_item in rule_spec.inputs['x']:
if rule_item not in choices:
yield (item.id,
'%s: %s' % (state_name.encode('utf-8'),
rule_item.encode('utf-8')))
def map(item):
from core.domain import exp_services
from core.domain import rights_manager
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
exp_summary = rights_manager.get_exploration_rights(item.id)
try:
if exp_summary.status == rights_manager.EXPLORATION_STATUS_PRIVATE:
exploration.validate()
else:
exploration.validate(strict=True)
except utils.ValidationError as e:
yield (item.id, unicode(e).encode('utf-8'))
def map(item):
if item.deleted:
return
exploration = exp_services.get_exploration_from_model(item)
for state_name, state in exploration.states.iteritems():
if state.interaction.id == 'ItemSelectionInput':
choices = (
state.interaction.customization_args['choices']['value'])
for group in state.interaction.answer_groups:
for rule_spec in group.rule_specs:
for rule_item in rule_spec.inputs['x']:
if rule_item not in choices:
yield (
item.id,
'%s: %s' % (
state_name.encode('utf-8'),
rule_item.encode('utf-8')))
def map(item):
from core.domain import exp_domain
from core.domain import exp_services
if item.deleted:
return
exp = exp_services.get_exploration_from_model(item)
change_list = []
for state_name, state in exp.states.iteritems():
if state.interaction.id == 'InteractiveMap':
old_value = {}
for handler in state.interaction.handlers:
handler_dict = handler.to_dict()
old_value[handler_dict['name']] = (
handler_dict['rule_specs'])
new_value = copy.deepcopy(old_value)
for handler in new_value['submit']:
if handler['definition']['rule_type'] != 'default':
handler['definition']['inputs']['d'] *= 110
change_list.append({
'cmd': exp_domain.CMD_EDIT_STATE_PROPERTY,
'property_name': 'widget_handlers',
'state_name': state_name,
'old_value': old_value,
'new_value': new_value,
})
if change_list:
yield (item.id, 'started')
try:
exp_services.update_exploration(
feconf.MIGRATION_BOT_USERNAME, item.id, change_list,
'Convert distances in rules for world map to kilometers.')
yield (item.id, 'completed')
except utils.ValidationError as e:
if 'An objective must be specified' in str(e):
yield ('ERROR-OBJECTIVE', item.id)
else:
raise
def map(item):
from core.domain import exp_services
if item.deleted:
return
exp = exp_services.get_exploration_from_model(item)
change_list = []
for state_name, state in exp.states.iteritems():
if state.interaction.id == 'InteractiveMap':
old_value = {}
for handler in state.interaction.handlers:
handler_dict = handler.to_dict()
old_value[handler_dict['name']] = (
handler_dict['rule_specs'])
new_value = copy.deepcopy(old_value)
for handler in new_value['submit']:
if handler['definition']['rule_type'] != 'default':
handler['definition']['inputs']['d'] *= 110
change_list.append({
'cmd': 'edit_state_property',
'property_name': 'widget_handlers',
'state_name': state_name,
'old_value': old_value,
'new_value': new_value,
})
if change_list:
yield (item.id, 'started')
try:
exp_services.update_exploration(
feconf.MIGRATION_BOT_USERNAME, item.id, change_list,
'Convert distances in rules for world map to kilometers.')
yield (item.id, 'completed')
except utils.ValidationError as e:
if 'An objective must be specified' in str(e):
yield ('ERROR-OBJECTIVE', item.id)
else:
raise
def map(exp_model):
current_version = exp_model.version
version_numbers = range(1, current_version + 1)
list_of_exploration_models = (exp_model.get_multi_versions(
exp_model.id, version_numbers))
filenames = []
for exploration_model in list_of_exploration_models:
exploration = exp_services.get_exploration_from_model(
exploration_model)
filenames_in_version = (
exp_services.get_image_filenames_from_exploration(exploration))
filenames = list(set().union(filenames, filenames_in_version))
for filename in filenames:
file_system_class = (fs_domain.ExplorationFileSystem if
feconf.DEV_MODE else fs_domain.GcsFileSystem)
fs = fs_domain.AbstractFileSystem(file_system_class(exp_model.id))
filepath = (filename if feconf.DEV_MODE else 'image/%s' % filename)
raw_data = fs.get(filepath)
exp_services.save_original_and_compressed_versions_of_image(
'ADMIN', filename, exp_model.id, raw_data)
yield (ADDED_COMPRESSED_VERSIONS_OF_IMAGES, exp_model.id)
def map(item):
if item.deleted:
return
err_dict = {}
try:
exploration = exp_services.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s in exploration' % str(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
try:
err_dict = html_cleaner.validate_rte_format(
html_list, feconf.RTE_FORMAT_CKEDITOR, run_migration=True)
except Exception as e:
yield ('Error %s in exploration %s' % (str(e), item.id), html_list)
return
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
if item.deleted:
return
err_dict = {}
try:
exploration = exp_services.get_exploration_from_model(item)
except Exception as e:
yield ('Error %s when loading exploration' % str(e), [item.id])
return
html_list = exploration.get_all_html_content_strings()
try:
err_dict = html_validation_service.validate_rte_format(
html_list, feconf.RTE_FORMAT_CKEDITOR, run_migration=True)
except Exception as e:
yield ('Error in validating rte format for exploration %s' %
item.id, [traceback.format_exc()])
return
for key in err_dict:
if err_dict[key]:
yield (key, err_dict[key])
def map(item):
from core.domain import exp_services
if ExpCopiesMRJobManager._entity_created_before_job_queued(item):
for count in range(10):
yield ("%scopy%d" % (item.id, count), exp_services.get_exploration_from_model(item).to_yaml())
def save_new_exp_with_states_schema_v0(self, exp_id, user_id, title):
"""Saves a new default exploration with a default version 0 states
dictionary.
This function should only be used for creating explorations in tests
involving migration of datastore explorations that use an old states
schema version.
Note that it makes an explicit commit to the datastore instead of using
the usual functions for updating and creating explorations. This is
because the latter approach would result in an exploration with the
*current* states schema version.
Args:
exp_id: str. The exploration ID.
user_id: str. The user_id of the creator.
title: str. The title of the exploration.
"""
exp_model = exp_models.ExplorationModel(
id=exp_id,
category='category',
title=title,
objective='Old objective',
language_code='en',
tags=[],
blurb='',
author_notes='',
states_schema_version=0,
init_state_name=feconf.DEFAULT_INIT_STATE_NAME,
states=self.VERSION_0_STATES_DICT,
param_specs={},
param_changes=[])
rights_manager.create_new_exploration_rights(exp_id, user_id)
commit_message = 'New exploration created with title \'%s\'.' % title
exp_model.commit(user_id, commit_message, [{
'cmd': 'create_new',
'title': 'title',
'category': 'category',
}])
exp_rights = exp_models.ExplorationRightsModel.get_by_id(exp_id)
exp_summary_model = exp_models.ExpSummaryModel(
id=exp_id,
title=title,
category='category',
objective='Old objective',
language_code='en',
tags=[],
ratings=feconf.get_empty_ratings(),
scaled_average_rating=feconf.EMPTY_SCALED_AVERAGE_RATING,
status=exp_rights.status,
community_owned=exp_rights.community_owned,
owner_ids=exp_rights.owner_ids,
contributor_ids=[],
contributors_summary={},
)
exp_summary_model.put()
# Note: Also save state id mappping model for new exploration. If not
# saved, it may cause errors in test cases.
exploration = exp_services.get_exploration_from_model(exp_model)
exp_services.create_and_save_state_id_mapping_model(exploration, [])
def map(item):
if item.deleted:
return
try:
exploration = exp_services.get_exploration_from_model(item)
except Exception as e:
yield ('ERROR get_exp_from_model: exp_id %s' % item.id, str(e))
return
# Commit commands which are required to generate state id mapping for
# given version of exploration from previous version of exploration.
RELEVANT_COMMIT_CMDS = [
exp_domain.CMD_ADD_STATE,
exp_domain.CMD_RENAME_STATE,
exp_domain.CMD_DELETE_STATE,
exp_models.ExplorationModel.CMD_REVERT_COMMIT
]
explorations = []
# Fetch all versions of the exploration, if they exist.
if exploration.version > 1:
# Ignore latest version since we already have corresponding
# exploration.
versions = range(1, exploration.version)
# Get all exploration versions for current exploration id.
try:
explorations = (
exp_services.get_multiple_explorations_by_version(
exploration.id, versions))
except Exception as e:
yield (
'ERROR get_multiple_exp_by_version exp_id %s' % item.id,
str(e))
return
# Append latest exploration to the list of explorations.
explorations.append(exploration)
# Retrieve list of snapshot models representing each version of the
# exploration.
versions = range(1, exploration.version + 1)
snapshots_by_version = (
exp_models.ExplorationModel.get_snapshots_metadata(
exploration.id, versions))
# Create and save state id mapping model for all exploration versions.
for exploration, snapshot in zip(explorations, snapshots_by_version):
if snapshot is None:
yield (
'ERROR with exp_id %s' % item.id,
'Error: No exploration snapshot metadata model instance '
'found for exploration %s, version %d' % (
exploration.id, exploration.version))
return
change_list = [
exp_domain.ExplorationChange(change_cmd)
for change_cmd in snapshot['commit_cmds']
if change_cmd['cmd'] in RELEVANT_COMMIT_CMDS
]
try:
# Check if commit is to revert the exploration.
if change_list and change_list[0].cmd == (
exp_models.ExplorationModel.CMD_REVERT_COMMIT):
reverted_version = change_list[0].version_number
# pylint: disable=line-too-long
state_id_mapping = exp_services.generate_state_id_mapping_model_for_reverted_exploration(
exploration.id, exploration.version - 1,
reverted_version)
# pylint: enable=line-too-long
else:
state_id_mapping = (
exp_services.generate_state_id_mapping_model(
exploration, change_list))
state_id_mapping_model = exp_models.StateIdMappingModel.create(
state_id_mapping.exploration_id,
state_id_mapping.exploration_version,
state_id_mapping.state_names_to_ids,
state_id_mapping.largest_state_id_used, overwrite=True)
state_id_mapping_model.put()
except Exception as e:
yield (
'ERROR with exp_id %s version %s' % (
item.id, exploration.version),
traceback.format_exc())
return
yield (exploration.id, exploration.version)
def map(item):
from core.domain import exp_services
if ExpCopiesMRJobManager._entity_created_before_job_queued(item):
for count in range(10):
yield ('%scopy%d' % (item.id, count),
exp_services.get_exploration_from_model(item).to_yaml())
