def test_yaml_dict_conversion(self):
"""Test yaml_from_dict and dict_from_yaml methods."""
test_dicts = [{}, {'a': 'b'}, {'a': 2}, {'a': ['b', 2, {'c': 3.5}]}]
for adict in test_dicts:
yaml_str = utils.yaml_from_dict(adict)
yaml_dict = utils.dict_from_yaml(yaml_str)
self.assertEqual(adict, yaml_dict)
with self.assertRaises(utils.InvalidInputException):
yaml_str = utils.dict_from_yaml('{')
def test_yaml_import_and_export_with_gadgets(self):
"""Test from_yaml() and to_yaml() methods including gadgets."""
EXP_ID = 'An exploration_id'
exploration_with_gadgets = exp_domain.Exploration.from_yaml(
EXP_ID, 'A title', 'Category', SAMPLE_YAML_CONTENT_WITH_GADGETS)
generated_yaml = exploration_with_gadgets.to_yaml()
generated_yaml_as_dict = utils.dict_from_yaml(generated_yaml)
sample_yaml_as_dict = utils.dict_from_yaml(
SAMPLE_YAML_CONTENT_WITH_GADGETS)
self.assertEqual(generated_yaml_as_dict, sample_yaml_as_dict)
def create_from_yaml(cls, yaml_file, user, title, category, exploration_id=None, image_id=None):
"""Creates an exploration from a YAML file."""
init_state_name = yaml_file[: yaml_file.index(":\n")]
if not init_state_name or "\n" in init_state_name:
raise Exception(
"Invalid YAML file: the name of the initial state "
"should be left-aligned on the first line and "
"followed by a colon"
)
exploration = cls.create(
user, title, category, exploration_id=exploration_id, init_state_name=init_state_name, image_id=image_id
)
init_state = State.get_by_name(init_state_name, exploration)
try:
exploration_dict = utils.dict_from_yaml(yaml_file)
state_list = []
for state_name, state_description in exploration_dict.iteritems():
state = init_state if state_name == init_state_name else exploration.add_state(state_name)
state_list.append({"state": state, "desc": state_description})
for index, state in enumerate(state_list):
State.modify_using_dict(exploration, state["state"], state["desc"])
except Exception:
exploration.delete()
raise
return exploration
def load_default_widgets(cls):
"""Loads the default widgets.
Assumes that everything is valid (directories exist, widget config files
are formatted correctly, etc.).
"""
widget_ids = os.listdir(feconf.INTERACTIVE_WIDGETS_DIR)
for widget_id in widget_ids:
widget_dir = os.path.join(feconf.INTERACTIVE_WIDGETS_DIR, widget_id)
widget_conf_filename = '%s.config.yaml' % widget_id
with open(os.path.join(widget_dir, widget_conf_filename)) as f:
conf = utils.dict_from_yaml(f.read().decode('utf-8'))
conf['id'] = '%s-%s' % (feconf.INTERACTIVE_PREFIX, widget_id)
conf['params'] = [Parameter(**param) for param in conf['params']]
conf['handlers'] = [AnswerHandler(**ah) for ah in conf['handlers']]
conf['template'] = utils.get_file_contents(
os.path.join(widget_dir, '%s.html' % widget_id))
conf['static_template'] = ''
static_path = os.path.join(widget_dir, '%s.static.html' % widget_id)
if os.path.exists(static_path):
conf['static_template'] = utils.get_file_contents(static_path)
widget = cls(**conf)
widget.put()
def test_default_explorations_are_valid(self):
"""Test the default explorations."""
exploration_files = os.listdir(
os.path.join(feconf.SAMPLE_EXPLORATIONS_DIR))
self.assertTrue(feconf.DEMO_EXPLORATIONS,
msg='There must be at least one demo exploration.')
derived_exploration_filenames = [
('%s.yaml' % item[0]) for item in feconf.DEMO_EXPLORATIONS]
self.assertItemsEqual(exploration_files, derived_exploration_filenames,
msg='Files in data/explorations do not match '
'the demo explorations in feconf.py.')
for filename in exploration_files:
filepath = os.path.join(feconf.SAMPLE_EXPLORATIONS_DIR, filename)
self.assertTrue(
os.path.isfile(filepath), msg='%s is not a file.' % filepath)
# Convert each exploration into a dict, and verify it.
with open(filepath) as f:
exploration_yaml = f.read().decode('utf-8')
exploration_dict = utils.dict_from_yaml(exploration_yaml)
try:
exp_services.verify_exploration_dict(exploration_dict)
except Exception as e:
raise Exception('%s: %s' % (filepath, e))
def _migrate_to_latest_yaml_version(cls, yaml_content):
try:
collection_dict = utils.dict_from_yaml(yaml_content)
except Exception as e:
raise Exception(
'Please ensure that you are uploading a YAML text file, not '
'a zip file. The YAML parser returned the following error: %s'
% e)
collection_schema_version = collection_dict.get('schema_version')
if collection_schema_version is None:
raise Exception('Invalid YAML file: no schema version specified.')
if not (1 <= collection_schema_version
<= feconf.CURRENT_COLLECTION_SCHEMA_VERSION):
raise Exception(
'Sorry, we can only process v1 to v%s collection YAML files at '
'present.' % feconf.CURRENT_COLLECTION_SCHEMA_VERSION)
while (collection_schema_version <
feconf.CURRENT_COLLECTION_SCHEMA_VERSION):
conversion_fn = getattr(
cls, '_convert_v%s_dict_to_v%s_dict' % (
collection_schema_version, collection_schema_version + 1))
collection_dict = conversion_fn(collection_dict)
collection_schema_version += 1
return collection_dict
def create_from_yaml(
yaml_content, user_id, title, category, exploration_id=None,
image_id=None):
"""Creates an exploration from a YAML text string."""
exploration_dict = utils.dict_from_yaml(yaml_content)
init_state_name = exploration_dict['states'][0]['name']
exploration = Exploration.get(create_new(
user_id, title, category, exploration_id=exploration_id,
init_state_name=init_state_name, image_id=image_id))
init_state = State.get_by_name(init_state_name, exploration)
try:
exploration.parameters = [Parameter(
name=param['name'], obj_type=param['obj_type'],
values=param['values']
) for param in exploration_dict['parameters']]
state_list = []
for state_description in exploration_dict['states']:
state_name = state_description['name']
state = (init_state if state_name == init_state_name
else exploration.add_state(state_name))
state_list.append({'state': state, 'desc': state_description})
for index, state in enumerate(state_list):
modify_using_dict(exploration.id, state['state'].id, state['desc'])
except Exception:
exploration.delete()
raise
return exploration.id
def verify_exploration_yaml(self, exploration_yaml):
"""Verifies an exploration YAML file."""
init_state_name = exploration_yaml[:exploration_yaml.find(':\n')]
if not init_state_name or '\n' in init_state_name:
raise Exception('Invalid YAML file: the name of the initial state '
'should be left-aligned on the first line and '
'followed by a colon')
exploration_dict = utils.dict_from_yaml(exploration_yaml)
self.verify_exploration_dict(exploration_dict, init_state_name)
def from_yaml(cls, collection_id, yaml_content):
try:
collection_dict = utils.dict_from_yaml(yaml_content)
except Exception as e:
raise Exception(
'Please ensure that you are uploading a YAML text file, not '
'a zip file. The YAML parser returned the following error: %s'
% e)
collection_dict['id'] = collection_id
return Collection.from_dict(collection_dict)
def get_readable_name(self, handler_name, rule_name):
"""Get the human-readable name for a rule."""
for handler in self.handlers:
if handler.name == handler_name:
classifier = handler.classifier
with open(os.path.join(
feconf.SAMPLE_CLASSIFIERS_DIR,
classifier,
'%sRules.yaml' % classifier)) as f:
rules = utils.dict_from_yaml(f.read().decode('utf-8'))
return rules[rule_name]['name']
raise Exception('No rule name found for %s' % rule_name)
def get_static_asset_filepath(self):
"""Returns filepath for referencing static files on disk.
examples: '' or 'build/1234'
"""
cache_slug_filepath = ''
if feconf.IS_MINIFIED or not feconf.DEV_MODE:
yaml_file_content = utils.dict_from_yaml(
utils.get_file_contents('cache_slug.yaml'))
cache_slug = yaml_file_content['cache_slug']
cache_slug_filepath = os.path.join('build', cache_slug)
return cache_slug_filepath
def load_default_classifiers(cls):
"""Loads the default classifiers."""
classifier_ids = cls.get_classifier_ids()
for classifier_id in classifier_ids:
rules_filepath = os.path.join(
feconf.SAMPLE_CLASSIFIERS_DIR, classifier_id,
'%sRules.yaml' % classifier_id)
with open(rules_filepath) as f:
rule_dict = utils.dict_from_yaml(f.read().decode('utf-8'))
rules = []
for rule in rule_dict:
r_spec = RuleSpec(rule=rule, name=rule_dict[rule]['name'])
if 'checks' in rule_dict[rule]:
r_spec.checks = rule_dict[rule]['checks']
rules.append(r_spec)
Classifier(id=classifier_id, rules=rules).put()
def load_default_widgets(cls):
"""Loads the default widgets.
Assumes that everything is valid (directories exist, widget config files
are formatted correctly, etc.).
"""
widget_ids = os.listdir(feconf.SAMPLE_WIDGETS_DIR)
for widget_id in widget_ids:
widget_dir = os.path.join(feconf.SAMPLE_WIDGETS_DIR, widget_id)
widget_conf_filename = "%s.config.yaml" % widget_id
with open(os.path.join(widget_dir, widget_conf_filename)) as f:
conf = utils.dict_from_yaml(f.read().decode("utf-8"))
conf["params"] = [Parameter(**param) for param in conf["params"]]
conf["handlers"] = [AnswerHandler(**ah) for ah in conf["handlers"]]
conf["template"] = utils.get_file_contents(os.path.join(widget_dir, "%s.html" % widget_id))
widget = cls(**conf)
widget.put()
def load_default_widgets(cls):
"""Loads the default widgets."""
widget_ids = os.listdir(feconf.NONINTERACTIVE_WIDGETS_DIR)
for widget_id in widget_ids:
widget_dir = os.path.join(
feconf.NONINTERACTIVE_WIDGETS_DIR, widget_id)
widget_conf_filename = '%s.config.yaml' % widget_id
with open(os.path.join(widget_dir, widget_conf_filename)) as f:
conf = utils.dict_from_yaml(f.read().decode('utf-8'))
conf['id'] = '%s-%s' % (feconf.NONINTERACTIVE_PREFIX, widget_id)
conf['params'] = [Parameter(**param) for param in conf['params']]
# TODO(sll): Should this be a template *location* instead?
conf['template'] = utils.get_file_contents(
os.path.join(widget_dir, '%s.html' % widget_id))
conf['static_template'] = ''
widget = cls(**conf)
widget.put()
def get_with_params(cls, widget_id, params=None):
"""Gets a parameterized widget."""
if params is None:
params = {}
widget = cls.get(widget_id)
result = copy.deepcopy(widget.to_dict())
result['id'] = widget_id
result['raw'] = cls.get_raw_code(widget_id, params)
# TODO(sll): Restructure this so that it is
# {key: {value: ..., obj_type: ...}}
result['params'] = dict((param.name, params.get(param.name, param.value))
for param in widget.params)
if 'handlers' in result:
actions = {}
for item in result['handlers']:
actions[item['name']] = {'classifier': item['classifier']}
result['actions'] = actions
del result['handlers']
for unused_action, properties in result['actions'].iteritems():
classifier = properties['classifier']
if classifier:
with open(os.path.join(
feconf.SAMPLE_CLASSIFIERS_DIR,
classifier,
'%sRules.yaml' % classifier)) as f:
rules = utils.dict_from_yaml(f.read().decode('utf-8'))
rule_dict = {}
for rule in rules:
rule_dict[rules[rule]['name']] = {'classifier': rule}
if 'checks' in rules[rule]:
rule_dict[rules[rule]['name']]['checks'] = (
rules[rule]['checks'])
properties['rules'] = rule_dict
return result
def put(self, exploration, state):
"""Saves updates to a state."""
yaml_file = self.payload.get('yaml_file')
if yaml_file and feconf.ALLOW_YAML_FILE_UPLOAD:
# The user has uploaded a YAML file. Process only this action.
state = exp_services.modify_using_dict(
exploration.id, state.id, utils.dict_from_yaml(yaml_file))
self.render_json(get_state_for_frontend(state, exploration))
return
state_name = self.payload.get('state_name')
param_changes = self.payload.get('param_changes')
interactive_widget = self.payload.get('interactive_widget')
interactive_params = self.payload.get('interactive_params')
interactive_rulesets = self.payload.get('interactive_rulesets')
sticky_interactive_widget = self.payload.get(
'sticky_interactive_widget')
content = self.payload.get('content')
unresolved_answers = self.payload.get('unresolved_answers')
if 'state_name' in self.payload:
# Replace the state name with this one, after checking validity.
if state_name == feconf.END_DEST:
raise self.InvalidInputException('Invalid state name: END')
exploration.rename_state(state.id, state_name)
if 'param_changes' in self.payload:
state.param_changes = []
for param_change in param_changes:
instance = exp_services.get_or_create_param(
exploration.id, param_change['name'])
instance.values = param_change['values']
state.param_changes.append(instance)
if interactive_widget:
state.widget.widget_id = interactive_widget
if interactive_params:
state.widget.params = interactive_params
if sticky_interactive_widget is not None:
state.widget.sticky = sticky_interactive_widget
if interactive_rulesets:
ruleset = interactive_rulesets['submit']
utils.recursively_remove_key(ruleset, u'$$hashKey')
state.widget.handlers = [AnswerHandlerInstance(
name='submit', rules=[])]
# This is part of the state. The rules should be put into it.
state_ruleset = state.widget.handlers[0].rules
# TODO(yanamal): Do additional calculations here to get the
# parameter changes, if necessary.
for rule_ind in range(len(ruleset)):
rule = ruleset[rule_ind]
state_rule = Rule()
state_rule.name = rule.get('name')
state_rule.inputs = rule.get('inputs')
state_rule.dest = rule.get('dest')
state_rule.feedback = rule.get('feedback')
# Generate the code to be executed.
if rule['rule'] == 'Default':
# This is the default rule.
assert rule_ind == len(ruleset) - 1
state_rule.name = 'Default'
state_ruleset.append(state_rule)
continue
# Normalize the params here, then store them.
classifier_func = state_rule.name.replace(' ', '')
first_bracket = classifier_func.find('(')
mutable_rule = rule['rule']
params = classifier_func[first_bracket + 1: -1].split(',')
for index, param in enumerate(params):
if param not in rule['inputs']:
raise self.InvalidInputException(
'Parameter %s could not be replaced.' % param)
typed_object = state.get_typed_object(mutable_rule, param)
# TODO(sll): Make the following check more robust.
if (not isinstance(rule['inputs'][param], basestring) or
'{{' not in rule['inputs'][param] or
'}}' not in rule['inputs'][param]):
normalized_param = typed_object.normalize(
rule['inputs'][param])
else:
normalized_param = rule['inputs'][param]
if normalized_param is None:
raise self.InvalidInputException(
'%s has the wrong type. Please replace it with a '
'%s.' % (rule['inputs'][param],
typed_object.__name__))
state_rule.inputs[param] = normalized_param
state_ruleset.append(state_rule)
if content:
state.content = [Content(type=item['type'], value=item['value'])
for item in content]
if 'unresolved_answers' in self.payload:
state.unresolved_answers = {}
for answer, count in unresolved_answers.iteritems():
if count > 0:
state.unresolved_answers[answer] = count
state.put()
self.render_json(get_state_for_frontend(state, exploration))
def test_default_widgets_are_valid(self):
"""Test the default widgets."""
widget_ids = os.listdir(os.path.join(feconf.INTERACTIVE_WIDGETS_DIR))
for widget_id in widget_ids:
# Check that the widget_id name is valid.
self.assertTrue(self.is_camel_cased(widget_id))
# Check that the widget directory exists.
widget_dir = os.path.join(feconf.INTERACTIVE_WIDGETS_DIR, widget_id)
self.assertTrue(os.path.isdir(widget_dir))
# In this directory there should only be a config.yaml file, an html
# entry-point file, and (optionally) a static.html and a directory
# named 'static'.
dir_contents = os.listdir(widget_dir)
self.assertLessEqual(len(dir_contents), 4)
if len(dir_contents) == 3:
self.assertIn('static', dir_contents)
static_dir = os.path.join(widget_dir, 'static')
self.assertTrue(os.path.isdir(static_dir))
config_filepath = os.path.join(
widget_dir, '%s.config.yaml' % widget_id)
html_entry_point = os.path.join(widget_dir, '%s.html' % widget_id)
self.assertTrue(os.path.isfile(config_filepath))
self.assertTrue(os.path.isfile(html_entry_point))
# Read the widget configuration from config.yaml.
with open(config_filepath) as f:
widget_config = utils.dict_from_yaml(f.read().decode('utf-8'))
WIDGET_CONFIG_SCHEMA = [
('id', basestring), ('name', basestring),
('category', basestring), ('description', basestring),
('handlers', list), ('params', list)
]
# Check that the configuration file contains the correct top-level
# keys, and that these keys have the correct types.
self.verify_dict_keys_and_types(widget_config, WIDGET_CONFIG_SCHEMA)
# Check that the specified widget id is the same as the directory
# name.
self.assertTrue(widget_config['id'], widget_id)
for handler in widget_config['handlers']:
HANDLER_KEYS = ['name', 'classifier']
self.assertItemsEqual(HANDLER_KEYS, handler.keys())
self.assertTrue(isinstance(handler['name'], basestring))
# The default classifier may be None (e.g. in the case of the
# 'Continue' widget).
if handler['classifier'] is not None:
self.assertTrue(isinstance(
handler['classifier'], basestring))
# Check that the classifier directory exists.
classifier_dir = os.path.join(
feconf.SAMPLE_CLASSIFIERS_DIR, handler['classifier'])
self.assertTrue(
os.path.isdir(classifier_dir),
msg='Classifier %s does not exist' % classifier_dir)
for param in widget_config['params']:
PARAM_KEYS = ['name', 'description', 'obj_type', 'values']
self.assertItemsEqual(PARAM_KEYS, param.keys())
self.assertTrue(isinstance(param['name'], basestring))
self.assertTrue(self.is_alphanumeric_string(param['name']))
self.assertTrue(isinstance(param['description'], basestring))
# Check that the default values have the correct types.
obj_class = get_object_class(param['obj_type'])
self.assertIsNotNone(obj_class)
self.assertTrue(isinstance(param['values'], list))
for value in param['values']:
obj_class.normalize(value)
def verify_exploration_yaml(self, exploration_yaml):
"""Verifies an exploration YAML file."""
exploration_dict = utils.dict_from_yaml(exploration_yaml)
self.verify_exploration_dict(exploration_dict)
def from_yaml(cls, exploration_id, title, category, yaml_content):
"""Creates and returns exploration from a YAML text string."""
exploration_dict = utils.dict_from_yaml(yaml_content)
exploration_schema_version = exploration_dict.get('schema_version')
if exploration_schema_version is None:
raise Exception('Invalid YAML file: no schema version specified.')
if not (1 <= exploration_schema_version
<= cls.CURRENT_EXPLORATION_SCHEMA_VERSION):
raise Exception(
'Sorry, we can only process v1 and v2 YAML files at present.')
if exploration_schema_version == 1:
exploration_dict = cls._convert_v1_dict_to_v2_dict(
exploration_dict)
exploration = cls.create_default_exploration(
exploration_id, title, category)
exploration.param_specs = {
ps_name: param_domain.ParamSpec.from_dict(ps_val) for
(ps_name, ps_val) in exploration_dict['param_specs'].iteritems()
}
init_state_name = exploration_dict['init_state_name']
exploration.rename_state(exploration.init_state_name, init_state_name)
exploration.add_states([
state_name for state_name in exploration_dict['states']
if state_name != init_state_name])
for (state_name, sdict) in exploration_dict['states'].iteritems():
state = exploration.states[state_name]
state.content = [
Content(item['type'], html_cleaner.clean(item['value']))
for item in sdict['content']
]
state.param_changes = [param_domain.ParamChange(
pc['name'], pc['generator_id'], pc['customization_args']
) for pc in sdict['param_changes']]
for pc in state.param_changes:
if pc.name not in exploration.param_specs:
raise Exception('Parameter %s was used in a state but not '
'declared in the exploration param_specs.'
% pc.name)
wdict = sdict['widget']
widget_handlers = [AnswerHandlerInstance.from_dict({
'name': handler['name'],
'rule_specs': [{
'definition': rule_spec['definition'],
'dest': rule_spec['dest'],
'feedback': [html_cleaner.clean(feedback)
for feedback in rule_spec['feedback']],
'param_changes': rule_spec.get('param_changes', []),
} for rule_spec in handler['rule_specs']],
}) for handler in wdict['handlers']]
state.widget = WidgetInstance(
wdict['widget_id'], wdict['customization_args'],
widget_handlers, wdict['sticky'])
exploration.states[state_name] = state
exploration.default_skin = exploration_dict['default_skin']
exploration.param_changes = [
param_domain.ParamChange.from_dict(pc)
for pc in exploration_dict['param_changes']]
return exploration
def mock_read_travis_yml_file():
travis_ci_file = python_utils.open_file(
os.path.join(DUMMY_CONF_FILES, '.dummy_travis.yml'),
'r').read()
travis_ci_dict = utils.dict_from_yaml(travis_ci_file)
return travis_ci_dict
def create_from_yaml(
yaml_content, user_id, title, category, exploration_id=None,
image_id=None):
"""Creates an exploration from a YAML text string."""
exploration_dict = utils.dict_from_yaml(yaml_content)
exploration_schema_version = exploration_dict.get('schema_version')
if exploration_schema_version != CURRENT_EXPLORATION_SCHEMA_VERSION:
raise Exception('Sorry, we can only process v1 YAML files at present.')
init_state_name = exploration_dict['states'][0]['name']
exploration_id = create_new(
user_id, title, category, exploration_id=exploration_id,
init_state_name=init_state_name, image_id=image_id)
try:
exploration_param_specs = {
ps_name: param_domain.ParamSpec.from_dict(ps_val)
for (ps_name, ps_val) in exploration_dict['param_specs'].iteritems()
}
for sdict in exploration_dict['states']:
if sdict['name'] != init_state_name:
add_state(user_id, exploration_id, sdict['name'])
for sdict in exploration_dict['states']:
state = get_state_by_name(exploration_id, sdict['name'])
state.content = [
exp_domain.Content(item['type'], item['value'])
for item in sdict['content']
]
state.param_changes = [param_domain.ParamChange(
pc['name'], pc['generator_id'], pc['customization_args']
) for pc in sdict['param_changes']]
for pc in state.param_changes:
if pc.name not in exploration_param_specs:
raise Exception('Parameter %s was used in a state but not '
'declared in the exploration param_specs.'
% pc.name)
wdict = sdict['widget']
widget_handlers = [exp_domain.AnswerHandlerInstance.from_dict({
'name': handler['name'],
'rule_specs': [{
'definition': rule_spec['definition'],
'dest': convert_state_name_to_id(
exploration_id, rule_spec['dest']),
'feedback': rule_spec['feedback'],
'param_changes': rule_spec.get('param_changes', []),
} for rule_spec in handler['rule_specs']],
}) for handler in wdict['handlers']]
state.widget = exp_domain.WidgetInstance(
wdict['widget_id'], wdict['customization_args'],
widget_handlers, wdict['sticky'])
save_state(user_id, exploration_id, state)
exploration = get_exploration_by_id(exploration_id)
exploration.default_skin = exploration_dict['default_skin']
exploration.param_changes = [param_domain.ParamChange(
pc['name'], pc['generator_id'], pc['customization_args']
) for pc in exploration_dict['param_changes']]
exploration.param_specs = exploration_param_specs
save_exploration(user_id, exploration)
except Exception:
delete_exploration(user_id, exploration_id, force_deletion=True)
raise
return exploration_id
