def parse_data_dictionary_from(raw_dictionary, root_folder):
d = make_absolute_paths(raw_dictionary, root_folder)
errors = OrderedDict()
for key, value in d.items():
data_type = get_data_type(key)
try:
value = data_type.parse(value)
except DataTypeError as e:
errors[key] = text_type(e)
except Exception as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[key] = 'could_not_parse'
d[key] = value
if not key.endswith('_path'):
continue
noun = key[:-5]
data_type = get_data_type(noun)
try:
data_type.load(value)
except DataTypeError as e:
errors[noun] = text_type(e)
except IOError as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[noun] = 'not_found'
except Exception as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[noun] = 'could_not_load'
if errors:
raise DataParseError(errors, d)
return d
def parse_data_dictionary_from(raw_dictionary, root_folder):
d = make_absolute_paths(raw_dictionary, root_folder)
errors = OrderedDict()
for key, value in d.items():
data_type = get_data_type(key)
try:
value = data_type.parse(value)
except DataTypeError as e:
errors[key] = text_type(e)
except Exception as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[key] = 'could_not_parse'
d[key] = value
if not key.endswith('_path'):
continue
noun = key[:-5]
data_type = get_data_type(noun)
try:
data_type.load(value)
except DataTypeError as e:
errors[noun] = text_type(e)
except IOError as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[noun] = 'not_found'
except Exception as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[noun] = 'could_not_load'
if errors:
raise DataParseError(errors, d)
return d
def parse_data_dictionary_from(raw_dictionary, root_folder, tool_definition):
d = make_absolute_paths(raw_dictionary, root_folder)
errors = OrderedDict()
for key, value in d.items():
data_type = get_data_type(key)
try:
value = data_type.parse(value)
except DataTypeError as e:
errors[key] = text_type(e)
except Exception as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[key] = 'could_not_parse'
try:
old_value = get_default_value(key, tool_definition)
except KeyError:
pass
else:
old_value = data_type.parse(old_value)
if old_value != value:
value = data_type.merge(old_value, value)
d[key] = value
if not key.endswith('_path'):
continue
noun = key[:-5]
data_type = get_data_type(noun)
try:
data_type.load(value)
except DataTypeError as e:
errors[noun] = text_type(e)
except (IOError, Exception) as e:
log_traceback(LOG, {'key': key, 'value': value})
errors[noun] = 'could_not_load'
if errors:
raise DataParseError(errors, d)
return d
def load_tool_definition_by_name(
tool_configuration_path, default_tool_name=None):
tool_definition_by_name = {}
configuration = RawCaseSensitiveConfigParser()
configuration.read(tool_configuration_path)
configuration_folder = dirname(tool_configuration_path)
d = {
u'configuration_path': tool_configuration_path,
u'configuration_folder': configuration_folder,
}
for section_name in configuration.sections():
try:
tool_name = TOOL_NAME_PATTERN.match(section_name).group(1).strip()
except AttributeError:
continue
if not tool_name:
tool_name = default_tool_name
tool_definition = {
unicode_safely(k): unicode_safely(v)
for k, v in configuration.items(section_name)}
for key in tool_definition:
if key in ('show_standard_output', 'show_standard_error'):
tool_definition[key] = asbool(tool_definition[key])
elif key.endswith('.dependencies'):
tool_definition[key] = aslist(tool_definition[key])
tool_definition[u'tool_name'] = tool_name
tool_definition[u'argument_names'] = parse_tool_argument_names(
tool_definition.get('command_template', u''))
tool_definition_by_name[tool_name] = dict(tool_definition, **d)
return make_absolute_paths(tool_definition_by_name, configuration_folder)
def prepare_script_folder(script_folder,
notebook_path,
tool_name,
with_debugging=False):
notebook = load_notebook(notebook_path)
notebook_folder = dirname(abspath(notebook_path))
copy_folder(script_folder, notebook_folder)
# Prepare arguments
tool_arguments = load_tool_arguments(notebook)
for k in RESERVED_ARGUMENT_NAMES:
tool_arguments[k] = ''
tool_arguments = make_absolute_paths(tool_arguments, script_folder)
for k, v in tool_arguments.items():
if k.endswith('_path') and not exists(v):
raise CrossComputeError(
{k: 'file not found (%s)' % relpath(v, script_folder)})
kw = {'with_debugging': with_debugging}
# Save script
script_content = prepare_script(tool_arguments, notebook)
script_name = 'run.py'
copy_text(join(script_folder, script_name), script_content)
# Save tool template
cell = notebook['cells'][0]
if cell['cell_type'] == u'markdown':
template_name = 'tool.md'
copy_text(join(script_folder, template_name), cell['source'])
kw['tool_template_path'] = template_name
# Save result template
cell = notebook['cells'][-1]
if cell['cell_type'] == u'markdown':
template_name = 'result.md'
copy_text(join(script_folder, template_name), cell['source'])
kw['result_template_path'] = template_name
# Save configuration
command_name = notebook['metadata']['kernelspec']['name']
configuration_content = prepare_configuration(
tool_name, command_name, script_name,
make_relative_paths(tool_arguments, script_folder), **kw)
configuration_name = 'cc.ini'
copy_text(join(script_folder, configuration_name), configuration_content)
