def __split_off_extra_attributes(self, mapping: CommentedMap,
known_attrs: List[str]) -> CommentedMap:
"""Separates the extra attributes in mapping into _yatiml_extra.
This returns a mapping containing all key-value pairs from
mapping whose key is in known_attrs, and an additional key
_yatiml_extra which maps to a dict containing the remaining
key-value pairs.
Args:
mapping: The mapping to split
known_attrs: Attributes that should be kept in the main
map, and not moved to _yatiml_extra.
Returns:
A map with attributes reorganised as described above.
"""
attr_names = list(mapping.keys())
main_attrs = mapping.copy() # type: CommentedMap
extra_attrs = OrderedDict(mapping.items())
for name in attr_names:
if name not in known_attrs or name == '_yatiml_extra':
del (main_attrs[name])
else:
del (extra_attrs[name])
main_attrs['_yatiml_extra'] = extra_attrs
return main_attrs
def _loadSpec(self, spec, path, repositories):
if "service_template" in spec:
toscaDef = spec["service_template"]
elif "tosca" in spec: # backward compat
toscaDef = spec["tosca"]
else:
toscaDef = {}
repositoriesTpl = toscaDef.setdefault("repositories", CommentedMap())
for name, value in repositories.items():
if name not in repositoriesTpl:
repositoriesTpl[name] = value
# make sure this is present
toscaDef["tosca_definitions_version"] = TOSCA_VERSION
if not isinstance(toscaDef, CommentedMap):
toscaDef = CommentedMap(toscaDef.items())
if getattr(toscaDef, "baseDir",
None) and (not path
or toscaDef.baseDir != os.path.dirname(path)):
# note: we only recorded the baseDir not the name of the included file
path = toscaDef.baseDir
return ToscaSpec(toscaDef, spec.get("inputs"), spec, path,
self.getImportResolver())
def _clean_yaml(self, recipe_yaml: CommentedMap):
recipe = deepcopy(recipe_yaml)
for key, value in recipe_yaml.items():
if key == "extra" or (key == "test" and value):
continue
if not isinstance(value, bool) and not value:
del recipe[key]
elif isinstance(value, (CommentedMap, dict)):
if key != "requirements":
self.__reduce_list(key, recipe)
return recipe
def deep_convert_dict(layer):
to_ret = layer
if isinstance(layer, dict):
to_ret = CommentedMap(layer)
try:
for key, value in to_ret.items():
to_ret[key] = deep_convert_dict(value)
except AttributeError:
pass
return to_ret
def _get_data_encoded(self, include_default=False):
data_encoded = {}
for section in self._fields:
if not include_default and (not section in self._data or len(self._data[section]) == 0):
continue
data_encoded[section] = CommentedMap()
for setting, field in self._fields[section].items():
if setting in self._data[section] or include_default:
data_encoded[section].insert(0, setting, field._encode(self._data[section].get(setting, field.default)), field.comment)
data_encoded = CommentedMap(sorted(data_encoded.items(), key=lambda t: t[0]))
return data_encoded
def main():
image_set_dir = "data/LINEMOD_6D/LM6d_converted/LM6d_render_v1/image_set/real/"
test_set_dict = CommentedMap({cls_idx: CommentedSeq([]) for cls_idx in IDX2CLASS.keys()})
for cls_idx, cls_name in IDX2CLASS.items():
print(cls_idx, cls_name)
index_file = osp.join(image_set_dir, "{}_test.txt".format(cls_name))
with open(index_file, "r") as f:
test_indices = [line.strip("\r\n") for line in f.readlines()]
for test_idx in test_indices:
sixd_idx = int(test_idx.split("/")[1]) - 1
test_set_dict[cls_idx].append(sixd_idx)
for cls_idx, indices in test_set_dict.items():
indices.fa.set_flow_style()
res_file = osp.join(osp.expanduser("~/Storage/SIXD_DATASETS/LM6d_origin/", "test_set_bb8_sixd.yml"))
with open(res_file, "w") as f:
yaml.dump(test_set_dict, f, Dumper=yaml.RoundTripDumper, width=10000)
print("done")
def __type_check_attributes(self, node: yaml.Node, mapping: CommentedMap,
argspec: inspect.FullArgSpec) -> None:
"""Ensure all attributes have a matching constructor argument.
This checks that there is a constructor argument with a
matching type for each existing attribute.
If the class has a _yatiml_extra attribute, then extra
attributes are okay and no error will be raised if they exist.
Args:
node: The node we're processing
mapping: The mapping with constructed subobjects
constructor_attrs: The attributes of the constructor,
including self and _yatiml_extra, if applicable
"""
logger.debug('Checking for extraneous attributes')
logger.debug('Constructor arguments: {}, mapping: {}'.format(
argspec.args, list(mapping.keys())))
for key, value in mapping.items():
if not isinstance(key, str):
raise RecognitionError(('{}{}YAtiML only supports strings'
' for mapping keys').format(
node.start_mark, os.linesep))
if key not in argspec.args and '_yatiml_extra' not in argspec.args:
raise RecognitionError(
('{}{}Found additional attributes ("{}")'
' and {} does not support those').format(
node.start_mark, os.linesep, key,
self.class_.__name__))
if key in argspec.args and key in argspec.annotations:
if not self.__type_matches(value, argspec.annotations[key]):
raise RecognitionError(
('{}{}Expected attribute "{}" to be of type {}'
' but it is a(n) {}').format(
node.start_mark, os.linesep, key,
argspec.annotations[key], type(value)))
def _load_spec(self,
spec,
path,
repositories,
more_spec,
skip_validation=False):
if "service_template" in spec:
toscaDef = spec["service_template"] or {}
elif "tosca" in spec: # backward compat
toscaDef = spec["tosca"] or {}
else:
toscaDef = {}
repositoriesTpl = toscaDef.setdefault("repositories", CommentedMap())
for name, value in repositories.items():
if name not in repositoriesTpl:
repositoriesTpl[name] = value
# make sure this is present
toscaDef["tosca_definitions_version"] = TOSCA_VERSION
if more_spec:
# don't merge individual templates
toscaDef = merge_dicts(
toscaDef,
more_spec,
replaceKeys=['node_templates', 'relationship_templates'])
if not isinstance(toscaDef, CommentedMap):
toscaDef = CommentedMap(toscaDef.items())
if getattr(toscaDef, "base_dir",
None) and (not path
or toscaDef.base_dir != os.path.dirname(path)):
# note: we only recorded the baseDir not the name of the included file
path = toscaDef.base_dir
return ToscaSpec(toscaDef, spec, path,
self.get_import_resolver(expand=True),
skip_validation)
class WorkflowGenerator(object):
"""Class for creating a CWL workflow.
The WorkflowGenerator class allows users to tie together inputs and outputs
of the steps that need to be executed to perform a data processing task.
The steps (i.e., command line tools and subworkflows) must be added to the
steps library of the WorkflowGenerator object before they can be added to
the workflow. To add steps to the steps library, the `load` method can be
called with either a path to a directory containing CWL files:
::
from scriptcwl import WorkflowGenerator
with WorkflowGenerator() as wf:
wf.load(steps_dir='/path/to/dir/with/cwl/steps/')
Or a single CWL file:
::
with WorkflowGenerator() as wf:
wf.load(step_file='/path/to/cwl/step/file')
``wf.load()`` can be called multiple times. Step files are added to the
steps library one after the other. For every step that is added to the
steps library, a method with the same name is added to the
WorkflowGenerator object. To add a step to the workflow, this method must
be called (examples below).
Next, the user should add one or more workflow inputs:
::
txt_dir = wf.add_input(txt_dir='Directory')
The ``add_input()`` method expects a ``name=type`` pair as input parameter.
The pair connects an input name (``txt_dir`` in the example) to a CWL type
(``'Directory'``). Optionally, a default value can be specified using
``default=value``.
The ``add_input()`` method returns a string containing the name
that can be used to connect this input parameter to step input parameter
names.
Next, workflow steps can be added. To add a workflow step, its method must
be called on the WorkflowGenerator object. This method expects a list of
(key, value) pairs as input parameters. (To find out what inputs a step
needs call ``wf.inputs(<step name>)``. This method prints all the inputs
and their types.) The method returns a list of strings containing output
names that can be used as input for later steps, or that can be connected
to workflow outputs.
For example, to add a step called ``frog-dir`` to the workflow, the
following method must be called:
::
frogout = wf.frog_dir(dir_in=txt_dir)
In a next step, ``frogout`` can be used as input:
::
saf = wf.frog_to_saf(in_files=frogout)
txt = wf.saf_to_txt(in_files=saf)
Etcetera.
When all steps of the workflow have been added, the user can specify
workflow outputs:
::
wf.add_outputs(txt=txt)
Finally, the workflow can be saved to file:
::
wf.save('workflow.cwl')
To list steps and signatures available in the steps library, call:
::
wf.list_steps()
"""
def __init__(self, steps_dir=None, working_dir=None):
self.working_dir = working_dir
if self.working_dir:
self.working_dir = os.path.abspath(self.working_dir)
if not os.path.exists(self.working_dir):
os.makedirs(self.working_dir)
self.wf_steps = CommentedMap()
self.wf_inputs = CommentedMap()
self.wf_outputs = CommentedMap()
self.step_output_types = {}
self.steps_library = StepsLibrary(working_dir=working_dir)
self.has_workflow_step = False
self.has_scatter_requirement = False
self.has_multiple_inputs = False
self._wf_closed = False
self.load(steps_dir)
def __enter__(self):
self._wf_closed = False
return self
def __exit__(self, *args):
self.wf_steps = None
self.wf_inputs = None
self.wf_outputs = None
self.step_output_types = None
self.steps_library = None
self.has_workflow_step = None
self.has_scatter_requirement = None
self.working_dir = None
self._wf_closed = True
def __getattr__(self, name, **kwargs):
name = self.steps_library.python_names2step_names.get(name, None)
step = self._get_step(name)
return partial(self._make_step, step, **kwargs)
def __str__(self):
# use absolute paths for printing
return yaml2string(self, pack=False, relpath=None, wd=False)
def _closed(self):
if self._wf_closed:
raise ValueError('Operation on closed WorkflowGenerator.')
def load(self, steps_dir=None, step_file=None, step_list=None):
"""Load CWL steps into the WorkflowGenerator's steps library.
Adds steps (command line tools and workflows) to the
``WorkflowGenerator``'s steps library. These steps can be used to
create workflows.
Args:
steps_dir (str): path to directory containing CWL files. All CWL in
the directory are loaded.
step_file (str): path to a file containing a CWL step that will be
added to the steps library.
"""
self._closed()
self.steps_library.load(steps_dir=steps_dir,
step_file=step_file,
step_list=step_list)
def list_steps(self):
"""Return string with the signature of all steps in the steps library.
"""
self._closed()
return self.steps_library.list_steps()
def _has_requirements(self):
"""Returns True if the workflow needs a requirements section.
Returns:
bool: True if the workflow needs a requirements section, False
otherwise.
"""
self._closed()
return any([
self.has_workflow_step, self.has_scatter_requirement,
self.has_multiple_inputs
])
def inputs(self, name):
"""List input names and types of a step in the steps library.
Args:
name (str): name of a step in the steps library.
"""
self._closed()
step = self._get_step(name, make_copy=False)
return step.list_inputs()
def _add_step(self, step):
"""Add a step to the workflow.
Args:
step (Step): a step from the steps library.
"""
self._closed()
self.has_workflow_step = self.has_workflow_step or step.is_workflow
self.wf_steps[step.name_in_workflow] = step
def add_input(self, **kwargs):
"""Add workflow input.
Args:
kwargs (dict): A dict with a `name: type` item
and optionally a `default: value` item, where name is the
name (id) of the workflow input (e.g., `dir_in`) and type is
the type of the input (e.g., `'Directory'`).
The type of input parameter can be learned from
`step.inputs(step_name=input_name)`.
Returns:
inputname
Raises:
ValueError: No or multiple parameter(s) have been specified.
"""
self._closed()
def _get_item(args):
"""Get a single item from args."""
if not args:
raise ValueError("No parameter specified.")
item = args.popitem()
if args:
raise ValueError("Too many parameters, not clear what to do "
"with {}".format(kwargs))
return item
symbols = None
input_dict = CommentedMap()
if 'default' in kwargs:
input_dict['default'] = kwargs.pop('default')
if 'label' in kwargs:
input_dict['label'] = kwargs.pop('label')
if 'symbols' in kwargs:
symbols = kwargs.pop('symbols')
name, input_type = _get_item(kwargs)
if input_type == 'enum':
typ = CommentedMap()
typ['type'] = 'enum'
# make sure symbols is set
if symbols is None:
raise ValueError("Please specify the enum's symbols.")
# make sure symbols is not empty
if symbols == []:
raise ValueError("The enum's symbols cannot be empty.")
# make sure the symbols are a list
if type(symbols) != list:
raise ValueError('Symbols should be a list.')
# make sure symbols is a list of strings
symbols = [str(s) for s in symbols]
typ['symbols'] = symbols
input_dict['type'] = typ
else:
# Set the 'type' if we can't use simple notation (because there is
# a default value or a label)
if bool(input_dict):
input_dict['type'] = input_type
msg = '"{}" is already used as a workflow input. Please use a ' +\
'different name.'
if name in self.wf_inputs:
raise ValueError(msg.format(name))
# Add 'type' for complex input types, so the user doesn't have to do it
if isinstance(input_type, dict):
input_dict['type'] = input_type
# Make sure we can use the notation without 'type' if the input allows
# it.
if bool(input_dict):
self.wf_inputs[name] = input_dict
else:
self.wf_inputs[name] = input_type
return Reference(input_name=name)
def add_outputs(self, **kwargs):
"""Add workflow outputs.
The output type is added automatically, based on the steps in the steps
library.
Args:
kwargs (dict): A dict containing ``name=source name`` pairs.
``name`` is the name of the workflow output (e.g.,
``txt_files``) and source name is the name of the step that
produced this output plus the output name (e.g.,
``saf-to-txt/out_files``).
"""
self._closed()
for name, source_name in kwargs.items():
obj = {}
obj['outputSource'] = source_name
obj['type'] = self.step_output_types[source_name]
self.wf_outputs[name] = obj
def set_documentation(self, doc):
"""Set workflow documentation.
Args:
doc (str): documentation string.
"""
self._closed()
self.documentation = doc
def set_label(self, label):
"""Set workflow label.
Args:
label (str): short description of workflow.
"""
self._closed()
self.label = label
def _get_step(self, name, make_copy=True):
"""Return step from steps library.
Optionally, the step returned is a deep copy from the step in the steps
library, so additional information (e.g., about whether the step was
scattered) can be stored in the copy.
Args:
name (str): name of the step in the steps library.
make_copy (bool): whether a deep copy of the step should be
returned or not (default: True).
Returns:
Step from steps library.
Raises:
ValueError: The requested step cannot be found in the steps
library.
"""
self._closed()
s = self.steps_library.get_step(name)
if s is None:
msg = '"{}" not found in steps library. Please check your ' \
'spelling or load additional steps'
raise ValueError(msg.format(name))
if make_copy:
s = copy.deepcopy(s)
return s
def _generate_step_name(self, step_name):
name = step_name
i = 1
while name in self.steps_library.step_ids:
name = '{}-{}'.format(step_name, i)
i += 1
return name
def to_obj(self, wd=False, pack=False, relpath=None):
"""Return the created workflow as a dict.
The dict can be written to a yaml file.
Returns:
A yaml-compatible dict representing the workflow.
"""
self._closed()
obj = CommentedMap()
obj['cwlVersion'] = 'v1.0'
obj['class'] = 'Workflow'
try:
obj['doc'] = self.documentation
except (AttributeError, ValueError):
pass
try:
obj['label'] = self.label
except (AttributeError, ValueError):
pass
if self._has_requirements():
obj['requirements'] = []
if self.has_workflow_step:
obj['requirements'].append(
{'class': 'SubworkflowFeatureRequirement'})
if self.has_scatter_requirement:
obj['requirements'].append({'class': 'ScatterFeatureRequirement'})
if self.has_multiple_inputs:
obj['requirements'].append(
{'class': 'MultipleInputFeatureRequirement'})
obj['inputs'] = self.wf_inputs
obj['outputs'] = self.wf_outputs
steps_obj = CommentedMap()
for key in self.wf_steps:
steps_obj[key] = self.wf_steps[key].to_obj(relpath=relpath,
pack=pack,
wd=wd)
obj['steps'] = steps_obj
return obj
def to_script(self, wf_name='wf'):
"""Generated and print the scriptcwl script for the currunt workflow.
Args:
wf_name (str): string used for the WorkflowGenerator object in the
generated script (default: ``wf``).
"""
self._closed()
script = []
# Workflow documentation
# if self.documentation:
# if is_multiline(self.documentation):
# print('doc = """')
# print(self.documentation)
# print('"""')
# print('{}.set_documentation(doc)'.format(wf_name))
# else:
# print('{}.set_documentation(\'{}\')'.format(wf_name,
# self.documentation))
# Workflow inputs
params = []
returns = []
for name, typ in self.wf_inputs.items():
params.append('{}=\'{}\''.format(name, typ))
returns.append(name)
script.append('{} = {}.add_inputs({})'.format(', '.join(returns),
wf_name,
', '.join(params)))
# Workflow steps
returns = []
for name, step in self.wf_steps.items():
pyname = step.python_name
returns = ['{}_{}'.format(pyname, o) for o in step['out']]
params = [
'{}={}'.format(name, python_name(param))
for name, param in step['in'].items()
]
script.append('{} = {}.{}({})'.format(', '.join(returns), wf_name,
pyname, ', '.join(params)))
# Workflow outputs
params = []
for name, details in self.wf_outputs.items():
params.append('{}={}'.format(name,
python_name(details['outputSource'])))
script.append('{}.add_outputs({})'.format(wf_name, ', '.join(params)))
return '\n'.join(script)
@staticmethod
def _get_input_type(step, input_name):
input_type = step.input_types.get(input_name)
if not input_type:
input_type = step.optional_input_types[input_name]
if step.is_scattered:
for scattered_input in step.scattered_inputs:
if scattered_input == input_name:
input_type += '[]'
return input_type
def _get_source_type(self, ref):
if isinstance(ref, list):
self.has_multiple_inputs = True
return [self._get_source_type_single(r) for r in ref]
else:
return self._get_source_type_single(ref)
def _get_source_type_single(self, ref):
if ref.refers_to_step_output():
step = self.wf_steps[ref.step_name]
return step.output_types[ref.output_name]
else:
input_def = self.wf_inputs[ref.input_name]
if isinstance(input_def, six.string_types):
return input_def
return input_def['type']
@staticmethod
def _types_match(type1, type2):
"""Returns False only if it can show that no value of type1
can possibly match type2.
Supports only a limited selection of types.
"""
if isinstance(type1, six.string_types) and \
isinstance(type2, six.string_types):
type1 = type1.rstrip('?')
type2 = type2.rstrip('?')
if type1 != type2:
return False
return True
def _type_check_reference(self, step, input_name, reference):
input_type = self._get_input_type(step, input_name)
source_type = self._get_source_type(reference)
if isinstance(source_type, list):
# all source_types must be equal
if len(set(source_type)) > 1:
inputs = [
'{} ({})'.format(n, t)
for n, t in zip(reference, source_type)
]
msg = 'The types of the workflow inputs/step outputs for ' \
'"{}" are not equal: {}.'.format(input_name,
', '.join(inputs))
raise ValueError(msg)
# continue type checking using the first item from the list
source_type = source_type[0]
input_type = input_type['items']
reference = reference[0]
if self._types_match(source_type, input_type):
return True
else:
if step.is_scattered:
scattered = ' (scattered)'
else:
scattered = ''
if reference.refers_to_wf_input():
msg = 'Workflow input "{}" of type "{}" is not'
msg += ' compatible with{} step input "{}" of type "{}"'
msg = msg.format(reference.input_name, source_type, scattered,
python_name(input_name), input_type)
else:
msg = 'Step output "{}" of type "{}" is not'
msg += ' compatible with{} step input "{}" of type "{}"'
msg = msg.format(reference, source_type, scattered,
python_name(input_name), input_type)
raise ValueError(msg)
def _make_step(self, step, **kwargs):
self._closed()
for k in step.get_input_names():
p_name = python_name(k)
if p_name in kwargs.keys():
if isinstance(kwargs[p_name], Reference):
step.set_input(p_name, six.text_type(kwargs[p_name]))
elif isinstance(kwargs[p_name], list):
if all(isinstance(n, Reference) for n in kwargs[p_name]):
step.set_input(p_name, kwargs[k])
else:
raise ValueError(
'List of inputs contains an input with an '
'incorrect type for keyword argument {} (should '
'be a value returned by set_input or from adding '
'a step).'.format(p_name))
else:
raise ValueError(
'Incorrect type (should be a value returned'
'by set_inputs() or from adding a step) for keyword '
'argument {}'.format(p_name))
elif k not in step.optional_input_names:
raise ValueError(
'Expecting "{}" as a keyword argument.'.format(p_name))
if 'scatter' in kwargs.keys() or 'scatter_method' in kwargs.keys():
# Check whether 'scatter' keyword is present
if not kwargs.get('scatter'):
raise ValueError('Expecting "scatter" as a keyword argument.')
# Check whether the scatter variables are valid for this step
scatter_vars = kwargs.get('scatter')
if isinstance(scatter_vars, six.string_types):
scatter_vars = [scatter_vars]
for var in scatter_vars:
if var not in step.get_input_names():
msg = 'Invalid variable "{}" for scatter.'
raise ValueError(msg.format(var))
step.scattered_inputs.append(var)
# Check whether 'scatter_method' keyword is present if there is
# more than 1 scatter variable
if not kwargs.get('scatter_method') and len(scatter_vars) > 1:
msg = 'Expecting "scatter_method" as a keyword argument.'
raise ValueError(msg)
# Check validity of scatterMethod
scatter_methods = [
'dotproduct', 'nested_crossproduct', 'flat_crossproduct'
]
m = kwargs.get('scatter_method')
if m and m not in scatter_methods:
msg = 'Invalid scatterMethod "{}". Please use one of ({}).'
raise ValueError(msg.format(m, ', '.join(scatter_methods)))
step.scatter_method = m
# Update step output types (outputs are now arrays)
for name, typ in step.output_types.items():
step.output_types[name] = {'type': 'array', 'items': typ}
self.has_scatter_requirement = True
step.is_scattered = True
# Check types of references
for k in step.get_input_names():
p_name = python_name(k)
if p_name in kwargs.keys():
self._type_check_reference(step, k, kwargs[p_name])
# Make sure the step has a unique name in the workflow (so command line
# tools can be added to the same workflow multiple times).
name_in_wf = self._generate_step_name(step.name)
step._set_name_in_workflow(name_in_wf)
self.steps_library.step_ids.append(name_in_wf)
# Create a reference for each output for use in subsequent
# steps' inputs.
outputs = []
for n in step.output_names:
ref = step.output_reference(n)
self.step_output_types[ref] = step.output_types[n]
outputs.append(ref)
self._add_step(step)
if len(outputs) == 1:
return outputs[0]
return outputs
def validate(self):
"""Validate workflow object.
This method currently validates the workflow object with the use of
cwltool. It writes the workflow to a tmp CWL file, reads it, validates
it and removes the tmp file again. By default, the workflow is written
to file using absolute paths to the steps.
"""
# define tmpfile
(fd, tmpfile) = tempfile.mkstemp()
os.close(fd)
try:
# save workflow object to tmpfile,
# do not recursively call validate function
self.save(tmpfile, mode='abs', validate=False)
# load workflow from tmpfile
document_loader, processobj, metadata, uri = load_cwl(tmpfile)
finally:
# cleanup tmpfile
os.remove(tmpfile)
def _pack(self, fname, encoding):
"""Save workflow with ``--pack`` option
This means that al tools and subworkflows are included in the workflow
file that is created. A packed workflow cannot be loaded and used in
scriptcwl.
"""
(fd, tmpfile) = tempfile.mkstemp()
os.close(fd)
try:
self.save(tmpfile, mode='abs', validate=False)
document_loader, processobj, metadata, uri = load_cwl(tmpfile)
finally:
# cleanup tmpfile
os.remove(tmpfile)
with codecs.open(fname, 'wb', encoding=encoding) as f:
f.write(print_pack(document_loader, processobj, uri, metadata))
def save(self,
fname,
mode=None,
validate=True,
encoding='utf-8',
wd=False,
inline=False,
relative=False,
pack=False):
"""Save the workflow to file.
Save the workflow to a CWL file that can be run with a CWL runner.
Args:
fname (str): file to save the workflow to.
mode (str): one of (rel, abs, wd, inline, pack)
encoding (str): file encoding to use (default: ``utf-8``).
"""
self._closed()
if mode is None:
mode = 'abs'
if pack:
mode = 'pack'
elif wd:
mode = 'wd'
elif relative:
mode = 'rel'
msg = 'Using deprecated save method. Please save the workflow ' \
'with: wf.save(\'{}\', mode=\'{}\'). Redirecting to new ' \
'save method.'.format(fname, mode)
warnings.warn(msg, DeprecationWarning)
modes = ('rel', 'abs', 'wd', 'inline', 'pack')
if mode not in modes:
msg = 'Illegal mode "{}". Choose one of ({}).'\
.format(mode, ','.join(modes))
raise ValueError(msg)
if validate:
self.validate()
dirname = os.path.dirname(os.path.abspath(fname))
if not os.path.exists(dirname):
os.makedirs(dirname)
if mode == 'inline':
msg = ('Inline saving is deprecated. Please save the workflow '
'using mode=\'pack\'. Setting mode to pack.')
warnings.warn(msg, DeprecationWarning)
mode = 'pack'
if mode == 'rel':
relpath = dirname
save_yaml(fname=fname,
wf=self,
pack=False,
relpath=relpath,
wd=False)
if mode == 'abs':
save_yaml(fname=fname, wf=self, pack=False, relpath=None, wd=False)
if mode == 'pack':
self._pack(fname, encoding)
if mode == 'wd':
if self.get_working_dir() is None:
raise ValueError('Working directory not set.')
else:
# save in working_dir
bn = os.path.basename(fname)
wd_file = os.path.join(self.working_dir, bn)
save_yaml(fname=wd_file,
wf=self,
pack=False,
relpath=None,
wd=True)
# and copy workflow file to other location (as though all steps
# are in the same directory as the workflow)
try:
shutil.copy2(wd_file, fname)
except shutil.Error:
pass
def get_working_dir(self):
return self.working_dir
def add_inputs(self, **kwargs):
"""Deprecated function, use add_input(self, **kwargs) instead.
Add workflow input.
Args:
kwargs (dict): A dict with a `name: type` item
and optionally a `default: value` item, where name is the
name (id) of the workflow input (e.g., `dir_in`) and type is
the type of the input (e.g., `'Directory'`).
The type of input parameter can be learned from
`step.inputs(step_name=input_name)`.
Returns:
inputname
Raises:
ValueError: No or multiple parameter(s) have been specified.
"""
msg = ('The add_inputs() function is deprecation in favour of the '
'add_input() function, redirecting...')
warnings.warn(msg, DeprecationWarning)
return self.add_input(**kwargs)
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin):
"""
Class for YAML-based (roundtrip) configurations
"""
def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]):
self.data = CommentedMap()
super().__init__(owner, manager, path, *args, **kwargs)
def load(self):
with open(self.path, "r") as fh:
self.data = yaml.round_trip_load(fh, version=(1, 2))
def reload(self):
self.unload()
self.load()
def unload(self):
self.data.clear()
def save(self):
if not self.mutable:
raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!")
with open(self.path, "w") as fh:
yaml.round_trip_dump(self.data, fh)
# region: CommentedMap functions
def insert(self, pos, key, value, *, comment=None):
"""
Insert a `key: value` pair at the given position, attaching a comment if provided
Wrapper for `CommentedMap.insert()`
"""
return self.data.insert(pos, key, value, comment)
def add_eol_comment(self, comment, *, key=NoComment, column=30):
"""
Add an end-of-line comment for a key at a particular column (30 by default)
Wrapper for `CommentedMap.yaml_add_eol_comment()`
"""
# Setting the column to None as the API actually defaults to will raise an exception, so we have to
# specify one unfortunately
return self.data.yaml_add_eol_comment(comment, key=key, column=column)
def set_comment_before_key(self, key, comment, *, indent=0):
"""
Set a comment before a given key
Wrapper for `CommentedMap.yaml_set_comment_before_after_key()`
"""
return self.data.yaml_set_comment_before_after_key(
key, before=comment, indent=indent, after=None, after_indent=None
)
def set_start_comment(self, comment, indent=0):
"""
Set the starting comment
Wrapper for `CommentedMap.yaml_set_start_comment()`
"""
return self.data.yaml_set_start_comment(comment, indent=indent)
# endregion
# region: Dict functions
def clear(self):
return self.data.clear()
def copy(self):
return self.data.copy()
def get(self, key, default=None):
return self.data.get(key, default)
def items(self):
return self.data.items()
def keys(self):
return self.data.keys()
def pop(self, key, default=None):
return self.data.pop(key, default)
def popitem(self):
return self.data.popitem()
def setdefault(self, key, default=None):
if key not in self.data:
self.data[key] = default
return default
return self.data[key]
def update(self, other):
return self.data.update(other)
def values(self):
return self.data.values()
# endregion
# Item access functions
def __contains__(self, key):
"""
Wrapper for `dict.__contains__()`
"""
return self.data.__contains__(key)
def __delitem__(self, key):
"""
Wrapper for `dict.__delitem__()`
"""
del self.data[key]
def __getitem__(self, key):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__getitem__(key)
def __iter__(self):
"""
Wrapper for `dict.__iter__()`
"""
return self.data.__iter__()
def __len__(self):
"""
Wrapper for `dict.__len__()`
"""
return self.data.__len__()
def __setitem__(self, key, value):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__setitem__(key, value)
def _diff_dicts(self, path: YAMLPath, lhs: CommentedMap,
rhs: CommentedMap) -> None:
"""
Diff two dicts.
Parameters:
1. path (YAMLPath) YAML Path to the document element under evaluation
2. lhs (Any) The left-hand-side (original) document
3. rhs (Any) The right-hand-side (altered) document
"""
self.logger.debug("Comparing LHS:",
prefix="Differ::_diff_dicts: ",
data=lhs)
self.logger.debug("Against RHS:",
prefix="Differ::_diff_dicts: ",
data=rhs)
# Check first for a difference in YAML Tag
lhs_tag = lhs.tag.value if hasattr(lhs, "tag") else None
rhs_tag = rhs.tag.value if hasattr(rhs, "tag") else None
if lhs_tag != rhs_tag:
self.logger.debug(
"Dictionaries have different YAML Tags; {} != {}:".format(
lhs_tag, rhs_tag),
prefix="Differ::_diff_dicts: ")
self._diffs.append(
DiffEntry(DiffActions.DELETE, path, lhs, None,
key_tag=lhs_tag))
self._diffs.append(
DiffEntry(DiffActions.ADD, path, None, rhs, key_tag=rhs_tag))
return
lhs_keys = set(lhs)
rhs_keys = set(rhs)
lhs_key_indicies = Differ._get_key_indicies(lhs)
rhs_key_indicies = Differ._get_key_indicies(rhs)
self.logger.debug("Got LHS key indicies:",
prefix="Differ::_diff_dicts: ",
data=lhs_key_indicies)
self.logger.debug("Got RHS key indicies:",
prefix="Differ::_diff_dicts: ",
data=rhs_key_indicies)
# Look for changes
for key, val in [(key, val) for key, val in rhs.items()
if key in lhs and key in rhs]:
next_path = (path +
YAMLPath.escape_path_section(key, path.seperator))
self._diff_between(next_path,
lhs[key],
val,
lhs_parent=lhs,
lhs_iteration=lhs_key_indicies[key],
rhs_parent=rhs,
rhs_iteration=rhs_key_indicies[key],
parentref=key)
# Look for deleted keys
for key in lhs_keys - rhs_keys:
next_path = (path +
YAMLPath.escape_path_section(key, path.seperator))
self._diffs.append(
DiffEntry(
DiffActions.DELETE,
next_path,
lhs[key],
None,
lhs_parent=lhs,
lhs_iteration=lhs_key_indicies[key],
rhs_parent=rhs,
key_tag=key.tag.value if hasattr(key, "tag") else None))
# Look for new keys
for key in rhs_keys - lhs_keys:
next_path = (path +
YAMLPath.escape_path_section(key, path.seperator))
self._diffs.append(
DiffEntry(
DiffActions.ADD,
next_path,
None,
rhs[key],
lhs_parent=lhs,
rhs_parent=rhs,
rhs_iteration=rhs_key_indicies[key],
key_tag=key.tag.value if hasattr(key, "tag") else None))
