def add_new_lines_after_section(recipe_yaml: CommentedMap) -> CommentedMap:
for section in recipe_yaml.keys():
if section == "package":
recipe_yaml.yaml_set_comment_before_after_key(section, "\n\n")
else:
recipe_yaml.yaml_set_comment_before_after_key(section, "\n")
return recipe_yaml
def __split_off_extra_attributes(self, mapping: CommentedMap,
known_attrs: List[str]) -> CommentedMap:
"""Separates the extra attributes in mapping into _yatiml_extra.
This returns a mapping containing all key-value pairs from
mapping whose key is in known_attrs, and an additional key
_yatiml_extra which maps to a dict containing the remaining
key-value pairs.
Args:
mapping: The mapping to split
known_attrs: Attributes that should be kept in the main
map, and not moved to _yatiml_extra.
Returns:
A map with attributes reorganised as described above.
"""
attr_names = list(mapping.keys())
main_attrs = mapping.copy() # type: CommentedMap
extra_attrs = OrderedDict(mapping.items())
for name in attr_names:
if name not in known_attrs or name == '_yatiml_extra':
del (main_attrs[name])
else:
del (extra_attrs[name])
main_attrs['_yatiml_extra'] = extra_attrs
return main_attrs
def _add_new_lines_after_section(
self, recipe_yaml: CommentedMap) -> CommentedMap:
for section in recipe_yaml.keys():
if section == "package":
continue
recipe_yaml.yaml_set_comment_before_after_key(section, "\n")
return recipe_yaml
def _get_key_indicies(cls, data: CommentedMap) -> Dict[Any, int]:
"""Get a dictionary mapping of keys to relative positions."""
key_map = {}
if isinstance(data, CommentedMap):
for idx, key in enumerate(data.keys()):
key_map[key] = idx
return key_map
def _normalize_fields(self, document: CommentedMap, loader: "Loader") -> None:
# Normalize fields which are prefixed or full URIn to vocabulary terms
for d in list(document.keys()):
if isinstance(d, str):
d2 = loader.expand_url(d, "", scoped_id=False, vocab_term=True)
if d != d2:
document[d2] = document[d]
document.lc.add_kv_line_col(d2, document.lc.data[d])
del document[d]
def _delete_mergeref_keys(self, data: CommentedMap) -> None:
"""
Delete all YAML merge reference keys from a CommentedMap.
This is necessary when using the insert() method of a CommentedMap
because doing so converts all YAML merge references (key-value pairs
merged into a YAML Hash via the `<<:` operator) to concrete key-value
pairs of the affected Hash.
"""
concrete_keys = []
for local_key, _ in data.non_merged_items():
concrete_keys.append(local_key)
reference_keys = set(data.keys()).difference(concrete_keys)
for key in reference_keys:
self.logger.debug("Deleting key from LHS:",
data=key,
prefix="Merger::_delete_mergeref_keys: ",
header="!" * 50)
del data[key]
def __type_check_attributes(self, node: yaml.Node, mapping: CommentedMap,
argspec: inspect.FullArgSpec) -> None:
"""Ensure all attributes have a matching constructor argument.
This checks that there is a constructor argument with a
matching type for each existing attribute.
If the class has a _yatiml_extra attribute, then extra
attributes are okay and no error will be raised if they exist.
Args:
node: The node we're processing
mapping: The mapping with constructed subobjects
constructor_attrs: The attributes of the constructor,
including self and _yatiml_extra, if applicable
"""
logger.debug('Checking for extraneous attributes')
logger.debug('Constructor arguments: {}, mapping: {}'.format(
argspec.args, list(mapping.keys())))
for key, value in mapping.items():
if not isinstance(key, str):
raise RecognitionError(('{}{}YAtiML only supports strings'
' for mapping keys').format(
node.start_mark, os.linesep))
if key not in argspec.args and '_yatiml_extra' not in argspec.args:
raise RecognitionError(
('{}{}Found additional attributes ("{}")'
' and {} does not support those').format(
node.start_mark, os.linesep, key,
self.class_.__name__))
if key in argspec.args and key in argspec.annotations:
if not self.__type_matches(value, argspec.annotations[key]):
raise RecognitionError(
('{}{}Expected attribute "{}" to be of type {}'
' but it is a(n) {}').format(
node.start_mark, os.linesep, key,
argspec.annotations[key], type(value)))
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin):
"""
Class for YAML-based (roundtrip) configurations
"""
def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]):
self.data = CommentedMap()
super().__init__(owner, manager, path, *args, **kwargs)
def load(self):
with open(self.path, "r") as fh:
self.data = yaml.round_trip_load(fh, version=(1, 2))
def reload(self):
self.unload()
self.load()
def unload(self):
self.data.clear()
def save(self):
if not self.mutable:
raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!")
with open(self.path, "w") as fh:
yaml.round_trip_dump(self.data, fh)
# region: CommentedMap functions
def insert(self, pos, key, value, *, comment=None):
"""
Insert a `key: value` pair at the given position, attaching a comment if provided
Wrapper for `CommentedMap.insert()`
"""
return self.data.insert(pos, key, value, comment)
def add_eol_comment(self, comment, *, key=NoComment, column=30):
"""
Add an end-of-line comment for a key at a particular column (30 by default)
Wrapper for `CommentedMap.yaml_add_eol_comment()`
"""
# Setting the column to None as the API actually defaults to will raise an exception, so we have to
# specify one unfortunately
return self.data.yaml_add_eol_comment(comment, key=key, column=column)
def set_comment_before_key(self, key, comment, *, indent=0):
"""
Set a comment before a given key
Wrapper for `CommentedMap.yaml_set_comment_before_after_key()`
"""
return self.data.yaml_set_comment_before_after_key(
key, before=comment, indent=indent, after=None, after_indent=None
)
def set_start_comment(self, comment, indent=0):
"""
Set the starting comment
Wrapper for `CommentedMap.yaml_set_start_comment()`
"""
return self.data.yaml_set_start_comment(comment, indent=indent)
# endregion
# region: Dict functions
def clear(self):
return self.data.clear()
def copy(self):
return self.data.copy()
def get(self, key, default=None):
return self.data.get(key, default)
def items(self):
return self.data.items()
def keys(self):
return self.data.keys()
def pop(self, key, default=None):
return self.data.pop(key, default)
def popitem(self):
return self.data.popitem()
def setdefault(self, key, default=None):
if key not in self.data:
self.data[key] = default
return default
return self.data[key]
def update(self, other):
return self.data.update(other)
def values(self):
return self.data.values()
# endregion
# Item access functions
def __contains__(self, key):
"""
Wrapper for `dict.__contains__()`
"""
return self.data.__contains__(key)
def __delitem__(self, key):
"""
Wrapper for `dict.__delitem__()`
"""
del self.data[key]
def __getitem__(self, key):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__getitem__(key)
def __iter__(self):
"""
Wrapper for `dict.__iter__()`
"""
return self.data.__iter__()
def __len__(self):
"""
Wrapper for `dict.__len__()`
"""
return self.data.__len__()
def __setitem__(self, key, value):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__setitem__(key, value)
def add_space_between_main_sections(cwl: CommentedMap):
for k in cwl.keys():
if k in ["inputs", "outputs", "steps", "requirements", "hints", "baseCommand"]:
cwl.yaml_set_comment_before_after_key(key=k, before="\n")
def _merge_dicts(
self, lhs: CommentedMap, rhs: CommentedMap, path: YAMLPath
) -> CommentedMap:
"""
Merge two YAML maps (CommentedMap-wrapped dicts).
Parameters:
1. lhs (CommentedMap) The merge target.
2. rhs (CommentedMap) The merge source.
3. path (YAMLPath) Location within the DOM where this merge is taking
place.
Returns: (CommentedMap) The merged result.
Raises:
- `MergeException` when a clean merge is impossible.
"""
if not isinstance(lhs, CommentedMap):
raise MergeException(
"Impossible to add Hash data to non-Hash destination.", path)
self.logger.debug(
"Merging INTO dict with keys: {}:".format(", ".join([
str(k.value) if isinstance(k, TaggedScalar)
else str(k)
for k in lhs.keys()])),
data=lhs, prefix="Merger::_merge_dicts: ",
header="--------------------")
self.logger.debug(
"Merging FROM dict with keys: {}:".format(", ".join([
str(k.value) if isinstance(k, TaggedScalar)
else str(k)
for k in rhs.keys()])),
data=rhs, prefix="Merger::_merge_dicts: ",
footer="====================")
# Delete all internal YAML merge reference keys lest any later
# .insert() operation on LHS inexplicably convert them from reference
# to concrete keys. This seems like a bug in ruamel.yaml...
self._delete_mergeref_keys(lhs)
# Assume deep merge until a node's merge rule indicates otherwise
buffer: List[Tuple[Any, Any]] = []
buffer_pos = 0
for key, val in rhs.non_merged_items():
path_next = (path +
YAMLPath.escape_path_section(key, path.seperator))
if key in lhs:
# Write the buffer if populated
for b_key, b_val in buffer:
self.logger.debug(
"Merger::_merge_dicts: Inserting key, {}, from"
" buffer to position, {}, at path, {}."
.format(b_key, buffer_pos, path_next),
header="INSERT " * 15)
self.logger.debug(
"Before INSERT, the LHS document was:",
data=lhs, prefix="Merger::_merge_dicts: ")
self.logger.debug(
"... and before INSERT, the incoming value will be:",
data=b_val, prefix="Merger::_merge_dicts: ")
lhs.insert(buffer_pos, b_key, b_val)
self.logger.debug(
"After INSERT, the LHS document became:",
data=lhs, prefix="Merger::_merge_dicts: ")
buffer_pos += 1
buffer = []
# Short-circuit the deep merge if a different merge rule
# applies to this node.
node_coord = NodeCoords(val, rhs, key)
merge_mode = (
self.config.hash_merge_mode(node_coord)
if isinstance(val, CommentedMap)
else self.config.set_merge_mode(node_coord)
if isinstance(val, CommentedSet)
else self.config.aoh_merge_mode(node_coord)
)
self.logger.debug("Merger::_merge_dicts: Got merge mode, {}."
.format(merge_mode))
if merge_mode in (
HashMergeOpts.LEFT, AoHMergeOpts.LEFT, SetMergeOpts.LEFT
):
continue
if merge_mode in (
HashMergeOpts.RIGHT, AoHMergeOpts.RIGHT, SetMergeOpts.RIGHT
):
self.logger.debug(
"Merger::_merge_dicts: Overwriting key, {}, at path,"
" {}.".format(key, path_next),
header="OVERWRITE " * 15)
lhs[key] = val
continue
if isinstance(val, CommentedMap):
lhs[key] = self._merge_dicts(lhs[key], val, path_next)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
self.logger.debug(
"Document BEFORE calling combine_merge_anchors:",
data=lhs, prefix="Merger::_merge_dicts: ",
header="+------------------+")
Anchors.combine_merge_anchors(lhs[key], val)
self.logger.debug(
"Document AFTER calling combine_merge_anchors:",
data=lhs, prefix="Merger::_merge_dicts: ",
footer="+==================+")
elif isinstance(val, CommentedSeq):
lhs[key] = self._merge_lists(
lhs[key], val, path_next, parent=rhs, parentref=key)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
elif isinstance(val, CommentedSet):
lhs[key] = self._merge_sets(
lhs[key], val, path_next, node_coord)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
else:
self.logger.debug(
"Merger::_merge_dicts: Updating key, {}, at path,"
" {}.".format(key, path_next), header="UPDATE " * 15)
self.logger.debug(
"Before UPDATE, the LHS document was:",
data=lhs, prefix="Merger::_merge_dicts: ")
self.logger.debug(
"... and before UPDATE, the incoming value will be:",
data=val, prefix="Merger::_merge_dicts: ")
lhs[key] = val
self.logger.debug(
"After UPDATE, the LHS document became:",
data=lhs, prefix="Merger::_merge_dicts: ")
else:
# LHS lacks the RHS key. Buffer this key-value pair in order
# to insert it ahead of whatever key(s) follow this one in RHS
# to keep anchor definitions before their aliases.
buffer.append((key, val))
buffer_pos += 1
# Write any remaining buffered content to the end of LHS
for b_key, b_val in buffer:
self.logger.debug(
"Merger::_merge_dicts: Appending key, {}, from buffer at"
" path, {}.".format(b_key, path), header="APPEND " * 15)
lhs[b_key] = b_val
self.logger.debug(
"Completed merge result for path, {}:".format(path),
data=lhs, prefix="Merger::_merge_dicts: ")
return lhs
class MVYaml(object):
protected_keys = (
'__current',
'__type',
)
def __init__(self, base64=False):
self._b64 = base64
self._raw = CommentedMap()
self._yaml = YAML()
self._curr_version = None
self._curr_data = None
self._create()
def _create(self):
tag = self._make_tag()
self._raw[tag] = CommentedMap()
self._raw.insert(0, '__current', tag, 'current version')
self._raw.insert(1, '__type', None, 'base64 if value are base64')
self._commit(tag=tag, comment='Initial version')
def import_yaml(self, file: AnyStr = None, stream: AnyStr = None):
data = None
if file:
with open(file, 'r') as fp:
data = fp.read()
imported_data = self._yaml.load(data or stream)
self.override(imported_data)
return self
def load(self, file_handler: AnyStr = None, stream_data: AnyStr = None):
data = None
if file_handler:
with open(file_handler, 'r') as fp:
data = fp.read()
self._raw = self._yaml.load(data or stream_data)
if self.protected_keys not in self._raw.keys():
raise MVYamlFileException(
f'Not a valid mvyaml file. Perhaps is a yaml you want to import with '
f'import_yaml()?')
return self
def write(self,
file_handler: IO = None,
comment: AnyStr = None) -> [AnyStr, None]:
if not self._raw:
return
if self._has_changes():
self._commit(comment=comment)
output = file_handler or StringIO()
self._yaml.dump(self._raw, output)
return output.getvalue() if not file_handler else None
@property
def versions(self):
if not self._raw:
return []
return [k for k in self._raw.keys() if k not in self.protected_keys]
@property
def current(self):
return self._raw['__current']
@property
def data(self):
if not self._curr_data:
self._curr_data = deepcopy(self._raw[self._curr_version
or self.current])
return self._curr_data
def with_version(self, version: str = '__current'):
if version not in self.versions:
raise MVYamlVersionNotFoundException(
f'version {version} not found')
self._curr_version = version
self._curr_data = None
return self
@staticmethod
def _make_tag() -> str:
d = datetime.utcnow().isoformat()
return d
def override(self, data: [Iterable]):
self._curr_data = CommentedMap()
self._curr_data.update(data)
self._commit(comment='Overridden')
return self
def _commit(self, *args, **kwargs):
return self._commit_head(*args, **kwargs)
def _commit_head(self, tag: AnyStr = None, comment: AnyStr = None):
"""
apply the modifications on curr_data to the underling opened version
and create a new tag
"""
commented_map = CommentedMap()
commented_map.update(self._curr_data or self.data)
if tag:
self._raw[tag] = commented_map
self._raw['__current'] = tag
else:
new_tag = self._make_tag()
self._raw.insert(2, new_tag, commented_map, comment=comment)
self._raw['__current'] = new_tag
self._curr_version = None
self._curr_data = None
return self
def _commit_tail(self, tag: AnyStr = None, comment: AnyStr = None):
"""
apply the modifications on curr_data to the underling opened version
and create a new tag
"""
commented_map = CommentedMap()
commented_map.update(self._curr_data or self.data)
if tag:
self._raw[tag] = commented_map
self._raw['__current'] = tag
else:
new_tag = self._make_tag()
self._raw.insert(len(self._raw.keys()),
new_tag,
commented_map,
comment=comment)
self._raw['__current'] = new_tag
self._curr_version = None
self._curr_data = None
return self
def _has_changes(self):
orig = self._raw[self._curr_version or self.current]
current = self._curr_data or self.data
try:
assert_equal(orig, current)
except AssertionError:
return True
return False
@property
def changes(self) -> AnyStr:
if not self._has_changes():
return ''
yaml_orig = as_yaml(self._raw[self._curr_version or self.current])
yaml_curr = as_yaml(self._curr_data)
differ = Differ()
result = list(
differ.compare(yaml_orig.splitlines(), yaml_curr.splitlines()))
return '\n'.join(result)
def set_current(self, version_label: AnyStr):
if version_label not in self.versions:
raise MVYamlVersionNotFoundException(
f'request version [{version_label}] not found')
self._raw['__current'] = version_label
self.with_version(version_label)
return self
