def add_new_lines_after_section(recipe_yaml: CommentedMap) -> CommentedMap: for section in recipe_yaml.keys(): if section == "package": recipe_yaml.yaml_set_comment_before_after_key(section, "\n\n") else: recipe_yaml.yaml_set_comment_before_after_key(section, "\n") return recipe_yaml
def __split_off_extra_attributes(self, mapping: CommentedMap, known_attrs: List[str]) -> CommentedMap: """Separates the extra attributes in mapping into _yatiml_extra. This returns a mapping containing all key-value pairs from mapping whose key is in known_attrs, and an additional key _yatiml_extra which maps to a dict containing the remaining key-value pairs. Args: mapping: The mapping to split known_attrs: Attributes that should be kept in the main map, and not moved to _yatiml_extra. Returns: A map with attributes reorganised as described above. """ attr_names = list(mapping.keys()) main_attrs = mapping.copy() # type: CommentedMap extra_attrs = OrderedDict(mapping.items()) for name in attr_names: if name not in known_attrs or name == '_yatiml_extra': del (main_attrs[name]) else: del (extra_attrs[name]) main_attrs['_yatiml_extra'] = extra_attrs return main_attrs
def _add_new_lines_after_section( self, recipe_yaml: CommentedMap) -> CommentedMap: for section in recipe_yaml.keys(): if section == "package": continue recipe_yaml.yaml_set_comment_before_after_key(section, "\n") return recipe_yaml
def _get_key_indicies(cls, data: CommentedMap) -> Dict[Any, int]: """Get a dictionary mapping of keys to relative positions.""" key_map = {} if isinstance(data, CommentedMap): for idx, key in enumerate(data.keys()): key_map[key] = idx return key_map
def _normalize_fields(self, document: CommentedMap, loader: "Loader") -> None: # Normalize fields which are prefixed or full URIn to vocabulary terms for d in list(document.keys()): if isinstance(d, str): d2 = loader.expand_url(d, "", scoped_id=False, vocab_term=True) if d != d2: document[d2] = document[d] document.lc.add_kv_line_col(d2, document.lc.data[d]) del document[d]
def _delete_mergeref_keys(self, data: CommentedMap) -> None: """ Delete all YAML merge reference keys from a CommentedMap. This is necessary when using the insert() method of a CommentedMap because doing so converts all YAML merge references (key-value pairs merged into a YAML Hash via the `<<:` operator) to concrete key-value pairs of the affected Hash. """ concrete_keys = [] for local_key, _ in data.non_merged_items(): concrete_keys.append(local_key) reference_keys = set(data.keys()).difference(concrete_keys) for key in reference_keys: self.logger.debug("Deleting key from LHS:", data=key, prefix="Merger::_delete_mergeref_keys: ", header="!" * 50) del data[key]
def __type_check_attributes(self, node: yaml.Node, mapping: CommentedMap, argspec: inspect.FullArgSpec) -> None: """Ensure all attributes have a matching constructor argument. This checks that there is a constructor argument with a matching type for each existing attribute. If the class has a _yatiml_extra attribute, then extra attributes are okay and no error will be raised if they exist. Args: node: The node we're processing mapping: The mapping with constructed subobjects constructor_attrs: The attributes of the constructor, including self and _yatiml_extra, if applicable """ logger.debug('Checking for extraneous attributes') logger.debug('Constructor arguments: {}, mapping: {}'.format( argspec.args, list(mapping.keys()))) for key, value in mapping.items(): if not isinstance(key, str): raise RecognitionError(('{}{}YAtiML only supports strings' ' for mapping keys').format( node.start_mark, os.linesep)) if key not in argspec.args and '_yatiml_extra' not in argspec.args: raise RecognitionError( ('{}{}Found additional attributes ("{}")' ' and {} does not support those').format( node.start_mark, os.linesep, key, self.class_.__name__)) if key in argspec.args and key in argspec.annotations: if not self.__type_matches(value, argspec.annotations[key]): raise RecognitionError( ('{}{}Expected attribute "{}" to be of type {}' ' but it is a(n) {}').format( node.start_mark, os.linesep, key, argspec.annotations[key], type(value)))
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin): """ Class for YAML-based (roundtrip) configurations """ def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]): self.data = CommentedMap() super().__init__(owner, manager, path, *args, **kwargs) def load(self): with open(self.path, "r") as fh: self.data = yaml.round_trip_load(fh, version=(1, 2)) def reload(self): self.unload() self.load() def unload(self): self.data.clear() def save(self): if not self.mutable: raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!") with open(self.path, "w") as fh: yaml.round_trip_dump(self.data, fh) # region: CommentedMap functions def insert(self, pos, key, value, *, comment=None): """ Insert a `key: value` pair at the given position, attaching a comment if provided Wrapper for `CommentedMap.insert()` """ return self.data.insert(pos, key, value, comment) def add_eol_comment(self, comment, *, key=NoComment, column=30): """ Add an end-of-line comment for a key at a particular column (30 by default) Wrapper for `CommentedMap.yaml_add_eol_comment()` """ # Setting the column to None as the API actually defaults to will raise an exception, so we have to # specify one unfortunately return self.data.yaml_add_eol_comment(comment, key=key, column=column) def set_comment_before_key(self, key, comment, *, indent=0): """ Set a comment before a given key Wrapper for `CommentedMap.yaml_set_comment_before_after_key()` """ return self.data.yaml_set_comment_before_after_key( key, before=comment, indent=indent, after=None, after_indent=None ) def set_start_comment(self, comment, indent=0): """ Set the starting comment Wrapper for `CommentedMap.yaml_set_start_comment()` """ return self.data.yaml_set_start_comment(comment, indent=indent) # endregion # region: Dict functions def clear(self): return self.data.clear() def copy(self): return self.data.copy() def get(self, key, default=None): return self.data.get(key, default) def items(self): return self.data.items() def keys(self): return self.data.keys() def pop(self, key, default=None): return self.data.pop(key, default) def popitem(self): return self.data.popitem() def setdefault(self, key, default=None): if key not in self.data: self.data[key] = default return default return self.data[key] def update(self, other): return self.data.update(other) def values(self): return self.data.values() # endregion # Item access functions def __contains__(self, key): """ Wrapper for `dict.__contains__()` """ return self.data.__contains__(key) def __delitem__(self, key): """ Wrapper for `dict.__delitem__()` """ del self.data[key] def __getitem__(self, key): """ Wrapper for `dict.__getitem__()` """ return self.data.__getitem__(key) def __iter__(self): """ Wrapper for `dict.__iter__()` """ return self.data.__iter__() def __len__(self): """ Wrapper for `dict.__len__()` """ return self.data.__len__() def __setitem__(self, key, value): """ Wrapper for `dict.__getitem__()` """ return self.data.__setitem__(key, value)
def add_space_between_main_sections(cwl: CommentedMap): for k in cwl.keys(): if k in ["inputs", "outputs", "steps", "requirements", "hints", "baseCommand"]: cwl.yaml_set_comment_before_after_key(key=k, before="\n")
def _merge_dicts( self, lhs: CommentedMap, rhs: CommentedMap, path: YAMLPath ) -> CommentedMap: """ Merge two YAML maps (CommentedMap-wrapped dicts). Parameters: 1. lhs (CommentedMap) The merge target. 2. rhs (CommentedMap) The merge source. 3. path (YAMLPath) Location within the DOM where this merge is taking place. Returns: (CommentedMap) The merged result. Raises: - `MergeException` when a clean merge is impossible. """ if not isinstance(lhs, CommentedMap): raise MergeException( "Impossible to add Hash data to non-Hash destination.", path) self.logger.debug( "Merging INTO dict with keys: {}:".format(", ".join([ str(k.value) if isinstance(k, TaggedScalar) else str(k) for k in lhs.keys()])), data=lhs, prefix="Merger::_merge_dicts: ", header="--------------------") self.logger.debug( "Merging FROM dict with keys: {}:".format(", ".join([ str(k.value) if isinstance(k, TaggedScalar) else str(k) for k in rhs.keys()])), data=rhs, prefix="Merger::_merge_dicts: ", footer="====================") # Delete all internal YAML merge reference keys lest any later # .insert() operation on LHS inexplicably convert them from reference # to concrete keys. This seems like a bug in ruamel.yaml... self._delete_mergeref_keys(lhs) # Assume deep merge until a node's merge rule indicates otherwise buffer: List[Tuple[Any, Any]] = [] buffer_pos = 0 for key, val in rhs.non_merged_items(): path_next = (path + YAMLPath.escape_path_section(key, path.seperator)) if key in lhs: # Write the buffer if populated for b_key, b_val in buffer: self.logger.debug( "Merger::_merge_dicts: Inserting key, {}, from" " buffer to position, {}, at path, {}." .format(b_key, buffer_pos, path_next), header="INSERT " * 15) self.logger.debug( "Before INSERT, the LHS document was:", data=lhs, prefix="Merger::_merge_dicts: ") self.logger.debug( "... and before INSERT, the incoming value will be:", data=b_val, prefix="Merger::_merge_dicts: ") lhs.insert(buffer_pos, b_key, b_val) self.logger.debug( "After INSERT, the LHS document became:", data=lhs, prefix="Merger::_merge_dicts: ") buffer_pos += 1 buffer = [] # Short-circuit the deep merge if a different merge rule # applies to this node. node_coord = NodeCoords(val, rhs, key) merge_mode = ( self.config.hash_merge_mode(node_coord) if isinstance(val, CommentedMap) else self.config.set_merge_mode(node_coord) if isinstance(val, CommentedSet) else self.config.aoh_merge_mode(node_coord) ) self.logger.debug("Merger::_merge_dicts: Got merge mode, {}." .format(merge_mode)) if merge_mode in ( HashMergeOpts.LEFT, AoHMergeOpts.LEFT, SetMergeOpts.LEFT ): continue if merge_mode in ( HashMergeOpts.RIGHT, AoHMergeOpts.RIGHT, SetMergeOpts.RIGHT ): self.logger.debug( "Merger::_merge_dicts: Overwriting key, {}, at path," " {}.".format(key, path_next), header="OVERWRITE " * 15) lhs[key] = val continue if isinstance(val, CommentedMap): lhs[key] = self._merge_dicts(lhs[key], val, path_next) # Synchronize any YAML Tag self.logger.debug( "Merger::_merge_dicts: Setting LHS tag from {} to {}." .format(lhs[key].tag.value, val.tag.value)) lhs[key].yaml_set_tag(val.tag.value) self.logger.debug( "Document BEFORE calling combine_merge_anchors:", data=lhs, prefix="Merger::_merge_dicts: ", header="+------------------+") Anchors.combine_merge_anchors(lhs[key], val) self.logger.debug( "Document AFTER calling combine_merge_anchors:", data=lhs, prefix="Merger::_merge_dicts: ", footer="+==================+") elif isinstance(val, CommentedSeq): lhs[key] = self._merge_lists( lhs[key], val, path_next, parent=rhs, parentref=key) # Synchronize any YAML Tag self.logger.debug( "Merger::_merge_dicts: Setting LHS tag from {} to {}." .format(lhs[key].tag.value, val.tag.value)) lhs[key].yaml_set_tag(val.tag.value) elif isinstance(val, CommentedSet): lhs[key] = self._merge_sets( lhs[key], val, path_next, node_coord) # Synchronize any YAML Tag self.logger.debug( "Merger::_merge_dicts: Setting LHS tag from {} to {}." .format(lhs[key].tag.value, val.tag.value)) lhs[key].yaml_set_tag(val.tag.value) else: self.logger.debug( "Merger::_merge_dicts: Updating key, {}, at path," " {}.".format(key, path_next), header="UPDATE " * 15) self.logger.debug( "Before UPDATE, the LHS document was:", data=lhs, prefix="Merger::_merge_dicts: ") self.logger.debug( "... and before UPDATE, the incoming value will be:", data=val, prefix="Merger::_merge_dicts: ") lhs[key] = val self.logger.debug( "After UPDATE, the LHS document became:", data=lhs, prefix="Merger::_merge_dicts: ") else: # LHS lacks the RHS key. Buffer this key-value pair in order # to insert it ahead of whatever key(s) follow this one in RHS # to keep anchor definitions before their aliases. buffer.append((key, val)) buffer_pos += 1 # Write any remaining buffered content to the end of LHS for b_key, b_val in buffer: self.logger.debug( "Merger::_merge_dicts: Appending key, {}, from buffer at" " path, {}.".format(b_key, path), header="APPEND " * 15) lhs[b_key] = b_val self.logger.debug( "Completed merge result for path, {}:".format(path), data=lhs, prefix="Merger::_merge_dicts: ") return lhs
class MVYaml(object): protected_keys = ( '__current', '__type', ) def __init__(self, base64=False): self._b64 = base64 self._raw = CommentedMap() self._yaml = YAML() self._curr_version = None self._curr_data = None self._create() def _create(self): tag = self._make_tag() self._raw[tag] = CommentedMap() self._raw.insert(0, '__current', tag, 'current version') self._raw.insert(1, '__type', None, 'base64 if value are base64') self._commit(tag=tag, comment='Initial version') def import_yaml(self, file: AnyStr = None, stream: AnyStr = None): data = None if file: with open(file, 'r') as fp: data = fp.read() imported_data = self._yaml.load(data or stream) self.override(imported_data) return self def load(self, file_handler: AnyStr = None, stream_data: AnyStr = None): data = None if file_handler: with open(file_handler, 'r') as fp: data = fp.read() self._raw = self._yaml.load(data or stream_data) if self.protected_keys not in self._raw.keys(): raise MVYamlFileException( f'Not a valid mvyaml file. Perhaps is a yaml you want to import with ' f'import_yaml()?') return self def write(self, file_handler: IO = None, comment: AnyStr = None) -> [AnyStr, None]: if not self._raw: return if self._has_changes(): self._commit(comment=comment) output = file_handler or StringIO() self._yaml.dump(self._raw, output) return output.getvalue() if not file_handler else None @property def versions(self): if not self._raw: return [] return [k for k in self._raw.keys() if k not in self.protected_keys] @property def current(self): return self._raw['__current'] @property def data(self): if not self._curr_data: self._curr_data = deepcopy(self._raw[self._curr_version or self.current]) return self._curr_data def with_version(self, version: str = '__current'): if version not in self.versions: raise MVYamlVersionNotFoundException( f'version {version} not found') self._curr_version = version self._curr_data = None return self @staticmethod def _make_tag() -> str: d = datetime.utcnow().isoformat() return d def override(self, data: [Iterable]): self._curr_data = CommentedMap() self._curr_data.update(data) self._commit(comment='Overridden') return self def _commit(self, *args, **kwargs): return self._commit_head(*args, **kwargs) def _commit_head(self, tag: AnyStr = None, comment: AnyStr = None): """ apply the modifications on curr_data to the underling opened version and create a new tag """ commented_map = CommentedMap() commented_map.update(self._curr_data or self.data) if tag: self._raw[tag] = commented_map self._raw['__current'] = tag else: new_tag = self._make_tag() self._raw.insert(2, new_tag, commented_map, comment=comment) self._raw['__current'] = new_tag self._curr_version = None self._curr_data = None return self def _commit_tail(self, tag: AnyStr = None, comment: AnyStr = None): """ apply the modifications on curr_data to the underling opened version and create a new tag """ commented_map = CommentedMap() commented_map.update(self._curr_data or self.data) if tag: self._raw[tag] = commented_map self._raw['__current'] = tag else: new_tag = self._make_tag() self._raw.insert(len(self._raw.keys()), new_tag, commented_map, comment=comment) self._raw['__current'] = new_tag self._curr_version = None self._curr_data = None return self def _has_changes(self): orig = self._raw[self._curr_version or self.current] current = self._curr_data or self.data try: assert_equal(orig, current) except AssertionError: return True return False @property def changes(self) -> AnyStr: if not self._has_changes(): return '' yaml_orig = as_yaml(self._raw[self._curr_version or self.current]) yaml_curr = as_yaml(self._curr_data) differ = Differ() result = list( differ.compare(yaml_orig.splitlines(), yaml_curr.splitlines())) return '\n'.join(result) def set_current(self, version_label: AnyStr): if version_label not in self.versions: raise MVYamlVersionNotFoundException( f'request version [{version_label}] not found') self._raw['__current'] = version_label self.with_version(version_label) return self