def _include(self, constructor, suffix, node, must_exist): # parse the suffix suffix_match = _include_suffix_re.match(suffix) if suffix_match is None: raise ConstructorError( problem='invalid include tag suffix "%s"' % suffix, problem_mark=node.start_mark, ) # get the scheme scheme = suffix_match.group("scheme") if scheme is None: scheme = "file" elif scheme not in self._include_schemes: raise ConstructorError( problem='unsupported include scheme "%s"' % scheme, problem_mark=node.start_mark, ) # get the opener open = self._include_schemes[scheme] # get the format format = suffix_match.group("format") if format is None: format = "yaml" elif format not in self._include_formats: raise ConstructorError( problem='unsupported include format "%s"' % format, problem_mark=node.start_mark, ) # get the loader load = self._include_formats[format] # get the argument value = constructor.construct_scalar(node) # expand environment variables value = os.path.expandvars(value) try: # open the document with open(self, value, must_exist) as f: # if None, then it doesn't exist if f is None: return None # load the document return load(self, f) # catch errors except Exception as e: raise_from( ConstructorError( context='while including %s %s "%s"' % (format, scheme, value), context_mark=node.start_mark, ), e, )
def construct_mapping(self, node, deep=False): """Store mappings as OrderedDicts instead of as regular python dictionaries to preserve file ordering.""" if not isinstance(node, MappingNode): raise ConstructorError( None, None, "expected a mapping node, but found %s" % node.id, node.start_mark) mapping = syaml_dict() for key_node, value_node in node.value: key = self.construct_object(key_node, deep=deep) try: hash(key) except TypeError as exc: raise ConstructorError("while constructing a mapping", node.start_mark, "found unacceptable key (%s)" % exc, key_node.start_mark) value = self.construct_object(value_node, deep=deep) if key in mapping: raise ConstructorError("while constructing a mapping", node.start_mark, "found already in-use key (%s)" % key, key_node.start_mark) mapping[key] = value mark(mapping, node) return mapping
def construct_mapping(self, node, deep=False): """Construct mapping but raise error when a section is defined twice. From: http://stackoverflow.com/questions/34358974/how-to-prevent-re-definition-of-keys-in-yaml """ if not isinstance(node, MappingNode): # pragma: no cover raise ConstructorError(problem="expected a mapping node, but found %s" % node.id, problem_mark=node.start_mark) mapping = {} for key_node, value_node in node.value: # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are if not isinstance(key, Hashable): # pragma: no cover if isinstance(key, list): key = tuple(key) if not isinstance(key, Hashable): # pragma: no cover raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) value = self.construct_object(value_node, deep=deep) # next two lines differ from original if key in mapping: raise KeyError("Key \"{}\" was defined multiple times in config {}". format(key, key_node.start_mark)) mapping[key] = value return mapping
def _check_order(dst, src): dst_is_ordered = isinstance(dst, OrderedDict) src_is_ordered = isinstance(src, OrderedDict) if dst_is_ordered and not src_is_ordered: raise ConstructorError( problem="expected an ordered dictionary, but found %r" % src) if not dst_is_ordered and src_is_ordered: raise ConstructorError( problem="expected an unordered dictionary, but found %r" % src)
def construct_mapping(self, node, maptyp, deep=False): if not isinstance(node, MappingNode): raise ConstructorError( None, None, "expected a mapping node, but found %s" % node.id, node.start_mark) merge_map = self.flatten_mapping(node) # mapping = {} if node.comment: maptyp._yaml_add_comment(node.comment[:2]) if len(node.comment) > 2: maptyp.yaml_end_comment_extend(node.comment[2], clear=True) if node.anchor: from ruamel.yaml.serializer import templated_id if not templated_id(node.anchor): maptyp.yaml_set_anchor(node.anchor) for key_node, value_node in node.value: # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are if not isinstance(key, collections.Hashable): if isinstance(key, list): key = tuple(key) if PY2: try: hash(key) except TypeError as exc: raise ConstructorError("while constructing a mapping", node.start_mark, "found unacceptable key (%s)" % exc, key_node.start_mark) else: if not isinstance(key, collections.Hashable): raise ConstructorError("while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) value = self.construct_object(value_node, deep=deep) if key_node.comment: maptyp._yaml_add_comment(key_node.comment, key=key) if value_node.comment: maptyp._yaml_add_comment(value_node.comment, value=key) maptyp._yaml_set_kv_line_col(key, [ key_node.start_mark.line, key_node.start_mark.column, value_node.start_mark.line, value_node.start_mark.column ]) if key in maptyp: raise exceptions.DuplicateKeysDisallowed( "While parsing", key_node.start_mark, "Duplicate key '{0}' found".format(key), key_node.end_mark, ) maptyp[key] = value # do this last, or <<: before a key will prevent insertion in instances # of collections.OrderedDict (as they have no __contains__ if merge_map: maptyp.add_yaml_merge(merge_map)
def declarative_from_yaml(cls, constructor, tag_suffix, node): """Convert YAML to declarative class instances.""" try: class_ = cls._get_class(tag_suffix) except ImportError: raise ConstructorError( "while constructing a Stone Soup component", node.start_mark, "unable to import component {!r}".format(tag_suffix), node.start_mark) properties = [data for data in constructor.construct_yaml_omap(node)][0] try: return class_(**properties) except Exception as e: raise ConstructorError("while constructing Stone Soup component", node.start_mark, str(e), node.start_mark)
def _merge_sequence(self, dst, src): # verify source is a sequence if not _is_sequence(src): raise ConstructorError(problem="expected a sequence, but found %r" % src) dst.extend(src)
def _add_set_item(self, dst, item): key, value = item if value is not None: raise ConstructorError(problem="expected a null value, but found: %r" % value) dst.add(key)
def declarative_from_yaml(constructor, tag_suffix, node): """Convert YAML to declarative class instances.""" try: class_ = get_class(f'!stonesoup.{tag_suffix}') except ImportError: raise ConstructorError( "while constructing a Stone Soup component", node.start_mark, f"unable to import component 'stonesoup.{tag_suffix}'", node.start_mark) # Must have deep construct here to ensure mutable sub-objects are fully created. constructor.deep_construct = True properties = [data for data in constructor.construct_yaml_omap(node)][0] try: return class_(**properties) except Exception as e: raise ConstructorError("while constructing Stone Soup component", node.start_mark, str(e), node.start_mark)
def expression_constructor(self, node): if isinstance(node, ScalarNode): return ExpressionNode(self.construct_scalar(node), node) else: raise ConstructorError( None, None, 'expected a scalar node, but found %s' % node.id, node.start_mark)
def construct_sequence(self, node, deep=False): if not isinstance(node, SequenceNode): raise ConstructorError( None, None, "expected a sequence node, but found %s" % node.id, node.start_mark) value = syaml_list( self.construct_object(child, deep=deep) for child in node.value) mark(value, node) return value
def _iter_sequence_nodes(node): # verify the node is a sequence if not isinstance(node, SequenceNode): raise ConstructorError( problem="expected a sequence node, but found %s" % node.id, problem_mark=node.start_mark, ) # iterate over sequence sub-nodes for subnode in node.value: yield subnode
def _iter_mapping_nodes(node): # verify node is a mapping if not isinstance(node, MappingNode): raise ConstructorError( problem="expected a mapping node, but found %s" % node.id, problem_mark=node.start_mark, ) # iterate over mapping sub-nodes for key_node, value_node in node.value: yield key_node, value_node
def _construct_vault(constructor, node, tag): value = constructor.construct_scalar(node) if not constructor.vault.secrets: raise ConstructorError( context=None, context_mark=None, problem="found %s but no vault password provided" % tag, problem_mark=node.start_mark, note=None, ) return value
def _merge_mapping(self, dst, src, recursive=False): # verify source is a mapping if not isinstance(src, Mapping): raise ConstructorError(problem="expected a mapping, but found %r" % src) # verify destination and source ordering matches # Note: don't case about ordering if source has a length of 1 if len(src) != 1: _check_order(dst, src) for key, src_value in iteritems(src): if recursive and key in dst: dst_value = dst[key] # is destination a sequence? if isinstance(dst_value, MutableSequence): _merge_sequence(self, dst_value, src_value) # is destination a set? elif isinstance(dst_value, MutableSet): _merge_set(self, dst_value, src_value) # if destination a mapping elif isinstance(dst_value, MutableMapping): _merge_mapping(self, dst_value, src_value, True) # its a scalar, so just replace key:value pair else: # verify the types are identical if _is_container(src_value): raise ConstructorError( problem="expected scalar, but found %r" % src_value) dst[key] = src_value else: # recursive merge not required, so just add key_value pair dst[key] = src_value
def _construct_vault(constructor, node, tag): value = constructor.construct_scalar(node) if os.getenv("UNFURL_VAULT_SKIP_DECRYPT"): return sensitive_str.redacted_str if not constructor.vault.secrets: raise ConstructorError( context=None, context_mark=None, problem=f"found {tag} but no vault password provided", problem_mark=node.start_mark, note=None, ) return value
def _apply_item(self, loader, dst, node, subnode, **ops): # extract key_value nodes node_pair = self._get_node_pair(subnode) if node_pair is not None: key_node, value_node = node_pair if key_node.tag in self._special_tags: context, op = self._special_tags[key_node.tag] operation = ops[op] if operation is None: raise ConstructorError( context="while constructing a %s" % node.tag, context_mark=node.start_mark, problem="%s not supported" % key_node.tag, problem_mark=key_node.start_mark, ) # construct value with context(self, key_node): value = loader.construct_object(value_node, deep=True) # ignore None values if value is not None: operation(self, dst, value) return # if its a single node, construct the item if isinstance(subnode, Node): item = loader.construct_object(subnode, deep=True) # otherwise, construct a key:value pair else: key_node, value_node = subnode key = loader.construct_object(key_node, deep=True) value = loader.construct_object(value_node, deep=True) item = key, value # add item # add_method = ops["add"] ops["add"](self, dst, item)
def _iter_pairs_nodes(node): # iterate over sequence sub-nodes for seq_subnode in _iter_sequence_nodes(node): # get mapping sub-nodes map_subnodes = list(_iter_mapping_nodes(seq_subnode)) # verify there is one mapping sub-node if len(map_subnodes) != 1: raise ConstructorError( problem="expected a single mapping item," " but found %d items" % len(map_subnodes), problem_mark=seq_subnode.start_mark, ) # ###### ORDERING ISSUE ##### # extract key:value nodes key_node, value_node = map_subnodes[0] yield key_node, value_node
def construct_mapping(self, node, maptyp, deep=False): if not isinstance(node, MappingNode): raise ConstructorError( None, None, "expected a mapping node, but found %s" % node.id, node.start_mark, ) merge_map = self.flatten_mapping(node) # mapping = {} if node.comment: maptyp._yaml_add_comment(node.comment[:2]) if len(node.comment) > 2: maptyp.yaml_end_comment_extend(node.comment[2], clear=True) if node.anchor: from ruamel.yaml.serializer import templated_id if not templated_id(node.anchor): maptyp.yaml_set_anchor(node.anchor) for key_node, value_node in node.value: # keys can be list -> deep key = self.construct_object(key_node, deep=True) # lists are not hashable, but tuples are if not isinstance(key, Hashable): if isinstance(key, list): key = tuple(key) if PY2: try: hash(key) except TypeError as exc: raise ConstructorError( "while constructing a mapping", node.start_mark, "found unacceptable key (%s)" % exc, key_node.start_mark, ) else: if not isinstance(key, Hashable): raise ConstructorError( "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark, ) value = self.construct_object(value_node, deep=deep) if key_node.comment: maptyp._yaml_add_comment(key_node.comment, key=key) if value_node.comment: maptyp._yaml_add_comment(value_node.comment, value=key) maptyp._yaml_set_kv_line_col( key, [ key_node.start_mark.line, key_node.start_mark.column, value_node.start_mark.line, value_node.start_mark.column, ], ) if key in maptyp: key_node.start_mark.name = self.label key_node.end_mark.name = self.label raise exceptions.DuplicateKeysDisallowed( "While parsing", key_node.start_mark, "Duplicate key '{0}' found".format(key), key_node.end_mark, ) maptyp[key] = value # do this last, or <<: before a key will prevent insertion in instances # of collections.OrderedDict (as they have no __contains__ if merge_map: maptyp.add_yaml_merge(merge_map) # Don't verify Mapping indentation when allowing flow, # as that disallows: # short_key: { x = 1 } # very_long_key: { x = 1 } if not self.allow_flow_style: previous_indentation = None for node in [ nodegroup[1] for nodegroup in node.value if isinstance(nodegroup[1], ruamelyaml.nodes.MappingNode) ]: if previous_indentation is None: previous_indentation = node.start_mark.column if node.start_mark.column != previous_indentation: raise exceptions.InconsistentIndentationDisallowed( "While parsing", node.start_mark, "Found mapping with indentation " "inconsistent with previous mapping", node.end_mark, )
def _merge_set(self, dst, src): # verify source is a set if not isinstance(src, Set): raise ConstructorError(problem="expected a set, but found %r" % src) dst.update(src)