def _include(self, constructor, suffix, node, must_exist):
# parse the suffix
suffix_match = _include_suffix_re.match(suffix)
if suffix_match is None:
raise ConstructorError(
problem='invalid include tag suffix "%s"' % suffix,
problem_mark=node.start_mark,
)
# get the scheme
scheme = suffix_match.group("scheme")
if scheme is None:
scheme = "file"
elif scheme not in self._include_schemes:
raise ConstructorError(
problem='unsupported include scheme "%s"' % scheme,
problem_mark=node.start_mark,
)
# get the opener
open = self._include_schemes[scheme]
# get the format
format = suffix_match.group("format")
if format is None:
format = "yaml"
elif format not in self._include_formats:
raise ConstructorError(
problem='unsupported include format "%s"' % format,
problem_mark=node.start_mark,
)
# get the loader
load = self._include_formats[format]
# get the argument
value = constructor.construct_scalar(node)
# expand environment variables
value = os.path.expandvars(value)
try:
# open the document
with open(self, value, must_exist) as f:
# if None, then it doesn't exist
if f is None:
return None
# load the document
return load(self, f)
# catch errors
except Exception as e:
raise_from(
ConstructorError(
context='while including %s %s "%s"' %
(format, scheme, value),
context_mark=node.start_mark,
),
e,
)
def construct_mapping(self, node, deep=False):
"""Store mappings as OrderedDicts instead of as regular python
dictionaries to preserve file ordering."""
if not isinstance(node, MappingNode):
raise ConstructorError(
None, None, "expected a mapping node, but found %s" % node.id,
node.start_mark)
mapping = syaml_dict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError as exc:
raise ConstructorError("while constructing a mapping",
node.start_mark,
"found unacceptable key (%s)" % exc,
key_node.start_mark)
value = self.construct_object(value_node, deep=deep)
if key in mapping:
raise ConstructorError("while constructing a mapping",
node.start_mark,
"found already in-use key (%s)" % key,
key_node.start_mark)
mapping[key] = value
mark(mapping, node)
return mapping
def construct_mapping(self, node, deep=False):
"""Construct mapping but raise error when a section is defined twice.
From: http://stackoverflow.com/questions/34358974/how-to-prevent-re-definition-of-keys-in-yaml
"""
if not isinstance(node, MappingNode): # pragma: no cover
raise ConstructorError(problem="expected a mapping node, but found %s" % node.id,
problem_mark=node.start_mark)
mapping = {}
for key_node, value_node in node.value:
# keys can be list -> deep
key = self.construct_object(key_node, deep=True)
# lists are not hashable, but tuples are
if not isinstance(key, Hashable): # pragma: no cover
if isinstance(key, list):
key = tuple(key)
if not isinstance(key, Hashable): # pragma: no cover
raise ConstructorError(
"while constructing a mapping", node.start_mark,
"found unhashable key", key_node.start_mark)
value = self.construct_object(value_node, deep=deep)
# next two lines differ from original
if key in mapping:
raise KeyError("Key \"{}\" was defined multiple times in config {}".
format(key, key_node.start_mark))
mapping[key] = value
return mapping
def _check_order(dst, src):
dst_is_ordered = isinstance(dst, OrderedDict)
src_is_ordered = isinstance(src, OrderedDict)
if dst_is_ordered and not src_is_ordered:
raise ConstructorError(
problem="expected an ordered dictionary, but found %r" % src)
if not dst_is_ordered and src_is_ordered:
raise ConstructorError(
problem="expected an unordered dictionary, but found %r" % src)
def construct_mapping(self, node, maptyp, deep=False):
if not isinstance(node, MappingNode):
raise ConstructorError(
None, None, "expected a mapping node, but found %s" % node.id,
node.start_mark)
merge_map = self.flatten_mapping(node)
# mapping = {}
if node.comment:
maptyp._yaml_add_comment(node.comment[:2])
if len(node.comment) > 2:
maptyp.yaml_end_comment_extend(node.comment[2], clear=True)
if node.anchor:
from ruamel.yaml.serializer import templated_id
if not templated_id(node.anchor):
maptyp.yaml_set_anchor(node.anchor)
for key_node, value_node in node.value:
# keys can be list -> deep
key = self.construct_object(key_node, deep=True)
# lists are not hashable, but tuples are
if not isinstance(key, collections.Hashable):
if isinstance(key, list):
key = tuple(key)
if PY2:
try:
hash(key)
except TypeError as exc:
raise ConstructorError("while constructing a mapping",
node.start_mark,
"found unacceptable key (%s)" % exc,
key_node.start_mark)
else:
if not isinstance(key, collections.Hashable):
raise ConstructorError("while constructing a mapping",
node.start_mark,
"found unhashable key",
key_node.start_mark)
value = self.construct_object(value_node, deep=deep)
if key_node.comment:
maptyp._yaml_add_comment(key_node.comment, key=key)
if value_node.comment:
maptyp._yaml_add_comment(value_node.comment, value=key)
maptyp._yaml_set_kv_line_col(key, [
key_node.start_mark.line, key_node.start_mark.column,
value_node.start_mark.line, value_node.start_mark.column
])
if key in maptyp:
raise exceptions.DuplicateKeysDisallowed(
"While parsing",
key_node.start_mark,
"Duplicate key '{0}' found".format(key),
key_node.end_mark,
)
maptyp[key] = value
# do this last, or <<: before a key will prevent insertion in instances
# of collections.OrderedDict (as they have no __contains__
if merge_map:
maptyp.add_yaml_merge(merge_map)
def declarative_from_yaml(cls, constructor, tag_suffix, node):
"""Convert YAML to declarative class instances."""
try:
class_ = cls._get_class(tag_suffix)
except ImportError:
raise ConstructorError(
"while constructing a Stone Soup component", node.start_mark,
"unable to import component {!r}".format(tag_suffix),
node.start_mark)
properties = [data
for data in constructor.construct_yaml_omap(node)][0]
try:
return class_(**properties)
except Exception as e:
raise ConstructorError("while constructing Stone Soup component",
node.start_mark, str(e), node.start_mark)
def _merge_sequence(self, dst, src):
# verify source is a sequence
if not _is_sequence(src):
raise ConstructorError(problem="expected a sequence, but found %r" %
src)
dst.extend(src)
def _add_set_item(self, dst, item):
key, value = item
if value is not None:
raise ConstructorError(problem="expected a null value, but found: %r" %
value)
dst.add(key)
def declarative_from_yaml(constructor, tag_suffix, node):
"""Convert YAML to declarative class instances."""
try:
class_ = get_class(f'!stonesoup.{tag_suffix}')
except ImportError:
raise ConstructorError(
"while constructing a Stone Soup component", node.start_mark,
f"unable to import component 'stonesoup.{tag_suffix}'",
node.start_mark)
# Must have deep construct here to ensure mutable sub-objects are fully created.
constructor.deep_construct = True
properties = [data for data in constructor.construct_yaml_omap(node)][0]
try:
return class_(**properties)
except Exception as e:
raise ConstructorError("while constructing Stone Soup component",
node.start_mark, str(e), node.start_mark)
def expression_constructor(self, node):
if isinstance(node, ScalarNode):
return ExpressionNode(self.construct_scalar(node), node)
else:
raise ConstructorError(
None, None,
'expected a scalar node, but found %s' % node.id,
node.start_mark)
def construct_sequence(self, node, deep=False):
if not isinstance(node, SequenceNode):
raise ConstructorError(
None, None, "expected a sequence node, but found %s" % node.id,
node.start_mark)
value = syaml_list(
self.construct_object(child, deep=deep) for child in node.value)
mark(value, node)
return value
def _iter_sequence_nodes(node):
# verify the node is a sequence
if not isinstance(node, SequenceNode):
raise ConstructorError(
problem="expected a sequence node, but found %s" % node.id,
problem_mark=node.start_mark,
)
# iterate over sequence sub-nodes
for subnode in node.value:
yield subnode
def _iter_mapping_nodes(node):
# verify node is a mapping
if not isinstance(node, MappingNode):
raise ConstructorError(
problem="expected a mapping node, but found %s" % node.id,
problem_mark=node.start_mark,
)
# iterate over mapping sub-nodes
for key_node, value_node in node.value:
yield key_node, value_node
def _construct_vault(constructor, node, tag):
value = constructor.construct_scalar(node)
if not constructor.vault.secrets:
raise ConstructorError(
context=None,
context_mark=None,
problem="found %s but no vault password provided" % tag,
problem_mark=node.start_mark,
note=None,
)
return value
def _merge_mapping(self, dst, src, recursive=False):
# verify source is a mapping
if not isinstance(src, Mapping):
raise ConstructorError(problem="expected a mapping, but found %r" %
src)
# verify destination and source ordering matches
# Note: don't case about ordering if source has a length of 1
if len(src) != 1:
_check_order(dst, src)
for key, src_value in iteritems(src):
if recursive and key in dst:
dst_value = dst[key]
# is destination a sequence?
if isinstance(dst_value, MutableSequence):
_merge_sequence(self, dst_value, src_value)
# is destination a set?
elif isinstance(dst_value, MutableSet):
_merge_set(self, dst_value, src_value)
# if destination a mapping
elif isinstance(dst_value, MutableMapping):
_merge_mapping(self, dst_value, src_value, True)
# its a scalar, so just replace key:value pair
else:
# verify the types are identical
if _is_container(src_value):
raise ConstructorError(
problem="expected scalar, but found %r" % src_value)
dst[key] = src_value
else:
# recursive merge not required, so just add key_value pair
dst[key] = src_value
def _construct_vault(constructor, node, tag):
value = constructor.construct_scalar(node)
if os.getenv("UNFURL_VAULT_SKIP_DECRYPT"):
return sensitive_str.redacted_str
if not constructor.vault.secrets:
raise ConstructorError(
context=None,
context_mark=None,
problem=f"found {tag} but no vault password provided",
problem_mark=node.start_mark,
note=None,
)
return value
def _apply_item(self, loader, dst, node, subnode, **ops):
# extract key_value nodes
node_pair = self._get_node_pair(subnode)
if node_pair is not None:
key_node, value_node = node_pair
if key_node.tag in self._special_tags:
context, op = self._special_tags[key_node.tag]
operation = ops[op]
if operation is None:
raise ConstructorError(
context="while constructing a %s" % node.tag,
context_mark=node.start_mark,
problem="%s not supported" % key_node.tag,
problem_mark=key_node.start_mark,
)
# construct value
with context(self, key_node):
value = loader.construct_object(value_node, deep=True)
# ignore None values
if value is not None:
operation(self, dst, value)
return
# if its a single node, construct the item
if isinstance(subnode, Node):
item = loader.construct_object(subnode, deep=True)
# otherwise, construct a key:value pair
else:
key_node, value_node = subnode
key = loader.construct_object(key_node, deep=True)
value = loader.construct_object(value_node, deep=True)
item = key, value
# add item
# add_method = ops["add"]
ops["add"](self, dst, item)
def _iter_pairs_nodes(node):
# iterate over sequence sub-nodes
for seq_subnode in _iter_sequence_nodes(node):
# get mapping sub-nodes
map_subnodes = list(_iter_mapping_nodes(seq_subnode))
# verify there is one mapping sub-node
if len(map_subnodes) != 1:
raise ConstructorError(
problem="expected a single mapping item,"
" but found %d items" % len(map_subnodes),
problem_mark=seq_subnode.start_mark,
)
# ###### ORDERING ISSUE #####
# extract key:value nodes
key_node, value_node = map_subnodes[0]
yield key_node, value_node
def construct_mapping(self, node, maptyp, deep=False):
if not isinstance(node, MappingNode):
raise ConstructorError(
None,
None,
"expected a mapping node, but found %s" % node.id,
node.start_mark,
)
merge_map = self.flatten_mapping(node)
# mapping = {}
if node.comment:
maptyp._yaml_add_comment(node.comment[:2])
if len(node.comment) > 2:
maptyp.yaml_end_comment_extend(node.comment[2], clear=True)
if node.anchor:
from ruamel.yaml.serializer import templated_id
if not templated_id(node.anchor):
maptyp.yaml_set_anchor(node.anchor)
for key_node, value_node in node.value:
# keys can be list -> deep
key = self.construct_object(key_node, deep=True)
# lists are not hashable, but tuples are
if not isinstance(key, Hashable):
if isinstance(key, list):
key = tuple(key)
if PY2:
try:
hash(key)
except TypeError as exc:
raise ConstructorError(
"while constructing a mapping",
node.start_mark,
"found unacceptable key (%s)" % exc,
key_node.start_mark,
)
else:
if not isinstance(key, Hashable):
raise ConstructorError(
"while constructing a mapping",
node.start_mark,
"found unhashable key",
key_node.start_mark,
)
value = self.construct_object(value_node, deep=deep)
if key_node.comment:
maptyp._yaml_add_comment(key_node.comment, key=key)
if value_node.comment:
maptyp._yaml_add_comment(value_node.comment, value=key)
maptyp._yaml_set_kv_line_col(
key,
[
key_node.start_mark.line,
key_node.start_mark.column,
value_node.start_mark.line,
value_node.start_mark.column,
],
)
if key in maptyp:
key_node.start_mark.name = self.label
key_node.end_mark.name = self.label
raise exceptions.DuplicateKeysDisallowed(
"While parsing",
key_node.start_mark,
"Duplicate key '{0}' found".format(key),
key_node.end_mark,
)
maptyp[key] = value
# do this last, or <<: before a key will prevent insertion in instances
# of collections.OrderedDict (as they have no __contains__
if merge_map:
maptyp.add_yaml_merge(merge_map)
# Don't verify Mapping indentation when allowing flow,
# as that disallows:
# short_key: { x = 1 }
# very_long_key: { x = 1 }
if not self.allow_flow_style:
previous_indentation = None
for node in [
nodegroup[1]
for nodegroup in node.value
if isinstance(nodegroup[1], ruamelyaml.nodes.MappingNode)
]:
if previous_indentation is None:
previous_indentation = node.start_mark.column
if node.start_mark.column != previous_indentation:
raise exceptions.InconsistentIndentationDisallowed(
"While parsing",
node.start_mark,
"Found mapping with indentation "
"inconsistent with previous mapping",
node.end_mark,
)
def _merge_set(self, dst, src):
# verify source is a set
if not isinstance(src, Set):
raise ConstructorError(problem="expected a set, but found %r" % src)
dst.update(src)