def test_aoh_merge_key_default(self, quiet_logger, tmp_path_factory):
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
merge_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = MergerConfig(quiet_logger, SimpleNamespace())
mc.prepare(lhs_data)
node = lhs_data["array_of_hashes"]
parent = lhs_data
parentref = "array_of_hashes"
record = node[0]
assert mc.aoh_merge_key(NodeCoords(node, parent, parentref),
record) == "name"
def _merge_lists(
self, lhs: CommentedSeq, rhs: CommentedSeq, path: YAMLPath,
**kwargs: Any
) -> CommentedSeq:
"""
Merge two lists; understands lists-of-dicts.
Parameters:
1. lhs (CommentedSeq) The list to merge into.
2. rhs (CommentedSeq) The list to merge from.
3. path (YAMLPath) Location of the `rhs` source list within its DOM.
Keyword Arguments:
* parent (Any) Parent node of `rhs`
* parentref (Any) Child Key or Index of `rhs` within `parent`.
Returns: (CommentedSeq) The merged result.
"""
parent: Any = kwargs.pop("parent", None)
parentref: Any = kwargs.pop("parentref", None)
node_coord = NodeCoords(rhs, parent, parentref)
if len(rhs) > 0:
if isinstance(rhs[0], CommentedMap):
# This list is an Array-of-Hashes
return self._merge_arrays_of_hashes(lhs, rhs, path, node_coord)
# This list is an Array-of-Arrays or a simple list of Scalars
return self._merge_simple_lists(lhs, rhs, path, node_coord)
# No RHS list
return lhs
def test_aoh_diff_key_default(self, quiet_logger, tmp_path_factory):
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
diff_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = DifferConfig(quiet_logger, SimpleNamespace())
mc.prepare(lhs_data)
parent = lhs_data["array_of_hashes"]
parentref = 0
node = parent[parentref]
nc = NodeCoords(node, parent, parentref)
(key_attr, is_user_defined) = mc.aoh_diff_key(nc)
assert key_attr == "name" and is_user_defined == False
def test_set_merge_mode_ini_rule_overrides_cli(self, quiet_logger,
tmp_path_factory, cli,
ini_default, ini_rule,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
sets = {}
[rules]
/hash/merge_targets/subset = {}
""".format(ini_default, ini_rule))
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
merge_targets:
subkey: lhs value 2
subset:
? one
? two
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = MergerConfig(quiet_logger,
SimpleNamespace(config=config_file, sets=cli))
mc.prepare(lhs_data)
node = lhs_data["hash"]["merge_targets"]["subset"]
parent = lhs_data["hash"]["merge_targets"]
parentref = "subset"
assert mc.set_merge_mode(NodeCoords(node, parent, parentref)) == mode
def _diff_lists(self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq,
**kwargs) -> None:
"""
Diff two lists.
Parameters:
1. path (YAMLPath) YAML Path to the document element under evaluation
2. lhs (Any) The left-hand-side (original) document
3. rhs (Any) The right-hand-side (altered) document
Keyword Arguments:
* rhs_parent (Any) Parent data node of rhs
* parentref (Any) Reference indicating rhs within rhs_parent
Returns: N/A
"""
self.logger.debug("Comparing LHS:",
prefix="Differ::_diff_lists: ",
data=lhs)
self.logger.debug("Against RHS:",
prefix="Differ::_diff_lists: ",
data=rhs)
parent: Any = kwargs.pop("rhs_parent", None)
parentref: Any = kwargs.pop("parentref", None)
node_coord = NodeCoords(rhs, parent, parentref)
if len(rhs) > 0:
if isinstance(rhs[0], CommentedMap):
# This list is an Array-of-Hashes
self._diff_arrays_of_hashes(path, lhs, rhs, node_coord)
else:
# This list is an Array-of-Arrays or a simple list of Scalars
self._diff_arrays_of_scalars(path, lhs, rhs, node_coord)
def _insert_dict(
self, insert_at: YAMLPath,
lhs: Union[CommentedMap, CommentedSeq, CommentedSet],
rhs: CommentedMap
) -> bool:
"""Insert an RHS dict merge result into the LHS document."""
merge_performed = False
merged_data: Union[
CommentedMap, CommentedSeq, CommentedSet
] = CommentedMap()
if isinstance(lhs, CommentedSeq):
# Merge a dict into a list
self.logger.debug(
"Merger::_insert_dict: Merging a dict into a list.")
merged_data = self._merge_lists(
lhs, CommentedSeq([rhs]), insert_at)
merge_performed = True
elif isinstance(lhs, CommentedSet):
# Merge a dict into a set; this is destructive
raise MergeException(
"Merging a Hash into a Set is destructive to the"
" source Hash because only the keys would be"
" preserved. Please adjust your merge to target a"
" suitable node.", insert_at)
else:
# Merge a dict into a dict
self.logger.debug(
"Merger::_insert_dict: Merging a dict into a dict.")
merge_mode = self.config.hash_merge_mode(
NodeCoords(rhs, None, None))
if merge_mode is HashMergeOpts.LEFT:
self.logger.debug(
"Configured mode short-circuits the merge; returning LHS:"
, prefix="Merger::_insert_dict: "
, data=lhs)
merged_data = lhs
elif merge_mode is HashMergeOpts.RIGHT:
self.logger.debug(
"Configured mode short-circuits the merge; returning RHS:"
, prefix="Merger::_insert_dict: "
, data=rhs)
merged_data = rhs
else:
merged_data = self._merge_dicts(lhs, rhs, insert_at)
merge_performed = True
# Synchronize YAML Tags
self.logger.debug(
"Merger::_insert_dict: Setting LHS tag from {} to {}."
.format(lhs.tag.value, rhs.tag.value))
lhs.yaml_set_tag(rhs.tag.value)
if insert_at.is_root:
self.data = merged_data
return merge_performed
def test_hash_merge_mode_ini(self, quiet_logger, tmp_path_factory, setting,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
hashes = {}
""".format(setting))
mc = MergerConfig(quiet_logger,
SimpleNamespace(config=config_file, hashes=None))
assert mc.hash_merge_mode(NodeCoords(None, None, None)) == mode
def test_array_diff_mode_ini(self, quiet_logger, tmp_path_factory, setting,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
arrays = {}
""".format(setting))
mc = DifferConfig(quiet_logger,
SimpleNamespace(config=config_file, arrays=None))
assert mc.array_diff_mode(NodeCoords(None, None, None)) == mode
def test_array_diff_mode_cli_overrides_ini_defaults(
self, quiet_logger, tmp_path_factory, cli, ini, mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
arrays = {}
""".format(ini))
mc = DifferConfig(quiet_logger,
SimpleNamespace(config=config_file, arrays=cli))
assert mc.array_diff_mode(NodeCoords(None, None, None)) == mode
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
yaml_path = YAMLPath(args.query, pathsep=args.pathsep)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
(yaml_data, doc_loaded) = Parsers.get_yaml_data(
yaml, log, args.yaml_file if args.yaml_file else "-")
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
# Seek the queried value(s)
discovered_nodes = []
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_eyaml_values(yaml_path, mustexist=True):
log.debug("Got node from {}:".format(yaml_path),
data=node,
prefix="yaml_get::main: ")
discovered_nodes.append(NodeCoords.unwrap_node_coords(node))
except YAMLPathException as ex:
log.critical(ex, 1)
except EYAMLCommandException as ex:
log.critical(ex, 2)
try:
for node in discovered_nodes:
if isinstance(node, (dict, list, CommentedSet)):
print(json.dumps(Parsers.jsonify_yaml_data(node)))
else:
if node is None:
node = "\x00"
print("{}".format(str(node).replace("\n", r"\n")))
except RecursionError:
log.critical(
"The YAML data contains an infinitely recursing YAML Alias!", 1)
def name(invert: bool, parameters: List[str], yaml_path: YAMLPath,
**kwargs: Any) -> Generator[NodeCoords, None, None]:
"""
Match only the key-name of the present node.
Parameters:
1. invert (bool) Invert the evaluation
2. parameters (List[str]) Parsed parameters
3. yaml_path (YAMLPath) YAML Path begetting this operation
Keyword Arguments:
* parent (ruamel.yaml node) The parent node from which this query
originates
* parentref (Any) The Index or Key of data within parent
* relay_segment (PathSegment) YAML Path segment presently under
evaluation
* translated_path (YAMLPath) YAML Path indicating precisely which node
is being evaluated
* ancestry (List[AncestryEntry]) Stack of ancestors preceding the
present node under evaluation
Returns: (Generator[NodeCoords, None, None]) each result as it is
generated
"""
parent: Any = kwargs.pop("parent", None)
parentref: Any = kwargs.pop("parentref", None)
translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath(""))
ancestry: List[AncestryEntry] = kwargs.pop("ancestry", [])
relay_segment: PathSegment = kwargs.pop("relay_segment", None)
# There are no parameters
param_count = len(parameters)
if param_count > 1:
raise YAMLPathException(
("Invalid parameter count to {}(); {} are permitted, "
" got {} in YAML Path").format(PathSearchKeywords.NAME, 0,
param_count), str(yaml_path))
if invert:
raise YAMLPathException(
("Inversion is meaningless to {}()").format(
PathSearchKeywords.NAME), str(yaml_path))
yield NodeCoords(parentref, parent, parentref, translated_path,
ancestry, relay_segment)
def _insert_set(
self, insert_at: YAMLPath,
lhs: Union[CommentedMap, CommentedSeq, CommentedSet],
rhs: CommentedSet
) -> bool:
"""Insert an RHS list merge result into the LHS document."""
merge_performed = False
merged_data: Union[
CommentedSeq, CommentedMap, CommentedSet, None] = None
if isinstance(lhs, CommentedSeq):
self.logger.debug(
"Merger::_insert_set: Merging a set into a list.")
merge_list = []
for ele in rhs:
merge_list.append(ele)
merged_data = self._merge_lists(
lhs, CommentedSeq(merge_list), insert_at)
merge_performed = True
elif isinstance(lhs, CommentedMap):
self.logger.debug(
"Merger::_insert_set: Merging a set into a dict.")
merge_dict: Dict[str, None] = {}
for ele in rhs:
merge_dict[ele] = None
merged_data = self._merge_dicts(
lhs, CommentedMap(merge_dict), insert_at)
merge_performed = True
else:
self.logger.debug(
"Merger::_insert_set: Merging a set into a set.")
merged_data = self._merge_sets(lhs, rhs, insert_at, NodeCoords(
rhs, None, None
))
merge_performed = True
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_insert_set: Setting LHS tag from {} to {}."
.format(lhs.tag.value, rhs.tag.value))
lhs.yaml_set_tag(rhs.tag.value)
if insert_at.is_root:
self.data = merged_data
return merge_performed
def test_array_diff_mode_ini_rule_overrides_cli(self, quiet_logger,
tmp_path_factory, cli,
ini_default, ini_rule,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
arrays = {}
[rules]
/hash/diff_targets/subarray = {}
""".format(ini_default, ini_rule))
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
diff_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = DifferConfig(quiet_logger,
SimpleNamespace(config=config_file, arrays=cli))
mc.prepare(lhs_data)
node = lhs_data["hash"]["diff_targets"]["subarray"]
parent = lhs_data["hash"]["diff_targets"]
parentref = "subarray"
assert mc.array_diff_mode(NodeCoords(node, parent, parentref)) == mode
def _insert_list(
self, insert_at: YAMLPath,
lhs: Union[CommentedMap, CommentedSeq, CommentedSet],
rhs: CommentedSeq
) -> bool:
"""Insert an RHS list merge result into the LHS document."""
merge_performed = False
merged_data: Union[CommentedSeq, CommentedSet, None] = None
if isinstance(lhs, CommentedSeq):
# Merge list into list
self.logger.debug(
"Merger::_insert_list: Merging a list into a list.")
merged_data = self._merge_lists(
lhs, rhs, insert_at)
merge_performed = True
elif isinstance(lhs, CommentedSet):
# Merge list into set
self.logger.debug(
"Merger::_insert_list: Merging a list into a set.")
mset = CommentedSet()
for ele in rhs:
mset.add(ele)
merged_data = self._merge_sets(
lhs, mset, insert_at, NodeCoords(rhs, None, None))
merge_performed = True
else:
# Merge list into hash
raise MergeException(
"Impossible to merge an Array into a Hash without a"
" key given to receive the new elements.", insert_at)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_insert_list: Setting LHS tag from {} to {}."
.format(lhs.tag.value, rhs.tag.value))
lhs.yaml_set_tag(rhs.tag.value)
if insert_at.is_root:
self.data = merged_data
return merge_performed
def _diff_lists(
self, path: YAMLPath, lhs: CommentedSeq, rhs: CommentedSeq, **kwargs
) -> None:
"""Diff two lists."""
self.logger.debug(
"Comparing LHS:",
prefix="Differ::_diff_lists: ",
data=lhs)
self.logger.debug(
"Against RHS:",
prefix="Differ::_diff_lists: ",
data=rhs)
parent: Any = kwargs.pop("rhs_parent", None)
parentref: Any = kwargs.pop("parentref", None)
node_coord = NodeCoords(rhs, parent, parentref)
if len(rhs) > 0:
if isinstance(rhs[0], CommentedMap):
# This list is an Array-of-Hashes
self._diff_arrays_of_hashes(path, lhs, rhs, node_coord)
else:
# This list is an Array-of-Arrays or a simple list of Scalars
self._diff_arrays_of_scalars(path, lhs, rhs, node_coord)
def _insert_scalar(
self, insert_at: YAMLPath, lhs: Any, lhs_proc: Processor, rhs: Any
) -> bool:
"""Insert an RHS scalar into the LHS document."""
merge_performed = False
if isinstance(lhs, CommentedSeq):
self.logger.debug(
"Merger::_insert_scalar: Merging a scalar into a list.")
Nodes.append_list_element(lhs, rhs)
merge_performed = True
elif isinstance(lhs, CommentedSet):
self.logger.debug(
"Merger::_insert_scalar: Merging a scalar into a set.")
self._merge_sets(
lhs, CommentedSet([rhs]), insert_at,
NodeCoords(rhs, None, None))
merge_performed = True
elif isinstance(lhs, CommentedMap):
ex_message = (
"Impossible to add Scalar value, {}, to a Hash without"
" a key. Change the value to a 'key: value' pair, a"
" '{{key: value}}' Hash, or change the merge target to"
" an Array or other Scalar value."
).format(rhs)
if len(str(rhs)) < 1 and not sys.stdin.isatty():
ex_message += (
" You may be seeing this because your workflow"
" inadvertently opened a STDIN handle to {}. If"
" this may be the case, try adding --nostdin or -S"
" so as to block unintentional STDIN reading."
).format(basename(sys.argv[0]))
raise MergeException(ex_message, insert_at)
else:
lhs_proc.set_value(insert_at, rhs)
merge_performed = True
return merge_performed
def test_generic(self):
node_coord = NodeCoords([], None, None)
def min(data: Any, invert: bool, parameters: List[str],
yaml_path: YAMLPath,
**kwargs: Any) -> Generator[NodeCoords, None, None]:
"""
Find whichever nodes/elements have a minimum value.
Parameters:
1. data (Any) The data to evaluate
2. invert (bool) Invert the evaluation
3. parameters (List[str]) Parsed parameters
4. yaml_path (YAMLPath) YAML Path begetting this operation
Keyword Arguments:
* parent (ruamel.yaml node) The parent node from which this query
originates
* parentref (Any) The Index or Key of data within parent
* relay_segment (PathSegment) YAML Path segment presently under
evaluation
* translated_path (YAMLPath) YAML Path indicating precisely which node
is being evaluated
* ancestry (List[AncestryEntry]) Stack of ancestors preceding the
present node under evaluation
Returns: (Generator[NodeCoords, None, None]) each result as it is
generated
"""
parent: Any = kwargs.pop("parent", None)
parentref: Any = kwargs.pop("parentref", None)
translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath(""))
ancestry: List[AncestryEntry] = kwargs.pop("ancestry", [])
relay_segment: PathSegment = kwargs.pop("relay_segment", None)
# There may be 0 or 1 parameters
param_count = len(parameters)
if param_count > 1:
raise YAMLPathException(
("Invalid parameter count to {}([NAME]); up to {} permitted, "
" got {} in YAML Path").format(PathSearchKeywords.MIN, 1,
param_count), str(yaml_path))
scan_node = parameters[0] if param_count > 0 else None
match_value: Any = None
match_nodes: List[NodeCoords] = []
discard_nodes: List[NodeCoords] = []
unwrapped_data: Any = NodeCoords.unwrap_node_coords(data)
if Nodes.node_is_aoh(unwrapped_data, accept_nulls=True):
# A named child node is mandatory
if scan_node is None:
raise YAMLPathException((
"The {}([NAME]) Search Keyword requires a key name to scan"
" when evaluating an Array-of-Hashes in YAML Path").format(
PathSearchKeywords.MIN), str(yaml_path))
for idx, wrapped_ele in enumerate(data):
ele = NodeCoords.unwrap_node_coords(wrapped_ele)
next_path = translated_path + "[{}]".format(idx)
next_ancestry = ancestry + [(data, idx)]
if ele is not None and scan_node in ele:
eval_val = ele[scan_node]
if (match_value is None or Searches.search_matches(
PathSearchMethods.LESS_THAN, match_value,
eval_val)):
match_value = eval_val
discard_nodes.extend(match_nodes)
match_nodes = [
NodeCoords(ele, data, idx, next_path,
next_ancestry, relay_segment)
]
continue
if (match_value is None or Searches.search_matches(
PathSearchMethods.EQUALS, match_value, eval_val)):
match_nodes.append(
NodeCoords(ele, data, idx, next_path,
next_ancestry, relay_segment))
continue
discard_nodes.append(
NodeCoords(ele, data, idx, next_path, next_ancestry,
relay_segment))
elif isinstance(data, dict):
# A named child node is mandatory
if scan_node is None:
raise YAMLPathException((
"The {}([NAME]) Search Keyword requires a key name to scan"
" when comparing Hash/map/dict children in YAML Path"
).format(PathSearchKeywords.MIN), str(yaml_path))
for key, val in data.items():
next_ancestry = ancestry + [(data, key)]
next_path = (translated_path + YAMLPath.escape_path_section(
key, translated_path.seperator))
if isinstance(val, dict):
if val is not None and scan_node in val:
eval_val = val[scan_node]
if (match_value is None or Searches.search_matches(
PathSearchMethods.LESS_THAN, match_value,
eval_val)):
match_value = eval_val
discard_nodes.extend(match_nodes)
match_nodes = [
NodeCoords(val, data, key, next_path,
next_ancestry, relay_segment)
]
continue
if (match_value is None or Searches.search_matches(
PathSearchMethods.EQUALS, match_value,
eval_val)):
match_nodes.append(
NodeCoords(val, data, key, next_path,
next_ancestry, relay_segment))
continue
elif scan_node in data:
# The user probably meant to operate against the parent
raise YAMLPathException(
("The {}([NAME]) Search Keyword operates against"
" collections of data which share a common attribute"
" yet there is only a single node to consider. Did"
" you mean to evaluate the parent of the selected"
" node? Please review your YAML Path").format(
PathSearchKeywords.MIN), str(yaml_path))
discard_nodes.append(
NodeCoords(val, data, key, next_path, next_ancestry,
relay_segment))
elif isinstance(data, list):
# A named child node is useless
if scan_node is not None:
raise YAMLPathException(
("The {}([NAME]) Search Keyword cannot utilize a key name"
" when comparing Array/sequence/list elements to one"
" another in YAML Path").format(PathSearchKeywords.MIN),
str(yaml_path))
for idx, ele in enumerate(data):
next_path = translated_path + "[{}]".format(idx)
next_ancestry = ancestry + [(data, idx)]
if (ele is not None
and (match_value is None or Searches.search_matches(
PathSearchMethods.LESS_THAN, match_value, ele))):
match_value = ele
discard_nodes.extend(match_nodes)
match_nodes = [
NodeCoords(ele, data, idx, next_path, next_ancestry,
relay_segment)
]
continue
if (ele is not None and Searches.search_matches(
PathSearchMethods.EQUALS, match_value, ele)):
match_nodes.append(
NodeCoords(ele, data, idx, next_path, next_ancestry,
relay_segment))
continue
discard_nodes.append(
NodeCoords(ele, data, idx, next_path, next_ancestry,
relay_segment))
else:
# Non-complex data is always its own maximum and does not invert
match_value = data
match_nodes = [
NodeCoords(data, parent, parentref, translated_path, ancestry,
relay_segment)
]
yield_nodes = discard_nodes if invert else match_nodes
for node_coord in yield_nodes:
yield node_coord
def parent(data: Any, invert: bool, parameters: List[str],
yaml_path: YAMLPath,
**kwargs: Any) -> Generator[NodeCoords, None, None]:
"""
Climb back up N parent levels in the data hierarchy.
Parameters:
1. data (Any) The data to evaluate
2. invert (bool) Invert the evaluation; not possible for parent()
3. parameters (List[str]) Parsed parameters
4. yaml_path (YAMLPath) YAML Path begetting this operation
Keyword Arguments:
* parent (ruamel.yaml node) The parent node from which this query
originates
* parentref (Any) The Index or Key of data within parent
* relay_segment (PathSegment) YAML Path segment presently under
evaluation
* translated_path (YAMLPath) YAML Path indicating precisely which node
is being evaluated
* ancestry (List[AncestryEntry]) Stack of ancestors preceding the
present node under evaluation
Returns: (Generator[NodeCoords, None, None]) each result as it is
generated
"""
parent: Any = kwargs.pop("parent", None)
parentref: Any = kwargs.pop("parentref", None)
translated_path: YAMLPath = kwargs.pop("translated_path", YAMLPath(""))
ancestry: List[AncestryEntry] = kwargs.pop("ancestry", [])
relay_segment: PathSegment = kwargs.pop("relay_segment", None)
# There may be 0 or 1 parameters
param_count = len(parameters)
if param_count > 1:
raise YAMLPathException(
("Invalid parameter count to {}([STEPS]); up to {} permitted, "
" got {} in YAML Path").format(PathSearchKeywords.PARENT, 1,
param_count), str(yaml_path))
if invert:
raise YAMLPathException(
("Inversion is meaningless to {}([STEPS])").format(
PathSearchKeywords.PARENT), str(yaml_path))
parent_levels: int = 1
ancestry_len: int = len(ancestry)
steps_max = ancestry_len
if param_count > 0:
try:
parent_levels = int(parameters[0])
except ValueError as ex:
raise YAMLPathException(
("Invalid parameter passed to {}([STEPS]), {}; must be"
" unset or an integer number indicating how may parent"
" STEPS to climb in YAML Path").format(
PathSearchKeywords.PARENT, parameters[0]),
str(yaml_path)) from ex
if parent_levels > steps_max:
raise YAMLPathException(
("Cannot {}([STEPS]) higher than the document root. {} steps"
" requested when {} available in YAML Path").format(
PathSearchKeywords.PARENT, parent_levels, steps_max),
str(yaml_path))
if parent_levels < 1:
# parent(0) is the present node
yield NodeCoords(data, parent, parentref, translated_path,
ancestry, relay_segment)
else:
for _ in range(parent_levels):
translated_path.pop()
(data, _) = ancestry.pop()
ancestry_len -= 1
parentref = ancestry[-1][1] if ancestry_len > 0 else None
parent = ancestry[-1][0] if ancestry_len > 0 else None
yield NodeCoords(data, parent, parentref, translated_path,
ancestry, relay_segment)
def test_array_diff_mode_cli(self, quiet_logger, setting, mode):
mc = DifferConfig(quiet_logger, SimpleNamespace(arrays=setting))
assert mc.array_diff_mode(NodeCoords(None, None, None)) == mode
def _merge_dicts(
self, lhs: CommentedMap, rhs: CommentedMap, path: YAMLPath
) -> CommentedMap:
"""
Merge two YAML maps (CommentedMap-wrapped dicts).
Parameters:
1. lhs (CommentedMap) The merge target.
2. rhs (CommentedMap) The merge source.
3. path (YAMLPath) Location within the DOM where this merge is taking
place.
Returns: (CommentedMap) The merged result.
Raises:
- `MergeException` when a clean merge is impossible.
"""
if not isinstance(lhs, CommentedMap):
raise MergeException(
"Impossible to add Hash data to non-Hash destination.", path)
self.logger.debug(
"Merging INTO dict with keys: {}:".format(", ".join([
str(k.value) if isinstance(k, TaggedScalar)
else str(k)
for k in lhs.keys()])),
data=lhs, prefix="Merger::_merge_dicts: ",
header="--------------------")
self.logger.debug(
"Merging FROM dict with keys: {}:".format(", ".join([
str(k.value) if isinstance(k, TaggedScalar)
else str(k)
for k in rhs.keys()])),
data=rhs, prefix="Merger::_merge_dicts: ",
footer="====================")
# Delete all internal YAML merge reference keys lest any later
# .insert() operation on LHS inexplicably convert them from reference
# to concrete keys. This seems like a bug in ruamel.yaml...
self._delete_mergeref_keys(lhs)
# Assume deep merge until a node's merge rule indicates otherwise
buffer: List[Tuple[Any, Any]] = []
buffer_pos = 0
for key, val in rhs.non_merged_items():
path_next = (path +
YAMLPath.escape_path_section(key, path.seperator))
if key in lhs:
# Write the buffer if populated
for b_key, b_val in buffer:
self.logger.debug(
"Merger::_merge_dicts: Inserting key, {}, from"
" buffer to position, {}, at path, {}."
.format(b_key, buffer_pos, path_next),
header="INSERT " * 15)
self.logger.debug(
"Before INSERT, the LHS document was:",
data=lhs, prefix="Merger::_merge_dicts: ")
self.logger.debug(
"... and before INSERT, the incoming value will be:",
data=b_val, prefix="Merger::_merge_dicts: ")
lhs.insert(buffer_pos, b_key, b_val)
self.logger.debug(
"After INSERT, the LHS document became:",
data=lhs, prefix="Merger::_merge_dicts: ")
buffer_pos += 1
buffer = []
# Short-circuit the deep merge if a different merge rule
# applies to this node.
node_coord = NodeCoords(val, rhs, key)
merge_mode = (
self.config.hash_merge_mode(node_coord)
if isinstance(val, CommentedMap)
else self.config.set_merge_mode(node_coord)
if isinstance(val, CommentedSet)
else self.config.aoh_merge_mode(node_coord)
)
self.logger.debug("Merger::_merge_dicts: Got merge mode, {}."
.format(merge_mode))
if merge_mode in (
HashMergeOpts.LEFT, AoHMergeOpts.LEFT, SetMergeOpts.LEFT
):
continue
if merge_mode in (
HashMergeOpts.RIGHT, AoHMergeOpts.RIGHT, SetMergeOpts.RIGHT
):
self.logger.debug(
"Merger::_merge_dicts: Overwriting key, {}, at path,"
" {}.".format(key, path_next),
header="OVERWRITE " * 15)
lhs[key] = val
continue
if isinstance(val, CommentedMap):
lhs[key] = self._merge_dicts(lhs[key], val, path_next)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
self.logger.debug(
"Document BEFORE calling combine_merge_anchors:",
data=lhs, prefix="Merger::_merge_dicts: ",
header="+------------------+")
Anchors.combine_merge_anchors(lhs[key], val)
self.logger.debug(
"Document AFTER calling combine_merge_anchors:",
data=lhs, prefix="Merger::_merge_dicts: ",
footer="+==================+")
elif isinstance(val, CommentedSeq):
lhs[key] = self._merge_lists(
lhs[key], val, path_next, parent=rhs, parentref=key)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
elif isinstance(val, CommentedSet):
lhs[key] = self._merge_sets(
lhs[key], val, path_next, node_coord)
# Synchronize any YAML Tag
self.logger.debug(
"Merger::_merge_dicts: Setting LHS tag from {} to {}."
.format(lhs[key].tag.value, val.tag.value))
lhs[key].yaml_set_tag(val.tag.value)
else:
self.logger.debug(
"Merger::_merge_dicts: Updating key, {}, at path,"
" {}.".format(key, path_next), header="UPDATE " * 15)
self.logger.debug(
"Before UPDATE, the LHS document was:",
data=lhs, prefix="Merger::_merge_dicts: ")
self.logger.debug(
"... and before UPDATE, the incoming value will be:",
data=val, prefix="Merger::_merge_dicts: ")
lhs[key] = val
self.logger.debug(
"After UPDATE, the LHS document became:",
data=lhs, prefix="Merger::_merge_dicts: ")
else:
# LHS lacks the RHS key. Buffer this key-value pair in order
# to insert it ahead of whatever key(s) follow this one in RHS
# to keep anchor definitions before their aliases.
buffer.append((key, val))
buffer_pos += 1
# Write any remaining buffered content to the end of LHS
for b_key, b_val in buffer:
self.logger.debug(
"Merger::_merge_dicts: Appending key, {}, from buffer at"
" path, {}.".format(b_key, path), header="APPEND " * 15)
lhs[b_key] = b_val
self.logger.debug(
"Completed merge result for path, {}:".format(path),
data=lhs, prefix="Merger::_merge_dicts: ")
return lhs
def test_array_diff_mode_default(self, quiet_logger):
mc = DifferConfig(quiet_logger, SimpleNamespace(arrays=None))
assert mc.array_diff_mode(NodeCoords(None, None,
None)) == ArrayDiffOpts.POSITION
def _merge_arrays_of_hashes(
self, lhs: CommentedSeq, rhs: CommentedSeq, path: YAMLPath,
node_coord: NodeCoords
) -> CommentedSeq:
"""
Merge two Arrays-of-Hashes.
This is a deep merge operation. Each dict is treated as a record with
an identity key. RHS records are merged with LHS records for which the
identity key matches. As such, an identity key is required in both LHS
and RHS records. This key is configurable. When there is no LHS match
for an RHS key, the RHS record is appended to the LHS list.
Parameters:
1. lhs (CommentedSeq) The merge target.
2. rhs (CommentedSeq) The merge source.
3. path (YAMLPath) Location within the DOM where this merge is taking
place.
4. node_coord (NodeCoords) The RHS root node, its parent, and reference
within its parent; used for config lookups.
Returns: (CommentedSeq) The merged result.
Raises:
- `MergeException` when a clean merge is impossible.
"""
if not isinstance(lhs, CommentedSeq):
raise MergeException(
"Impossible to add Array-of-Hash data to non-Array"
" destination."
, path)
self.logger.debug(
"Merging {} Hash(es) at {}.".format(len(rhs), path),
prefix="Merger::_merge_arrays_of_hashes: ", data=rhs)
id_key: str = ""
if len(rhs) > 0 and isinstance(rhs[0], CommentedMap):
id_key = self.config.aoh_merge_key(
NodeCoords(rhs[0], rhs, 0), rhs[0])
self.logger.debug(
"Merger::_merge_arrays_of_hashes: RHS AoH yielded id_key:"
" {}.".format(id_key))
merge_mode = self.config.aoh_merge_mode(node_coord)
for idx, ele in enumerate(rhs):
path_next = path + "[{}]".format(idx)
self.logger.debug(
"Processing element #{} at {}.".format(idx, path_next),
prefix="Merger::_merge_arrays_of_hashes: ", data=ele)
if merge_mode is AoHMergeOpts.DEEP:
if id_key in ele:
id_val = Nodes.tagless_value(ele[id_key])
else:
raise MergeException(
"Mandatory identity key, {}, not present in Hash with"
" keys: {}."
.format(id_key, ", ".join(ele.keys()))
, path_next
)
merged_hash = False
for lhs_hash in (
lhs_hash for lhs_hash in lhs
if isinstance(lhs_hash, CommentedMap)
and id_key in lhs_hash
and Nodes.tagless_value(lhs_hash[id_key]) == id_val
):
self._merge_dicts(lhs_hash, ele, path_next)
merged_hash = True
# Synchronize YAML Tags
lhs_hash.yaml_set_tag(ele.tag.value)
break
if not merged_hash:
Nodes.append_list_element(lhs, ele,
ele.anchor.value if hasattr(ele, "anchor") else None)
elif merge_mode is AoHMergeOpts.UNIQUE:
if ele not in lhs:
Nodes.append_list_element(
lhs, ele,
ele.anchor.value if hasattr(ele, "anchor") else None)
else:
Nodes.append_list_element(lhs, ele,
ele.anchor.value if hasattr(ele, "anchor") else None)
return lhs
def test_set_merge_mode_cli(self, quiet_logger, setting, mode):
mc = MergerConfig(quiet_logger, SimpleNamespace(sets=setting))
assert mc.set_merge_mode(NodeCoords(None, None, None)) == mode
def test_set_merge_mode_default(self, quiet_logger):
mc = MergerConfig(quiet_logger, SimpleNamespace(sets=None))
assert mc.set_merge_mode(NodeCoords(None, None,
None)) == SetMergeOpts.UNIQUE
def test_aoh_merge_mode_default(self, quiet_logger):
mc = MergerConfig(quiet_logger, SimpleNamespace(aoh=None))
assert mc.aoh_merge_mode(NodeCoords(None, None,
None)) == AoHMergeOpts.ALL
def test_repr(self):
node_coord = NodeCoords([], None, None)
assert repr(node_coord) == "NodeCoords('[]', 'None', 'None')"
def test_hash_merge_mode_default(self, quiet_logger):
mc = MergerConfig(quiet_logger, SimpleNamespace(hashes=None))
assert mc.hash_merge_mode(NodeCoords(None, None,
None)) == HashMergeOpts.DEEP
def test_debug_noisy(self, capsys):
args = SimpleNamespace(verbose=False, quiet=False, debug=True)
logger = ConsolePrinter(args)
anchoredkey = PlainScalarString("TestKey", anchor="KeyAnchor")
anchoredval = PlainScalarString("TestVal", anchor="Anchor")
foldedstr = "123456789 123456789 123456789"
foldedstrfolds = [10, 20]
foldedval = FoldedScalarString(foldedstr)
foldedval.fold_pos = foldedstrfolds
logger.debug(anchoredval)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: (&Anchor)TestVal",
]) + "\n" == console.out
logger.debug(["test", anchoredval])
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [0]test<class 'str'>",
"DEBUG: [1](&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, anchoredkey: anchoredval})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [TestKey](&KeyAnchor,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, anchoredkey: "non-anchored value"})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [TestKey](&KeyAnchor,_)non-anchored value<class 'str'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, "non-anchored-key": anchoredval})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [non-anchored-key](_,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
tagged_value = "value"
tagged_value_node = TaggedScalar(tagged_value, tag="!tag")
tagged_sequence = CommentedSeq(["a", "b"])
tagged_sequence.yaml_set_tag("!raz")
selfref_value = "self_referring"
selfref_value_node = TaggedScalar(selfref_value, tag="!self_referring")
logger.debug(
"test_wrappers_consoleprinter:",
prefix="test_debug_noisy: ",
header="--- HEADER ---",
footer="=== FOOTER ===",
data_header="+++ DATA HEADER +++",
data_footer="::: DATA FOOTER :::",
data=CommentedMap({
"key": "value",
"tagged": tagged_value_node,
tagged_value_node: "untagged value",
selfref_value_node: selfref_value_node,
"array": ["ichi", "ni", "san"],
"tagged_array": tagged_sequence,
"aoh": [{"id": 1},{"id": 2},{"id": 3}],
"aoa": [[True, True], [True, False], [False, True]],
"dod": {"a": {"b": {"c": "d"}}},
"set": CommentedSet(["one", "two"]),
})
)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: test_debug_noisy: --- HEADER ---",
"DEBUG: test_debug_noisy: test_wrappers_consoleprinter:",
"DEBUG: test_debug_noisy: +++ DATA HEADER +++",
"DEBUG: test_debug_noisy: [key]value<class 'str'>",
"DEBUG: test_debug_noisy: [tagged]<_,!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
"DEBUG: test_debug_noisy: [value]<!tag,_>untagged value<class 'str'>",
"DEBUG: test_debug_noisy: [self_referring]<!self_referring,!self_referring>self_referring<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
"DEBUG: test_debug_noisy: [array][0]ichi<class 'str'>",
"DEBUG: test_debug_noisy: [array][1]ni<class 'str'>",
"DEBUG: test_debug_noisy: [array][2]san<class 'str'>",
"DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[0]a<class 'str'>",
"DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[1]b<class 'str'>",
"DEBUG: test_debug_noisy: [aoh][0][id]1<class 'int'>",
"DEBUG: test_debug_noisy: [aoh][1][id]2<class 'int'>",
"DEBUG: test_debug_noisy: [aoh][2][id]3<class 'int'>",
"DEBUG: test_debug_noisy: [aoa][0][0]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][0][1]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][1][0]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][1][1]False<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][2][0]False<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][2][1]True<class 'bool'>",
"DEBUG: test_debug_noisy: [dod][a][b][c]d<class 'str'>",
"DEBUG: test_debug_noisy: [set]{one}<class 'str'>",
"DEBUG: test_debug_noisy: [set]{two}<class 'str'>",
"DEBUG: test_debug_noisy: ::: DATA FOOTER :::",
"DEBUG: test_debug_noisy: === FOOTER ===",
]) + "\n" == console.out
logger.debug(tagged_value_node)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: <!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
])
logger.debug(tagged_sequence)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [tagged_array]<!raz>[0]a<class 'str'>",
"DEBUG: [tagged_array]<!raz>[1]b<class 'str'>",
])
nc = NodeCoords(
"value",
dict(key="value"),
"key",
YAMLPath("doc_root.key"),
[ (dict(doc_root=dict(key="value")), "doc_root"),
(dict(key="value"), "key")],
(PathSegmentTypes.KEY, "key")
)
logger.debug(
"A node coordinate:", prefix="test_debug_noisy: ", data=nc)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: test_debug_noisy: A node coordinate:",
"DEBUG: test_debug_noisy: (path)doc_root.key",
"DEBUG: test_debug_noisy: (segment)[0]PathSegmentTypes.KEY<enum 'PathSegmentTypes'>",
"DEBUG: test_debug_noisy: (segment)[1]key<class 'str'>",
"DEBUG: test_debug_noisy: (node)value",
"DEBUG: test_debug_noisy: (parent)[key]value<class 'str'>",
"DEBUG: test_debug_noisy: (parentref)key",
"DEBUG: test_debug_noisy: (ancestry)[0][0][doc_root][key]value<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[0][1]doc_root<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[1][0][key]value<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[1][1]key<class 'str'>",
]) + "\n" == console.out
logger.debug(foldedval)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: {}<class 'ruamel.yaml.scalarstring.FoldedScalarString'>,folded@{}".format(foldedstr, foldedstrfolds)
])
def test_str(self):
node_coord = NodeCoords([], None, None)
assert str(node_coord) == "[]"
