def _prepare_user_rules(self, proc: Processor, section: str,
collector: dict) -> None:
"""
Identify DOM nodes matching user-defined diff rules.
Parameters:
1. proc (Processor) Reference to the DOM Processor.
2. section (str) User-configuration file section defining the diff
rules to apply.
3. collector (dict) Storage collector for matching nodes.
Returns: N/A
"""
if self.config is None or not section in self.config:
self.log.warning(
"User-specified configuration file has no {} section.".format(
section))
return
for rule_key in self.config[section]:
rule_value = self.config[section][rule_key]
if "=" in rule_value:
# There were at least two = signs on the configuration line
conf_line = rule_key + "=" + rule_value
delim_pos = conf_line.rfind("=")
rule_key = conf_line[0:delim_pos].strip()
rule_value = conf_line[delim_pos + 1:].strip()
self.log.debug(
"DifferConfig::_prepare_user_rules: Reconstituted"
" configuration line '{}' to extract adjusted key '{}'"
" with value '{}'".format(conf_line, rule_key, rule_value))
rule_path = YAMLPath(rule_key)
yaml_path = YAMLPath(rule_path)
self.log.debug(
"DifferConfig::_prepare_user_rules: Matching '{}' nodes to"
" YAML Path '{}' from key, {}.".format(section, yaml_path,
rule_key))
try:
for node_coord in proc.get_nodes(yaml_path, mustexist=True):
self.log.debug(
"Node will have comparisons rule, {}:".format(
rule_value),
prefix="DifferConfig::_prepare_user_rules: ",
data=node_coord.node)
collector[node_coord] = rule_value
except YAMLPathException:
self.log.warning("{} YAML Path matches no nodes: {}".format(
section, yaml_path))
self.log.debug("Matched rules to nodes:",
prefix="DifferConfig::_prepare_user_rules: ")
for node_coord, diff_rule in collector.items():
self.log.debug("... RULE: {}".format(diff_rule),
prefix="DifferConfig::_prepare_user_rules: ")
self.log.debug("... NODE:",
data=node_coord,
prefix="DifferConfig::_prepare_user_rules: ")
def test_seperator_change(self):
# IMPORTANT: The YAML Path is only lazily parsed! This means parsing
# ONLY happens when the path is in some way used. Casting it to string
# qualifies as one form of use, so this test will instigate parsing via
# stringification. THIS MATTERS WHEN YOUR INTENTION IS TO **CHANGE**
# THE PATH SEPERATOR! So, if an original path uses dot-notation and
# you wish to change it to forward-slash-notation, you must first cause
# the original to become parsed, AND THEN change the seperator.
testpath = YAMLPath("abc.def")
dotted = str(testpath)
testpath.seperator = PathSeperators.FSLASH
assert "/abc/def" == str(testpath) != dotted
def get_insertion_point(self) -> YAMLPath:
"""
Get the YAML Path at which merging shall be performed.
Parameters: N/A
Returns: (YAMLPath) User-specified point(s) within the document where
the RHS document is directed to be merged-in.
"""
if hasattr(self.args, "mergeat"):
return YAMLPath(self.args.mergeat)
return YAMLPath("/")
def _purge_document(self, path: YAMLPath, data: Any) -> None:
"""
Record changes necessary to delete every node in the document.
Parameters:
1. path (YAMLPath) YAML Path to the document element under evaluation
2. data (Any) The DOM element under evaluation
Returns: N/A
"""
if isinstance(data, CommentedMap):
lhs_iteration = -1
for key, val in data.items():
lhs_iteration += 1
next_path = (path +
YAMLPath.escape_path_section(key, path.seperator))
self._diffs.append(
DiffEntry(DiffActions.DELETE,
next_path,
val,
None,
lhs_parent=data,
lhs_iteration=lhs_iteration))
elif isinstance(data, CommentedSeq):
for idx, ele in enumerate(data):
next_path = path + "[{}]".format(idx)
self._diffs.append(
DiffEntry(DiffActions.DELETE,
next_path,
ele,
None,
lhs_parent=data,
lhs_iteration=idx))
elif isinstance(data, CommentedSet):
for idx, ele in enumerate(data):
next_path = (path +
YAMLPath.escape_path_section(ele, path.seperator))
self._diffs.append(
DiffEntry(DiffActions.DELETE,
next_path,
ele,
None,
lhs_parent=data,
lhs_iteration=idx))
else:
if data is not None:
self._diffs.append(
DiffEntry(DiffActions.DELETE, path, data, None))
def get_search_term(logger: ConsolePrinter,
expression: str) -> Optional[SearchTerms]:
"""
Attempts to cast a search expression into a SearchTerms instance. Returns
None on failure.
"""
# The leading character must be a known search operator
check_operator = expression[0] if expression else ""
if not (PathSearchMethods.is_operator(check_operator)
or check_operator == '!'):
logger.error(("Invalid search expression, '{}'. The first symbol of" +
" every search expression must be one of: {}").format(
expression,
", ".join(PathSearchMethods.get_operators())))
return None
if not len(expression) > 1:
# Empty expressions do nothing
logger.error(
"An EXPRESSION with only a search operator has no effect, '{}'.".
format(expression))
return None
try:
exterm = create_searchterms_from_pathattributes(
YAMLPath("[*{}]".format(expression)).escaped[0][1])
except YAMLPathException as ex:
logger.error(("Invalid search expression, '{}', due to: {}").format(
expression, ex))
return None
return exterm
def test_key_anchor_changes(self, quiet_logger, yamlpath, value, tally, mustexist, vformat, pathsep):
yamldata = """---
anchorKeys:
&keyOne aliasOne: 11A1
&keyTwo aliasTwo: 22B2
&recursiveAnchorKey subjectKey: *recursiveAnchorKey
hash:
*keyOne :
subval: 1.1
*keyTwo :
subval: 2.2
*recursiveAnchorKey :
subval: 3.3
"""
yaml = YAML()
data = yaml.load(yamldata)
processor = Processor(quiet_logger, data)
yamlpath = YAMLPath(yamlpath)
processor.set_value(yamlpath, value, mustexist=mustexist, value_format=vformat, pathsep=pathsep)
matchtally = 0
for node in processor.get_nodes(yamlpath):
assert unwrap_node_coords(node) == value
matchtally += 1
assert matchtally == tally
def test_get_every_data_type(self, quiet_logger):
# Contributed by https://github.com/AndydeCleyre
yamldata = """---
intthing: 6
floatthing: 6.8
yesthing: yes
nothing: no
truething: true
falsething: false
nullthing: null
nothingthing:
emptystring: ""
nullstring: "null"
"""
results = [6, 6.8, "yes", "no", True, False, None, None, "", "null"]
yaml = YAML()
data = yaml.load(yamldata)
processor = Processor(quiet_logger, data)
yamlpath = YAMLPath("*")
match_index = 0
for node in processor.get_nodes(yamlpath):
assert unwrap_node_coords(node) == results[match_index]
match_index += 1
def get_nodes(self, yaml_path: Union[YAMLPath, str],
**kwargs: Any) -> Generator[Any, None, None]:
"""
Retrieves zero or more node at YAML Path in YAML data.
Parameters:
1. yaml_path (Union[Path, str]) The YAML Path to evaluate
Keyword Parameters:
* mustexist (bool) Indicate whether yaml_path must exist
in data prior to this query (lest an Exception be raised);
default=False
* default_value (Any) The value to set at yaml_path should
it not already exist in data and mustexist is False;
default=None
* pathsep (PathSeperators) Forced YAML Path segment seperator; set
only when automatic inference fails;
default = PathSeperators.AUTO
Returns: (Generator) The requested YAML nodes as they are matched
Raises:
- `YAMLPathException` when YAML Path is invalid
"""
mustexist: bool = kwargs.pop("mustexist", False)
default_value: Any = kwargs.pop("default_value", None)
pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO)
node: Any = None
if self.data is None:
return
if isinstance(yaml_path, str):
yaml_path = YAMLPath(yaml_path, pathsep)
elif pathsep is not PathSeperators.AUTO:
yaml_path.seperator = pathsep
if mustexist:
matched_nodes: int = 0
for node in self._get_required_nodes(self.data, yaml_path):
matched_nodes += 1
self.logger.debug(
"Processor::get_nodes: Relaying required node <{}>:".
format(type(node)))
self.logger.debug(node)
yield node
if matched_nodes < 1:
raise YAMLPathException(
"Required YAML Path does not match any nodes",
str(yaml_path))
else:
for node in self._get_optional_nodes(self.data, yaml_path,
default_value):
self.logger.debug(
"Processor::get_nodes: Relaying optional node <{}>:".
format(type(node)))
self.logger.debug(node)
yield node
def test_unknown_search_keyword(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.search_matches(
SearchKeywordTerms(False, None, ""),
{},
YAMLPath("/")
))
assert -1 < str(ex.value).find("Unsupported search keyword")
def test_name_invalid_inversion(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.name(
True,
[],
YAMLPath("/")
))
assert -1 < str(ex.value).find("Inversion is meaningless to ")
def test_name_invalid_param_count(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.name(
False,
["1", "2"],
YAMLPath("/")
))
assert -1 < str(ex.value).find("Invalid parameter count to ")
def test_has_child_invalid_param_count(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.search_matches(
SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, []),
{},
YAMLPath("/")
))
assert -1 < str(ex.value).find("Invalid parameter count to ")
def test_max_missing_aoh_param(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.max(
[{'a': 1},{'a': 2}],
False,
[],
YAMLPath("/")
))
assert -1 < str(ex.value).find("when evaluating an Array-of-Hashes")
def test_min_incorrect_node(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.min(
{'b': 2},
False,
['b'],
YAMLPath("/*[max(b)]")
))
assert -1 < str(ex.value).find("operates against collections of data")
def test_parent_invalid_parameter(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.parent(
{},
False,
["abc"],
YAMLPath("/")
))
assert -1 < str(ex.value).find("Invalid parameter passed to ")
def test_max_missing_hash_param(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.max(
{'a': {'b': 1}, 'c': {'d': 2}},
False,
[],
YAMLPath("/")
))
assert -1 < str(ex.value).find("when comparing Hash/map/dict children")
def test_parent_invalid_step_count(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.parent(
{},
False,
["5"],
YAMLPath("/")
))
assert -1 < str(ex.value).find("higher than the document root")
def test_has_child_invalid_node(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.has_child(
"abc: xyz",
False,
["wwk"],
YAMLPath("")
))
assert -1 < str(ex.value).find("has no child nodes")
def test_min_invalid_array_param(self):
with pytest.raises(YAMLPathException) as ex:
nodes = list(KeywordSearches.min(
[1, 2, 3],
False,
['3'],
YAMLPath("/")
))
assert -1 < str(ex.value).find("when comparing Array/sequence/list elements to one another")
def test_none_data_to_get_nodes_by_path_segment(self, capsys, quiet_logger):
import sys
yamldata = ""
yaml = YAML()
data = yaml.load(yamldata)
processor = Processor(quiet_logger, data)
nodes = list(processor._get_nodes_by_path_segment(data, YAMLPath("abc"), 0))
yaml.dump(data, sys.stdout)
assert -1 == capsys.readouterr().out.find("abc")
def set_value(self, yaml_path: Union[YAMLPath, str], value: Any,
**kwargs) -> None:
"""
Sets the value of zero or more nodes at YAML Path in YAML data.
Parameters:
1. yaml_path (Union[Path, str]) The YAML Path to evaluate
2. value (Any) The value to set
Keyword Parameters:
* mustexist (bool) Indicate whether yaml_path must exist
in data prior to this query (lest an Exception be raised);
default=False
* value_format (YAMLValueFormats) The demarcation or visual
representation to use when writing the data;
default=YAMLValueFormats.DEFAULT
* pathsep (PathSeperators) Forced YAML Path segment seperator; set
only when automatic inference fails;
default = PathSeperators.AUTO
Returns: N/A
Raises:
- `YAMLPathException` when YAML Path is invalid
"""
if self.data is None:
return
mustexist: bool = kwargs.pop("mustexist", False)
value_format: YAMLValueFormats = kwargs.pop("value_format",
YAMLValueFormats.DEFAULT)
pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO)
node: Any = None
if isinstance(yaml_path, str):
yaml_path = YAMLPath(yaml_path, pathsep)
elif pathsep is not PathSeperators.AUTO:
yaml_path.seperator = pathsep
if mustexist:
self.logger.debug(
"Processor::set_value: Seeking required node at {}.".format(
yaml_path))
found_nodes: int = 0
for node in self._get_required_nodes(self.data, yaml_path):
found_nodes += 1
self._update_node(node, value, value_format)
if found_nodes < 1:
raise YAMLPathException("No nodes matched required YAML Path",
str(yaml_path))
else:
self.logger.debug(
"Processor::set_value: Seeking optional node at {}.".format(
yaml_path))
for node in self._get_optional_nodes(self.data, yaml_path, value):
self._update_node(node, value, value_format)
def _find_eyaml_paths(
self,
data: Any,
build_path: str = "") -> Generator[YAMLPath, None, None]:
"""
Find every encrypted value and report each as a YAML Path.
Recursively generates a set of stringified YAML Paths, each entry
leading to an EYAML value within the evaluated YAML data.
Parameters:
1. data (Any) The parsed YAML data to process
2. build_path (str) A YAML Path under construction
Returns: (Generator[Path, None, None]) each YAML Path entry as they
are discovered
Raises: N/A
"""
if isinstance(data, CommentedSeq):
build_path += "["
for idx, ele in enumerate(data):
if hasattr(ele, "anchor") and ele.anchor.value is not None:
tmp_path = build_path + "&" + ele.anchor.value + "]"
else:
tmp_path = build_path + str(idx) + "]"
if self.is_eyaml_value(ele):
yield YAMLPath(tmp_path)
else:
for subpath in self._find_eyaml_paths(ele, tmp_path):
yield subpath
elif isinstance(data, CommentedMap):
if build_path:
build_path += "."
for key, val in data.non_merged_items():
tmp_path = build_path + str(key)
if self.is_eyaml_value(val):
yield YAMLPath(tmp_path)
else:
for subpath in self._find_eyaml_paths(val, tmp_path):
yield subpath
def _find_eyaml_paths(
self, data: Any,
build_path: YAMLPath) -> Generator[YAMLPath, None, None]:
"""
Find every encrypted value and report each as a YAML Path.
Recursively generates a set of stringified YAML Paths, each entry
leading to an EYAML value within the evaluated YAML data.
Parameters:
1. data (Any) The parsed YAML data to process
2. build_path (YAMLPath) A YAML Path under construction
Returns: (Generator[Path, None, None]) each YAML Path entry as they
are discovered
Raises: N/A
"""
if isinstance(data, CommentedSeq):
for idx, ele in enumerate(data):
node_anchor = Anchors.get_node_anchor(ele)
if node_anchor is not None:
escaped_section = YAMLPath.escape_path_section(
node_anchor, PathSeperators.DOT)
tmp_path_segment = f"[&{escaped_section}]"
else:
tmp_path_segment = f"[{idx}]"
tmp_path = build_path + tmp_path_segment
if self.is_eyaml_value(ele):
yield tmp_path
else:
for subpath in self._find_eyaml_paths(ele, tmp_path):
yield subpath
elif isinstance(data, CommentedMap):
for key, val in data.non_merged_items():
tmp_path = build_path + YAMLPath.escape_path_section(
key, PathSeperators.DOT)
if self.is_eyaml_value(val):
yield tmp_path
else:
for subpath in self._find_eyaml_paths(val, tmp_path):
yield subpath
def test_enforce_pathsep(self, quiet_logger):
yamldata = """---
aliases:
- &aliasAnchorOne Anchored Scalar Value
"""
yaml = YAML()
processor = Processor(quiet_logger, yaml.load(yamldata))
yamlpath = YAMLPath("aliases[&aliasAnchorOne]")
for node in processor.get_nodes(yamlpath, pathsep=PathSeperators.FSLASH):
assert unwrap_node_coords(node) == "Anchored Scalar Value"
def test_get_none_data_nodes(self, quiet_logger):
processor = Processor(quiet_logger, None)
yamlpath = YAMLPath("abc")
matches = 0
for node in processor.get_nodes(yamlpath, mustexist=False):
matches += 1
for node in processor.get_nodes(yamlpath, mustexist=True):
matches += 1
for node in processor._get_required_nodes(None, yamlpath):
matches += 1
assert matches == 0
def compare_to(self, document: Any) -> None:
"""
Perform the diff calculation.
Parameers:
1. document (Any) The document to compare against
Returns: N/A
"""
self._diffs.clear()
self.config.prepare(document)
self._diff_between(YAMLPath(), self._data, document)
def test_non_int_array_index_error(self, quiet_logger):
from collections import deque
yamldata = """---
- 1
"""
yaml = YAML()
data = yaml.load(yamldata)
path = YAMLPath("[0]")
processor = Processor(quiet_logger, data)
strp = str(path)
path._escaped = deque([
(PathSegmentTypes.INDEX, "0F"),
])
path._unescaped = deque([
(PathSegmentTypes.INDEX, "0F"),
])
with pytest.raises(YAMLPathException) as ex:
nodes = list(processor._get_nodes_by_index(data, path, 0))
assert -1 < str(ex.value).find("is not an integer array index")
def test_get_nodes_by_unknown_path_segment_error(self, quiet_logger):
from collections import deque
from enum import Enum
from yamlpath.enums import PathSegmentTypes
names = [m.name for m in PathSegmentTypes] + ['DNF']
PathSegmentTypes = Enum('PathSegmentTypes', names)
yamldata = """---
key: value
"""
yaml = YAML()
data = yaml.load(yamldata)
processor = Processor(quiet_logger, data)
path = YAMLPath("abc")
stringified = str(path) # Force Path to parse
path._escaped = deque([
(PathSegmentTypes.DNF, "abc"),
])
with pytest.raises(NotImplementedError):
nodes = list(processor._get_nodes_by_path_segment(data, path, 0))
def find_eyaml_paths(self) -> Generator[YAMLPath, None, None]:
"""
Find every encrypted value and reports its YAML Path.
Parameters: N/A
Returns: (Generator[Path, None, None]) each YAML Path entry as they
are discovered
Raises: N/A
"""
# Initiate the scan from the data root
for path in self._find_eyaml_paths(self.data, YAMLPath()):
yield path
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
yaml_path = YAMLPath(args.query, pathsep=args.pathsep)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
(yaml_data, doc_loaded) = Parsers.get_yaml_data(
yaml, log, args.yaml_file if args.yaml_file else "-")
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
# Seek the queried value(s)
discovered_nodes = []
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_eyaml_values(yaml_path, mustexist=True):
log.debug("Got node from {}:".format(yaml_path),
data=node,
prefix="yaml_get::main: ")
discovered_nodes.append(NodeCoords.unwrap_node_coords(node))
except YAMLPathException as ex:
log.critical(ex, 1)
except EYAMLCommandException as ex:
log.critical(ex, 2)
try:
for node in discovered_nodes:
if isinstance(node, (dict, list, CommentedSet)):
print(json.dumps(Parsers.jsonify_yaml_data(node)))
else:
if node is None:
node = "\x00"
print("{}".format(str(node).replace("\n", r"\n")))
except RecursionError:
log.critical(
"The YAML data contains an infinitely recursing YAML Alias!", 1)
