def _prepare_user_rules(self, proc: Processor, section: str, collector: dict) -> None: """ Identify DOM nodes matching user-defined diff rules. Parameters: 1. proc (Processor) Reference to the DOM Processor. 2. section (str) User-configuration file section defining the diff rules to apply. 3. collector (dict) Storage collector for matching nodes. Returns: N/A """ if self.config is None or not section in self.config: self.log.warning( "User-specified configuration file has no {} section.".format( section)) return for rule_key in self.config[section]: rule_value = self.config[section][rule_key] if "=" in rule_value: # There were at least two = signs on the configuration line conf_line = rule_key + "=" + rule_value delim_pos = conf_line.rfind("=") rule_key = conf_line[0:delim_pos].strip() rule_value = conf_line[delim_pos + 1:].strip() self.log.debug( "DifferConfig::_prepare_user_rules: Reconstituted" " configuration line '{}' to extract adjusted key '{}'" " with value '{}'".format(conf_line, rule_key, rule_value)) rule_path = YAMLPath(rule_key) yaml_path = YAMLPath(rule_path) self.log.debug( "DifferConfig::_prepare_user_rules: Matching '{}' nodes to" " YAML Path '{}' from key, {}.".format(section, yaml_path, rule_key)) try: for node_coord in proc.get_nodes(yaml_path, mustexist=True): self.log.debug( "Node will have comparisons rule, {}:".format( rule_value), prefix="DifferConfig::_prepare_user_rules: ", data=node_coord.node) collector[node_coord] = rule_value except YAMLPathException: self.log.warning("{} YAML Path matches no nodes: {}".format( section, yaml_path)) self.log.debug("Matched rules to nodes:", prefix="DifferConfig::_prepare_user_rules: ") for node_coord, diff_rule in collector.items(): self.log.debug("... RULE: {}".format(diff_rule), prefix="DifferConfig::_prepare_user_rules: ") self.log.debug("... NODE:", data=node_coord, prefix="DifferConfig::_prepare_user_rules: ")
def test_seperator_change(self): # IMPORTANT: The YAML Path is only lazily parsed! This means parsing # ONLY happens when the path is in some way used. Casting it to string # qualifies as one form of use, so this test will instigate parsing via # stringification. THIS MATTERS WHEN YOUR INTENTION IS TO **CHANGE** # THE PATH SEPERATOR! So, if an original path uses dot-notation and # you wish to change it to forward-slash-notation, you must first cause # the original to become parsed, AND THEN change the seperator. testpath = YAMLPath("abc.def") dotted = str(testpath) testpath.seperator = PathSeperators.FSLASH assert "/abc/def" == str(testpath) != dotted
def get_insertion_point(self) -> YAMLPath: """ Get the YAML Path at which merging shall be performed. Parameters: N/A Returns: (YAMLPath) User-specified point(s) within the document where the RHS document is directed to be merged-in. """ if hasattr(self.args, "mergeat"): return YAMLPath(self.args.mergeat) return YAMLPath("/")
def _purge_document(self, path: YAMLPath, data: Any) -> None: """ Record changes necessary to delete every node in the document. Parameters: 1. path (YAMLPath) YAML Path to the document element under evaluation 2. data (Any) The DOM element under evaluation Returns: N/A """ if isinstance(data, CommentedMap): lhs_iteration = -1 for key, val in data.items(): lhs_iteration += 1 next_path = (path + YAMLPath.escape_path_section(key, path.seperator)) self._diffs.append( DiffEntry(DiffActions.DELETE, next_path, val, None, lhs_parent=data, lhs_iteration=lhs_iteration)) elif isinstance(data, CommentedSeq): for idx, ele in enumerate(data): next_path = path + "[{}]".format(idx) self._diffs.append( DiffEntry(DiffActions.DELETE, next_path, ele, None, lhs_parent=data, lhs_iteration=idx)) elif isinstance(data, CommentedSet): for idx, ele in enumerate(data): next_path = (path + YAMLPath.escape_path_section(ele, path.seperator)) self._diffs.append( DiffEntry(DiffActions.DELETE, next_path, ele, None, lhs_parent=data, lhs_iteration=idx)) else: if data is not None: self._diffs.append( DiffEntry(DiffActions.DELETE, path, data, None))
def get_search_term(logger: ConsolePrinter, expression: str) -> Optional[SearchTerms]: """ Attempts to cast a search expression into a SearchTerms instance. Returns None on failure. """ # The leading character must be a known search operator check_operator = expression[0] if expression else "" if not (PathSearchMethods.is_operator(check_operator) or check_operator == '!'): logger.error(("Invalid search expression, '{}'. The first symbol of" + " every search expression must be one of: {}").format( expression, ", ".join(PathSearchMethods.get_operators()))) return None if not len(expression) > 1: # Empty expressions do nothing logger.error( "An EXPRESSION with only a search operator has no effect, '{}'.". format(expression)) return None try: exterm = create_searchterms_from_pathattributes( YAMLPath("[*{}]".format(expression)).escaped[0][1]) except YAMLPathException as ex: logger.error(("Invalid search expression, '{}', due to: {}").format( expression, ex)) return None return exterm
def test_key_anchor_changes(self, quiet_logger, yamlpath, value, tally, mustexist, vformat, pathsep): yamldata = """--- anchorKeys: &keyOne aliasOne: 11A1 &keyTwo aliasTwo: 22B2 &recursiveAnchorKey subjectKey: *recursiveAnchorKey hash: *keyOne : subval: 1.1 *keyTwo : subval: 2.2 *recursiveAnchorKey : subval: 3.3 """ yaml = YAML() data = yaml.load(yamldata) processor = Processor(quiet_logger, data) yamlpath = YAMLPath(yamlpath) processor.set_value(yamlpath, value, mustexist=mustexist, value_format=vformat, pathsep=pathsep) matchtally = 0 for node in processor.get_nodes(yamlpath): assert unwrap_node_coords(node) == value matchtally += 1 assert matchtally == tally
def test_get_every_data_type(self, quiet_logger): # Contributed by https://github.com/AndydeCleyre yamldata = """--- intthing: 6 floatthing: 6.8 yesthing: yes nothing: no truething: true falsething: false nullthing: null nothingthing: emptystring: "" nullstring: "null" """ results = [6, 6.8, "yes", "no", True, False, None, None, "", "null"] yaml = YAML() data = yaml.load(yamldata) processor = Processor(quiet_logger, data) yamlpath = YAMLPath("*") match_index = 0 for node in processor.get_nodes(yamlpath): assert unwrap_node_coords(node) == results[match_index] match_index += 1
def get_nodes(self, yaml_path: Union[YAMLPath, str], **kwargs: Any) -> Generator[Any, None, None]: """ Retrieves zero or more node at YAML Path in YAML data. Parameters: 1. yaml_path (Union[Path, str]) The YAML Path to evaluate Keyword Parameters: * mustexist (bool) Indicate whether yaml_path must exist in data prior to this query (lest an Exception be raised); default=False * default_value (Any) The value to set at yaml_path should it not already exist in data and mustexist is False; default=None * pathsep (PathSeperators) Forced YAML Path segment seperator; set only when automatic inference fails; default = PathSeperators.AUTO Returns: (Generator) The requested YAML nodes as they are matched Raises: - `YAMLPathException` when YAML Path is invalid """ mustexist: bool = kwargs.pop("mustexist", False) default_value: Any = kwargs.pop("default_value", None) pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) node: Any = None if self.data is None: return if isinstance(yaml_path, str): yaml_path = YAMLPath(yaml_path, pathsep) elif pathsep is not PathSeperators.AUTO: yaml_path.seperator = pathsep if mustexist: matched_nodes: int = 0 for node in self._get_required_nodes(self.data, yaml_path): matched_nodes += 1 self.logger.debug( "Processor::get_nodes: Relaying required node <{}>:". format(type(node))) self.logger.debug(node) yield node if matched_nodes < 1: raise YAMLPathException( "Required YAML Path does not match any nodes", str(yaml_path)) else: for node in self._get_optional_nodes(self.data, yaml_path, default_value): self.logger.debug( "Processor::get_nodes: Relaying optional node <{}>:". format(type(node))) self.logger.debug(node) yield node
def test_unknown_search_keyword(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.search_matches( SearchKeywordTerms(False, None, ""), {}, YAMLPath("/") )) assert -1 < str(ex.value).find("Unsupported search keyword")
def test_name_invalid_inversion(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.name( True, [], YAMLPath("/") )) assert -1 < str(ex.value).find("Inversion is meaningless to ")
def test_name_invalid_param_count(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.name( False, ["1", "2"], YAMLPath("/") )) assert -1 < str(ex.value).find("Invalid parameter count to ")
def test_has_child_invalid_param_count(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.search_matches( SearchKeywordTerms(False, PathSearchKeywords.HAS_CHILD, []), {}, YAMLPath("/") )) assert -1 < str(ex.value).find("Invalid parameter count to ")
def test_max_missing_aoh_param(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.max( [{'a': 1},{'a': 2}], False, [], YAMLPath("/") )) assert -1 < str(ex.value).find("when evaluating an Array-of-Hashes")
def test_min_incorrect_node(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.min( {'b': 2}, False, ['b'], YAMLPath("/*[max(b)]") )) assert -1 < str(ex.value).find("operates against collections of data")
def test_parent_invalid_parameter(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.parent( {}, False, ["abc"], YAMLPath("/") )) assert -1 < str(ex.value).find("Invalid parameter passed to ")
def test_max_missing_hash_param(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.max( {'a': {'b': 1}, 'c': {'d': 2}}, False, [], YAMLPath("/") )) assert -1 < str(ex.value).find("when comparing Hash/map/dict children")
def test_parent_invalid_step_count(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.parent( {}, False, ["5"], YAMLPath("/") )) assert -1 < str(ex.value).find("higher than the document root")
def test_has_child_invalid_node(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.has_child( "abc: xyz", False, ["wwk"], YAMLPath("") )) assert -1 < str(ex.value).find("has no child nodes")
def test_min_invalid_array_param(self): with pytest.raises(YAMLPathException) as ex: nodes = list(KeywordSearches.min( [1, 2, 3], False, ['3'], YAMLPath("/") )) assert -1 < str(ex.value).find("when comparing Array/sequence/list elements to one another")
def test_none_data_to_get_nodes_by_path_segment(self, capsys, quiet_logger): import sys yamldata = "" yaml = YAML() data = yaml.load(yamldata) processor = Processor(quiet_logger, data) nodes = list(processor._get_nodes_by_path_segment(data, YAMLPath("abc"), 0)) yaml.dump(data, sys.stdout) assert -1 == capsys.readouterr().out.find("abc")
def set_value(self, yaml_path: Union[YAMLPath, str], value: Any, **kwargs) -> None: """ Sets the value of zero or more nodes at YAML Path in YAML data. Parameters: 1. yaml_path (Union[Path, str]) The YAML Path to evaluate 2. value (Any) The value to set Keyword Parameters: * mustexist (bool) Indicate whether yaml_path must exist in data prior to this query (lest an Exception be raised); default=False * value_format (YAMLValueFormats) The demarcation or visual representation to use when writing the data; default=YAMLValueFormats.DEFAULT * pathsep (PathSeperators) Forced YAML Path segment seperator; set only when automatic inference fails; default = PathSeperators.AUTO Returns: N/A Raises: - `YAMLPathException` when YAML Path is invalid """ if self.data is None: return mustexist: bool = kwargs.pop("mustexist", False) value_format: YAMLValueFormats = kwargs.pop("value_format", YAMLValueFormats.DEFAULT) pathsep: PathSeperators = kwargs.pop("pathsep", PathSeperators.AUTO) node: Any = None if isinstance(yaml_path, str): yaml_path = YAMLPath(yaml_path, pathsep) elif pathsep is not PathSeperators.AUTO: yaml_path.seperator = pathsep if mustexist: self.logger.debug( "Processor::set_value: Seeking required node at {}.".format( yaml_path)) found_nodes: int = 0 for node in self._get_required_nodes(self.data, yaml_path): found_nodes += 1 self._update_node(node, value, value_format) if found_nodes < 1: raise YAMLPathException("No nodes matched required YAML Path", str(yaml_path)) else: self.logger.debug( "Processor::set_value: Seeking optional node at {}.".format( yaml_path)) for node in self._get_optional_nodes(self.data, yaml_path, value): self._update_node(node, value, value_format)
def _find_eyaml_paths( self, data: Any, build_path: str = "") -> Generator[YAMLPath, None, None]: """ Find every encrypted value and report each as a YAML Path. Recursively generates a set of stringified YAML Paths, each entry leading to an EYAML value within the evaluated YAML data. Parameters: 1. data (Any) The parsed YAML data to process 2. build_path (str) A YAML Path under construction Returns: (Generator[Path, None, None]) each YAML Path entry as they are discovered Raises: N/A """ if isinstance(data, CommentedSeq): build_path += "[" for idx, ele in enumerate(data): if hasattr(ele, "anchor") and ele.anchor.value is not None: tmp_path = build_path + "&" + ele.anchor.value + "]" else: tmp_path = build_path + str(idx) + "]" if self.is_eyaml_value(ele): yield YAMLPath(tmp_path) else: for subpath in self._find_eyaml_paths(ele, tmp_path): yield subpath elif isinstance(data, CommentedMap): if build_path: build_path += "." for key, val in data.non_merged_items(): tmp_path = build_path + str(key) if self.is_eyaml_value(val): yield YAMLPath(tmp_path) else: for subpath in self._find_eyaml_paths(val, tmp_path): yield subpath
def _find_eyaml_paths( self, data: Any, build_path: YAMLPath) -> Generator[YAMLPath, None, None]: """ Find every encrypted value and report each as a YAML Path. Recursively generates a set of stringified YAML Paths, each entry leading to an EYAML value within the evaluated YAML data. Parameters: 1. data (Any) The parsed YAML data to process 2. build_path (YAMLPath) A YAML Path under construction Returns: (Generator[Path, None, None]) each YAML Path entry as they are discovered Raises: N/A """ if isinstance(data, CommentedSeq): for idx, ele in enumerate(data): node_anchor = Anchors.get_node_anchor(ele) if node_anchor is not None: escaped_section = YAMLPath.escape_path_section( node_anchor, PathSeperators.DOT) tmp_path_segment = f"[&{escaped_section}]" else: tmp_path_segment = f"[{idx}]" tmp_path = build_path + tmp_path_segment if self.is_eyaml_value(ele): yield tmp_path else: for subpath in self._find_eyaml_paths(ele, tmp_path): yield subpath elif isinstance(data, CommentedMap): for key, val in data.non_merged_items(): tmp_path = build_path + YAMLPath.escape_path_section( key, PathSeperators.DOT) if self.is_eyaml_value(val): yield tmp_path else: for subpath in self._find_eyaml_paths(val, tmp_path): yield subpath
def test_enforce_pathsep(self, quiet_logger): yamldata = """--- aliases: - &aliasAnchorOne Anchored Scalar Value """ yaml = YAML() processor = Processor(quiet_logger, yaml.load(yamldata)) yamlpath = YAMLPath("aliases[&aliasAnchorOne]") for node in processor.get_nodes(yamlpath, pathsep=PathSeperators.FSLASH): assert unwrap_node_coords(node) == "Anchored Scalar Value"
def test_get_none_data_nodes(self, quiet_logger): processor = Processor(quiet_logger, None) yamlpath = YAMLPath("abc") matches = 0 for node in processor.get_nodes(yamlpath, mustexist=False): matches += 1 for node in processor.get_nodes(yamlpath, mustexist=True): matches += 1 for node in processor._get_required_nodes(None, yamlpath): matches += 1 assert matches == 0
def compare_to(self, document: Any) -> None: """ Perform the diff calculation. Parameers: 1. document (Any) The document to compare against Returns: N/A """ self._diffs.clear() self.config.prepare(document) self._diff_between(YAMLPath(), self._data, document)
def test_non_int_array_index_error(self, quiet_logger): from collections import deque yamldata = """--- - 1 """ yaml = YAML() data = yaml.load(yamldata) path = YAMLPath("[0]") processor = Processor(quiet_logger, data) strp = str(path) path._escaped = deque([ (PathSegmentTypes.INDEX, "0F"), ]) path._unescaped = deque([ (PathSegmentTypes.INDEX, "0F"), ]) with pytest.raises(YAMLPathException) as ex: nodes = list(processor._get_nodes_by_index(data, path, 0)) assert -1 < str(ex.value).find("is not an integer array index")
def test_get_nodes_by_unknown_path_segment_error(self, quiet_logger): from collections import deque from enum import Enum from yamlpath.enums import PathSegmentTypes names = [m.name for m in PathSegmentTypes] + ['DNF'] PathSegmentTypes = Enum('PathSegmentTypes', names) yamldata = """--- key: value """ yaml = YAML() data = yaml.load(yamldata) processor = Processor(quiet_logger, data) path = YAMLPath("abc") stringified = str(path) # Force Path to parse path._escaped = deque([ (PathSegmentTypes.DNF, "abc"), ]) with pytest.raises(NotImplementedError): nodes = list(processor._get_nodes_by_path_segment(data, path, 0))
def find_eyaml_paths(self) -> Generator[YAMLPath, None, None]: """ Find every encrypted value and reports its YAML Path. Parameters: N/A Returns: (Generator[Path, None, None]) each YAML Path entry as they are discovered Raises: N/A """ # Initiate the scan from the data root for path in self._find_eyaml_paths(self.data, YAMLPath()): yield path
def main(): """Main code.""" args = processcli() log = ConsolePrinter(args) validateargs(args, log) yaml_path = YAMLPath(args.query, pathsep=args.pathsep) # Prep the YAML parser yaml = Parsers.get_yaml_editor() # Attempt to open the YAML file; check for parsing errors (yaml_data, doc_loaded) = Parsers.get_yaml_data( yaml, log, args.yaml_file if args.yaml_file else "-") if not doc_loaded: # An error message has already been logged sys.exit(1) # Seek the queried value(s) discovered_nodes = [] processor = EYAMLProcessor(log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) try: for node in processor.get_eyaml_values(yaml_path, mustexist=True): log.debug("Got node from {}:".format(yaml_path), data=node, prefix="yaml_get::main: ") discovered_nodes.append(NodeCoords.unwrap_node_coords(node)) except YAMLPathException as ex: log.critical(ex, 1) except EYAMLCommandException as ex: log.critical(ex, 2) try: for node in discovered_nodes: if isinstance(node, (dict, list, CommentedSet)): print(json.dumps(Parsers.jsonify_yaml_data(node))) else: if node is None: node = "\x00" print("{}".format(str(node).replace("\n", r"\n"))) except RecursionError: log.critical( "The YAML data contains an infinitely recursing YAML Alias!", 1)