def test_set_merge_mode_ini_rule_overrides_cli(self, quiet_logger, tmp_path_factory, cli, ini_default, ini_rule, mode): config_file = create_temp_yaml_file( tmp_path_factory, """ [defaults] sets = {} [rules] /hash/merge_targets/subset = {} """.format(ini_default, ini_rule)) lhs_yaml_file = create_temp_yaml_file( tmp_path_factory, """--- hash: lhs_exclusive: lhs value 1 merge_targets: subkey: lhs value 2 subset: ? one ? two """) lhs_yaml = get_yaml_editor() (lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger, lhs_yaml_file) mc = MergerConfig(quiet_logger, SimpleNamespace(config=config_file, sets=cli)) mc.prepare(lhs_data) node = lhs_data["hash"]["merge_targets"]["subset"] parent = lhs_data["hash"]["merge_targets"] parentref = "subset" assert mc.set_merge_mode(NodeCoords(node, parent, parentref)) == mode
def test_aoh_merge_key_default(self, quiet_logger, tmp_path_factory): lhs_yaml_file = create_temp_yaml_file( tmp_path_factory, """--- hash: lhs_exclusive: lhs value 1 merge_targets: subkey: lhs value 2 subarray: - one - two array_of_hashes: - name: LHS Record 1 id: 1 prop: LHS value AoH 1 - name: LHS Record 2 id: 2 prop: LHS value AoH 2 """) lhs_yaml = get_yaml_editor() (lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger, lhs_yaml_file) mc = MergerConfig(quiet_logger, SimpleNamespace()) mc.prepare(lhs_data) node = lhs_data["array_of_hashes"] parent = lhs_data parentref = "array_of_hashes" record = node[0] assert mc.aoh_merge_key(NodeCoords(node, parent, parentref), record) == "name"
def test_warn_when_rules_matches_zero_nodes(self, capsys, info_warn_logger, tmp_path_factory): config_file = create_temp_yaml_file( tmp_path_factory, """ [rules] /does_not_exist = left /array_of_hashes[name = "Does Not Compute"] = right """) lhs_yaml_file = create_temp_yaml_file( tmp_path_factory, """--- hash: lhs_exclusive: lhs value 1 diff_targets: subkey: lhs value 2 subarray: - one - two array_of_hashes: - name: LHS Record 1 id: 1 prop: LHS value AoH 1 - name: LHS Record 2 id: 2 prop: LHS value AoH 2 """) lhs_yaml = get_yaml_editor() lhs_data = get_yaml_data(lhs_yaml, info_warn_logger, lhs_yaml_file) mc = DifferConfig(info_warn_logger, SimpleNamespace(config=config_file)) mc.prepare(lhs_data) console = capsys.readouterr() assert "YAML Path matches no nodes" in console.out
def test_aoh_diff_key_default(self, quiet_logger, tmp_path_factory): lhs_yaml_file = create_temp_yaml_file( tmp_path_factory, """--- hash: lhs_exclusive: lhs value 1 diff_targets: subkey: lhs value 2 subarray: - one - two array_of_hashes: - name: LHS Record 1 id: 1 prop: LHS value AoH 1 - name: LHS Record 2 id: 2 prop: LHS value AoH 2 """) lhs_yaml = get_yaml_editor() (lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger, lhs_yaml_file) mc = DifferConfig(quiet_logger, SimpleNamespace()) mc.prepare(lhs_data) parent = lhs_data["array_of_hashes"] parentref = 0 node = parent[parentref] nc = NodeCoords(node, parent, parentref) (key_attr, is_user_defined) = mc.aoh_diff_key(nc) assert key_attr == "name" and is_user_defined == False
def test_yield_map_children_direct(self, tmp_path_factory, quiet_logger, include_aliases, assertions): from yamlpath.enums import PathSeperators, PathSearchMethods from yamlpath.path import SearchTerms from yamlpath.func import get_yaml_data, get_yaml_editor from yamlpath.commands.yaml_paths import yield_children from itertools import zip_longest content = """--- aliases: - &aValue val2 hash: key1: val1 key2: *aValue key3: val3 """ processor = get_yaml_editor() yaml_file = create_temp_yaml_file(tmp_path_factory, content) (yaml_data, doc_loaded) = get_yaml_data(processor, quiet_logger, yaml_file) seen_anchors = [] results = [] for assertion, path in zip_longest( assertions, yield_children(quiet_logger, yaml_data, SearchTerms(False, PathSearchMethods.EQUALS, "*", "anchor"), PathSeperators.FSLASH, "", seen_anchors, search_anchors=True, include_value_aliases=include_aliases)): assert assertion == str(path)
def test_yield_seq_children_direct(self, tmp_path_factory, quiet_logger): from yamlpath.enums import PathSeperators, PathSearchMethods from yamlpath.path import SearchTerms from yamlpath.func import get_yaml_data, get_yaml_editor from yamlpath.commands.yaml_paths import yield_children from itertools import zip_longest content = """--- - &value Test value - value - *value """ processor = get_yaml_editor() yaml_file = create_temp_yaml_file(tmp_path_factory, content) (yaml_data, doc_loaded) = get_yaml_data(processor, quiet_logger, yaml_file) seen_anchors = [] assertions = ["/&value", "/[1]"] results = [] for assertion, path in zip_longest( assertions, yield_children(quiet_logger, yaml_data, SearchTerms(False, PathSearchMethods.EQUALS, "*", "value"), PathSeperators.FSLASH, "", seen_anchors, search_anchors=True, include_aliases=False)): assert assertion == str(path)
def test_yield_raw_children_direct(self, tmp_path_factory, quiet_logger): from yamlpath.enums import PathSeperators, PathSearchMethods from yamlpath.path import SearchTerms from yamlpath.func import get_yaml_data, get_yaml_editor from yamlpath.commands.yaml_paths import yield_children from itertools import zip_longest content = """some raw text value """ processor = get_yaml_editor() yaml_file = create_temp_yaml_file(tmp_path_factory, content) yaml_data = get_yaml_data(processor, quiet_logger, yaml_file) seen_anchors = [] assertions = ["/"] results = [] for assertion, path in zip_longest( assertions, yield_children(quiet_logger, yaml_data, SearchTerms(False, PathSearchMethods.STARTS_WITH, "*", "some"), PathSeperators.FSLASH, "", seen_anchors, search_anchors=False, include_key_aliases=False, include_value_aliases=False)): assert assertion == str(path)
def test_get_yaml_data_parser_error(self, capsys, quiet_logger, imparsible_yaml_file): yp = get_yaml_editor() (yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger, imparsible_yaml_file) assert doc_loaded == False captured = capsys.readouterr() assert -1 < captured.err.find("YAML parsing error")
def test_get_yaml_data_syntax_error(self, capsys, quiet_logger, tmp_path_factory, badsyntax_yaml_file): yp = get_yaml_editor() (yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger, badsyntax_yaml_file) assert doc_loaded == False captured = capsys.readouterr() assert -1 < captured.err.find("YAML syntax error")
def test_get_yaml_data_composition_error(self, capsys, quiet_logger, badcmp_yaml_file): yp = get_yaml_editor() (yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger, badcmp_yaml_file) assert doc_loaded == False captured = capsys.readouterr() assert -1 < captured.err.find("YAML composition error")
def test_get_yaml_data_filenotfound_error( self, capsys, quiet_logger, force_ruamel_load_keyboardinterrupt): yp = get_yaml_editor() (yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger, "no-such.file") assert doc_loaded == False captured = capsys.readouterr() assert -1 < captured.err.find("File not found")
def test_get_yaml_multidoc_data_filenotfound_error( self, capsys, quiet_logger, force_ruamel_load_keyboardinterrupt): yp = get_yaml_editor() docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, "no-such.file"): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("File not found")
def test_get_yaml_data_duplicatekey_error(self, capsys, quiet_logger, tmp_path_factory): yp = get_yaml_editor() content = """--- key: value1 key: value2 """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) assert None == get_yaml_data(yp, quiet_logger, yaml_file) captured = capsys.readouterr() assert -1 < captured.err.find("Duplicate Hash key detected")
def test_get_yaml_multidoc_data_composition_error(self, capsys, quiet_logger, badcmp_yaml_file): yp = get_yaml_editor() docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, badcmp_yaml_file): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("YAML composition error")
def test_get_yaml_data_construction_error(self, capsys, quiet_logger, tmp_path_factory): yp = get_yaml_editor() content = """--- missing: <<: """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) assert None == get_yaml_data(yp, quiet_logger, yaml_file) captured = capsys.readouterr() assert -1 < captured.err.find("YAML construction error")
def test_get_yaml_data_keyboardinterrupt_error( self, capsys, quiet_logger, tmp_path_factory, force_ruamel_load_keyboardinterrupt): yp = get_yaml_editor() content = """--- no: '' """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) assert None == get_yaml_data(yp, quiet_logger, yaml_file) captured = capsys.readouterr() assert -1 < captured.err.find("keyboard interrupt")
def test_get_yaml_data_duplicateanchor_error(self, capsys, quiet_logger, tmp_path_factory): yp = get_yaml_editor() content = """--- aliases: - &anchor value1 - &anchor value2 """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) assert None == get_yaml_data(yp, quiet_logger, yaml_file) captured = capsys.readouterr() assert -1 < captured.err.find("Duplicate YAML Anchor detected")
def test_get_yaml_multidoc_data_syntax_error(self, capsys, quiet_logger, tmp_path_factory, badsyntax_yaml_file): yp = get_yaml_editor() docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, badsyntax_yaml_file): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("YAML syntax error")
def test_get_yaml_multidoc_data_keyboardinterrupt_error( self, capsys, quiet_logger, tmp_path_factory, force_ruamel_load_keyboardinterrupt): yp = get_yaml_editor() content = """--- no: '' """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("keyboard interrupt")
def test_get_yaml_multidoc_data_construction_error(self, capsys, quiet_logger, tmp_path_factory): yp = get_yaml_editor() content = """--- missing: <<: """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("YAML construction error")
def test_get_yaml_multidoc_data_duplicateanchor_error( self, capsys, quiet_logger, tmp_path_factory): yp = get_yaml_editor() content = """--- aliases: - &anchor value1 - &anchor value2 """ yaml_file = create_temp_yaml_file(tmp_path_factory, content) docs_yielded = 0 for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file): if not doc and not isinstance(doc, bool): docs_yielded += 1 assert docs_yielded == 0 captured = capsys.readouterr() assert -1 < captured.err.find("Duplicate YAML Anchor detected")
def test_array_diff_mode_ini_rule_overrides_cli(self, quiet_logger, tmp_path_factory, cli, ini_default, ini_rule, mode): config_file = create_temp_yaml_file( tmp_path_factory, """ [defaults] arrays = {} [rules] /hash/diff_targets/subarray = {} """.format(ini_default, ini_rule)) lhs_yaml_file = create_temp_yaml_file( tmp_path_factory, """--- hash: lhs_exclusive: lhs value 1 diff_targets: subkey: lhs value 2 subarray: - one - two array_of_hashes: - name: LHS Record 1 id: 1 prop: LHS value AoH 1 - name: LHS Record 2 id: 2 prop: LHS value AoH 2 """) lhs_yaml = get_yaml_editor() (lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger, lhs_yaml_file) mc = DifferConfig(quiet_logger, SimpleNamespace(config=config_file, arrays=cli)) mc.prepare(lhs_data) node = lhs_data["hash"]["diff_targets"]["subarray"] parent = lhs_data["hash"]["diff_targets"] parentref = "subarray" assert mc.array_diff_mode(NodeCoords(node, parent, parentref)) == mode
def main(): """Main code.""" # Process any command-line arguments args = processcli() log = ConsolePrinter(args) validateargs(args, log) search_values = True search_keys = False include_key_aliases = False include_value_aliases = False if args.onlykeynames: search_values = False search_keys = True elif args.keynames: search_keys = True if args.include_aliases is IncludeAliases.INCLUDE_ALL_ALIASES: include_key_aliases = True include_value_aliases = True elif args.include_aliases is IncludeAliases.INCLUDE_KEY_ALIASES: include_key_aliases = True elif args.include_aliases is IncludeAliases.INCLUDE_VALUE_ALIASES: include_value_aliases = True # Prepare the YAML processor yaml = get_yaml_editor() processor = EYAMLProcessor(log, None, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) # Process the input file(s) exit_state = 0 # pylint: disable=too-many-nested-blocks for yaml_file in args.yaml_files: # Try to open the file yaml_data = get_yaml_data(yaml, log, yaml_file) if yaml_data is None: # An error message has already been logged exit_state = 3 continue # Process all searches processor.data = yaml_data yaml_paths = [] for expression in args.search: exterm = get_search_term(log, expression) log.debug(("yaml_paths::main:" + "converting search expression '{}' into '{}'").format( expression, exterm)) if exterm is None: exit_state = 1 continue for result in search_for_paths( log, processor, yaml_data, exterm, args.pathsep, search_values=search_values, search_keys=search_keys, search_anchors=args.refnames, include_key_aliases=include_key_aliases, include_value_aliases=include_value_aliases, decrypt_eyaml=args.decrypt, expand_children=args.expand): # Record only unique results add_entry = True for entry in yaml_paths: if str(result) == str(entry[1]): add_entry = False break if add_entry: yaml_paths.append((expression, result)) if not yaml_paths: # Nothing further to do when there are no results continue if args.except_expression: for expression in args.except_expression: exterm = get_search_term(log, expression) log.debug( ("yaml_paths::main:" + "converted except expression '{}' into '{}'").format( expression, exterm)) if exterm is None: exit_state = 1 continue for result in search_for_paths( log, processor, yaml_data, exterm, args.pathsep, search_values=search_values, search_keys=search_keys, search_anchors=args.refnames, include_key_aliases=include_key_aliases, include_value_aliases=include_value_aliases, decrypt_eyaml=args.decrypt, expand_children=args.expand): for entry in yaml_paths: if str(result) == str(entry[1]): yaml_paths.remove(entry) break # Entries are already unique print_results(args, processor, yaml_file, yaml_paths) exit(exit_state)
def test_get_yaml_editor(self): assert get_yaml_editor()
def test_get_json_editor(self): assert get_yaml_editor(explode_aliases=True)
def main(): """Main code.""" args = processcli() log = ConsolePrinter(args) validateargs(args, log) change_path = YAMLPath(args.change, pathsep=args.pathsep) backup_file = args.yaml_file + ".bak" # Obtain the replacement value if args.value: new_value = args.value elif args.stdin: new_value = ''.join(sys.stdin.readlines()) elif args.file: with open(args.file, 'r') as fhnd: new_value = fhnd.read().rstrip() elif args.random is not None: new_value = ''.join( secrets.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(args.random)) # Prep the YAML parser yaml = get_yaml_editor() # Attempt to open the YAML file; check for parsing errors yaml_data = get_yaml_data(yaml, log, args.yaml_file) if yaml_data is None: # An error message has already been logged exit(1) # Load the present value at the specified YAML Path change_nodes = [] old_format = YAMLValueFormats.DEFAULT processor = EYAMLProcessor(log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) try: for node in processor.get_nodes( change_path, mustexist=(args.mustexist or args.saveto), default_value=("" if new_value else " ")): log.debug('Got "{}" from {}.'.format(node, change_path)) change_nodes.append(node) except YAMLPathException as ex: log.critical(ex, 1) if len(change_nodes) == 1: # When there is exactly one result, its old format can be known. This # is necessary to retain whether the replacement value should be # represented later as a multi-line string when the new value is to be # encrypted. old_format = YAMLValueFormats.from_node(change_nodes[0]) log.debug("Collected nodes:") log.debug(change_nodes) # Check the value(s), if desired if args.check: for node in change_nodes: if processor.is_eyaml_value(node): # Sanity check: If either --publickey or --privatekey were set # then they must both be set in order to decrypt this value. # This is enforced only when the value must be decrypted due to # a --check request. if ((args.publickey and not args.privatekey) or (args.privatekey and not args.publickey)): log.error( "Neither or both private and public EYAML keys must be" + " set when --check is required to decrypt the old" + " value.") exit(1) try: check_value = processor.decrypt_eyaml(node) except EYAMLCommandException as ex: log.critical(ex, 1) else: check_value = node if not args.check == check_value: log.critical( '"{}" does not match the check value.'.format(args.check), 20) # Save the old value, if desired and possible if args.saveto: # Only one can be saved; otherwise it is impossible to meaningfully # convey to the end-user from exactly which other YAML node each saved # value came. if len(change_nodes) > 1: log.critical( "It is impossible to meaningly save more than one matched" + " value. Please omit --saveto or set --change to affect" + " exactly one value.", 1) saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep) log.verbose("Saving the old value to {}.".format(saveto_path)) # Folded EYAML values have their embedded newlines converted to spaces # when read. As such, writing them back out breaks their original # format, despite being properly typed. To restore the original # written form, reverse the conversion, here. old_value = change_nodes[0] if ((old_format is YAMLValueFormats.FOLDED or old_format is YAMLValueFormats.LITERAL) and EYAMLProcessor.is_eyaml_value(old_value)): old_value = old_value.replace(" ", "\n") try: processor.set_value(saveto_path, clone_node(old_value), value_format=old_format) except YAMLPathException as ex: log.critical(ex, 1) # Set the requested value log.verbose("Setting the new value for {}.".format(change_path)) if args.eyamlcrypt: # If the user hasn't specified a format, use the same format as the # value being replaced, if known. format_type = YAMLValueFormats.from_str(args.format) if format_type is YAMLValueFormats.DEFAULT: format_type = old_format output_type = EYAMLOutputFormats.STRING if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]: output_type = EYAMLOutputFormats.BLOCK try: processor.set_eyaml_value(change_path, new_value, output=output_type, mustexist=False) except EYAMLCommandException as ex: log.critical(ex, 2) else: processor.set_value(change_path, new_value, value_format=args.format) # Save a backup of the original file, if requested if args.backup: log.verbose("Saving a backup of {} to {}.".format( args.yaml_file, backup_file)) if exists(backup_file): remove(backup_file) copy2(args.yaml_file, backup_file) # Save the changed file log.verbose("Writing changed data to {}.".format(args.yaml_file)) with open(args.yaml_file, 'w') as yaml_dump: yaml.dump(yaml_data, yaml_dump)
def main(): """Main code.""" # Process any command-line arguments args = processcli() log = ConsolePrinter(args) validateargs(args, log) processor = EYAMLProcessor(log, None, binary=args.eyaml) # Prep the YAML parser yaml = get_yaml_editor() # Process the input file(s) in_file_count = len(args.yaml_files) exit_state = 0 for yaml_file in args.yaml_files: file_changed = False backup_file = yaml_file + ".bak" seen_anchors = [] # Each YAML_FILE must actually be a file if not isfile(yaml_file): log.error("Not a file: {}".format(yaml_file)) exit_state = 2 continue # Don't bother with the file change update when there's only one input # file. if in_file_count > 1: log.info("Processing {}...".format(yaml_file)) # Try to open the file yaml_data = get_yaml_data(yaml, log, yaml_file) if yaml_data is None: # An error message has already been logged exit_state = 3 continue # Process all EYAML values processor.data = yaml_data for yaml_path in processor.find_eyaml_paths(): # Use ::get_nodes() instead of ::get_eyaml_values() here in order # to ignore values that have already been decrypted via their # Anchors. for node in processor.get_nodes(yaml_path): # Ignore values which are Aliases for those already decrypted anchor_name = (node.anchor.value if hasattr(node, "anchor") else None) if anchor_name is not None: if anchor_name in seen_anchors: continue else: seen_anchors.append(anchor_name) log.verbose("Decrypting value(s) at {}.".format(yaml_path)) processor.publickey = args.oldpublickey processor.privatekey = args.oldprivatekey try: txtval = processor.decrypt_eyaml(node) except EYAMLCommandException as ex: log.error(ex) exit_state = 3 continue # Prefer block (folded) values unless the original YAML value # was already a massivly long (string) line. output = "block" if not isinstance(node, FoldedScalarString): output = "string" # Re-encrypt the value with new EYAML keys processor.publickey = args.newpublickey processor.privatekey = args.newprivatekey try: processor.set_eyaml_value(yaml_path, txtval, output=output) except EYAMLCommandException as ex: log.error(ex) exit_state = 3 continue file_changed = True # Save the changes if file_changed: if args.backup: log.verbose("Saving a backup of {} to {}.".format( yaml_file, backup_file)) if exists(backup_file): remove(backup_file) copy2(yaml_file, backup_file) log.verbose("Writing changed data to {}.".format(yaml_file)) with open(yaml_file, 'w') as yaml_dump: yaml.dump(yaml_data, yaml_dump) exit(exit_state)