def test_set_merge_mode_ini_rule_overrides_cli(self, quiet_logger,
tmp_path_factory, cli,
ini_default, ini_rule,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
sets = {}
[rules]
/hash/merge_targets/subset = {}
""".format(ini_default, ini_rule))
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
merge_targets:
subkey: lhs value 2
subset:
? one
? two
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = MergerConfig(quiet_logger,
SimpleNamespace(config=config_file, sets=cli))
mc.prepare(lhs_data)
node = lhs_data["hash"]["merge_targets"]["subset"]
parent = lhs_data["hash"]["merge_targets"]
parentref = "subset"
assert mc.set_merge_mode(NodeCoords(node, parent, parentref)) == mode
def test_aoh_merge_key_default(self, quiet_logger, tmp_path_factory):
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
merge_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = MergerConfig(quiet_logger, SimpleNamespace())
mc.prepare(lhs_data)
node = lhs_data["array_of_hashes"]
parent = lhs_data
parentref = "array_of_hashes"
record = node[0]
assert mc.aoh_merge_key(NodeCoords(node, parent, parentref),
record) == "name"
def test_warn_when_rules_matches_zero_nodes(self, capsys, info_warn_logger,
tmp_path_factory):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[rules]
/does_not_exist = left
/array_of_hashes[name = "Does Not Compute"] = right
""")
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
diff_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
lhs_data = get_yaml_data(lhs_yaml, info_warn_logger, lhs_yaml_file)
mc = DifferConfig(info_warn_logger,
SimpleNamespace(config=config_file))
mc.prepare(lhs_data)
console = capsys.readouterr()
assert "YAML Path matches no nodes" in console.out
def test_aoh_diff_key_default(self, quiet_logger, tmp_path_factory):
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
diff_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = DifferConfig(quiet_logger, SimpleNamespace())
mc.prepare(lhs_data)
parent = lhs_data["array_of_hashes"]
parentref = 0
node = parent[parentref]
nc = NodeCoords(node, parent, parentref)
(key_attr, is_user_defined) = mc.aoh_diff_key(nc)
assert key_attr == "name" and is_user_defined == False
def test_yield_map_children_direct(self, tmp_path_factory, quiet_logger,
include_aliases, assertions):
from yamlpath.enums import PathSeperators, PathSearchMethods
from yamlpath.path import SearchTerms
from yamlpath.func import get_yaml_data, get_yaml_editor
from yamlpath.commands.yaml_paths import yield_children
from itertools import zip_longest
content = """---
aliases:
- &aValue val2
hash:
key1: val1
key2: *aValue
key3: val3
"""
processor = get_yaml_editor()
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
(yaml_data, doc_loaded) = get_yaml_data(processor, quiet_logger,
yaml_file)
seen_anchors = []
results = []
for assertion, path in zip_longest(
assertions,
yield_children(quiet_logger,
yaml_data,
SearchTerms(False, PathSearchMethods.EQUALS,
"*", "anchor"),
PathSeperators.FSLASH,
"",
seen_anchors,
search_anchors=True,
include_value_aliases=include_aliases)):
assert assertion == str(path)
def test_yield_seq_children_direct(self, tmp_path_factory, quiet_logger):
from yamlpath.enums import PathSeperators, PathSearchMethods
from yamlpath.path import SearchTerms
from yamlpath.func import get_yaml_data, get_yaml_editor
from yamlpath.commands.yaml_paths import yield_children
from itertools import zip_longest
content = """---
- &value Test value
- value
- *value
"""
processor = get_yaml_editor()
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
(yaml_data, doc_loaded) = get_yaml_data(processor, quiet_logger,
yaml_file)
seen_anchors = []
assertions = ["/&value", "/[1]"]
results = []
for assertion, path in zip_longest(
assertions,
yield_children(quiet_logger,
yaml_data,
SearchTerms(False, PathSearchMethods.EQUALS,
"*", "value"),
PathSeperators.FSLASH,
"",
seen_anchors,
search_anchors=True,
include_aliases=False)):
assert assertion == str(path)
def test_yield_raw_children_direct(self, tmp_path_factory, quiet_logger):
from yamlpath.enums import PathSeperators, PathSearchMethods
from yamlpath.path import SearchTerms
from yamlpath.func import get_yaml_data, get_yaml_editor
from yamlpath.commands.yaml_paths import yield_children
from itertools import zip_longest
content = """some raw text value
"""
processor = get_yaml_editor()
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
yaml_data = get_yaml_data(processor, quiet_logger, yaml_file)
seen_anchors = []
assertions = ["/"]
results = []
for assertion, path in zip_longest(
assertions,
yield_children(quiet_logger,
yaml_data,
SearchTerms(False,
PathSearchMethods.STARTS_WITH, "*",
"some"),
PathSeperators.FSLASH,
"",
seen_anchors,
search_anchors=False,
include_key_aliases=False,
include_value_aliases=False)):
assert assertion == str(path)
def test_get_yaml_data_parser_error(self, capsys, quiet_logger,
imparsible_yaml_file):
yp = get_yaml_editor()
(yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger,
imparsible_yaml_file)
assert doc_loaded == False
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML parsing error")
def test_get_yaml_data_syntax_error(self, capsys, quiet_logger,
tmp_path_factory, badsyntax_yaml_file):
yp = get_yaml_editor()
(yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger,
badsyntax_yaml_file)
assert doc_loaded == False
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML syntax error")
def test_get_yaml_data_composition_error(self, capsys, quiet_logger,
badcmp_yaml_file):
yp = get_yaml_editor()
(yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger,
badcmp_yaml_file)
assert doc_loaded == False
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML composition error")
def test_get_yaml_data_filenotfound_error(
self, capsys, quiet_logger, force_ruamel_load_keyboardinterrupt):
yp = get_yaml_editor()
(yaml_data, doc_loaded) = get_yaml_data(yp, quiet_logger,
"no-such.file")
assert doc_loaded == False
captured = capsys.readouterr()
assert -1 < captured.err.find("File not found")
def test_get_yaml_multidoc_data_filenotfound_error(
self, capsys, quiet_logger, force_ruamel_load_keyboardinterrupt):
yp = get_yaml_editor()
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger, "no-such.file"):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("File not found")
def test_get_yaml_data_duplicatekey_error(self, capsys, quiet_logger,
tmp_path_factory):
yp = get_yaml_editor()
content = """---
key: value1
key: value2
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
assert None == get_yaml_data(yp, quiet_logger, yaml_file)
captured = capsys.readouterr()
assert -1 < captured.err.find("Duplicate Hash key detected")
def test_get_yaml_multidoc_data_composition_error(self, capsys,
quiet_logger,
badcmp_yaml_file):
yp = get_yaml_editor()
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger, badcmp_yaml_file):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML composition error")
def test_get_yaml_data_construction_error(self, capsys, quiet_logger,
tmp_path_factory):
yp = get_yaml_editor()
content = """---
missing:
<<:
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
assert None == get_yaml_data(yp, quiet_logger, yaml_file)
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML construction error")
def test_get_yaml_data_keyboardinterrupt_error(
self, capsys, quiet_logger, tmp_path_factory,
force_ruamel_load_keyboardinterrupt):
yp = get_yaml_editor()
content = """---
no: ''
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
assert None == get_yaml_data(yp, quiet_logger, yaml_file)
captured = capsys.readouterr()
assert -1 < captured.err.find("keyboard interrupt")
def test_get_yaml_data_duplicateanchor_error(self, capsys, quiet_logger,
tmp_path_factory):
yp = get_yaml_editor()
content = """---
aliases:
- &anchor value1
- &anchor value2
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
assert None == get_yaml_data(yp, quiet_logger, yaml_file)
captured = capsys.readouterr()
assert -1 < captured.err.find("Duplicate YAML Anchor detected")
def test_get_yaml_multidoc_data_syntax_error(self, capsys, quiet_logger,
tmp_path_factory,
badsyntax_yaml_file):
yp = get_yaml_editor()
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger,
badsyntax_yaml_file):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML syntax error")
def test_get_yaml_multidoc_data_keyboardinterrupt_error(
self, capsys, quiet_logger, tmp_path_factory,
force_ruamel_load_keyboardinterrupt):
yp = get_yaml_editor()
content = """---
no: ''
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("keyboard interrupt")
def test_get_yaml_multidoc_data_construction_error(self, capsys,
quiet_logger,
tmp_path_factory):
yp = get_yaml_editor()
content = """---
missing:
<<:
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("YAML construction error")
def test_get_yaml_multidoc_data_duplicateanchor_error(
self, capsys, quiet_logger, tmp_path_factory):
yp = get_yaml_editor()
content = """---
aliases:
- &anchor value1
- &anchor value2
"""
yaml_file = create_temp_yaml_file(tmp_path_factory, content)
docs_yielded = 0
for doc in get_yaml_multidoc_data(yp, quiet_logger, yaml_file):
if not doc and not isinstance(doc, bool):
docs_yielded += 1
assert docs_yielded == 0
captured = capsys.readouterr()
assert -1 < captured.err.find("Duplicate YAML Anchor detected")
def test_array_diff_mode_ini_rule_overrides_cli(self, quiet_logger,
tmp_path_factory, cli,
ini_default, ini_rule,
mode):
config_file = create_temp_yaml_file(
tmp_path_factory, """
[defaults]
arrays = {}
[rules]
/hash/diff_targets/subarray = {}
""".format(ini_default, ini_rule))
lhs_yaml_file = create_temp_yaml_file(
tmp_path_factory, """---
hash:
lhs_exclusive: lhs value 1
diff_targets:
subkey: lhs value 2
subarray:
- one
- two
array_of_hashes:
- name: LHS Record 1
id: 1
prop: LHS value AoH 1
- name: LHS Record 2
id: 2
prop: LHS value AoH 2
""")
lhs_yaml = get_yaml_editor()
(lhs_data, lhs_loaded) = get_yaml_data(lhs_yaml, quiet_logger,
lhs_yaml_file)
mc = DifferConfig(quiet_logger,
SimpleNamespace(config=config_file, arrays=cli))
mc.prepare(lhs_data)
node = lhs_data["hash"]["diff_targets"]["subarray"]
parent = lhs_data["hash"]["diff_targets"]
parentref = "subarray"
assert mc.array_diff_mode(NodeCoords(node, parent, parentref)) == mode
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
search_values = True
search_keys = False
include_key_aliases = False
include_value_aliases = False
if args.onlykeynames:
search_values = False
search_keys = True
elif args.keynames:
search_keys = True
if args.include_aliases is IncludeAliases.INCLUDE_ALL_ALIASES:
include_key_aliases = True
include_value_aliases = True
elif args.include_aliases is IncludeAliases.INCLUDE_KEY_ALIASES:
include_key_aliases = True
elif args.include_aliases is IncludeAliases.INCLUDE_VALUE_ALIASES:
include_value_aliases = True
# Prepare the YAML processor
yaml = get_yaml_editor()
processor = EYAMLProcessor(log,
None,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
# Process the input file(s)
exit_state = 0
# pylint: disable=too-many-nested-blocks
for yaml_file in args.yaml_files:
# Try to open the file
yaml_data = get_yaml_data(yaml, log, yaml_file)
if yaml_data is None:
# An error message has already been logged
exit_state = 3
continue
# Process all searches
processor.data = yaml_data
yaml_paths = []
for expression in args.search:
exterm = get_search_term(log, expression)
log.debug(("yaml_paths::main:" +
"converting search expression '{}' into '{}'").format(
expression, exterm))
if exterm is None:
exit_state = 1
continue
for result in search_for_paths(
log,
processor,
yaml_data,
exterm,
args.pathsep,
search_values=search_values,
search_keys=search_keys,
search_anchors=args.refnames,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=args.decrypt,
expand_children=args.expand):
# Record only unique results
add_entry = True
for entry in yaml_paths:
if str(result) == str(entry[1]):
add_entry = False
break
if add_entry:
yaml_paths.append((expression, result))
if not yaml_paths:
# Nothing further to do when there are no results
continue
if args.except_expression:
for expression in args.except_expression:
exterm = get_search_term(log, expression)
log.debug(
("yaml_paths::main:" +
"converted except expression '{}' into '{}'").format(
expression, exterm))
if exterm is None:
exit_state = 1
continue
for result in search_for_paths(
log,
processor,
yaml_data,
exterm,
args.pathsep,
search_values=search_values,
search_keys=search_keys,
search_anchors=args.refnames,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=args.decrypt,
expand_children=args.expand):
for entry in yaml_paths:
if str(result) == str(entry[1]):
yaml_paths.remove(entry)
break # Entries are already unique
print_results(args, processor, yaml_file, yaml_paths)
exit(exit_state)
def test_get_yaml_editor(self):
assert get_yaml_editor()
def test_get_json_editor(self):
assert get_yaml_editor(explode_aliases=True)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
change_path = YAMLPath(args.change, pathsep=args.pathsep)
backup_file = args.yaml_file + ".bak"
# Obtain the replacement value
if args.value:
new_value = args.value
elif args.stdin:
new_value = ''.join(sys.stdin.readlines())
elif args.file:
with open(args.file, 'r') as fhnd:
new_value = fhnd.read().rstrip()
elif args.random is not None:
new_value = ''.join(
secrets.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(args.random))
# Prep the YAML parser
yaml = get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
yaml_data = get_yaml_data(yaml, log, args.yaml_file)
if yaml_data is None:
# An error message has already been logged
exit(1)
# Load the present value at the specified YAML Path
change_nodes = []
old_format = YAMLValueFormats.DEFAULT
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_nodes(
change_path,
mustexist=(args.mustexist or args.saveto),
default_value=("" if new_value else " ")):
log.debug('Got "{}" from {}.'.format(node, change_path))
change_nodes.append(node)
except YAMLPathException as ex:
log.critical(ex, 1)
if len(change_nodes) == 1:
# When there is exactly one result, its old format can be known. This
# is necessary to retain whether the replacement value should be
# represented later as a multi-line string when the new value is to be
# encrypted.
old_format = YAMLValueFormats.from_node(change_nodes[0])
log.debug("Collected nodes:")
log.debug(change_nodes)
# Check the value(s), if desired
if args.check:
for node in change_nodes:
if processor.is_eyaml_value(node):
# Sanity check: If either --publickey or --privatekey were set
# then they must both be set in order to decrypt this value.
# This is enforced only when the value must be decrypted due to
# a --check request.
if ((args.publickey and not args.privatekey)
or (args.privatekey and not args.publickey)):
log.error(
"Neither or both private and public EYAML keys must be"
+ " set when --check is required to decrypt the old" +
" value.")
exit(1)
try:
check_value = processor.decrypt_eyaml(node)
except EYAMLCommandException as ex:
log.critical(ex, 1)
else:
check_value = node
if not args.check == check_value:
log.critical(
'"{}" does not match the check value.'.format(args.check),
20)
# Save the old value, if desired and possible
if args.saveto:
# Only one can be saved; otherwise it is impossible to meaningfully
# convey to the end-user from exactly which other YAML node each saved
# value came.
if len(change_nodes) > 1:
log.critical(
"It is impossible to meaningly save more than one matched" +
" value. Please omit --saveto or set --change to affect" +
" exactly one value.", 1)
saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep)
log.verbose("Saving the old value to {}.".format(saveto_path))
# Folded EYAML values have their embedded newlines converted to spaces
# when read. As such, writing them back out breaks their original
# format, despite being properly typed. To restore the original
# written form, reverse the conversion, here.
old_value = change_nodes[0]
if ((old_format is YAMLValueFormats.FOLDED
or old_format is YAMLValueFormats.LITERAL)
and EYAMLProcessor.is_eyaml_value(old_value)):
old_value = old_value.replace(" ", "\n")
try:
processor.set_value(saveto_path,
clone_node(old_value),
value_format=old_format)
except YAMLPathException as ex:
log.critical(ex, 1)
# Set the requested value
log.verbose("Setting the new value for {}.".format(change_path))
if args.eyamlcrypt:
# If the user hasn't specified a format, use the same format as the
# value being replaced, if known.
format_type = YAMLValueFormats.from_str(args.format)
if format_type is YAMLValueFormats.DEFAULT:
format_type = old_format
output_type = EYAMLOutputFormats.STRING
if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]:
output_type = EYAMLOutputFormats.BLOCK
try:
processor.set_eyaml_value(change_path,
new_value,
output=output_type,
mustexist=False)
except EYAMLCommandException as ex:
log.critical(ex, 2)
else:
processor.set_value(change_path, new_value, value_format=args.format)
# Save a backup of the original file, if requested
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
args.yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(args.yaml_file, backup_file)
# Save the changed file
log.verbose("Writing changed data to {}.".format(args.yaml_file))
with open(args.yaml_file, 'w') as yaml_dump:
yaml.dump(yaml_data, yaml_dump)
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
processor = EYAMLProcessor(log, None, binary=args.eyaml)
# Prep the YAML parser
yaml = get_yaml_editor()
# Process the input file(s)
in_file_count = len(args.yaml_files)
exit_state = 0
for yaml_file in args.yaml_files:
file_changed = False
backup_file = yaml_file + ".bak"
seen_anchors = []
# Each YAML_FILE must actually be a file
if not isfile(yaml_file):
log.error("Not a file: {}".format(yaml_file))
exit_state = 2
continue
# Don't bother with the file change update when there's only one input
# file.
if in_file_count > 1:
log.info("Processing {}...".format(yaml_file))
# Try to open the file
yaml_data = get_yaml_data(yaml, log, yaml_file)
if yaml_data is None:
# An error message has already been logged
exit_state = 3
continue
# Process all EYAML values
processor.data = yaml_data
for yaml_path in processor.find_eyaml_paths():
# Use ::get_nodes() instead of ::get_eyaml_values() here in order
# to ignore values that have already been decrypted via their
# Anchors.
for node in processor.get_nodes(yaml_path):
# Ignore values which are Aliases for those already decrypted
anchor_name = (node.anchor.value
if hasattr(node, "anchor") else None)
if anchor_name is not None:
if anchor_name in seen_anchors:
continue
else:
seen_anchors.append(anchor_name)
log.verbose("Decrypting value(s) at {}.".format(yaml_path))
processor.publickey = args.oldpublickey
processor.privatekey = args.oldprivatekey
try:
txtval = processor.decrypt_eyaml(node)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
# Prefer block (folded) values unless the original YAML value
# was already a massivly long (string) line.
output = "block"
if not isinstance(node, FoldedScalarString):
output = "string"
# Re-encrypt the value with new EYAML keys
processor.publickey = args.newpublickey
processor.privatekey = args.newprivatekey
try:
processor.set_eyaml_value(yaml_path, txtval, output=output)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
file_changed = True
# Save the changes
if file_changed:
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(yaml_file, backup_file)
log.verbose("Writing changed data to {}.".format(yaml_file))
with open(yaml_file, 'w') as yaml_dump:
yaml.dump(yaml_data, yaml_dump)
exit(exit_state)
