def __init__(
self, logger: ConsolePrinter, lhs: Any, config: MergerConfig
) -> None:
"""
Instantiate this class into an object.
Parameters:
1. logger (ConsolePrinter) Instance of ConsoleWriter or subclass
2. lhs (Any) The prime left-hand-side parsed YAML data
3. config (MergerConfig) User-defined document merging rules
Returns: N/A
Raises: N/A
"""
self.logger: ConsolePrinter = logger
self.config: MergerConfig = config
self.data: Any = lhs
# ryamel.yaml unfortunately tracks comments AFTER each YAML node. As
# such, it is impossible to copy comments from RHS to LHS in any
# sensible way. Trying leads to absurd merge results that are data-
# accurate but comment-insane. This ruamel.yaml design decision forces
# me to simply delete all comments from all merge documents to produce
# a sensible result. That said, enable users to attempt to preserve
# LHS comments.
if not self.config.is_preserving_lhs_comments():
Parsers.delete_all_comments(self.data)
def _parse_yaml(source: str, config_string: bool):
logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False)
log = ConsolePrinter(logging_args)
yaml = Parsers.get_yaml_editor()
# for better backward compatibility with PyYAML (that supports only YAML 1.1) used in the previous
# GitLabForm versions, let's force ruamel.yaml to use YAML version 1.1 by default too
yaml.version = (1, 1)
if config_string:
config_string = textwrap.dedent(source)
verbose("Reading config from the provided string.")
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml,
log,
config_string,
literal=True)
else:
config_path = source
verbose(f"Reading config from file: {config_path}")
(yaml_data,
doc_loaded) = Parsers.get_yaml_data(yaml, log, config_path)
if doc_loaded:
debug("Config parsed successfully as YAML.")
else:
# an error message has already been printed via ConsolePrinter
exit(EXIT_INVALID_INPUT)
return yaml_data
def test_get_yaml_multidoc_data_literally(self, quiet_logger):
serialized_yaml = """---
document: 1st
has: data
...
---
document: 2nd
has: different data
"""
yaml = Parsers.get_yaml_editor()
doc_id = 0
for (data, loaded) in Parsers.get_yaml_multidoc_data(yaml,
quiet_logger,
serialized_yaml,
literal=True):
assert loaded == True
if doc_id == 0:
document = "1st"
has = "data"
else:
document = "2nd"
has = "different data"
doc_id = doc_id + 1
assert data["document"] == document
assert data["has"] == has
def write_output_document(args, log, merger, yaml_editor):
"""Save a backup of the overwrite file, if requested."""
if args.backup:
backup_file = args.overwrite + ".bak"
log.verbose(
"Saving a backup of {} to {}."
.format(args.overwrite, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(args.overwrite, backup_file)
document_is_json = (
merger.prepare_for_dump(yaml_editor, args.output)
is OutputDocTypes.JSON)
if args.output:
with open(args.output, 'w') as out_fhnd:
if document_is_json:
json.dump(Parsers.jsonify_yaml_data(merger.data), out_fhnd)
else:
yaml_editor.dump(merger.data, out_fhnd)
else:
if document_is_json:
json.dump(Parsers.jsonify_yaml_data(merger.data), sys.stdout)
else:
yaml_editor.dump(merger.data, sys.stdout)
def write_output_document(
args: argparse.Namespace, log: ConsolePrinter, yaml_editor: YAML,
docs: List[Merger]
) -> None:
"""Save a backup of the overwrite file, if requested."""
if args.backup:
backup_file = args.overwrite + ".bak"
log.verbose(
"Saving a backup of {} to {}."
.format(args.overwrite, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(args.overwrite, backup_file)
document_is_json = (
docs[0].prepare_for_dump(yaml_editor, args.output)
is OutputDocTypes.JSON)
dumps = []
for doc in docs:
doc.prepare_for_dump(yaml_editor, args.output)
dumps.append(doc.data)
if args.output:
with open(args.output, 'w', encoding='utf-8') as out_fhnd:
if document_is_json:
if len(dumps) > 1:
for dump in dumps:
print(
json.dumps(Parsers.jsonify_yaml_data(dump)),
file=out_fhnd)
else:
json.dump(Parsers.jsonify_yaml_data(dumps[0]), out_fhnd)
else:
if len(dumps) > 1:
yaml_editor.explicit_end = True # type: ignore
yaml_editor.dump_all(dumps, out_fhnd)
else:
yaml_editor.dump(dumps[0], out_fhnd)
else:
if document_is_json:
if len(dumps) > 1:
for dump in dumps:
print(json.dumps(Parsers.jsonify_yaml_data(dump)))
else:
json.dump(Parsers.jsonify_yaml_data(dumps[0]), sys.stdout)
else:
if len(dumps) > 1:
yaml_editor.explicit_end = True # type: ignore
yaml_editor.dump_all(dumps, sys.stdout)
else:
yaml_editor.dump(dumps[0], sys.stdout)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
exit_state = 0
lhs_file = args.yaml_files[0]
rhs_file = args.yaml_files[1]
lhs_yaml = Parsers.get_yaml_editor()
rhs_yaml = Parsers.get_yaml_editor()
(lhs_docs, lhs_loaded) = get_docs(log, lhs_yaml, lhs_file)
(rhs_docs, rhs_loaded) = get_docs(log, rhs_yaml, rhs_file)
lhs_doc_count = len(lhs_docs) if lhs_loaded else 0
rhs_doc_count = len(rhs_docs) if rhs_loaded else 0
lhs_idx_set = (hasattr(args, "left_document_index")
and args.left_document_index is not None)
rhs_idx_set = (hasattr(args, "right_document_index")
and args.right_document_index is not None)
if not (lhs_loaded and rhs_loaded):
# An error message has already been logged
sys.exit(1)
if lhs_doc_count > 1 and not lhs_idx_set:
log.critical(
("--left-document-index|-L must be set; the source contains {}"
" documents.").format(lhs_doc_count), 1)
lhs_index = args.left_document_index if lhs_idx_set else 0
lhs_document = get_doc(log, lhs_docs, lhs_index)
if rhs_doc_count > 1 and not rhs_idx_set:
log.critical(
("--right-document-index|-R must be set; the source contains {}"
" documents.").format(rhs_doc_count), 1)
rhs_index = args.right_document_index if rhs_idx_set else 0
rhs_document = get_doc(log, rhs_docs, rhs_index)
diff = Differ(DifferConfig(log, args),
log,
lhs_document,
ignore_eyaml_values=args.ignore_eyaml_values,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
diff.compare_to(rhs_document)
except EYAMLCommandException as ex:
log.critical(ex, 1)
exit_state = 1 if print_report(log, args, diff) else 0
sys.exit(exit_state)
def prepare_for_dump(
self, yaml_writer: Any, output_file: str = ""
) -> OutputDocTypes:
"""
Prepare this merged document and its writer for final rendering.
This coalesces the YAML writer's settings to, in particular,
distinguish between YAML and JSON. It will also force demarcation of
every String key and value within the document when the output will be
JSON.
Parameters:
1. yaml_writer (ruamel.yaml.YAML) The YAML document writer
Returns: (OutputDocTypes) One of:
* OutputDocTypes.JSON: The document and yaml_writer are JSON format.
* OutputDocTypes.YAML: The document and yaml_writer are YAML format.
"""
# Check whether the user is forcing an output format
doc_format = self.config.get_document_format()
if doc_format is OutputDocTypes.AUTO:
# Identify by file-extension, if it indicates a known type
file_extension = (Path(output_file).suffix.lower()
if output_file else "")
if file_extension in [".json", ".yaml", ".yml"]:
is_flow = file_extension == ".json"
else:
# Check whether the document root is in flow or block format
is_flow = True
if hasattr(self.data, "fa"):
is_flow = self.data.fa.flow_style()
else:
is_flow = doc_format is OutputDocTypes.JSON
if is_flow:
# Dump the document as true JSON and reload it; this automatically
# exlodes all aliases.
xfer_buffer = StringIO()
json.dump(Parsers.jsonify_yaml_data(self.data), xfer_buffer)
xfer_buffer.seek(0)
self.data = yaml_writer.load(xfer_buffer)
# Ensure the writer doesn't emit a YAML Start-of-Document marker
yaml_writer.explicit_start = False
else:
# Ensure block style output
Parsers.set_flow_style(self.data, False)
# When writing YAML, ensure the document start mark is emitted
yaml_writer.explicit_start = True
return OutputDocTypes.JSON if is_flow else OutputDocTypes.YAML
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
# For the remainder of processing, overwrite overwrites output
if args.overwrite:
args.output = args.overwrite
# Merge all input files
yaml_editor = Parsers.get_yaml_editor()
merge_config = MergerConfig(log, args)
exit_state = 0
consumed_stdin = False
mergers: List[Merger] = []
merge_count = 0
for yaml_file in args.yaml_files:
if yaml_file.strip() == '-':
consumed_stdin = True
log.debug(
"yaml_merge::main: Processing file, {}".format(
"STDIN" if yaml_file.strip() == "-" else yaml_file))
if len(mergers) < 1:
(mergers, mergers_loaded) = get_doc_mergers(
log, yaml_editor, merge_config, yaml_file)
if not mergers_loaded:
exit_state = 4
break
else:
# Merge RHS into LHS
exit_state = merge_docs(
log, yaml_editor, merge_config, mergers, yaml_file)
if not exit_state == 0:
break
merge_count += 1
# Check for a waiting STDIN document
if (exit_state == 0
and not consumed_stdin
and not args.nostdin
and not sys.stdin.isatty()
):
exit_state = merge_docs(log, yaml_editor, merge_config, mergers, "-")
merge_count += 1
# When no merges have occurred, check for a single-doc merge request
if (exit_state == 0
and merge_count == 0
and merge_config.get_multidoc_mode() is MultiDocModes.CONDENSE_ALL
):
exit_state = merge_condense_all(log, mergers, [])
# Output the final document
if exit_state == 0:
write_output_document(args, log, yaml_editor, mergers)
sys.exit(exit_state)
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
exit_state = 0
consumed_stdin = False
yaml = Parsers.get_yaml_editor()
for yaml_file in args.yaml_files:
if yaml_file.strip() == '-':
consumed_stdin = True
log.debug("yaml_merge::main: Processing file, {}".format(
"STDIN" if yaml_file.strip() == "-" else yaml_file))
proc_state = process_file(log, yaml, yaml_file)
if proc_state != 0:
exit_state = proc_state
# Check for a waiting STDIN document
if (exit_state == 0 and not consumed_stdin and not args.nostdin
and not sys.stdin.isatty()):
exit_state = process_file(log, yaml, "-")
sys.exit(exit_state)
def merge_multidoc(yaml_file, yaml_editor, log, merger, merger_primed):
"""Merge all documents within a multi-document source."""
exit_state = 0
for (yaml_data, doc_loaded) in Parsers.get_yaml_multidoc_data(
yaml_editor, log, yaml_file
):
if not doc_loaded:
# An error message has already been logged
exit_state = 3
break
try:
if merger_primed:
merger.merge_with(yaml_data)
else:
merger.data = yaml_data
merger_primed = True
except MergeException as mex:
log.error(mex)
exit_state = 6
break
except YAMLPathException as yex:
log.error(yex)
exit_state = 7
break
log.debug("yaml_merge::merge_multidoc: Reporting status, {}."
.format(exit_state))
return exit_state
def test_jsonify_complex_data(self):
tagged_tag = "!tagged"
tagged_value = "tagged value"
tagged_scalar = ry.scalarstring.PlainScalarString(tagged_value)
tagged_node = ry.comments.TaggedScalar(tagged_scalar, tag=tagged_tag)
null_tag = "!null"
null_value = None
null_node = ry.comments.TaggedScalar(None, tag=null_tag)
cdata = ry.comments.CommentedMap({
"tagged":
tagged_node,
"null":
null_node,
"dates":
ry.comments.CommentedSeq(
[dt.date(2020, 10, 31),
dt.date(2020, 11, 3)])
})
jdata = Parsers.jsonify_yaml_data(cdata)
assert jdata["tagged"] == tagged_value
assert jdata["null"] == null_value
assert jdata["dates"][0] == "2020-10-31"
assert jdata["dates"][1] == "2020-11-03"
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
yaml_path = YAMLPath(args.query, pathsep=args.pathsep)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
(yaml_data, doc_loaded) = Parsers.get_yaml_data(
yaml, log, args.yaml_file if args.yaml_file else "-")
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
# Seek the queried value(s)
discovered_nodes = []
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_eyaml_values(yaml_path, mustexist=True):
log.debug("Got node from {}:".format(yaml_path),
data=node,
prefix="yaml_get::main: ")
discovered_nodes.append(NodeCoords.unwrap_node_coords(node))
except YAMLPathException as ex:
log.critical(ex, 1)
except EYAMLCommandException as ex:
log.critical(ex, 2)
try:
for node in discovered_nodes:
if isinstance(node, (dict, list, CommentedSet)):
print(json.dumps(Parsers.jsonify_yaml_data(node)))
else:
if node is None:
node = "\x00"
print("{}".format(str(node).replace("\n", r"\n")))
except RecursionError:
log.critical(
"The YAML data contains an infinitely recursing YAML Alias!", 1)
def _try_load_input_file(args, log, yaml, change_path, new_value):
"""Attempt to load the input data file or abend on error."""
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, args.yaml_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
elif yaml_data is None:
yaml_data = Nodes.build_next_node(change_path, 0, new_value)
return yaml_data
def test_get_yaml_data_literally(self, quiet_logger):
serialized_yaml = """---
hash:
key: value
list:
- ichi
- ni
- san
"""
yaml = Parsers.get_yaml_editor()
(data, loaded) = Parsers.get_yaml_data(
yaml, quiet_logger, serialized_yaml,
literal=True)
assert loaded == True
assert data["hash"]["key"] == "value"
assert data["list"][0] == "ichi"
assert data["list"][1] == "ni"
assert data["list"][2] == "san"
def test_stringify_complex_data_with_dates(self):
cdata = ry.comments.CommentedMap({
"dates":
ry.comments.CommentedSeq(
[dt.date(2020, 10, 31),
dt.date(2020, 11, 3)])
})
sdata = Parsers.stringify_dates(cdata)
assert sdata["dates"][0] == "2020-10-31"
assert sdata["dates"][1] == "2020-11-03"
def _present_data(cls, data: Any, prefix: str) -> str:
"""Stringify data."""
json_safe_data = Parsers.jsonify_yaml_data(data)
formatted_data = json_safe_data
if isinstance(json_safe_data, str):
formatted_data = json_safe_data.strip()
json_data = json.dumps(formatted_data).replace("\\n",
"\n{} ".format(prefix))
data_tag = ""
if isinstance(data, TaggedScalar) and data.tag.value:
data_tag = "{} ".format(data.tag.value)
return "{} {}{}".format(prefix, data_tag, json_data)
def print_results(args: Any, processor: EYAMLProcessor, yaml_file: str,
yaml_paths: List[Tuple[str, YAMLPath]],
document_index: int) -> None:
"""Dump search results to STDOUT with optional and dynamic formatting."""
in_expressions = len(args.search)
print_file_path = not args.nofile
print_expression = in_expressions > 1 and not args.noexpression
print_yaml_path = not args.noyamlpath
print_value = args.values
buffers = [
": " if print_file_path or print_expression and
(print_yaml_path or print_value) else "",
": " if print_yaml_path and print_value else "",
]
for entry in yaml_paths:
expression, result = entry
resline = ""
if print_file_path:
display_file_name = ("STDIN"
if yaml_file.strip() == "-" else yaml_file)
resline += "{}/{}".format(display_file_name, document_index)
if print_expression:
resline += "[{}]".format(expression)
resline += buffers[0]
if print_yaml_path:
if args.noescape:
use_flash = args.pathsep is PathSeperators.FSLASH
seglines = []
join_mark = "/" if use_flash else "."
path_prefix = "/" if use_flash else ""
for (_, segment) in result.escaped:
seglines.append(str(segment))
resline += "{}{}".format(path_prefix, join_mark.join(seglines))
else:
resline += "{}".format(result)
resline += buffers[1]
if print_value:
# These results can have only one match, but make sure lest the
# output become messy.
for node_coordinate in processor.get_nodes(result, mustexist=True):
node = node_coordinate.node
if isinstance(node, (dict, list, CommentedSet)):
resline += "{}".format(
json.dumps(Parsers.jsonify_yaml_data(node)))
else:
resline += "{}".format(str(node).replace("\n", r"\n"))
break
print(resline)
def test_jsonify_complex_python_data(self):
cdata = {
"dates": [
dt.date(2020, 10, 31),
dt.date(2020, 11, 3)
],
"bytes": b"abc"
}
jdata = Parsers.jsonify_yaml_data(cdata)
assert jdata["dates"][0] == "2020-10-31"
assert jdata["dates"][1] == "2020-11-03"
jstr = json.dumps(jdata)
assert jstr == """{"dates": ["2020-10-31", "2020-11-03"], "bytes": "b'abc'"}"""
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
exit_state = 0
lhs_file = args.yaml_files[0]
rhs_file = args.yaml_files[1]
lhs_yaml = Parsers.get_yaml_editor()
rhs_yaml = Parsers.get_yaml_editor()
(lhs_document, doc_loaded) = Parsers.get_yaml_data(lhs_yaml, log, lhs_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
(rhs_document, doc_loaded) = Parsers.get_yaml_data(rhs_yaml, log, rhs_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
diff = Differ(DifferConfig(log, args),
log,
lhs_document,
ignore_eyaml_values=args.ignore_eyaml_values,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
diff.compare_to(rhs_document)
except EYAMLCommandException as ex:
log.critical(ex, 1)
exit_state = 1 if print_report(log, args, diff) else 0
sys.exit(exit_state)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
# For the remainder of processing, overwrite overwrites output
if args.overwrite:
args.output = args.overwrite
# Merge all input files
merger = Merger(log, None, MergerConfig(log, args))
yaml_editor = Parsers.get_yaml_editor()
exit_state = 0
consumed_stdin = False
merger_primed = False
for yaml_file in args.yaml_files:
if yaml_file.strip() == '-':
consumed_stdin = True
log.debug(
"yaml_merge::main: Processing file, {}".format(
"STDIN" if yaml_file.strip() == "-" else yaml_file))
proc_state = process_yaml_file(
merger, log, yaml_editor, yaml_file, merger_primed)
if proc_state == 0:
merger_primed = True
else:
exit_state = proc_state
break
# Check for a waiting STDIN document
if (exit_state == 0
and not consumed_stdin
and not args.nostdin
and not sys.stdin.isatty()
):
exit_state = process_yaml_file(
merger, log, yaml_editor, '-', merger_primed)
# Output the final document
if exit_state == 0:
write_output_document(args, log, merger, yaml_editor)
sys.exit(exit_state)
def write_output_document(args, log, yaml, yaml_data):
"""Write the updated document to file or STDOUT."""
# Save a backup of the original file, if requested
backup_file = args.yaml_file + ".bak"
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
args.yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(args.yaml_file, backup_file)
# Save the changed file
if args.yaml_file.strip() == "-":
if write_document_as_yaml(args.yaml_file, yaml_data):
yaml.dump(yaml_data, sys.stdout)
else:
json.dump(Parsers.jsonify_yaml_data(yaml_data), sys.stdout)
else:
save_to_file(args, log, yaml, yaml_data, backup_file)
def process_file(log, yaml, yaml_file):
"""Process a (potentially multi-doc) YAML file."""
logcap = LogErrorCap()
subdoc_index = 0
exit_state = 0
file_name = "STDIN" if yaml_file.strip() == "-" else yaml_file
for (_,
doc_loaded) in Parsers.get_yaml_multidoc_data(yaml, logcap,
yaml_file):
if doc_loaded:
log.verbose("{}/{} is valid.".format(file_name, subdoc_index))
else:
# An error message has been captured
exit_state = 2
log.info("{}/{} is invalid due to:".format(file_name,
subdoc_index))
for line in logcap.lines:
log.info(" * {}".format(line))
logcap.lines.clear()
subdoc_index += 1
return exit_state
def get_doc_mergers(
log: ConsolePrinter, yaml_editor: YAML, config: MergerConfig,
yaml_file: str
) -> Tuple[List[Merger], bool]:
"""Create a list of Mergers, one for each source document."""
docs_loaded = True
if yaml_file != "-" and not isfile(yaml_file):
log.error("Not a file: {}".format(yaml_file))
return ([], False)
doc_mergers: List[Merger] = []
for (yaml_data, doc_loaded) in Parsers.get_yaml_multidoc_data(
yaml_editor, log, yaml_file
):
if not doc_loaded:
# An error message has already been logged
doc_mergers.clear()
docs_loaded = False
break
doc_mergers.append(Merger(log, yaml_data, config))
return (doc_mergers, docs_loaded)
def get_docs(log, yaml_editor, yaml_file):
"""Get all documents from a YAML/JSON/Compatible file."""
docs_loaded = True
docs = []
if yaml_file != "-" and not isfile(yaml_file):
log.error("File not found: {}".format(yaml_file))
return ([], False)
for (yaml_data,
doc_loaded) in Parsers.get_yaml_multidoc_data(yaml_editor, log,
yaml_file):
if not doc_loaded:
# An error message has already been logged
docs.clear()
docs_loaded = False
break
if (not isinstance(yaml_data, (list, dict))
and len(str(yaml_data)) < 1):
yaml_data = None
docs.append(yaml_data)
return (docs, docs_loaded)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
change_path = YAMLPath(args.change, pathsep=args.pathsep)
must_exist = args.mustexist or args.saveto
# Obtain the replacement value
consumed_stdin = False
new_value = None
has_new_value = False
if args.value or args.value == "":
new_value = args.value
has_new_value = True
elif args.stdin:
new_value = ''.join(sys.stdin.readlines())
consumed_stdin = True
has_new_value = True
elif args.file:
with open(args.file, 'r') as fhnd:
new_value = fhnd.read().rstrip()
has_new_value = True
elif args.null:
new_value = None
has_new_value = True
elif args.random is not None:
new_value = ''.join(
secrets.choice(args.random_from) for _ in range(args.random))
has_new_value = True
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
if args.yaml_file:
yaml_data = _try_load_input_file(args, log, yaml, change_path,
new_value)
if args.yaml_file.strip() == '-':
consumed_stdin = True
# Check for a waiting STDIN document
if (not consumed_stdin and not args.yaml_file and not args.nostdin
and not sys.stdin.isatty()):
args.yaml_file = "-"
yaml_data = _try_load_input_file(args, log, yaml, change_path,
new_value)
# Load the present nodes at the specified YAML Path
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
change_node_coordinates = _get_nodes(
log,
processor,
change_path,
must_exist=must_exist,
default_value=("" if new_value else " "))
old_format = YAMLValueFormats.DEFAULT
if len(change_node_coordinates) == 1:
# When there is exactly one result, its old format can be known. This
# is necessary to retain whether the replacement value should be
# represented later as a multi-line string when the new value is to be
# encrypted.
old_format = YAMLValueFormats.from_node(
change_node_coordinates[0].node)
# Check the value(s), if desired
if args.check:
for node_coordinate in change_node_coordinates:
if processor.is_eyaml_value(node_coordinate.node):
# Sanity check: If either --publickey or --privatekey were set
# then they must both be set in order to decrypt this value.
# This is enforced only when the value must be decrypted due to
# a --check request.
if ((args.publickey and not args.privatekey)
or (args.privatekey and not args.publickey)):
log.error(
"Neither or both private and public EYAML keys must be"
+ " set when --check is required to decrypt the old" +
" value.")
sys.exit(1)
try:
check_value = processor.decrypt_eyaml(node_coordinate.node)
except EYAMLCommandException as ex:
log.critical(ex, 1)
else:
check_value = node_coordinate.node
if not args.check == check_value:
log.critical(
'"{}" does not match the check value.'.format(args.check),
20)
# Save the old value, if desired and possible
if args.saveto:
# Only one can be saved; otherwise it is impossible to meaningfully
# convey to the end-user from exactly which other YAML node each saved
# value came.
if len(change_node_coordinates) > 1:
log.critical(
"It is impossible to meaningly save more than one matched" +
" value. Please omit --saveto or set --change to affect" +
" exactly one value.", 1)
saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep)
log.verbose("Saving the old value to {}.".format(saveto_path))
# Folded EYAML values have their embedded newlines converted to spaces
# when read. As such, writing them back out breaks their original
# format, despite being properly typed. To restore the original
# written form, reverse the conversion, here.
old_value = change_node_coordinates[0].node
if ((old_format is YAMLValueFormats.FOLDED
or old_format is YAMLValueFormats.LITERAL)
and EYAMLProcessor.is_eyaml_value(old_value)):
old_value = old_value.replace(" ", "\n")
try:
processor.set_value(saveto_path,
Nodes.clone_node(old_value),
value_format=old_format,
tag=args.tag)
except YAMLPathException as ex:
log.critical(ex, 1)
# Set the requested value
log.verbose("Applying changes to {}.".format(change_path))
if args.delete:
# Destroy the collected nodes (from their parents) in the reverse order
# they were discovered. This is necessary lest Array elements be
# improperly handled, leading to unwanted data loss.
_delete_nodes(log, processor, change_node_coordinates)
elif args.aliasof:
# Assign the change nodes as Aliases of whatever --aliasof points to
_alias_nodes(log, processor, change_node_coordinates, args.aliasof,
args.anchor)
elif args.eyamlcrypt:
# If the user hasn't specified a format, use the same format as the
# value being replaced, if known.
format_type = YAMLValueFormats.from_str(args.format)
if format_type is YAMLValueFormats.DEFAULT:
format_type = old_format
output_type = EYAMLOutputFormats.STRING
if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]:
output_type = EYAMLOutputFormats.BLOCK
try:
processor.set_eyaml_value(change_path,
new_value,
output=output_type,
mustexist=False)
except EYAMLCommandException as ex:
log.critical(ex, 2)
elif has_new_value:
try:
processor.set_value(change_path,
new_value,
value_format=args.format,
mustexist=must_exist,
tag=args.tag)
except YAMLPathException as ex:
log.critical(ex, 1)
elif args.tag:
_tag_nodes(processor.data, args.tag, change_node_coordinates)
# Write out the result
write_output_document(args, log, yaml, yaml_data)
def save_to_json_file(args, log, yaml_data):
"""Save to a JSON file."""
log.verbose("Writing changed data as JSON to {}.".format(args.yaml_file))
with open(args.yaml_file, 'w') as out_fhnd:
json.dump(Parsers.jsonify_yaml_data(yaml_data), out_fhnd)
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
search_values = True
search_keys = False
include_key_aliases = False
include_value_aliases = False
if args.onlykeynames:
search_values = False
search_keys = True
elif args.keynames:
search_keys = True
if args.include_aliases is IncludeAliases.INCLUDE_ALL_ALIASES:
include_key_aliases = True
include_value_aliases = True
elif args.include_aliases is IncludeAliases.INCLUDE_KEY_ALIASES:
include_key_aliases = True
elif args.include_aliases is IncludeAliases.INCLUDE_VALUE_ALIASES:
include_value_aliases = True
# Prepare the YAML processor
yaml = Parsers.get_yaml_editor()
processor = EYAMLProcessor(
log, None, binary=args.eyaml,
publickey=args.publickey, privatekey=args.privatekey)
# Process the input file(s)
exit_state = 0
file_tally = -1
consumed_stdin = False
for yaml_file in args.yaml_files:
file_tally += 1
if yaml_file.strip() == "-":
consumed_stdin = True
log.debug(
"yaml_merge::main: Processing file, {}".format(
"STDIN" if yaml_file.strip() == "-" else yaml_file))
proc_state = process_yaml_file(
args, yaml, log, yaml_file, processor, search_values, search_keys,
include_key_aliases, include_value_aliases, file_tally
)
if proc_state != 0:
exit_state = proc_state
# Check for a waiting STDIN document
if (exit_state == 0
and not consumed_stdin
and not args.nostdin
and not sys.stdin.isatty()
):
file_tally += 1
exit_state = process_yaml_file(
args, yaml, log, "-", processor, search_values, search_keys,
include_key_aliases, include_value_aliases, file_tally
)
sys.exit(exit_state)
def process_yaml_file(
args, yaml, log, yaml_file, processor, search_values, search_keys,
include_key_aliases, include_value_aliases, file_tally = 0
):
"""Process a (potentially multi-doc) YAML file."""
# Try to open the file
exit_state = 0
subdoc_index = -1
# pylint: disable=too-many-nested-blocks
for (yaml_data, doc_loaded) in Parsers.get_yaml_multidoc_data(
yaml, log, yaml_file
):
file_tally += 1
subdoc_index += 1
if not doc_loaded:
# An error message has already been logged
exit_state = 3
continue
# Process all searches
processor.data = yaml_data
yaml_paths = []
for expression in args.search:
exterm = get_search_term(log, expression)
log.debug(("yaml_paths::process_yaml_file:"
+ "converting search expression '{}' into '{}'"
).format(expression, exterm))
if exterm is None:
exit_state = 1
continue
for result in search_for_paths(
log, processor, yaml_data, exterm, args.pathsep,
search_values=search_values, search_keys=search_keys,
search_anchors=args.refnames,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=args.decrypt,
expand_children=args.expand):
# Record only unique results
add_entry = True
for entry in yaml_paths:
if str(result) == str(entry[1]):
add_entry = False
break
if add_entry:
yaml_paths.append((expression, result))
if not yaml_paths:
# Nothing further to do when there are no results
continue
if args.except_expression:
for expression in args.except_expression:
exterm = get_search_term(log, expression)
log.debug(("yaml_paths::process_yaml_file:"
+ "converted except expression '{}' into '{}'"
).format(expression, exterm))
if exterm is None:
exit_state = 1
continue
for result in search_for_paths(
log, processor, yaml_data, exterm, args.pathsep,
search_values=search_values,
search_keys=search_keys,
search_anchors=args.refnames,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=args.decrypt,
expand_children=args.expand):
for entry in yaml_paths:
if str(result) == str(entry[1]):
yaml_paths.remove(entry)
break # Entries are already unique
print_results(
args, processor, yaml_file, yaml_paths, subdoc_index)
return exit_state
def test_good_multi_replacements(self, script_runner, tmp_path_factory, old_eyaml_keys, new_eyaml_keys, quiet_logger):
from yamlpath.func import unwrap_node_coords
from yamlpath.common import Parsers
from yamlpath import Processor
from yamlpath.eyaml import EYAMLProcessor
simple_content = """---
encrypted_string: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAHA4rPcTzvgzPLtnGz3yoyX/kVlQ5TnPXcScXK2bwjguGZLkuzv/JVPAsOm4t6GlnROpy4zb/lUMHRJDChJhPLrSj919B8//huoMgw0EU5XTcaN6jeDDjL+vhjswjvLFOux66UwvMo8sRci/e2tlFiam8VgxzV0hpF2qRrL/l84V04gL45kq4PCYDWrJNynOwYVbSIF+qc5HaF25H8kHq1lD3RB6Ob/J942Q7k5Qt7W9mNm9cKZmxwgtUgIZWXW6mcPJ2dXDB/RuPJJSrLsb1VU/DkhdgxaNzvLCA+MViyoFUkCfHFNZbaHKNkoYXBy7dLmoh/E5tKv99FeG/7CzL3DBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCVU5Mjt8+4dLkoqB9YArfkgCDkdIhXR9T1M4YYa1qTE6by61VPU3g1aMExRmo4tNZ8FQ==]
encrypted_block: >
ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw
DQYJKoZIhvcNAQEBBQAEggEAnxQVqyIgRTb/+VP4Q+DLJcnlS8YPouXEW8+z
it9uwUA02CEPxCEU944GcHpgTY3EEtkm+2Z/jgXI119VMML+OOQ1NkwUiAw/
wq0vwz2D16X31XzhedQN5FZbfZ1C+2tWSQfCjE0bu7IeHfyR+k2ssD11kNZh
JDEr2bM2dwOdT0y7VGcQ06vI9gw6UXcwYAgS6FoLm7WmFftjcYiNB+0EJSW0
VcTn2gveaw9iOQcum/Grby+9Ybs28fWd8BoU+ZWDpoIMEceujNa9okIXNPJO
jcvv1sgauwJ3RX6WFQIy/beS2RT5EOLhWIZCAQCcgJWgovu3maB7dEUZ0NLG
OYUR7zA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAbO16EzQ5/cdcvgB0g
tpKIgBAEgTLT5n9Jtc9venK0CKso]
"""
anchored_content = """---
aliases:
- &blockStyle >
ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEw
DQYJKoZIhvcNAQEBBQAEggEArvk6OYa1gACTdrWq2SpCrtGRlc61la5AGU7L
aLTyKfqD9vqx71RDjobfOF96No07kLsEpoAJ+LKKHNjdG6kjvpGPmttj9Dkm
XVoU6A+YCmm4iYFKD/NkoSOEyAkoDOXSqdjrgt0f37GefEsXt6cqAavDpUJm
pmc0KI4TCG5zpfCxqttMs+stOY3Y+0WokkulQujZ7K3SdWUSHIysgMrWiect
Wdg5unxN1A/aeyvhgvYSNPjU9KBco7SDnigSs9InW/QghJFrZRrDhTp1oTUc
qK5lKvaseHkVGi91vPWeLQxZt1loJB5zL6j5BxMbvRfJK+wc3ax2u4x8WTAB
EurCwzBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAwcy7jvcOGcMfLEtug
LEXbgCBkocdckuDe14mVGmUmM++xN34OEVRCeGVWWUnWq1DJ4Q==]
- &stringStyle ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAIu44u62q5sVfzC7kytLi2Z/EzH2DKr4vDsoqDBeSZ71aRku/uSrjyiO4lyoq9Kva+eBAyjBay5fnqPVBaU3Rud2pdEoZEoyofi02jn4hxUKpAO1W0AUgsQolGe53qOdM4U8RbwnTR0gr3gp2mCd18pH3SRMP9ryrsBAxGzJ6mR3RgdZnlTlqVGXCeWUeVpbH+lcHw3uvd+o/xkvJ/3ypxz+rWILiAZ3QlCirzn/qb2fHuKf3VBh8RVFuQDaM5voajZlgjD6KzNCsbATOqOA6eJI4j0ngPdDlIjGHAnahuyluQ5f5SIaIjLC+ZeCOfIYni0MQ+BHO0JNbccjq2Unb7TBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCYmAI0Ao3Ok1cSmVw0SgQGgCBK62z1r5RfRjf1xKfqDxTsGUHfsUmM3EjGJfnWzCRvuQ==]
block: *blockStyle
string: *stringStyle
yet_another:
'more.complex.child': *blockStyle
"""
simple_file = create_temp_yaml_file(tmp_path_factory, simple_content)
anchored_file = create_temp_yaml_file(tmp_path_factory, anchored_content)
result = script_runner.run(
self.command,
"--newprivatekey={}".format(new_eyaml_keys[0]),
"--newpublickey={}".format(new_eyaml_keys[1]),
"--oldprivatekey={}".format(old_eyaml_keys[0]),
"--oldpublickey={}".format(old_eyaml_keys[1]),
simple_file,
anchored_file
)
assert result.success, result.stderr
with open(simple_file, 'r') as fhnd:
simple_data = fhnd.read()
with open(anchored_file, 'r') as fhnd:
anchored_data = fhnd.read()
assert not simple_data == simple_content
assert not anchored_data == anchored_content
# Verify that block and string formatting is correct
yaml = Parsers.get_yaml_editor()
(yaml_rotated_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_data, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Rotated anchored data failed to load"
source_processor = Processor(quiet_logger, yaml_rotated_data)
for node in source_processor.get_nodes('/block', mustexist=True):
assert not ' ' in unwrap_node_coords(node)
# Test that the pre- and post-rotated values are identical
(yaml_anchored_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_content, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Original anchored data failed to load"
(yaml_rotated_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_data, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Rotated anchored data failed to load"
source_processor = EYAMLProcessor(
quiet_logger, yaml_anchored_data,
privatekey=old_eyaml_keys[0],
publickey=old_eyaml_keys[1])
for node in source_processor.get_eyaml_values(
'/block', True
):
assert unwrap_node_coords(node) == 'This is a test value.'
rotated_processor = EYAMLProcessor(
quiet_logger, yaml_rotated_data,
privatekey=new_eyaml_keys[0],
publickey=new_eyaml_keys[1])
for node in rotated_processor.get_eyaml_values(
'/block', True
):
assert unwrap_node_coords(node) == 'This is a test value.'
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
processor = EYAMLProcessor(log, None, binary=args.eyaml)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Process the input file(s)
in_file_count = len(args.yaml_files)
exit_state = 0
for yaml_file in args.yaml_files:
file_changed = False
backup_file = yaml_file + ".bak"
seen_anchors = []
# Each YAML_FILE must actually be a file
if not isfile(yaml_file):
log.error("Not a file: {}".format(yaml_file))
exit_state = 2
continue
# Don't bother with the file change update when there's only one input
# file.
if in_file_count > 1:
log.info("Processing {}...".format(yaml_file))
# Try to open the file
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, yaml_file)
if not doc_loaded:
# An error message has already been logged
exit_state = 3
continue
# Process all EYAML values
processor.data = yaml_data
for yaml_path in processor.find_eyaml_paths():
# Use ::get_nodes() instead of ::get_eyaml_values() here in order
# to ignore values that have already been rotated via their
# Anchors.
for node_coordinate in processor.get_nodes(yaml_path,
mustexist=True):
# Ignore values which are Aliases for those already decrypted
node = node_coordinate.node
anchor_name = Anchors.get_node_anchor(node)
if anchor_name is not None:
if anchor_name in seen_anchors:
continue
seen_anchors.append(anchor_name)
log.verbose("Decrypting value(s) at {}.".format(yaml_path))
processor.publickey = args.oldpublickey
processor.privatekey = args.oldprivatekey
try:
txtval = processor.decrypt_eyaml(node)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
# Prefer block (folded) values unless the original YAML value
# was already a massivly long (string) line.
output = EYAMLOutputFormats.BLOCK
if not isinstance(node, FoldedScalarString):
output = EYAMLOutputFormats.STRING
# Re-encrypt the value with new EYAML keys
processor.publickey = args.newpublickey
processor.privatekey = args.newprivatekey
try:
processor.set_eyaml_value(yaml_path, txtval, output=output)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
file_changed = True
# Save the changes
if file_changed:
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(yaml_file, backup_file)
log.verbose("Writing changed data to {}.".format(yaml_file))
with open(yaml_file, 'w', encoding='utf-8') as yaml_dump:
yaml.dump(yaml_data, yaml_dump)
sys.exit(exit_state)