def _parse_yaml(source: str, config_string: bool):
logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False)
log = ConsolePrinter(logging_args)
yaml = Parsers.get_yaml_editor()
# for better backward compatibility with PyYAML (that supports only YAML 1.1) used in the previous
# GitLabForm versions, let's force ruamel.yaml to use YAML version 1.1 by default too
yaml.version = (1, 1)
if config_string:
config_string = textwrap.dedent(source)
verbose("Reading config from the provided string.")
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml,
log,
config_string,
literal=True)
else:
config_path = source
verbose(f"Reading config from file: {config_path}")
(yaml_data,
doc_loaded) = Parsers.get_yaml_data(yaml, log, config_path)
if doc_loaded:
debug("Config parsed successfully as YAML.")
else:
# an error message has already been printed via ConsolePrinter
exit(EXIT_INVALID_INPUT)
return yaml_data
def _try_load_input_file(args, log, yaml, change_path, new_value):
"""Attempt to load the input data file or abend on error."""
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, args.yaml_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
elif yaml_data is None:
yaml_data = Nodes.build_next_node(change_path, 0, new_value)
return yaml_data
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
exit_state = 0
lhs_file = args.yaml_files[0]
rhs_file = args.yaml_files[1]
lhs_yaml = Parsers.get_yaml_editor()
rhs_yaml = Parsers.get_yaml_editor()
(lhs_document, doc_loaded) = Parsers.get_yaml_data(lhs_yaml, log, lhs_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
(rhs_document, doc_loaded) = Parsers.get_yaml_data(rhs_yaml, log, rhs_file)
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
diff = Differ(DifferConfig(log, args),
log,
lhs_document,
ignore_eyaml_values=args.ignore_eyaml_values,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
diff.compare_to(rhs_document)
except EYAMLCommandException as ex:
log.critical(ex, 1)
exit_state = 1 if print_report(log, args, diff) else 0
sys.exit(exit_state)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
yaml_path = YAMLPath(args.query, pathsep=args.pathsep)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
(yaml_data, doc_loaded) = Parsers.get_yaml_data(
yaml, log, args.yaml_file if args.yaml_file else "-")
if not doc_loaded:
# An error message has already been logged
sys.exit(1)
# Seek the queried value(s)
discovered_nodes = []
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_eyaml_values(yaml_path, mustexist=True):
log.debug("Got node from {}:".format(yaml_path),
data=node,
prefix="yaml_get::main: ")
discovered_nodes.append(NodeCoords.unwrap_node_coords(node))
except YAMLPathException as ex:
log.critical(ex, 1)
except EYAMLCommandException as ex:
log.critical(ex, 2)
try:
for node in discovered_nodes:
if isinstance(node, (dict, list, CommentedSet)):
print(json.dumps(Parsers.jsonify_yaml_data(node)))
else:
if node is None:
node = "\x00"
print("{}".format(str(node).replace("\n", r"\n")))
except RecursionError:
log.critical(
"The YAML data contains an infinitely recursing YAML Alias!", 1)
def test_get_yaml_data_literally(self, quiet_logger):
serialized_yaml = """---
hash:
key: value
list:
- ichi
- ni
- san
"""
yaml = Parsers.get_yaml_editor()
(data, loaded) = Parsers.get_yaml_data(
yaml, quiet_logger, serialized_yaml,
literal=True)
assert loaded == True
assert data["hash"]["key"] == "value"
assert data["list"][0] == "ichi"
assert data["list"][1] == "ni"
assert data["list"][2] == "san"
def main():
"""Main code."""
# Process any command-line arguments
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
processor = EYAMLProcessor(log, None, binary=args.eyaml)
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Process the input file(s)
in_file_count = len(args.yaml_files)
exit_state = 0
for yaml_file in args.yaml_files:
file_changed = False
backup_file = yaml_file + ".bak"
seen_anchors = []
# Each YAML_FILE must actually be a file
if not isfile(yaml_file):
log.error("Not a file: {}".format(yaml_file))
exit_state = 2
continue
# Don't bother with the file change update when there's only one input
# file.
if in_file_count > 1:
log.info("Processing {}...".format(yaml_file))
# Try to open the file
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, yaml_file)
if not doc_loaded:
# An error message has already been logged
exit_state = 3
continue
# Process all EYAML values
processor.data = yaml_data
for yaml_path in processor.find_eyaml_paths():
# Use ::get_nodes() instead of ::get_eyaml_values() here in order
# to ignore values that have already been rotated via their
# Anchors.
for node_coordinate in processor.get_nodes(yaml_path,
mustexist=True):
# Ignore values which are Aliases for those already decrypted
node = node_coordinate.node
anchor_name = Anchors.get_node_anchor(node)
if anchor_name is not None:
if anchor_name in seen_anchors:
continue
seen_anchors.append(anchor_name)
log.verbose("Decrypting value(s) at {}.".format(yaml_path))
processor.publickey = args.oldpublickey
processor.privatekey = args.oldprivatekey
try:
txtval = processor.decrypt_eyaml(node)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
# Prefer block (folded) values unless the original YAML value
# was already a massivly long (string) line.
output = EYAMLOutputFormats.BLOCK
if not isinstance(node, FoldedScalarString):
output = EYAMLOutputFormats.STRING
# Re-encrypt the value with new EYAML keys
processor.publickey = args.newpublickey
processor.privatekey = args.newprivatekey
try:
processor.set_eyaml_value(yaml_path, txtval, output=output)
except EYAMLCommandException as ex:
log.error(ex)
exit_state = 3
continue
file_changed = True
# Save the changes
if file_changed:
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(yaml_file, backup_file)
log.verbose("Writing changed data to {}.".format(yaml_file))
with open(yaml_file, 'w', encoding='utf-8') as yaml_dump:
yaml.dump(yaml_data, yaml_dump)
sys.exit(exit_state)
def test_good_multi_replacements(self, script_runner, tmp_path_factory, old_eyaml_keys, new_eyaml_keys, quiet_logger):
from yamlpath.func import unwrap_node_coords
from yamlpath.common import Parsers
from yamlpath import Processor
from yamlpath.eyaml import EYAMLProcessor
simple_content = """---
encrypted_string: ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAHA4rPcTzvgzPLtnGz3yoyX/kVlQ5TnPXcScXK2bwjguGZLkuzv/JVPAsOm4t6GlnROpy4zb/lUMHRJDChJhPLrSj919B8//huoMgw0EU5XTcaN6jeDDjL+vhjswjvLFOux66UwvMo8sRci/e2tlFiam8VgxzV0hpF2qRrL/l84V04gL45kq4PCYDWrJNynOwYVbSIF+qc5HaF25H8kHq1lD3RB6Ob/J942Q7k5Qt7W9mNm9cKZmxwgtUgIZWXW6mcPJ2dXDB/RuPJJSrLsb1VU/DkhdgxaNzvLCA+MViyoFUkCfHFNZbaHKNkoYXBy7dLmoh/E5tKv99FeG/7CzL3DBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCVU5Mjt8+4dLkoqB9YArfkgCDkdIhXR9T1M4YYa1qTE6by61VPU3g1aMExRmo4tNZ8FQ==]
encrypted_block: >
ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw
DQYJKoZIhvcNAQEBBQAEggEAnxQVqyIgRTb/+VP4Q+DLJcnlS8YPouXEW8+z
it9uwUA02CEPxCEU944GcHpgTY3EEtkm+2Z/jgXI119VMML+OOQ1NkwUiAw/
wq0vwz2D16X31XzhedQN5FZbfZ1C+2tWSQfCjE0bu7IeHfyR+k2ssD11kNZh
JDEr2bM2dwOdT0y7VGcQ06vI9gw6UXcwYAgS6FoLm7WmFftjcYiNB+0EJSW0
VcTn2gveaw9iOQcum/Grby+9Ybs28fWd8BoU+ZWDpoIMEceujNa9okIXNPJO
jcvv1sgauwJ3RX6WFQIy/beS2RT5EOLhWIZCAQCcgJWgovu3maB7dEUZ0NLG
OYUR7zA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAbO16EzQ5/cdcvgB0g
tpKIgBAEgTLT5n9Jtc9venK0CKso]
"""
anchored_content = """---
aliases:
- &blockStyle >
ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEw
DQYJKoZIhvcNAQEBBQAEggEArvk6OYa1gACTdrWq2SpCrtGRlc61la5AGU7L
aLTyKfqD9vqx71RDjobfOF96No07kLsEpoAJ+LKKHNjdG6kjvpGPmttj9Dkm
XVoU6A+YCmm4iYFKD/NkoSOEyAkoDOXSqdjrgt0f37GefEsXt6cqAavDpUJm
pmc0KI4TCG5zpfCxqttMs+stOY3Y+0WokkulQujZ7K3SdWUSHIysgMrWiect
Wdg5unxN1A/aeyvhgvYSNPjU9KBco7SDnigSs9InW/QghJFrZRrDhTp1oTUc
qK5lKvaseHkVGi91vPWeLQxZt1loJB5zL6j5BxMbvRfJK+wc3ax2u4x8WTAB
EurCwzBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBAwcy7jvcOGcMfLEtug
LEXbgCBkocdckuDe14mVGmUmM++xN34OEVRCeGVWWUnWq1DJ4Q==]
- &stringStyle ENC[PKCS7,MIIBiQYJKoZIhvcNAQcDoIIBejCCAXYCAQAxggEhMIIBHQIBADAFMAACAQEwDQYJKoZIhvcNAQEBBQAEggEAIu44u62q5sVfzC7kytLi2Z/EzH2DKr4vDsoqDBeSZ71aRku/uSrjyiO4lyoq9Kva+eBAyjBay5fnqPVBaU3Rud2pdEoZEoyofi02jn4hxUKpAO1W0AUgsQolGe53qOdM4U8RbwnTR0gr3gp2mCd18pH3SRMP9ryrsBAxGzJ6mR3RgdZnlTlqVGXCeWUeVpbH+lcHw3uvd+o/xkvJ/3ypxz+rWILiAZ3QlCirzn/qb2fHuKf3VBh8RVFuQDaM5voajZlgjD6KzNCsbATOqOA6eJI4j0ngPdDlIjGHAnahuyluQ5f5SIaIjLC+ZeCOfIYni0MQ+BHO0JNbccjq2Unb7TBMBgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBCYmAI0Ao3Ok1cSmVw0SgQGgCBK62z1r5RfRjf1xKfqDxTsGUHfsUmM3EjGJfnWzCRvuQ==]
block: *blockStyle
string: *stringStyle
yet_another:
'more.complex.child': *blockStyle
"""
simple_file = create_temp_yaml_file(tmp_path_factory, simple_content)
anchored_file = create_temp_yaml_file(tmp_path_factory, anchored_content)
result = script_runner.run(
self.command,
"--newprivatekey={}".format(new_eyaml_keys[0]),
"--newpublickey={}".format(new_eyaml_keys[1]),
"--oldprivatekey={}".format(old_eyaml_keys[0]),
"--oldpublickey={}".format(old_eyaml_keys[1]),
simple_file,
anchored_file
)
assert result.success, result.stderr
with open(simple_file, 'r') as fhnd:
simple_data = fhnd.read()
with open(anchored_file, 'r') as fhnd:
anchored_data = fhnd.read()
assert not simple_data == simple_content
assert not anchored_data == anchored_content
# Verify that block and string formatting is correct
yaml = Parsers.get_yaml_editor()
(yaml_rotated_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_data, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Rotated anchored data failed to load"
source_processor = Processor(quiet_logger, yaml_rotated_data)
for node in source_processor.get_nodes('/block', mustexist=True):
assert not ' ' in unwrap_node_coords(node)
# Test that the pre- and post-rotated values are identical
(yaml_anchored_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_content, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Original anchored data failed to load"
(yaml_rotated_data, doc_loaded) = Parsers.get_yaml_data(
yaml, quiet_logger,
anchored_data, literal=True)
if not doc_loaded:
# An error message has already been logged
assert False, "Rotated anchored data failed to load"
source_processor = EYAMLProcessor(
quiet_logger, yaml_anchored_data,
privatekey=old_eyaml_keys[0],
publickey=old_eyaml_keys[1])
for node in source_processor.get_eyaml_values(
'/block', True
):
assert unwrap_node_coords(node) == 'This is a test value.'
rotated_processor = EYAMLProcessor(
quiet_logger, yaml_rotated_data,
privatekey=new_eyaml_keys[0],
publickey=new_eyaml_keys[1])
for node in rotated_processor.get_eyaml_values(
'/block', True
):
assert unwrap_node_coords(node) == 'This is a test value.'
def check_playbook_file_removed_and_added(playbook_path):
playbook_ok = True
yaml_parser = Parsers.get_yaml_editor()
logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False)
log = ConsolePrinter(logging_args)
# Find every path removed by a file Task (also matches tasks within blocks)
files_absent_string = "tasks.**.file[state=absent][parent()].path"
files_absent_yamlpath = YAMLPath(files_absent_string)
path_editing_tasks_yamlpath = ""
log.info("Info: Evaluating playbook '{}'".format(playbook_path))
(yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml_parser, log,
playbook_path)
if not doc_loaded:
# There was an issue loading the file; an error message has already been
# printed via ConsolePrinter.
return False
processor = Processor(log, yaml_data)
try:
for node in processor.get_nodes(files_absent_yamlpath,
mustexist=False):
path = str(node)
# 'node' is a NodeCoords.
if path == 'None':
continue
elif "{{" in path:
# Identified path is a Jinja expression, unfortunately there is no easy way to get
# the actual path without making this test very complicated
continue
# Check if this paths is used in any of the following ansible modules
ansible_modules = ["lineinfile", "blockinfile", "copy"]
path_editing_tasks_string = "tasks.**.[.=~/{modules}/][*='{path}'][parent()].name"
path_editing_tasks_yamlpath = YAMLPath(
path_editing_tasks_string.format(
modules="|".join(ansible_modules), path=node))
for task in processor.get_nodes(path_editing_tasks_yamlpath,
mustexist=False):
log.info(
"Error: Task '{}' manipulates a file that is removed by another task"
.format(task))
playbook_ok = False
except YAMLPathException as ex:
no_file_msg = (
"Cannot add PathSegmentTypes.TRAVERSE subreference to lists at 'None' "
"in '{}'.")
if str(ex) == no_file_msg.format(files_absent_string):
log.info(
"Info: Playbook {} has no 'file' tasks.".format(playbook_path))
elif path_editing_tasks_yamlpath and str(ex) == no_file_msg.format(
path_editing_tasks_yamlpath):
log.info("Info: Playbook {} has no '{}' tasks.".format(
playbook_path, " ".join(ansible_modules)))
else:
log.info("Error: {}.".format(ex))
return playbook_ok
def get_yaml_data(*args, **kwargs):
"""Relay function call to static method."""
return Parsers.get_yaml_data(*args, **kwargs)