def test_preserve_old_blockiness(self, quiet_logger, eyamldata_f, old_eyaml_keys, yaml_path, newval, eoformat, yvformat):
processor = EYAMLProcessor(quiet_logger, eyamldata_f, privatekey=old_eyaml_keys[0], publickey=old_eyaml_keys[1])
processor.set_eyaml_value(yaml_path, newval, output=eoformat)
encvalue = None
encformat = YAMLValueFormats.DEFAULT
for encnode in processor.get_nodes(yaml_path):
encvalue = encnode
encformat = YAMLValueFormats.from_node(encvalue)
break
assert EYAMLProcessor.is_eyaml_value(encvalue) and yvformat == encformat
def test_happy_set_eyaml_value(self, quiet_logger, eyamldata_f, old_eyaml_keys, yaml_path, compare, mustexist, output_format):
processor = EYAMLProcessor(quiet_logger, eyamldata_f, privatekey=old_eyaml_keys[0], publickey=old_eyaml_keys[1])
# Set the test value
processor.set_eyaml_value(yaml_path, compare, output_format, mustexist)
# Ensure the new value is encrypted
encvalue = None
for encnode in processor.get_nodes(yaml_path):
encvalue = encnode
break
assert EYAMLProcessor.is_eyaml_value(encvalue)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
change_path = YAMLPath(args.change, pathsep=args.pathsep)
must_exist = args.mustexist or args.saveto
# Obtain the replacement value
consumed_stdin = False
new_value = None
has_new_value = False
if args.value or args.value == "":
new_value = args.value
has_new_value = True
elif args.stdin:
new_value = ''.join(sys.stdin.readlines())
consumed_stdin = True
has_new_value = True
elif args.file:
with open(args.file, 'r') as fhnd:
new_value = fhnd.read().rstrip()
has_new_value = True
elif args.null:
new_value = None
has_new_value = True
elif args.random is not None:
new_value = ''.join(
secrets.choice(args.random_from) for _ in range(args.random))
has_new_value = True
# Prep the YAML parser
yaml = Parsers.get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
if args.yaml_file:
yaml_data = _try_load_input_file(args, log, yaml, change_path,
new_value)
if args.yaml_file.strip() == '-':
consumed_stdin = True
# Check for a waiting STDIN document
if (not consumed_stdin and not args.yaml_file and not args.nostdin
and not sys.stdin.isatty()):
args.yaml_file = "-"
yaml_data = _try_load_input_file(args, log, yaml, change_path,
new_value)
# Load the present nodes at the specified YAML Path
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
change_node_coordinates = _get_nodes(
log,
processor,
change_path,
must_exist=must_exist,
default_value=("" if new_value else " "))
old_format = YAMLValueFormats.DEFAULT
if len(change_node_coordinates) == 1:
# When there is exactly one result, its old format can be known. This
# is necessary to retain whether the replacement value should be
# represented later as a multi-line string when the new value is to be
# encrypted.
old_format = YAMLValueFormats.from_node(
change_node_coordinates[0].node)
# Check the value(s), if desired
if args.check:
for node_coordinate in change_node_coordinates:
if processor.is_eyaml_value(node_coordinate.node):
# Sanity check: If either --publickey or --privatekey were set
# then they must both be set in order to decrypt this value.
# This is enforced only when the value must be decrypted due to
# a --check request.
if ((args.publickey and not args.privatekey)
or (args.privatekey and not args.publickey)):
log.error(
"Neither or both private and public EYAML keys must be"
+ " set when --check is required to decrypt the old" +
" value.")
sys.exit(1)
try:
check_value = processor.decrypt_eyaml(node_coordinate.node)
except EYAMLCommandException as ex:
log.critical(ex, 1)
else:
check_value = node_coordinate.node
if not args.check == check_value:
log.critical(
'"{}" does not match the check value.'.format(args.check),
20)
# Save the old value, if desired and possible
if args.saveto:
# Only one can be saved; otherwise it is impossible to meaningfully
# convey to the end-user from exactly which other YAML node each saved
# value came.
if len(change_node_coordinates) > 1:
log.critical(
"It is impossible to meaningly save more than one matched" +
" value. Please omit --saveto or set --change to affect" +
" exactly one value.", 1)
saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep)
log.verbose("Saving the old value to {}.".format(saveto_path))
# Folded EYAML values have their embedded newlines converted to spaces
# when read. As such, writing them back out breaks their original
# format, despite being properly typed. To restore the original
# written form, reverse the conversion, here.
old_value = change_node_coordinates[0].node
if ((old_format is YAMLValueFormats.FOLDED
or old_format is YAMLValueFormats.LITERAL)
and EYAMLProcessor.is_eyaml_value(old_value)):
old_value = old_value.replace(" ", "\n")
try:
processor.set_value(saveto_path,
Nodes.clone_node(old_value),
value_format=old_format,
tag=args.tag)
except YAMLPathException as ex:
log.critical(ex, 1)
# Set the requested value
log.verbose("Applying changes to {}.".format(change_path))
if args.delete:
# Destroy the collected nodes (from their parents) in the reverse order
# they were discovered. This is necessary lest Array elements be
# improperly handled, leading to unwanted data loss.
_delete_nodes(log, processor, change_node_coordinates)
elif args.aliasof:
# Assign the change nodes as Aliases of whatever --aliasof points to
_alias_nodes(log, processor, change_node_coordinates, args.aliasof,
args.anchor)
elif args.eyamlcrypt:
# If the user hasn't specified a format, use the same format as the
# value being replaced, if known.
format_type = YAMLValueFormats.from_str(args.format)
if format_type is YAMLValueFormats.DEFAULT:
format_type = old_format
output_type = EYAMLOutputFormats.STRING
if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]:
output_type = EYAMLOutputFormats.BLOCK
try:
processor.set_eyaml_value(change_path,
new_value,
output=output_type,
mustexist=False)
except EYAMLCommandException as ex:
log.critical(ex, 2)
elif has_new_value:
try:
processor.set_value(change_path,
new_value,
value_format=args.format,
mustexist=must_exist,
tag=args.tag)
except YAMLPathException as ex:
log.critical(ex, 1)
elif args.tag:
_tag_nodes(processor.data, args.tag, change_node_coordinates)
# Write out the result
write_output_document(args, log, yaml, yaml_data)
def search_for_paths(logger: ConsolePrinter, processor: EYAMLProcessor,
data: Any, terms: SearchTerms,
pathsep: PathSeperators = PathSeperators.DOT,
build_path: str = "",
seen_anchors: Optional[List[str]] = None,
**kwargs: bool) -> Generator[YAMLPath, None, None]:
"""
Recursively search a data structure for nodes matching an expression.
The nodes can be keys, values, and/or elements. When dealing with anchors
and their aliases, the caller indicates whether to include only the
original anchor or the anchor and all of its (duplicate) aliases.
"""
search_values: bool = kwargs.pop("search_values", True)
search_keys: bool = kwargs.pop("search_keys", False)
search_anchors: bool = kwargs.pop("search_anchors", False)
include_key_aliases: bool = kwargs.pop("include_key_aliases", True)
include_value_aliases: bool = kwargs.pop("include_value_aliases", False)
decrypt_eyaml: bool = kwargs.pop("decrypt_eyaml", False)
expand_children: bool = kwargs.pop("expand_children", False)
strsep = str(pathsep)
invert = terms.inverted
method = terms.method
term = terms.term
if seen_anchors is None:
seen_anchors = []
if isinstance(data, CommentedSeq):
# Build the path
if not build_path and pathsep is PathSeperators.FSLASH:
build_path = strsep
build_path += "["
for idx, ele in enumerate(data):
# Any element may or may not have an Anchor/Alias
anchor_matched = Searches.search_anchor(
ele, terms, seen_anchors, search_anchors=search_anchors,
include_aliases=include_value_aliases)
logger.debug(
("yaml_paths::search_for_paths<list>:"
+ "anchor search => {}.")
.format(anchor_matched)
)
# Build the temporary YAML Path using either Anchor or Index
if anchor_matched is AnchorMatches.NO_ANCHOR:
# Not an anchor/alias, so ref this node by its index
tmp_path = build_path + str(idx) + "]"
else:
tmp_path = "{}&{}]".format(
build_path,
YAMLPath.escape_path_section(ele.anchor.value, pathsep)
)
if anchor_matched is AnchorMatches.ALIAS_EXCLUDED:
continue
if anchor_matched in [AnchorMatches.MATCH,
AnchorMatches.ALIAS_INCLUDED]:
logger.debug(
("yaml_paths::search_for_paths<list>:"
+ "yielding an Anchor/Alias match, {}.")
.format(tmp_path)
)
if expand_children:
for path in yield_children(
logger, ele, terms, pathsep, tmp_path,
seen_anchors, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases):
yield path
else:
yield YAMLPath(tmp_path)
continue
if isinstance(ele, (CommentedSeq, CommentedMap)):
logger.debug(
"Recursing into complex data:", data=ele,
prefix="yaml_paths::search_for_paths<list>: ",
footer=">>>> >>>> >>>> >>>> >>>> >>>> >>>>")
for subpath in search_for_paths(
logger, processor, ele, terms, pathsep, tmp_path,
seen_anchors, search_values=search_values,
search_keys=search_keys, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=decrypt_eyaml,
expand_children=expand_children
):
logger.debug(
"Yielding RECURSED match, {}.".format(subpath),
prefix="yaml_paths::search_for_paths<list>: ",
footer="<<<< <<<< <<<< <<<< <<<< <<<< <<<<"
)
yield subpath
elif search_values:
if (anchor_matched is AnchorMatches.UNSEARCHABLE_ALIAS
and not include_value_aliases):
continue
check_value = ele
if decrypt_eyaml and processor.is_eyaml_value(ele):
check_value = processor.decrypt_eyaml(ele)
matches = Searches.search_matches(method, term, check_value)
if (matches and not invert) or (invert and not matches):
logger.debug(
("yaml_paths::search_for_paths<list>:"
+ "yielding VALUE match, {}: {}."
).format(check_value, tmp_path)
)
yield YAMLPath(tmp_path)
# pylint: disable=too-many-nested-blocks
elif isinstance(data, CommentedMap):
if build_path:
build_path += strsep
elif pathsep is PathSeperators.FSLASH:
build_path = strsep
pool = data.non_merged_items()
if include_key_aliases or include_value_aliases:
pool = data.items()
for key, val in pool:
tmp_path = build_path + YAMLPath.escape_path_section(key, pathsep)
# Search the value anchor to have it on record, in case the key
# anchor match would otherwise block the value anchor from
# appearing in seen_anchors (which is important).
val_anchor_matched = Searches.search_anchor(
val, terms, seen_anchors, search_anchors=search_anchors,
include_aliases=include_value_aliases)
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "VALUE anchor search => {}.")
.format(val_anchor_matched)
)
# Search the key when the caller wishes it.
if search_keys:
# The key itself may be an Anchor or Alias. Search it when the
# caller wishes.
key_anchor_matched = Searches.search_anchor(
key, terms, seen_anchors, search_anchors=search_anchors,
include_aliases=include_key_aliases)
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "KEY anchor search, {}: {}.")
.format(key, key_anchor_matched)
)
if key_anchor_matched in [AnchorMatches.MATCH,
AnchorMatches.ALIAS_INCLUDED]:
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "yielding a KEY-ANCHOR match, {}."
).format(key, tmp_path)
)
if expand_children:
for path in yield_children(
logger, val, terms, pathsep, tmp_path,
seen_anchors, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases):
yield path
else:
yield YAMLPath(tmp_path)
continue
# Search the name of the key, itself
matches = Searches.search_matches(method, term, key)
if (matches and not invert) or (invert and not matches):
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "yielding KEY name match, {}: {}."
).format(key, tmp_path)
)
if expand_children:
# Include every non-excluded child node under this
# matched parent node.
for path in yield_children(
logger, val, terms, pathsep, tmp_path,
seen_anchors, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases):
yield path
else:
# No other matches within this node matter because they
# are already in the result.
yield YAMLPath(tmp_path)
continue
# The value may itself be anchored; search it if requested
if val_anchor_matched is AnchorMatches.ALIAS_EXCLUDED:
continue
if val_anchor_matched in [AnchorMatches.MATCH,
AnchorMatches.ALIAS_INCLUDED]:
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "yielding a VALUE-ANCHOR match, {}.")
.format(tmp_path)
)
if expand_children:
for path in yield_children(
logger, val, terms, pathsep, tmp_path,
seen_anchors, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases):
yield path
else:
yield YAMLPath(tmp_path)
continue
if isinstance(val, (CommentedSeq, CommentedMap)):
logger.debug(
"Recursing into complex data:", data=val,
prefix="yaml_paths::search_for_paths<dict>: ",
footer=">>>> >>>> >>>> >>>> >>>> >>>> >>>>"
)
for subpath in search_for_paths(
logger, processor, val, terms, pathsep, tmp_path,
seen_anchors, search_values=search_values,
search_keys=search_keys, search_anchors=search_anchors,
include_key_aliases=include_key_aliases,
include_value_aliases=include_value_aliases,
decrypt_eyaml=decrypt_eyaml,
expand_children=expand_children
):
logger.debug(
"Yielding RECURSED match, {}.".format(subpath),
prefix="yaml_paths::search_for_paths<dict>: ",
footer="<<<< <<<< <<<< <<<< <<<< <<<< <<<<"
)
yield subpath
elif search_values:
if (val_anchor_matched is AnchorMatches.UNSEARCHABLE_ALIAS
and not include_value_aliases):
continue
check_value = val
if decrypt_eyaml and processor.is_eyaml_value(val):
check_value = processor.decrypt_eyaml(val)
matches = Searches.search_matches(method, term, check_value)
if (matches and not invert) or (invert and not matches):
logger.debug(
("yaml_paths::search_for_paths<dict>:"
+ "yielding VALUE match, {}: {}."
).format(check_value, tmp_path)
)
yield YAMLPath(tmp_path)
def test_none_eyaml_value(self):
assert False == EYAMLProcessor.is_eyaml_value(None)
def main():
"""Main code."""
args = processcli()
log = ConsolePrinter(args)
validateargs(args, log)
change_path = YAMLPath(args.change, pathsep=args.pathsep)
backup_file = args.yaml_file + ".bak"
# Obtain the replacement value
if args.value:
new_value = args.value
elif args.stdin:
new_value = ''.join(sys.stdin.readlines())
elif args.file:
with open(args.file, 'r') as fhnd:
new_value = fhnd.read().rstrip()
elif args.random is not None:
new_value = ''.join(
secrets.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(args.random))
# Prep the YAML parser
yaml = get_yaml_editor()
# Attempt to open the YAML file; check for parsing errors
yaml_data = get_yaml_data(yaml, log, args.yaml_file)
if yaml_data is None:
# An error message has already been logged
exit(1)
# Load the present value at the specified YAML Path
change_nodes = []
old_format = YAMLValueFormats.DEFAULT
processor = EYAMLProcessor(log,
yaml_data,
binary=args.eyaml,
publickey=args.publickey,
privatekey=args.privatekey)
try:
for node in processor.get_nodes(
change_path,
mustexist=(args.mustexist or args.saveto),
default_value=("" if new_value else " ")):
log.debug('Got "{}" from {}.'.format(node, change_path))
change_nodes.append(node)
except YAMLPathException as ex:
log.critical(ex, 1)
if len(change_nodes) == 1:
# When there is exactly one result, its old format can be known. This
# is necessary to retain whether the replacement value should be
# represented later as a multi-line string when the new value is to be
# encrypted.
old_format = YAMLValueFormats.from_node(change_nodes[0])
log.debug("Collected nodes:")
log.debug(change_nodes)
# Check the value(s), if desired
if args.check:
for node in change_nodes:
if processor.is_eyaml_value(node):
# Sanity check: If either --publickey or --privatekey were set
# then they must both be set in order to decrypt this value.
# This is enforced only when the value must be decrypted due to
# a --check request.
if ((args.publickey and not args.privatekey)
or (args.privatekey and not args.publickey)):
log.error(
"Neither or both private and public EYAML keys must be"
+ " set when --check is required to decrypt the old" +
" value.")
exit(1)
try:
check_value = processor.decrypt_eyaml(node)
except EYAMLCommandException as ex:
log.critical(ex, 1)
else:
check_value = node
if not args.check == check_value:
log.critical(
'"{}" does not match the check value.'.format(args.check),
20)
# Save the old value, if desired and possible
if args.saveto:
# Only one can be saved; otherwise it is impossible to meaningfully
# convey to the end-user from exactly which other YAML node each saved
# value came.
if len(change_nodes) > 1:
log.critical(
"It is impossible to meaningly save more than one matched" +
" value. Please omit --saveto or set --change to affect" +
" exactly one value.", 1)
saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep)
log.verbose("Saving the old value to {}.".format(saveto_path))
# Folded EYAML values have their embedded newlines converted to spaces
# when read. As such, writing them back out breaks their original
# format, despite being properly typed. To restore the original
# written form, reverse the conversion, here.
old_value = change_nodes[0]
if ((old_format is YAMLValueFormats.FOLDED
or old_format is YAMLValueFormats.LITERAL)
and EYAMLProcessor.is_eyaml_value(old_value)):
old_value = old_value.replace(" ", "\n")
try:
processor.set_value(saveto_path,
clone_node(old_value),
value_format=old_format)
except YAMLPathException as ex:
log.critical(ex, 1)
# Set the requested value
log.verbose("Setting the new value for {}.".format(change_path))
if args.eyamlcrypt:
# If the user hasn't specified a format, use the same format as the
# value being replaced, if known.
format_type = YAMLValueFormats.from_str(args.format)
if format_type is YAMLValueFormats.DEFAULT:
format_type = old_format
output_type = EYAMLOutputFormats.STRING
if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]:
output_type = EYAMLOutputFormats.BLOCK
try:
processor.set_eyaml_value(change_path,
new_value,
output=output_type,
mustexist=False)
except EYAMLCommandException as ex:
log.critical(ex, 2)
else:
processor.set_value(change_path, new_value, value_format=args.format)
# Save a backup of the original file, if requested
if args.backup:
log.verbose("Saving a backup of {} to {}.".format(
args.yaml_file, backup_file))
if exists(backup_file):
remove(backup_file)
copy2(args.yaml_file, backup_file)
# Save the changed file
log.verbose("Writing changed data to {}.".format(args.yaml_file))
with open(args.yaml_file, 'w') as yaml_dump:
yaml.dump(yaml_data, yaml_dump)