def test_error_quiet_exit(self, capsys): args = SimpleNamespace(verbose=False, quiet=True, debug=False) logger = ConsolePrinter(args) with pytest.raises(SystemExit): logger.error("Test", 27) console = capsys.readouterr() assert console.err == "ERROR: Test\n"
def merge_across( log: ConsolePrinter, lhs_docs: List[Merger], rhs_docs: List[Merger] ) -> int: """Condense LHS and RHS multi-docs together into one.""" return_state = 0 lhs_len = len(lhs_docs) rhs_len = len(rhs_docs) max_len = lhs_len if lhs_len > rhs_len else rhs_len lhs_limit = lhs_len - 1 rhs_limit = rhs_len - 1 for i in range(0, max_len): if i > rhs_limit: break if i > lhs_limit: lhs_docs.append(rhs_docs[i]) continue try: lhs_docs[i].merge_with(rhs_docs[i].data) except MergeException as mex: log.error(mex) return_state = 31 break except YAMLPathException as yex: log.error(yex) return_state = 32 break return return_state
def merge_condense_all( log: ConsolePrinter, lhs_docs: List[Merger], rhs_docs: List[Merger] ) -> int: """Condense LHS and RHS multi-docs together into one.""" return_state = 0 lhs_prime = lhs_docs[0] if len(lhs_docs) > 1: for lhs_doc in lhs_docs[1:]: try: lhs_prime.merge_with(lhs_doc.data) except MergeException as mex: log.error(mex) return_state = 11 except YAMLPathException as yex: log.error(yex) return_state = 12 # With all subdocs merged into the first, eliminate all subdocs for i in reversed(range(1, len(lhs_docs))): del lhs_docs[i] # Merge every RHS doc into the prime LHS doc for rhs_doc in rhs_docs: try: lhs_prime.merge_with(rhs_doc.data) except MergeException as mex: log.error(mex) return_state = 13 except YAMLPathException as yex: log.error(yex) return_state = 14 return return_state
def process_yaml_file( merger: Merger, log: ConsolePrinter, rhs_yaml: Any, rhs_file: str, merger_primed: bool ): """Merge RHS document(s) into the prime document.""" # Except for - (STDIN), each YAML_FILE must actually be a file; because # merge data is expected, this is a fatal failure. if rhs_file != "-" and not isfile(rhs_file): log.error("Not a file: {}".format(rhs_file)) return 2 log.info( "Processing {}...".format( "STDIN" if rhs_file.strip() == "-" else rhs_file)) return merge_multidoc(rhs_file, rhs_yaml, log, merger, merger_primed)
def merge_matrix( log: ConsolePrinter, lhs_docs: List[Merger], rhs_docs: List[Merger] ) -> int: """Condense LHS and RHS multi-docs together into one.""" return_state = 0 for lhs_doc in lhs_docs: for rhs_doc in rhs_docs: try: lhs_doc.merge_with(rhs_doc.data) except MergeException as mex: log.error(mex) return_state = 41 break except YAMLPathException as yex: log.error(yex) return_state = 42 break return return_state
def get_search_term(logger: ConsolePrinter, expression: str) -> Optional[SearchTerms]: """ Attempts to cast a search expression into a SearchTerms instance. Returns None on failure. """ # The leading character must be a known search operator check_operator = expression[0] if expression else "" if not (PathSearchMethods.is_operator(check_operator) or check_operator == '!'): logger.error(("Invalid search expression, '{}'. The first symbol of" + " every search expression must be one of: {}").format( expression, ", ".join(PathSearchMethods.get_operators()))) return None if not len(expression) > 1: # Empty expressions do nothing logger.error( "An EXPRESSION with only a search operator has no effect, '{}'.". format(expression)) return None try: exterm = create_searchterms_from_pathattributes( YAMLPath("[*{}]".format(expression)).escaped[0][1]) except YAMLPathException as ex: logger.error(("Invalid search expression, '{}', due to: {}").format( expression, ex)) return None return exterm
def get_doc_mergers( log: ConsolePrinter, yaml_editor: YAML, config: MergerConfig, yaml_file: str ) -> Tuple[List[Merger], bool]: """Create a list of Mergers, one for each source document.""" docs_loaded = True if yaml_file != "-" and not isfile(yaml_file): log.error("Not a file: {}".format(yaml_file)) return ([], False) doc_mergers: List[Merger] = [] for (yaml_data, doc_loaded) in Parsers.get_yaml_multidoc_data( yaml_editor, log, yaml_file ): if not doc_loaded: # An error message has already been logged doc_mergers.clear() docs_loaded = False break doc_mergers.append(Merger(log, yaml_data, config)) return (doc_mergers, docs_loaded)
def main(): """Main code.""" args = processcli() log = ConsolePrinter(args) validateargs(args, log) change_path = YAMLPath(args.change, pathsep=args.pathsep) must_exist = args.mustexist or args.saveto # Obtain the replacement value consumed_stdin = False new_value = None has_new_value = False if args.value or args.value == "": new_value = args.value has_new_value = True elif args.stdin: new_value = ''.join(sys.stdin.readlines()) consumed_stdin = True has_new_value = True elif args.file: with open(args.file, 'r') as fhnd: new_value = fhnd.read().rstrip() has_new_value = True elif args.null: new_value = None has_new_value = True elif args.random is not None: new_value = ''.join( secrets.choice(args.random_from) for _ in range(args.random)) has_new_value = True # Prep the YAML parser yaml = Parsers.get_yaml_editor() # Attempt to open the YAML file; check for parsing errors if args.yaml_file: yaml_data = _try_load_input_file(args, log, yaml, change_path, new_value) if args.yaml_file.strip() == '-': consumed_stdin = True # Check for a waiting STDIN document if (not consumed_stdin and not args.yaml_file and not args.nostdin and not sys.stdin.isatty()): args.yaml_file = "-" yaml_data = _try_load_input_file(args, log, yaml, change_path, new_value) # Load the present nodes at the specified YAML Path processor = EYAMLProcessor(log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) change_node_coordinates = _get_nodes( log, processor, change_path, must_exist=must_exist, default_value=("" if new_value else " ")) old_format = YAMLValueFormats.DEFAULT if len(change_node_coordinates) == 1: # When there is exactly one result, its old format can be known. This # is necessary to retain whether the replacement value should be # represented later as a multi-line string when the new value is to be # encrypted. old_format = YAMLValueFormats.from_node( change_node_coordinates[0].node) # Check the value(s), if desired if args.check: for node_coordinate in change_node_coordinates: if processor.is_eyaml_value(node_coordinate.node): # Sanity check: If either --publickey or --privatekey were set # then they must both be set in order to decrypt this value. # This is enforced only when the value must be decrypted due to # a --check request. if ((args.publickey and not args.privatekey) or (args.privatekey and not args.publickey)): log.error( "Neither or both private and public EYAML keys must be" + " set when --check is required to decrypt the old" + " value.") sys.exit(1) try: check_value = processor.decrypt_eyaml(node_coordinate.node) except EYAMLCommandException as ex: log.critical(ex, 1) else: check_value = node_coordinate.node if not args.check == check_value: log.critical( '"{}" does not match the check value.'.format(args.check), 20) # Save the old value, if desired and possible if args.saveto: # Only one can be saved; otherwise it is impossible to meaningfully # convey to the end-user from exactly which other YAML node each saved # value came. if len(change_node_coordinates) > 1: log.critical( "It is impossible to meaningly save more than one matched" + " value. Please omit --saveto or set --change to affect" + " exactly one value.", 1) saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep) log.verbose("Saving the old value to {}.".format(saveto_path)) # Folded EYAML values have their embedded newlines converted to spaces # when read. As such, writing them back out breaks their original # format, despite being properly typed. To restore the original # written form, reverse the conversion, here. old_value = change_node_coordinates[0].node if ((old_format is YAMLValueFormats.FOLDED or old_format is YAMLValueFormats.LITERAL) and EYAMLProcessor.is_eyaml_value(old_value)): old_value = old_value.replace(" ", "\n") try: processor.set_value(saveto_path, Nodes.clone_node(old_value), value_format=old_format, tag=args.tag) except YAMLPathException as ex: log.critical(ex, 1) # Set the requested value log.verbose("Applying changes to {}.".format(change_path)) if args.delete: # Destroy the collected nodes (from their parents) in the reverse order # they were discovered. This is necessary lest Array elements be # improperly handled, leading to unwanted data loss. _delete_nodes(log, processor, change_node_coordinates) elif args.aliasof: # Assign the change nodes as Aliases of whatever --aliasof points to _alias_nodes(log, processor, change_node_coordinates, args.aliasof, args.anchor) elif args.eyamlcrypt: # If the user hasn't specified a format, use the same format as the # value being replaced, if known. format_type = YAMLValueFormats.from_str(args.format) if format_type is YAMLValueFormats.DEFAULT: format_type = old_format output_type = EYAMLOutputFormats.STRING if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]: output_type = EYAMLOutputFormats.BLOCK try: processor.set_eyaml_value(change_path, new_value, output=output_type, mustexist=False) except EYAMLCommandException as ex: log.critical(ex, 2) elif has_new_value: try: processor.set_value(change_path, new_value, value_format=args.format, mustexist=must_exist, tag=args.tag) except YAMLPathException as ex: log.critical(ex, 1) elif args.tag: _tag_nodes(processor.data, args.tag, change_node_coordinates) # Write out the result write_output_document(args, log, yaml, yaml_data)
def get_yaml_data(parser: Any, logger: ConsolePrinter, source: str, **kwargs) -> Tuple[Any, bool]: """ Parse YAML/Compatible data and return the ruamel.yaml object result. All known issues are caught and distinctively logged. Parameters: 1. parser (ruamel.yaml.YAML) The YAML data parser 2. logger (ConsolePrinter) The logging facility 3. source (str) The source file or serialized literal to load; can be - for reading from STDIN (implies literal=True) Keyword Parameters: * literal (bool) `source` is literal serialized YAML data rather than a file-spec, so load it directly Returns: Tuple[Any, bool] A tuple containing the document and its success/fail state. The first field is the parsed document; will be None for empty documents and for documents which could not be read. The second field will be True when there were no errors during parsing and False, otherwise. """ literal = kwargs.pop("literal", False) yaml_data = None data_available = True # This code traps errors and warnings from ruamel.yaml, substituting # lengthy stack-dumps with specific, meaningful feedback. Further, # some warnings are treated as errors by ruamel.yaml, so these are also # coallesced into cleaner feedback. try: with warnings.catch_warnings(): warnings.filterwarnings("error") if source == "-": yaml_data = parser.load(stdin.read()) else: if literal: yaml_data = parser.load(source) else: with open(source, 'r') as fhnd: yaml_data = parser.load(fhnd) except KeyboardInterrupt: logger.error("Aborting data load due to keyboard interrupt!") data_available = False except FileNotFoundError: logger.error("File not found: {}".format(source)) data_available = False except ParserError as ex: logger.error("YAML parsing error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) data_available = False except ComposerError as ex: logger.error("YAML composition error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) data_available = False except ConstructorError as ex: logger.error("YAML construction error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) data_available = False except ScannerError as ex: logger.error("YAML syntax error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) data_available = False except DuplicateKeyError as dke: omits = [ "while constructing", "To suppress this", "readthedocs", "future releases", "the new API", ] message = str(dke).split("\n") newmsg = "" for line in message: line = line.strip() if not line: continue write_line = True for omit in omits: if omit in line: write_line = False break if write_line: newmsg += "\n " + line logger.error("Duplicate Hash key detected: {}".format(newmsg)) data_available = False except ReusedAnchorWarning as raw: logger.error("Duplicate YAML Anchor detected: {}".format( str(raw).replace("occurrence ", "occurrence ").replace("\n", "\n "))) data_available = False return (yaml_data, data_available)
def get_yaml_multidoc_data( parser: Any, logger: ConsolePrinter, source: str, **kwargs) -> Generator[Tuple[Any, bool], None, None]: """ Parse YAML/Compatible multi-docs and yield each ruamel.yaml object. All known issues are caught and distinctively logged. Parameters: 1. parser (ruamel.yaml.YAML) The YAML data parser 2. logger (ConsolePrinter) The logging facility 3. source (str) The source file to load; can be - for reading from STDIN Keyword Parameters: * literal (bool) `source` is literal serialized YAML data rather than a file-spec, so load it directly Returns: Generator[Tuple[Any, bool], None, None] A tuple for each document as it is parsed. The first field is the parsed document; will be None for empty documents and for documents which could not be read. The second field will be True when there were no errors during parsing and False, otherwise. """ literal = kwargs.pop("literal", False) # This code traps errors and warnings from ruamel.yaml, substituting # lengthy stack-dumps with specific, meaningful feedback. Further, # some warnings are treated as errors by ruamel.yaml, so these are also # coallesced into cleaner feedback. has_error = False try: with warnings.catch_warnings(): warnings.filterwarnings("error") if source == "-": doc_yielded = False for document in parser.load_all(stdin.read()): doc_yielded = True logger.debug( "Yielding document from {}:".format(source), prefix="get_yaml_multidoc_data: ", data=document) yield (document, True) # The user sent a deliberately empty document via STDIN if not doc_yielded: yield ("", True) else: if literal: for document in parser.load_all(source): yield (document, True) else: with open(source, 'r') as fhnd: for document in parser.load_all(fhnd): logger.debug( "Yielding document from {}:".format( source), prefix="get_yaml_multidoc_data: ", data=document) yield (document, True) except KeyboardInterrupt: has_error = True logger.error("Aborting data load due to keyboard interrupt!") except FileNotFoundError: has_error = True logger.error("File not found: {}".format(source)) except ParserError as ex: has_error = True logger.error("YAML parsing error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) except ComposerError as ex: has_error = True logger.error("YAML composition error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) except ConstructorError as ex: has_error = True logger.error("YAML construction error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) except ScannerError as ex: has_error = True logger.error("YAML syntax error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) except DuplicateKeyError as dke: has_error = True omits = [ "while constructing", "To suppress this", "readthedocs", "future releases", "the new API", ] message = str(dke).split("\n") newmsg = "" for line in message: line = line.strip() if not line: continue write_line = True for omit in omits: if omit in line: write_line = False break if write_line: newmsg += "\n " + line logger.error("Duplicate Hash key detected: {}".format(newmsg)) except ReusedAnchorWarning as raw: has_error = True logger.error("Duplicate YAML Anchor detected: {}".format( str(raw).replace("occurrence ", "occurrence ").replace("\n", "\n "))) if has_error: yield (None, False)
def main(): """Main code.""" # Process any command-line arguments args = processcli() log = ConsolePrinter(args) validateargs(args, log) processor = EYAMLProcessor(log, None, binary=args.eyaml) # Prep the YAML parser yaml = Parsers.get_yaml_editor() # Process the input file(s) in_file_count = len(args.yaml_files) exit_state = 0 for yaml_file in args.yaml_files: file_changed = False backup_file = yaml_file + ".bak" seen_anchors = [] # Each YAML_FILE must actually be a file if not isfile(yaml_file): log.error("Not a file: {}".format(yaml_file)) exit_state = 2 continue # Don't bother with the file change update when there's only one input # file. if in_file_count > 1: log.info("Processing {}...".format(yaml_file)) # Try to open the file (yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml, log, yaml_file) if not doc_loaded: # An error message has already been logged exit_state = 3 continue # Process all EYAML values processor.data = yaml_data for yaml_path in processor.find_eyaml_paths(): # Use ::get_nodes() instead of ::get_eyaml_values() here in order # to ignore values that have already been rotated via their # Anchors. for node_coordinate in processor.get_nodes(yaml_path, mustexist=True): # Ignore values which are Aliases for those already decrypted node = node_coordinate.node anchor_name = Anchors.get_node_anchor(node) if anchor_name is not None: if anchor_name in seen_anchors: continue seen_anchors.append(anchor_name) log.verbose("Decrypting value(s) at {}.".format(yaml_path)) processor.publickey = args.oldpublickey processor.privatekey = args.oldprivatekey try: txtval = processor.decrypt_eyaml(node) except EYAMLCommandException as ex: log.error(ex) exit_state = 3 continue # Prefer block (folded) values unless the original YAML value # was already a massivly long (string) line. output = EYAMLOutputFormats.BLOCK if not isinstance(node, FoldedScalarString): output = EYAMLOutputFormats.STRING # Re-encrypt the value with new EYAML keys processor.publickey = args.newpublickey processor.privatekey = args.newprivatekey try: processor.set_eyaml_value(yaml_path, txtval, output=output) except EYAMLCommandException as ex: log.error(ex) exit_state = 3 continue file_changed = True # Save the changes if file_changed: if args.backup: log.verbose("Saving a backup of {} to {}.".format( yaml_file, backup_file)) if exists(backup_file): remove(backup_file) copy2(yaml_file, backup_file) log.verbose("Writing changed data to {}.".format(yaml_file)) with open(yaml_file, 'w', encoding='utf-8') as yaml_dump: yaml.dump(yaml_data, yaml_dump) sys.exit(exit_state)
def test_error_quiet_nonexit(self, capsys): args = SimpleNamespace(verbose=False, quiet=True, debug=False) logger = ConsolePrinter(args) logger.error("Test") console = capsys.readouterr() assert console.err == "ERROR: Test\n"
def main(): """Main code.""" args = processcli() log = ConsolePrinter(args) validateargs(args, log) change_path = YAMLPath(args.change, pathsep=args.pathsep) backup_file = args.yaml_file + ".bak" # Obtain the replacement value if args.value: new_value = args.value elif args.stdin: new_value = ''.join(sys.stdin.readlines()) elif args.file: with open(args.file, 'r') as fhnd: new_value = fhnd.read().rstrip() elif args.random is not None: new_value = ''.join( secrets.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(args.random)) # Prep the YAML parser yaml = get_yaml_editor() # Attempt to open the YAML file; check for parsing errors yaml_data = get_yaml_data(yaml, log, args.yaml_file) if yaml_data is None: # An error message has already been logged exit(1) # Load the present value at the specified YAML Path change_nodes = [] old_format = YAMLValueFormats.DEFAULT processor = EYAMLProcessor(log, yaml_data, binary=args.eyaml, publickey=args.publickey, privatekey=args.privatekey) try: for node in processor.get_nodes( change_path, mustexist=(args.mustexist or args.saveto), default_value=("" if new_value else " ")): log.debug('Got "{}" from {}.'.format(node, change_path)) change_nodes.append(node) except YAMLPathException as ex: log.critical(ex, 1) if len(change_nodes) == 1: # When there is exactly one result, its old format can be known. This # is necessary to retain whether the replacement value should be # represented later as a multi-line string when the new value is to be # encrypted. old_format = YAMLValueFormats.from_node(change_nodes[0]) log.debug("Collected nodes:") log.debug(change_nodes) # Check the value(s), if desired if args.check: for node in change_nodes: if processor.is_eyaml_value(node): # Sanity check: If either --publickey or --privatekey were set # then they must both be set in order to decrypt this value. # This is enforced only when the value must be decrypted due to # a --check request. if ((args.publickey and not args.privatekey) or (args.privatekey and not args.publickey)): log.error( "Neither or both private and public EYAML keys must be" + " set when --check is required to decrypt the old" + " value.") exit(1) try: check_value = processor.decrypt_eyaml(node) except EYAMLCommandException as ex: log.critical(ex, 1) else: check_value = node if not args.check == check_value: log.critical( '"{}" does not match the check value.'.format(args.check), 20) # Save the old value, if desired and possible if args.saveto: # Only one can be saved; otherwise it is impossible to meaningfully # convey to the end-user from exactly which other YAML node each saved # value came. if len(change_nodes) > 1: log.critical( "It is impossible to meaningly save more than one matched" + " value. Please omit --saveto or set --change to affect" + " exactly one value.", 1) saveto_path = YAMLPath(args.saveto, pathsep=args.pathsep) log.verbose("Saving the old value to {}.".format(saveto_path)) # Folded EYAML values have their embedded newlines converted to spaces # when read. As such, writing them back out breaks their original # format, despite being properly typed. To restore the original # written form, reverse the conversion, here. old_value = change_nodes[0] if ((old_format is YAMLValueFormats.FOLDED or old_format is YAMLValueFormats.LITERAL) and EYAMLProcessor.is_eyaml_value(old_value)): old_value = old_value.replace(" ", "\n") try: processor.set_value(saveto_path, clone_node(old_value), value_format=old_format) except YAMLPathException as ex: log.critical(ex, 1) # Set the requested value log.verbose("Setting the new value for {}.".format(change_path)) if args.eyamlcrypt: # If the user hasn't specified a format, use the same format as the # value being replaced, if known. format_type = YAMLValueFormats.from_str(args.format) if format_type is YAMLValueFormats.DEFAULT: format_type = old_format output_type = EYAMLOutputFormats.STRING if format_type in [YAMLValueFormats.FOLDED, YAMLValueFormats.LITERAL]: output_type = EYAMLOutputFormats.BLOCK try: processor.set_eyaml_value(change_path, new_value, output=output_type, mustexist=False) except EYAMLCommandException as ex: log.critical(ex, 2) else: processor.set_value(change_path, new_value, value_format=args.format) # Save a backup of the original file, if requested if args.backup: log.verbose("Saving a backup of {} to {}.".format( args.yaml_file, backup_file)) if exists(backup_file): remove(backup_file) copy2(args.yaml_file, backup_file) # Save the changed file log.verbose("Writing changed data to {}.".format(args.yaml_file)) with open(args.yaml_file, 'w') as yaml_dump: yaml.dump(yaml_data, yaml_dump)
def get_yaml_data(parser: Any, logger: ConsolePrinter, source: str) -> Any: """ Attempts to parse YAML/Compatible data and return the ruamel.yaml object result. All known issues are caught and distinctively logged. Returns None when the data could not be loaded. """ import warnings yaml_data = None # This code traps errors and warnings from ruamel.yaml, substituting # lengthy stack-dumps with specific, meaningful feedback. Further, some # warnings are treated as errors by ruamel.yaml, so these are also # coallesced into cleaner feedback. try: with open(source, 'r') as fhnd: with warnings.catch_warnings(): warnings.filterwarnings("error") yaml_data = parser.load(fhnd) except KeyboardInterrupt: logger.error("Aborting data load due to keyboard interrupt!") yaml_data = None except FileNotFoundError: logger.error("File not found: {}".format(source)) yaml_data = None except ParserError as ex: logger.error("YAML parsing error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) yaml_data = None except ComposerError as ex: logger.error("YAML composition error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) yaml_data = None except ConstructorError as ex: logger.error("YAML construction error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) yaml_data = None except ScannerError as ex: logger.error("YAML syntax error {}: {}".format( str(ex.problem_mark).lstrip(), ex.problem)) yaml_data = None except DuplicateKeyError as dke: omits = [ "while constructing", "To suppress this", "readthedocs", "future releases", "the new API", ] message = str(dke).split("\n") newmsg = "" for line in message: line = line.strip() if not line: continue write_line = True for omit in omits: if omit in line: write_line = False break if write_line: newmsg += "\n " + line logger.error("Duplicate Hash key detected: {}".format(newmsg)) yaml_data = None except ReusedAnchorWarning as raw: logger.error("Duplicate YAML Anchor detected: {}".format( str(raw).replace("occurrence ", "occurrence ").replace("\n", "\n "))) yaml_data = None return yaml_data