def explore(data): if isinstance(data, (dict, OrderedDict)): return PrintableDict([(key, explore(value)) for key, value in data.items()]) elif isinstance(data, (list, tuple, set)): return data.__class__([explore(value) for value in data]) return data
def _convert_to_unicode(data): """ Helper function that makes sure all items in the dictionary are unicode for comparing the existing state with the desired state. This function is only needed for Python 2 and can be removed once we've migrated to Python 3. The data returned by the current settings sometimes has a mix of unicode and string values (these don't matter in Py3). This causes the comparison to say it's not in the correct state even though it is. They basically compares apples to apples, etc. Also, in Python 2, the utf-16 encoded strings remain utf-16 encoded (each character separated by `/x00`) In Python 3 it returns a utf-8 string. This will just remove all the null bytes (`/x00`), again comparing apples to apples. """ if isinstance(data, str): data = data.replace("\x00", "") return salt.utils.stringutils.to_unicode(data) elif isinstance(data, dict): return { _convert_to_unicode(k): _convert_to_unicode(v) for k, v in data.items() } elif isinstance(data, list): return list(_convert_to_unicode(v) for v in data) else: return data
def output(data, **kwargs): # pylint: disable=unused-argument """ The HighState Outputter is only meant to be used with the state.highstate function, or a function that returns highstate return data. """ # If additional information is passed through via the "data" dictionary to # the highstate outputter, such as "outputter" or "retcode", discard it. # We only want the state data that was passed through, if it is wrapped up # in the "data" key, as the orchestrate runner does. See Issue #31330, # pull request #27838, and pull request #27175 for more information. # account for envelope data if being passed lookup_jid ret if isinstance(data, dict) and "return" in data: data = data["return"] if isinstance(data, dict) and "data" in data: data = data["data"] # account for envelope data if being passed lookup_jid ret if isinstance(data, dict) and len(data.keys()) == 1: _data = next(iter(data.values())) if isinstance(_data, dict): if "jid" in _data and "fun" in _data: data = _data.get("return", {}).get("data", data) # output() is recursive, if we aren't passed a dict just return it if isinstance(data, int) or isinstance(data, str): return data if data is None: return "None" # Discard retcode in dictionary as present in orchestrate data local_masters = [key for key in data.keys() if key.endswith("_master")] orchestrator_output = "retcode" in data.keys() and len(local_masters) == 1 if orchestrator_output: del data["retcode"] # pre-process data if state_compress_ids is set if __opts__.get("state_compress_ids", False): data = _compress_ids(data) indent_level = kwargs.get("indent_level", 1) ret = [ _format_host(host, hostdata, indent_level=indent_level)[0] for host, hostdata in data.items() ] if ret: return "\n".join(ret) log.error( "Data passed to highstate outputter is not a valid highstate return: %s", data) # We should not reach here, but if we do return empty string return ""
def format_call(fun, data, initial_ret=None, expected_extra_kws=(), is_class_method=None): """ Build the required arguments and keyword arguments required for the passed function. :param fun: The function to get the argspec from :param data: A dictionary containing the required data to build the arguments and keyword arguments. :param initial_ret: The initial return data pre-populated as dictionary or None :param expected_extra_kws: Any expected extra keyword argument names which should not trigger a :ref:`SaltInvocationError` :param is_class_method: Pass True if you are sure that the function being passed is a class method. The reason for this is that on Python 3 ``inspect.ismethod`` only returns ``True`` for bound methods, while on Python 2, it returns ``True`` for bound and unbound methods. So, on Python 3, in case of a class method, you'd need the class to which the function belongs to be instantiated and this is not always wanted. :returns: A dictionary with the function required arguments and keyword arguments. """ ret = initial_ret is not None and initial_ret or {} ret["args"] = [] ret["kwargs"] = {} aspec = get_function_argspec(fun, is_class_method=is_class_method) arg_data = arg_lookup(fun, aspec) args = arg_data["args"] kwargs = arg_data["kwargs"] # Since we WILL be changing the data dictionary, let's change a copy of it data = data.copy() missing_args = [] for key in kwargs: try: kwargs[key] = data.pop(key) except KeyError: # Let's leave the default value in place pass while args: arg = args.pop(0) try: ret["args"].append(data.pop(arg)) except KeyError: missing_args.append(arg) if missing_args: used_args_count = len(ret["args"]) + len(args) args_count = used_args_count + len(missing_args) raise SaltInvocationError( "{} takes at least {} argument{} ({} given)".format( fun.__name__, args_count, args_count > 1 and "s" or "", used_args_count)) ret["kwargs"].update(kwargs) if aspec.keywords: # The function accepts **kwargs, any non expected extra keyword # arguments will made available. for key, value in data.items(): if key in expected_extra_kws: continue ret["kwargs"][key] = value # No need to check for extra keyword arguments since they are all # **kwargs now. Return return ret # Did not return yet? Lets gather any remaining and unexpected keyword # arguments extra = {} for key, value in data.items(): if key in expected_extra_kws: continue extra[key] = copy.deepcopy(value) if extra: # Found unexpected keyword arguments, raise an error to the user if len(extra) == 1: msg = "'{0[0]}' is an invalid keyword argument for '{1}'".format( list(extra.keys()), ret.get( # In case this is being called for a state module "full", # Not a state module, build the name "{}.{}".format(fun.__module__, fun.__name__), ), ) else: msg = "{} and '{}' are invalid keyword arguments for '{}'".format( ", ".join(["'{}'".format(e) for e in extra][:-1]), list(extra.keys())[-1], ret.get( # In case this is being called for a state module "full", # Not a state module, build the name "{}.{}".format(fun.__module__, fun.__name__), ), ) raise SaltInvocationError(msg) return ret
def _format_host(host, data, indent_level=1): """ Main highstate formatter. can be called recursively if a nested highstate contains other highstates (ie in an orchestration) """ host = salt.utils.data.decode(host) colors = salt.utils.color.get_colors(__opts__.get("color"), __opts__.get("color_theme")) tabular = __opts__.get("state_tabular", False) rcounts = {} rdurations = [] hcolor = colors["GREEN"] hstrs = [] nchanges = 0 strip_colors = __opts__.get("strip_colors", True) if isinstance(data, int): nchanges = 1 hstrs.append("{0} {1}{2[ENDC]}".format(hcolor, data, colors)) hcolor = colors["CYAN"] # Print the minion name in cyan elif isinstance(data, str): # Data in this format is from saltmod.function, # so it is always a 'change' nchanges = 1 for data in data.splitlines(): hstrs.append("{0} {1}{2[ENDC]}".format(hcolor, data, colors)) hcolor = colors["CYAN"] # Print the minion name in cyan elif isinstance(data, list): # Errors have been detected, list them in RED! hcolor = colors["LIGHT_RED"] hstrs.append(" {0}Data failed to compile:{1[ENDC]}".format( hcolor, colors)) for err in data: if strip_colors: err = salt.output.strip_esc_sequence( salt.utils.data.decode(err)) hstrs.append("{0}----------\n {1}{2[ENDC]}".format( hcolor, err, colors)) elif isinstance(data, dict): # Verify that the needed data is present data_tmp = {} for tname, info in data.items(): if (isinstance(info, dict) and tname != "changes" and info and "__run_num__" not in info): err = ("The State execution failed to record the order " "in which all states were executed. The state " "return missing data is:") hstrs.insert(0, pprint.pformat(info)) hstrs.insert(0, err) if isinstance(info, dict) and "result" in info: data_tmp[tname] = info data = data_tmp # Everything rendered as it should display the output for tname in sorted(data, key=lambda k: data[k].get("__run_num__", 0)): ret = data[tname] # Increment result counts rcounts.setdefault(ret["result"], 0) # unpack state compression counts compressed_count = 1 if (__opts__.get("state_compress_ids", False) and "_|-state_compressed_" in tname): _, _id, _, _ = tname.split("_|-") count_match = re.search(r"\((\d+)\)$", _id) if count_match: compressed_count = int(count_match.group(1)) rcounts[ret["result"]] += compressed_count rduration = ret.get("duration", 0) try: rdurations.append(float(rduration)) except ValueError: rduration, _, _ = rduration.partition(" ms") try: rdurations.append(float(rduration)) except ValueError: log.error("Cannot parse a float from duration %s", ret.get("duration", 0)) tcolor = colors["GREEN"] if ret.get("name") in [ "state.orch", "state.orchestrate", "state.sls" ]: nested = output(ret["changes"], indent_level=indent_level + 1) ctext = re.sub("^", " " * 14 * indent_level, "\n" + nested, flags=re.MULTILINE) schanged = True nchanges += 1 else: schanged, ctext = _format_changes(ret["changes"]) # if compressed, the changes are keyed by name if schanged and compressed_count > 1: nchanges += len(ret["changes"].get("compressed changes", {})) or 1 else: nchanges += 1 if schanged else 0 # Skip this state if it was successful & diff output was requested if (__opts__.get("state_output_diff", False) and ret["result"] and not schanged): continue # Skip this state if state_verbose is False, the result is True and # there were no changes made if (not __opts__.get("state_verbose", False) and ret["result"] and not schanged): continue if schanged: tcolor = colors["CYAN"] if ret["result"] is False: hcolor = colors["RED"] tcolor = colors["RED"] if ret["result"] is None: hcolor = colors["LIGHT_YELLOW"] tcolor = colors["LIGHT_YELLOW"] state_output = __opts__.get("state_output", "full").lower() comps = tname.split("_|-") if state_output.endswith("_id"): # Swap in the ID for the name. Refs #35137 comps[2] = comps[1] if state_output.startswith("filter"): # By default, full data is shown for all types. However, return # data may be excluded by setting state_output_exclude to a # comma-separated list of True, False or None, or including the # same list with the exclude option on the command line. For # now, this option must include a comma. For example: # exclude=True, # The same functionality is also available for making return # data terse, instead of excluding it. cliargs = __opts__.get("arg", []) clikwargs = {} for item in cliargs: if isinstance(item, dict) and "__kwarg__" in item: clikwargs = item.copy() exclude = clikwargs.get( "exclude", __opts__.get("state_output_exclude", [])) if isinstance(exclude, str): exclude = str(exclude).split(",") terse = clikwargs.get("terse", __opts__.get("state_output_terse", [])) if isinstance(terse, str): terse = str(terse).split(",") if str(ret["result"]) in terse: msg = _format_terse(tcolor, comps, ret, colors, tabular) hstrs.append(msg) continue if str(ret["result"]) in exclude: continue elif any(( state_output.startswith("terse"), state_output.startswith("mixed") and ret["result"] is not False, # only non-error'd state_output.startswith("changes") and ret["result"] and not schanged, # non-error'd non-changed )): # Print this chunk in a terse way and continue in the loop msg = _format_terse(tcolor, comps, ret, colors, tabular) hstrs.append(msg) continue state_lines = [ "{tcolor}----------{colors[ENDC]}", " {tcolor} ID: {comps[1]}{colors[ENDC]}", " {tcolor}Function: {comps[0]}.{comps[3]}{colors[ENDC]}", " {tcolor} Result: {ret[result]!s}{colors[ENDC]}", " {tcolor} Comment: {comment}{colors[ENDC]}", ] if __opts__.get("state_output_profile") and "start_time" in ret: state_lines.extend([ " {tcolor} Started: {ret[start_time]!s}{colors[ENDC]}", " {tcolor}Duration: {ret[duration]!s}{colors[ENDC]}", ]) # This isn't the prettiest way of doing this, but it's readable. if comps[1] != comps[2]: state_lines.insert( 3, " {tcolor} Name: {comps[2]}{colors[ENDC]}") # be sure that ret['comment'] is utf-8 friendly try: if not isinstance(ret["comment"], str): ret["comment"] = str(ret["comment"]) except UnicodeDecodeError: # If we got here, we're on Python 2 and ret['comment'] somehow # contained a str type with unicode content. ret["comment"] = salt.utils.stringutils.to_unicode( ret["comment"]) try: comment = salt.utils.data.decode(ret["comment"]) comment = comment.strip().replace("\n", "\n" + " " * 14) except AttributeError: # Assume comment is a list try: comment = ret["comment"].join(" ").replace( "\n", "\n" + " " * 13) except AttributeError: # Comment isn't a list either, just convert to string comment = str(ret["comment"]) comment = comment.strip().replace("\n", "\n" + " " * 14) # If there is a data attribute, append it to the comment if "data" in ret: if isinstance(ret["data"], list): for item in ret["data"]: comment = "{} {}".format(comment, item) elif isinstance(ret["data"], dict): for key, value in ret["data"].items(): comment = "{}\n\t\t{}: {}".format(comment, key, value) else: comment = "{} {}".format(comment, ret["data"]) for detail in ["start_time", "duration"]: ret.setdefault(detail, "") if ret["duration"] != "": ret["duration"] = "{} ms".format(ret["duration"]) svars = { "tcolor": tcolor, "comps": comps, "ret": ret, "comment": salt.utils.data.decode(comment), # This nukes any trailing \n and indents the others. "colors": colors, } hstrs.extend([sline.format(**svars) for sline in state_lines]) changes = " Changes: " + ctext hstrs.append("{0}{1}{2[ENDC]}".format(tcolor, changes, colors)) if "warnings" in ret: rcounts.setdefault("warnings", 0) rcounts["warnings"] += 1 wrapper = textwrap.TextWrapper(width=80, initial_indent=" " * 14, subsequent_indent=" " * 14) hstrs.append( " {colors[LIGHT_RED]} Warnings: {0}{colors[ENDC]}". format(wrapper.fill("\n".join(ret["warnings"])).lstrip(), colors=colors)) # Append result counts to end of output colorfmt = "{0}{1}{2[ENDC]}" rlabel = { True: "Succeeded", False: "Failed", None: "Not Run", "warnings": "Warnings", } count_max_len = max([len(str(x)) for x in rcounts.values()] or [0]) label_max_len = max([len(x) for x in rlabel.values()] or [0]) line_max_len = label_max_len + count_max_len + 2 # +2 for ': ' hstrs.append( colorfmt.format( colors["CYAN"], "\nSummary for {}\n{}".format(host, "-" * line_max_len), colors, )) def _counts(label, count): return "{0}: {1:>{2}}".format(label, count, line_max_len - (len(label) + 2)) # Successful states changestats = [] if None in rcounts and rcounts.get(None, 0) > 0: # test=True states changestats.append( colorfmt.format( colors["LIGHT_YELLOW"], "unchanged={}".format(rcounts.get(None, 0)), colors, )) if nchanges > 0: changestats.append( colorfmt.format(colors["GREEN"], "changed={}".format(nchanges), colors)) if changestats: changestats = " ({})".format(", ".join(changestats)) else: changestats = "" hstrs.append( colorfmt.format( colors["GREEN"], _counts(rlabel[True], rcounts.get(True, 0) + rcounts.get(None, 0)), colors, ) + changestats) # Failed states num_failed = rcounts.get(False, 0) hstrs.append( colorfmt.format( colors["RED"] if num_failed else colors["CYAN"], _counts(rlabel[False], num_failed), colors, )) if __opts__.get("state_output_pct", False): # Add success percentages to the summary output try: success_pct = round( ((rcounts.get(True, 0) + rcounts.get(None, 0)) / (sum(rcounts.values()) - rcounts.get("warnings", 0))) * 100, 2, ) hstrs.append( colorfmt.format( colors["GREEN"], _counts("Success %", success_pct), colors, )) except ZeroDivisionError: pass # Add failure percentages to the summary output try: failed_pct = round( (num_failed / (sum(rcounts.values()) - rcounts.get("warnings", 0))) * 100, 2, ) hstrs.append( colorfmt.format( colors["RED"] if num_failed else colors["CYAN"], _counts("Failure %", failed_pct), colors, )) except ZeroDivisionError: pass num_warnings = rcounts.get("warnings", 0) if num_warnings: hstrs.append( colorfmt.format( colors["LIGHT_RED"], _counts(rlabel["warnings"], num_warnings), colors, )) totals = "{0}\nTotal states run: {1:>{2}}".format( "-" * line_max_len, sum(rcounts.values()) - rcounts.get("warnings", 0), line_max_len - 7, ) hstrs.append(colorfmt.format(colors["CYAN"], totals, colors)) if __opts__.get("state_output_profile"): sum_duration = sum(rdurations) duration_unit = "ms" # convert to seconds if duration is 1000ms or more if sum_duration > 999: sum_duration /= 1000 duration_unit = "s" total_duration = "Total run time: {} {}".format( "{:.3f}".format(sum_duration).rjust(line_max_len - 5), duration_unit) hstrs.append( colorfmt.format(colors["CYAN"], total_duration, colors)) if strip_colors: host = salt.output.strip_esc_sequence(host) hstrs.insert(0, "{0}{1}:{2[ENDC]}".format(hcolor, host, colors)) return "\n".join(hstrs), nchanges > 0
def _compress_ids(data): """ Function to take incoming raw state data and roll IDs with multiple names into a single state block for reporting purposes. This functionality is most useful for any "_id" state_output options, such as ``terse_id``. The following example state has one ID and four names. .. code-block:: yaml mix-matched results: cmd.run: - names: - "true" - "false" - "/bin/true" - "/bin/false" With ``state_output: terse_id`` set, this can create many lines of output which are not unique enough to be worth the screen real estate they occupy. .. code-block:: text 19:10:10.969049 [ 8.546 ms] cmd.run Changed Name: mix-matched results 19:10:10.977998 [ 8.606 ms] cmd.run Failed Name: mix-matched results 19:10:10.987116 [ 7.618 ms] cmd.run Changed Name: mix-matched results 19:10:10.995172 [ 9.344 ms] cmd.run Failed Name: mix-matched results Enabling ``state_compress_ids: True`` consolidates the state data by ID and result (e.g. success or failure). The earliest start time is chosen for display, duration is aggregated, and the total number of names if shown in parentheses to the right of the ID. .. code-block:: text 19:10:46.283323 [ 16.236 ms] cmd.run Changed Name: mix-matched results (2) 19:10:46.292181 [ 16.255 ms] cmd.run Failed Name: mix-matched results (2) A better real world use case would be passing dozens of files and directories to the ``names`` parameter of the ``file.absent`` state. The amount of lines consolidated in that case would be substantial. """ if not isinstance(data, dict): return data compressed = {} # any failures to compress result in passing the original data # to the highstate outputter without modification try: for host, hostdata in data.items(): compressed[host] = {} # count the number of unique IDs. use sls name and result in the key # so differences can be shown separately in the output id_count = collections.Counter([ "_".join( map( str, [ tname.split("_|-")[0], info["__id__"], info["__sls__"], info["result"], ], )) for tname, info in hostdata.items() ]) for tname, info in hostdata.items(): comps = tname.split("_|-") _id = "_".join( map(str, [ comps[0], info["__id__"], info["__sls__"], info["result"] ])) # state does not need to be compressed if id_count[_id] == 1: compressed[host][tname] = info continue # replace name to create a single key by sls and result comps[2] = "_".join( map( str, [ "state_compressed", info["__sls__"], info["__id__"], info["result"], ], )) comps[1] = "{} ({})".format(info["__id__"], id_count[_id]) tname = "_|-".join(comps) # store the first entry as-is if tname not in compressed[host]: compressed[host][tname] = info continue # subsequent entries for compression will use the lowest # __run_num__ value, the sum of the duration, and the earliest # start time found compressed[host][tname]["__run_num__"] = min( info["__run_num__"], compressed[host][tname]["__run_num__"]) compressed[host][tname]["duration"] = round( sum([ info["duration"], compressed[host][tname]["duration"] ]), 3) compressed[host][tname]["start_time"] = sorted([ info["start_time"], compressed[host][tname]["start_time"] ])[0] # changes are turned into a dict of changes keyed by name if compressed[host][tname].get("changes") and info.get( "changes"): if not compressed[host][tname]["changes"].get( "compressed changes"): compressed[host][tname]["changes"] = { "compressed changes": { compressed[host][tname]["name"]: compressed[host][tname]["changes"] } } compressed[host][tname]["changes"][ "compressed changes"].update( {info["name"]: info["changes"]}) elif info.get("changes"): compressed[host][tname]["changes"] = { "compressed changes": { info["name"]: info["changes"] } } except Exception: # pylint: disable=broad-except log.warning( "Unable to compress state output by ID! Returning output normally." ) return data return compressed