def status_text(self, max_depth=0, ignore_custom=False, **kwargs): """ status_text(max_depth=0, ignore_custom=True, colored=True) """ _colored = kwargs.get("colored", True) count = self.count(ignore_custom=ignore_custom) exists = count >= self._nMin() if exists: text = "existent" if _colored: text = colored(text, "green", style="bright") else: text = "absent" if _colored: text = colored(text, "red", style="bright") text = "%s (%s/%s)" % (text, count, len(self)) if max_depth > 0: if isinstance(self.targets, (list, tuple)): gen = enumerate(self.targets) else: # dict gen = six.iteritems(self.targets) for key, target in gen: text += "\n%s: " % key if isinstance(target, TargetCollection): text += "\n ".join( target.status_text(max_depth - 1).split("\n")) elif isinstance(target, Target): text += "%s (%s)" % (target.status_text(colored=_colored), target.colored_repr()) return text
def format(self, record): """""" # get and style the level level = self.format_level(record) if callable( self.format_level) else record.levelname level = colored(level, **self.level_styles.get(record.levelname, {})) # get and style the name name = self.format_name(record) if callable( self.format_name) else record.name name = colored(name, **self.name_styles.get(record.levelname, {})) # get and style the message msg = self.format_msg(record) if callable( self.format_msg) else record.getMessage() msg = colored(msg, **self.msg_styles.get(record.levelname, {})) # build template data tmpl = self.log_template data = dict(level=level, name=name, msg=msg) # add traceback and change the template when the record contains exception info if record.exc_info: tmpl = self.err_template data["traceback"] = self.formatException(record.exc_info) return tmpl.format(**data)
def remove_task_output(task, max_depth=0, mode=None, include_external=False): max_depth = int(max_depth) print("remove task output with max_depth {}".format(max_depth)) include_external = check_bool_flag(include_external) if include_external: print("include external tasks") # determine the mode, i.e., all, dry, interactive modes = ["i", "a", "d"] mode_names = ["interactive", "all", "dry"] if mode is None: mode = query_choice("removal mode?", modes, default="i", descriptions=mode_names) elif isinstance(mode, int): mode = modes[mode] else: mode = mode[0].lower() if mode not in modes: raise Exception("unknown removal mode '{}'".format(mode)) mode_name = mode_names[modes.index(mode)] print("selected " + colored(mode_name + " mode", "blue", style="bright")) done = [] ind = "| " for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"): offset = depth * ind print(offset) print("{}> remove output of {}".format(offset, dep.colored_repr())) offset += ind if not include_external and isinstance(dep, ExternalTask): print(offset + "- " + colored("task is external, skip", "yellow")) continue if mode == "i": task_mode = query_choice(offset + " walk through outputs?", ("y", "n"), default="y") if task_mode == "n": continue if dep in done: print(offset + "- " + colored("outputs already removed", "yellow")) continue done.append(dep) for outp in luigi.task.flatten(dep.output()): print("{}- remove {}".format(offset, outp.colored_repr())) if mode == "d": continue elif mode == "i": if query_choice(offset + " remove?", ("y", "n"), default="n") == "n": print(offset + colored(" skipped", "yellow")) continue outp.remove() print(offset + " " + colored("removed", "red", style="bright"))
def print_stats(profiler, text=None): print(colored("-" * 100, "light_blue")) print("line profiling of method {} of task {}".format( colored(fn.__name__, style="bright"), get_task(task).repr())) if text: print(text) print("") profiler.print_stats(output_unit=opts["output_unit"], stripzeros=opts["stripzeros"]) print(colored("-" * 100, "light_blue"))
def status_line(cls, counts, last_counts=None, skip=None, timestamp=True, align=False, color=False): status_names = cls.status_names if skip: status_names = [name for name in status_names if name not in skip] # check last counts if last_counts and len(last_counts) != len(status_names): raise Exception("{} last status counts expected, got {}".format( len(status_names), len(last_counts))) # check current counts if len(counts) != len(status_names): raise Exception("{} status counts expected, got {}".format( len(status_names), len(counts))) # calculate differences if last_counts: diffs = tuple(n - m for n, m in zip(counts, last_counts)) # number formatting if isinstance(align, bool) or not isinstance(align, six.integer_types): align = 4 if align else 0 count_fmt = "%d" if not align else "%{}d".format(align) diff_fmt = "%+d" if not align else "%+{}d".format(align) # build the status line line = "" if timestamp: line += "{}: ".format(time.strftime("%H:%M:%S")) line += "all: " + count_fmt % (sum(counts), ) for i, (status, count) in enumerate(zip(status_names, counts)): count = count_fmt % count if color: count = colored(count, style="bright") line += ", {}: {}".format(status, count) if last_counts: diff = diff_fmt % diffs[i] if color: # 0 if negative, 1 if zero, 2 if positive style_idx = (diffs[i] > 0) + (diffs[i] >= 0) diff = colored(diff, **cls.status_diff_styles[status][style_idx]) line += " ({})".format(diff) return line
def _purge_output(self, max_depth=0): print("purge output with max_depth %s\n" % max_depth) mode = query_choice("continue?", ("y", "n", "d", "i"), default="i") if mode == "n": return elif mode == "d": print("selected " + colored("dry mode", "blue", style="bright") + "\n") elif mode == "i": print("selected " + colored("interactive mode", "blue", style="bright") + "\n") else: print("") done = [] for task, _, depth in self.walk_deps(max_depth=max_depth, order="pre"): tpl = (depth * "| ", task.colored_repr()) print("%s> remove output of %s" % tpl) if mode == "i": msg = tpl[0] + " walk through outputs?" task_mode = query_choice(msg, ("y", "n", "d"), default="y") if task_mode == "n": continue if task in done: print((depth + 1) * "| " + "- " + colored("outputs already removed", "yellow")) else: done.append(task) for outp in luigi.task.flatten(task.output()): tpl = ((depth + 1) * "| ", outp.colored_repr()) print("%s- remove %s" % tpl) if mode == "d": continue if mode == "i" and task_mode != "d": msg = tpl[0] + " remove?" if query_choice(msg, ("y", "n"), default="n") == "n": print(tpl[0] + " skipped") continue outp.remove() print(tpl[0] + " " + law.util.colored("removed", "red", style="bright")) print("")
def status_text(self, max_depth=0, ignore_custom=True, **kwargs): """ status_text(max_depth=0, ignore_custom=True, colored=True) """ _colored = kwargs.get("colored", True) if self.exists(ignore_custom=ignore_custom): text = "existent" if _colored: text = colored(text, "green", style="bright") return text else: text = "absent" if _colored: text = colored(text, "red", style="bright") return text
def _print_status(self, max_depth=0): print("print status with max_depth %s\n" % max_depth) col_depth = six.moves.input("target collection depth? [0*, int] ") col_depth = 0 if col_depth == "" else int(col_depth) print("") done = [] for task, _, depth in self.walk_deps(max_depth=max_depth, order="pre"): tpl = (depth * "| ", task.colored_repr()) print("%s> check status of %s" % tpl) if task in done: print((depth + 1) * "| " + "- " + colored("outputs already checked", "yellow")) else: done.append(task) for outp in luigi.task.flatten(task.output()): tpl = ((depth + 1) * "| ", outp.colored_repr()) print("%s- check %s" % tpl) status_lines = outp.status_text( max_depth=col_depth).split("\n") status_text = status_lines[0] for line in status_lines[1:]: status_text += "\n" + (depth + 1) * "| " + " " + line tpl = ((depth + 1) * "| ", status_text) print("%s -> %s" % tpl) print("")
def format(self, record): """""" return self.tmpl.format( level=colored(record.levelname, **self.level_styles.get(record.levelname, {})), name=record.name, msg=record.msg, )
def print_task_status(task, max_depth=0, target_depth=0, flags=None): max_depth = int(max_depth) target_depth = int(target_depth) if flags: flags = tuple(flags.lower().split("-")) print("print task status with max_depth {} and target_depth {}".format( max_depth, target_depth)) done = [] ind = "| " for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"): offset = depth * ind print(offset) print("{}> check status of {}".format(offset, dep.colored_repr())) offset += ind if dep in done: print(offset + "- " + colored("outputs already checked", "yellow")) continue done.append(dep) for outp in luigi.task.flatten(dep.output()): print("{}- check {}".format(offset, outp.colored_repr())) status_lines = outp.status_text(max_depth=target_depth, flags=flags).split("\n") status_text = status_lines[0] for line in status_lines[1:]: status_text += "\n" + offset + " " + line print("{} -> {}".format(offset, status_text))
def colored_repr(self): params = self.get_params() param_values = self.get_param_values(params, [], self.param_kwargs) # build the parameter signature sig_parts = [] param_objs = dict(params) for param_name, param_value in param_values: if param_objs[param_name].significant: n = colored(param_name, "blue", style="bright") v = param_objs[param_name].serialize(param_value) sig_parts.append("{}={}".format(n, v)) task_str = "{}({})".format(colored(self.task_family, "green"), ", ".join(sig_parts)) return task_str
def status_text(self, max_depth=0, color=True): count = self.count() exists = count >= self._threshold() if exists: text = "existent" _color = "green" else: text = "absent" _color = "red" if not self.optional else "grey" text = colored(text, _color, style="bright") if color else text text += " ({}/{})".format(count, len(self)) if max_depth > 0: if isinstance(self.targets, (list, tuple)): gen = enumerate(self.targets) else: # dict gen = six.iteritems(self.targets) for key, item in gen: text += "\n{}: ".format(key) if isinstance(item, TargetCollection): text += "\n ".join(item.status_text(max_depth - 1, color=color).split("\n")) elif isinstance(item, Target): text += "{} ({})".format(item.status_text(color=color), item.colored_repr(color=color)) else: text += "\n ".join( self.__class__(item).status_text(max_depth - 1, color=color).split("\n")) return text
def format(self, record): return self.tmpl.format( level=colored(record.levelname, **self.level_styles.get(record.levelname, {})), spaces=" " * (self.max_level_len - len(record.levelname)), name=record.name, msg=record.msg, )
def _repr_param(self, name, value, color=False, serialize=True, **kwargs): # try to serialize first unless explicitly disabled if serialize: param = getattr(self.__class__, name, no_value) if param != no_value: value = param.serialize(value) return "{}={}".format(colored(name, color="blue", style="bright") if color else name, value)
def _run_log(self, cmd=None, color="pink"): # start banner print("") line = " entering sandbox '{}' ".format(self.sandbox_inst.key).center(100, "=") print(colored(line, color) if color else line) print("") # log the command if cmd: print("sandbox command:\n{}\n".format(cmd)) try: yield finally: # end banner line = " leaving sandbox '{}' ".format(self.sandbox_inst.key).center(100, "=") print(colored(line, color) if color else line) print("")
def status_text(self, max_depth=0, flags=None, color=True): if self.exists(): text = "existent" _color = "green" else: text = "absent" _color = "red" if not self.optional else "grey" return colored(text, _color, style="bright") if color else text
def status_text(self, max_depth=0, flags=None, color=False, exists=None): if exists is None: exists = self.exists() if exists: text = "existent" _color = "green" else: text = "absent" _color = "grey" if self.optional else "red" return colored(text, _color, style="bright") if color else text
def print_banner(msg, color): print("") print(colored(" {} ".format(msg).center(80, "="), color=color)) print(colored("sandbox: ", color=color) + colored(self.sandbox_inst.key, style="bright")) print(colored("task : ", color=color) + colored(self.task.task_id, style="bright")) print(colored(80 * "=", color=color)) print("")
def format(self, record): """""" data = dict( level=colored(record.levelname, **self.level_styles.get(record.levelname, {})), name=record.name, msg=record.getMessage(), ) tmpl = self.tmpl # add traceback and change the template when the record contains exception info if record.exc_info: data["traceback"] = self.formatException(record.exc_info) tmpl = self.tmpl_error return tmpl.format(**data)
def print_task_status(task, max_depth=0, target_depth=0, flags=None): from law.workflow.base import BaseWorkflow max_depth = int(max_depth) target_depth = int(target_depth) if flags: flags = tuple(flags.lower().split("-")) print("print task status with max_depth {} and target_depth {}".format( max_depth, target_depth)) # helper to print the actual output status text during output traversal def print_status_text(output, key, offset): print("{}{} {}".format(offset, key, output.repr(color=True))) status_text = output.status_text(max_depth=target_depth, flags=flags, color=True) status_lines = status_text.split("\n") status_text = status_lines[0] for line in status_lines[1:]: status_text += "\n{}{}{}".format(offset, ind, line) print("{}{}{}".format(offset, ind, status_text)) # walk through deps done = [] for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"): offset = depth * ("|" + ind) print(offset) # when the dep is a workflow, preload its branch map which updates branch parameters if isinstance(dep, BaseWorkflow): dep.get_branch_map() print("{}> check status of {}".format(offset, dep.repr(color=True))) offset += "|" + ind if dep in done: print(offset + colored("outputs already checked", "yellow")) continue done.append(dep) # start the traversing for output, okey, _, ooffset, _ in _iter_output(dep.output(), offset): print_status_text(output, okey, ooffset)
def status_text(self, max_depth=0, flags=None, color=False, exists=None): count, existing_keys = self.count(keys=True) exists = count >= self._abs_threshold() if exists: text = "existent" _color = "green" else: text = "absent" _color = "red" if not self.optional else "dark_grey" text = colored(text, _color, style="bright") if color else text text += " ({}/{})".format(count, len(self)) if flags and "missing" in flags and count != len(self): missing_keys = [ str(key) for key in self.keys() if key not in existing_keys ] text += ", missing: " + ",".join(missing_keys) if max_depth > 0: if isinstance(self.targets, (list, tuple)): gen = enumerate(self.targets) else: # dict gen = six.iteritems(self.targets) for key, item in gen: text += "\n{}: ".format(key) if isinstance(item, TargetCollection): t = item.status_text(max_depth=max_depth - 1, color=color) text += "\n ".join(t.split("\n")) elif isinstance(item, Target): t = item.status_text(color=color, exists=key in existing_keys) text += "{} ({})".format(t, item.repr(color=color)) else: t = self.__class__(item).status_text(max_depth=max_depth - 1, color=color) text += "\n ".join(t.split("\n")) return text
def _iter_output(output, offset): lookup = _flatten_output(output, 0) while lookup: output, odepth, oprefix = lookup.pop(0) ooffset = offset + odepth * ind if isinstance(output, Target): yield output, odepth, oprefix, ooffset, lookup else: # before updating the lookup list, but check if the output changes by this _lookup = _flatten_output(output, odepth + 1) if len(_lookup) > 0 and _lookup[0][0] == output: print("{} {}{}".format(ooffset, oprefix, colored("not a target", color="red"))) else: # print the key of the current structure print("{} {}".format(ooffset, oprefix)) # update the lookup list lookup[:0] = _lookup
def print_task_status(task, max_depth=0, target_depth=0, flags=None): from law.workflow.base import BaseWorkflow max_depth = int(max_depth) target_depth = int(target_depth) if flags: flags = tuple(flags.lower().split("-")) print("print task status with max_depth {} and target_depth {}".format( max_depth, target_depth)) done = [] ind = "| " for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"): offset = depth * ind print(offset) # when the dep is a workflow, preload its branch map which updates branch parameters if isinstance(dep, BaseWorkflow): dep.get_branch_map() print("{}> check status of {}".format(offset, dep.repr(color=True))) offset += ind if dep in done: print(offset + "- " + colored("outputs already checked", "yellow")) continue done.append(dep) for outp in flatten(dep.output()): print("{}- {}".format(offset, outp.repr(color=True))) status_text = outp.status_text(max_depth=target_depth, flags=flags, color=True) status_lines = status_text.split("\n") status_text = status_lines[0] for line in status_lines[1:]: status_text += "\n{} {}".format(offset, line) print("{} {}".format(offset, status_text))
def status_line(self, counts, last_counts=None, sum_counts=None, timestamp=True, align=False, color=False): """ Returns a job status line containing job counts per status. When *last_counts* is *True*, the status line also contains the differences in job counts with respect to the counts from the previous call to this method. When you pass a list or tuple, those values are used intead to compute the differences. The status line starts with the sum of jobs which is inferred from *counts*. When you want to use a custom value, set *sum_counts*. The length of *counts* should match the length of *status_names* of this instance. When *timestamp* is *True*, the status line begins with the current timestamp. When *timestamp* is a non-empty string, it is used as the ``strftime`` format. *align* handles the alignment of the values in the status line by using a maximum width. *True* will result in the default width of 4. When *align* evaluates to *False*, no alignment is used. By default, some elements of the status line are colored. Set *color* to *False* to disable this feature. Example: .. code-block:: python status_line((2, 0, 0, 0, 0)) # 12:45:18: all: 2, pending: 2, running: 0, finished: 0, retry: 0, failed: 0 status_line((0, 2, 0, 0), last_counts=(2, 0, 0, 0), skip=["retry"], timestamp=False) # all: 2, pending: 0 (-2), running: 2 (+2), finished: 2 (+0), failed: 0 (+0) """ # check and or set last counts use_last_counts = bool(last_counts) if use_last_counts and not isinstance(last_counts, (list, tuple)): last_counts = self.last_counts or ([0] * len(self.status_names)) if last_counts and len(last_counts) != len(self.status_names): raise Exception("{} last status counts expected, got {}".format( len(self.status_names), len(last_counts))) # check current counts if len(counts) != len(self.status_names): raise Exception("{} status counts expected, got {}".format( len(self.status_names), len(counts))) # store current counts for next call self.last_counts = counts # calculate differences if last_counts: diffs = tuple(n - m for n, m in zip(counts, last_counts)) # number formatting if isinstance(align, bool) or not isinstance(align, six.integer_types): align = 4 if align else 0 count_fmt = "%d" if not align else "%{}d".format(align) diff_fmt = "%+d" if not align else "%+{}d".format(align) # build the status line line = "" if timestamp: time_format = timestamp if isinstance( timestamp, six.string_types) else "%H:%M:%S" line += "{}: ".format(time.strftime(time_format)) if sum_counts is None: sum_counts = sum(counts) line += "all: " + count_fmt % (sum_counts, ) for i, (status, count) in enumerate(zip(self.status_names, counts)): count = count_fmt % count if color: count = colored(count, style="bright") line += ", {}: {}".format(status, count) if last_counts: diff = diff_fmt % diffs[i] if color: # 0 if negative, 1 if zero, 2 if positive style_idx = (diffs[i] > 0) + (diffs[i] >= 0) diff = colored( diff, **self.status_diff_styles[status][style_idx]) line += " ({})".format(diff) return line
def colored_repr(self): tpl = (colored(self.__class__.__name__, "cyan"), colored(self.path, style="bright"), hex(id(self))) return "%s(path=%s, %s)" % tpl
def execute(args): """ Executes the *index* subprogram with parsed commandline *args*. """ cfg = Config.instance() index_file = cfg.get_expanded("core", "index_file") # just print the file location? if args.location: print(index_file) return # just show the file content? if args.show: if os.path.exists(index_file): with open(index_file, "r") as f: print(f.read()) return else: abort("index file {} does not exist".format(index_file)) # just remove the index file? if args.remove: if os.path.exists(index_file): os.remove(index_file) print("removed index file {}".format(index_file)) return # get modules to lookup lookup = [m.strip() for m in cfg.options("modules")] if args.modules: lookup += args.modules print("indexing tasks in {} module(s)".format(len(lookup))) # loop through modules, import everything to load tasks for modid in lookup: if not modid: continue if args.verbose: sys.stdout.write("loading module '{}'".format(modid)) try: import_module(modid) except Exception as e: if not args.verbose: print("error in module '{}': {}".format( colored(modid, "red"), str(e))) else: print("\n\nerror in module '{}':".format(colored(modid, "red"))) traceback.print_exc() continue if args.verbose: print(", {}".format(colored("done", style="bright"))) # determine tasks to write into the index file seen_families = [] task_classes = [] lookup = [Task] while lookup: cls = lookup.pop(0) lookup.extend(cls.__subclasses__()) # skip already seen task families task_family = cls.get_task_family() if task_family in seen_families: continue seen_families.append(task_family) # skip when explicitly excluded if cls.exclude_index: continue # skip external tasks is_external_task = issubclass(cls, ExternalTask) if args.no_externals and is_external_task: continue # skip non-external tasks without run implementation run_is_callable = callable(getattr(cls, "run", None)) run_is_abstract = getattr(cls.run, "__isabstractmethod__", False) if not is_external_task and (not run_is_callable or run_is_abstract): continue # show an error when there is a "-" in the task family as the luigi command line parser will # automatically map it to "_", i.e., it will fail to lookup the actual task class # skip the task if "-" in task_family: logger.critical( "skipping task '{}' as its family '{}' contains a '-' which cannot be " "interpreted by luigi's command line parser, please use '_' or alike" .format(cls, task_family)) continue # show an error when there is a "_" after a "." in the task family, i.e., when there is a # "_" in the class name (which is bad python practice anyway), as the shell autocompletion # is not able to decide whether it should complete the task family or a task-level parameter # skip the task if "_" in task_family.rsplit(".", 1)[-1]: logger.error( "skipping task '{}' as its family '{}' contains a '_' after the namespace " "definition which would lead to ambiguities between task families and task-level " "parameters in the law shell autocompletion".format( cls, task_family)) continue task_classes.append(cls) def get_task_params(cls): params = [] for attr in dir(cls): member = getattr(cls, attr) if isinstance(member, luigi.Parameter): exclude = getattr(cls, "exclude_params_index", set()) if not multi_match(attr, exclude, any): params.append(attr.replace("_", "-")) return params def index_line(cls, params): # format: "module_id:task_family:param param ..." return "{}:{}:{}".format(cls.__module__, cls.get_task_family(), " ".join(params)) stats = OrderedDict() # write the index file if not os.path.exists(os.path.dirname(index_file)): os.makedirs(os.path.dirname(index_file)) with open(index_file, "w") as f: for cls in task_classes: # get prams params = get_task_params(cls) # fill stats if cls.__module__ not in stats: stats[cls.__module__] = [] stats[cls.__module__].append((cls.get_task_family(), params)) f.write(index_line(cls, params) + "\n") # print stats if args.verbose: for mod, data in six.iteritems(stats): print("\nmodule '{}', {} task(s):".format( colored(mod, style="bright"), len(data))) for task_family, _ in data: print(" - {}".format(colored(task_family, "green"))) print("") print("written {} task(s) to index file '{}'".format( len(task_classes), index_file))
def colored_repr(self): tpl = (colored(self.__class__.__name__, "cyan"), colored(len(self), style="bright"), colored(self.threshold, style="bright"), hex(id(self))) return "%s(len=%s, threshold=%s, %s)" % tpl
def _repr_class_name(cls, name, color=False): return colored(name, "cyan") if color else name
def _repr_flag(cls, name, color=False): return colored(name, color="magenta") if color else name
def _repr_pair(cls, key, value, color=False): return "{}={}".format( colored(key, color="blue", style="bright") if color else key, value)