def status_text(self, max_depth=0, ignore_custom=False, **kwargs):
""" status_text(max_depth=0, ignore_custom=True, colored=True)
"""
_colored = kwargs.get("colored", True)
count = self.count(ignore_custom=ignore_custom)
exists = count >= self._nMin()
if exists:
text = "existent"
if _colored:
text = colored(text, "green", style="bright")
else:
text = "absent"
if _colored:
text = colored(text, "red", style="bright")
text = "%s (%s/%s)" % (text, count, len(self))
if max_depth > 0:
if isinstance(self.targets, (list, tuple)):
gen = enumerate(self.targets)
else: # dict
gen = six.iteritems(self.targets)
for key, target in gen:
text += "\n%s: " % key
if isinstance(target, TargetCollection):
text += "\n ".join(
target.status_text(max_depth - 1).split("\n"))
elif isinstance(target, Target):
text += "%s (%s)" % (target.status_text(colored=_colored),
target.colored_repr())
return text
def format(self, record):
""""""
# get and style the level
level = self.format_level(record) if callable(
self.format_level) else record.levelname
level = colored(level, **self.level_styles.get(record.levelname, {}))
# get and style the name
name = self.format_name(record) if callable(
self.format_name) else record.name
name = colored(name, **self.name_styles.get(record.levelname, {}))
# get and style the message
msg = self.format_msg(record) if callable(
self.format_msg) else record.getMessage()
msg = colored(msg, **self.msg_styles.get(record.levelname, {}))
# build template data
tmpl = self.log_template
data = dict(level=level, name=name, msg=msg)
# add traceback and change the template when the record contains exception info
if record.exc_info:
tmpl = self.err_template
data["traceback"] = self.formatException(record.exc_info)
return tmpl.format(**data)
def remove_task_output(task, max_depth=0, mode=None, include_external=False):
max_depth = int(max_depth)
print("remove task output with max_depth {}".format(max_depth))
include_external = check_bool_flag(include_external)
if include_external:
print("include external tasks")
# determine the mode, i.e., all, dry, interactive
modes = ["i", "a", "d"]
mode_names = ["interactive", "all", "dry"]
if mode is None:
mode = query_choice("removal mode?", modes, default="i", descriptions=mode_names)
elif isinstance(mode, int):
mode = modes[mode]
else:
mode = mode[0].lower()
if mode not in modes:
raise Exception("unknown removal mode '{}'".format(mode))
mode_name = mode_names[modes.index(mode)]
print("selected " + colored(mode_name + " mode", "blue", style="bright"))
done = []
ind = "| "
for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"):
offset = depth * ind
print(offset)
print("{}> remove output of {}".format(offset, dep.colored_repr()))
offset += ind
if not include_external and isinstance(dep, ExternalTask):
print(offset + "- " + colored("task is external, skip", "yellow"))
continue
if mode == "i":
task_mode = query_choice(offset + " walk through outputs?", ("y", "n"), default="y")
if task_mode == "n":
continue
if dep in done:
print(offset + "- " + colored("outputs already removed", "yellow"))
continue
done.append(dep)
for outp in luigi.task.flatten(dep.output()):
print("{}- remove {}".format(offset, outp.colored_repr()))
if mode == "d":
continue
elif mode == "i":
if query_choice(offset + " remove?", ("y", "n"), default="n") == "n":
print(offset + colored(" skipped", "yellow"))
continue
outp.remove()
print(offset + " " + colored("removed", "red", style="bright"))
def print_stats(profiler, text=None):
print(colored("-" * 100, "light_blue"))
print("line profiling of method {} of task {}".format(
colored(fn.__name__, style="bright"), get_task(task).repr()))
if text:
print(text)
print("")
profiler.print_stats(output_unit=opts["output_unit"], stripzeros=opts["stripzeros"])
print(colored("-" * 100, "light_blue"))
def status_line(cls,
counts,
last_counts=None,
skip=None,
timestamp=True,
align=False,
color=False):
status_names = cls.status_names
if skip:
status_names = [name for name in status_names if name not in skip]
# check last counts
if last_counts and len(last_counts) != len(status_names):
raise Exception("{} last status counts expected, got {}".format(
len(status_names), len(last_counts)))
# check current counts
if len(counts) != len(status_names):
raise Exception("{} status counts expected, got {}".format(
len(status_names), len(counts)))
# calculate differences
if last_counts:
diffs = tuple(n - m for n, m in zip(counts, last_counts))
# number formatting
if isinstance(align, bool) or not isinstance(align, six.integer_types):
align = 4 if align else 0
count_fmt = "%d" if not align else "%{}d".format(align)
diff_fmt = "%+d" if not align else "%+{}d".format(align)
# build the status line
line = ""
if timestamp:
line += "{}: ".format(time.strftime("%H:%M:%S"))
line += "all: " + count_fmt % (sum(counts), )
for i, (status, count) in enumerate(zip(status_names, counts)):
count = count_fmt % count
if color:
count = colored(count, style="bright")
line += ", {}: {}".format(status, count)
if last_counts:
diff = diff_fmt % diffs[i]
if color:
# 0 if negative, 1 if zero, 2 if positive
style_idx = (diffs[i] > 0) + (diffs[i] >= 0)
diff = colored(diff,
**cls.status_diff_styles[status][style_idx])
line += " ({})".format(diff)
return line
def _purge_output(self, max_depth=0):
print("purge output with max_depth %s\n" % max_depth)
mode = query_choice("continue?", ("y", "n", "d", "i"), default="i")
if mode == "n":
return
elif mode == "d":
print("selected " + colored("dry mode", "blue", style="bright") +
"\n")
elif mode == "i":
print("selected " +
colored("interactive mode", "blue", style="bright") + "\n")
else:
print("")
done = []
for task, _, depth in self.walk_deps(max_depth=max_depth, order="pre"):
tpl = (depth * "| ", task.colored_repr())
print("%s> remove output of %s" % tpl)
if mode == "i":
msg = tpl[0] + " walk through outputs?"
task_mode = query_choice(msg, ("y", "n", "d"), default="y")
if task_mode == "n":
continue
if task in done:
print((depth + 1) * "| " + "- " +
colored("outputs already removed", "yellow"))
else:
done.append(task)
for outp in luigi.task.flatten(task.output()):
tpl = ((depth + 1) * "| ", outp.colored_repr())
print("%s- remove %s" % tpl)
if mode == "d":
continue
if mode == "i" and task_mode != "d":
msg = tpl[0] + " remove?"
if query_choice(msg, ("y", "n"), default="n") == "n":
print(tpl[0] + " skipped")
continue
outp.remove()
print(tpl[0] + " " +
law.util.colored("removed", "red", style="bright"))
print("")
def status_text(self, max_depth=0, ignore_custom=True, **kwargs):
""" status_text(max_depth=0, ignore_custom=True, colored=True)
"""
_colored = kwargs.get("colored", True)
if self.exists(ignore_custom=ignore_custom):
text = "existent"
if _colored:
text = colored(text, "green", style="bright")
return text
else:
text = "absent"
if _colored:
text = colored(text, "red", style="bright")
return text
def _print_status(self, max_depth=0):
print("print status with max_depth %s\n" % max_depth)
col_depth = six.moves.input("target collection depth? [0*, int] ")
col_depth = 0 if col_depth == "" else int(col_depth)
print("")
done = []
for task, _, depth in self.walk_deps(max_depth=max_depth, order="pre"):
tpl = (depth * "| ", task.colored_repr())
print("%s> check status of %s" % tpl)
if task in done:
print((depth + 1) * "| " + "- " +
colored("outputs already checked", "yellow"))
else:
done.append(task)
for outp in luigi.task.flatten(task.output()):
tpl = ((depth + 1) * "| ", outp.colored_repr())
print("%s- check %s" % tpl)
status_lines = outp.status_text(
max_depth=col_depth).split("\n")
status_text = status_lines[0]
for line in status_lines[1:]:
status_text += "\n" + (depth +
1) * "| " + " " + line
tpl = ((depth + 1) * "| ", status_text)
print("%s -> %s" % tpl)
print("")
def format(self, record):
""""""
return self.tmpl.format(
level=colored(record.levelname, **self.level_styles.get(record.levelname, {})),
name=record.name,
msg=record.msg,
)
def print_task_status(task, max_depth=0, target_depth=0, flags=None):
max_depth = int(max_depth)
target_depth = int(target_depth)
if flags:
flags = tuple(flags.lower().split("-"))
print("print task status with max_depth {} and target_depth {}".format(
max_depth, target_depth))
done = []
ind = "| "
for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"):
offset = depth * ind
print(offset)
print("{}> check status of {}".format(offset, dep.colored_repr()))
offset += ind
if dep in done:
print(offset + "- " + colored("outputs already checked", "yellow"))
continue
done.append(dep)
for outp in luigi.task.flatten(dep.output()):
print("{}- check {}".format(offset, outp.colored_repr()))
status_lines = outp.status_text(max_depth=target_depth,
flags=flags).split("\n")
status_text = status_lines[0]
for line in status_lines[1:]:
status_text += "\n" + offset + " " + line
print("{} -> {}".format(offset, status_text))
def colored_repr(self):
params = self.get_params()
param_values = self.get_param_values(params, [], self.param_kwargs)
# build the parameter signature
sig_parts = []
param_objs = dict(params)
for param_name, param_value in param_values:
if param_objs[param_name].significant:
n = colored(param_name, "blue", style="bright")
v = param_objs[param_name].serialize(param_value)
sig_parts.append("{}={}".format(n, v))
task_str = "{}({})".format(colored(self.task_family, "green"), ", ".join(sig_parts))
return task_str
def status_text(self, max_depth=0, color=True):
count = self.count()
exists = count >= self._threshold()
if exists:
text = "existent"
_color = "green"
else:
text = "absent"
_color = "red" if not self.optional else "grey"
text = colored(text, _color, style="bright") if color else text
text += " ({}/{})".format(count, len(self))
if max_depth > 0:
if isinstance(self.targets, (list, tuple)):
gen = enumerate(self.targets)
else: # dict
gen = six.iteritems(self.targets)
for key, item in gen:
text += "\n{}: ".format(key)
if isinstance(item, TargetCollection):
text += "\n ".join(item.status_text(max_depth - 1, color=color).split("\n"))
elif isinstance(item, Target):
text += "{} ({})".format(item.status_text(color=color),
item.colored_repr(color=color))
else:
text += "\n ".join(
self.__class__(item).status_text(max_depth - 1, color=color).split("\n"))
return text
def format(self, record):
return self.tmpl.format(
level=colored(record.levelname,
**self.level_styles.get(record.levelname, {})),
spaces=" " * (self.max_level_len - len(record.levelname)),
name=record.name,
msg=record.msg,
)
def _repr_param(self, name, value, color=False, serialize=True, **kwargs):
# try to serialize first unless explicitly disabled
if serialize:
param = getattr(self.__class__, name, no_value)
if param != no_value:
value = param.serialize(value)
return "{}={}".format(colored(name, color="blue", style="bright") if color else name, value)
def _run_log(self, cmd=None, color="pink"):
# start banner
print("")
line = " entering sandbox '{}' ".format(self.sandbox_inst.key).center(100, "=")
print(colored(line, color) if color else line)
print("")
# log the command
if cmd:
print("sandbox command:\n{}\n".format(cmd))
try:
yield
finally:
# end banner
line = " leaving sandbox '{}' ".format(self.sandbox_inst.key).center(100, "=")
print(colored(line, color) if color else line)
print("")
def status_text(self, max_depth=0, flags=None, color=True):
if self.exists():
text = "existent"
_color = "green"
else:
text = "absent"
_color = "red" if not self.optional else "grey"
return colored(text, _color, style="bright") if color else text
def status_text(self, max_depth=0, flags=None, color=False, exists=None):
if exists is None:
exists = self.exists()
if exists:
text = "existent"
_color = "green"
else:
text = "absent"
_color = "grey" if self.optional else "red"
return colored(text, _color, style="bright") if color else text
def print_banner(msg, color):
print("")
print(colored(" {} ".format(msg).center(80, "="), color=color))
print(colored("sandbox: ", color=color) + colored(self.sandbox_inst.key, style="bright"))
print(colored("task : ", color=color) + colored(self.task.task_id, style="bright"))
print(colored(80 * "=", color=color))
print("")
def format(self, record):
""""""
data = dict(
level=colored(record.levelname,
**self.level_styles.get(record.levelname, {})),
name=record.name,
msg=record.getMessage(),
)
tmpl = self.tmpl
# add traceback and change the template when the record contains exception info
if record.exc_info:
data["traceback"] = self.formatException(record.exc_info)
tmpl = self.tmpl_error
return tmpl.format(**data)
def print_task_status(task, max_depth=0, target_depth=0, flags=None):
from law.workflow.base import BaseWorkflow
max_depth = int(max_depth)
target_depth = int(target_depth)
if flags:
flags = tuple(flags.lower().split("-"))
print("print task status with max_depth {} and target_depth {}".format(
max_depth, target_depth))
# helper to print the actual output status text during output traversal
def print_status_text(output, key, offset):
print("{}{} {}".format(offset, key, output.repr(color=True)))
status_text = output.status_text(max_depth=target_depth,
flags=flags,
color=True)
status_lines = status_text.split("\n")
status_text = status_lines[0]
for line in status_lines[1:]:
status_text += "\n{}{}{}".format(offset, ind, line)
print("{}{}{}".format(offset, ind, status_text))
# walk through deps
done = []
for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"):
offset = depth * ("|" + ind)
print(offset)
# when the dep is a workflow, preload its branch map which updates branch parameters
if isinstance(dep, BaseWorkflow):
dep.get_branch_map()
print("{}> check status of {}".format(offset, dep.repr(color=True)))
offset += "|" + ind
if dep in done:
print(offset + colored("outputs already checked", "yellow"))
continue
done.append(dep)
# start the traversing
for output, okey, _, ooffset, _ in _iter_output(dep.output(), offset):
print_status_text(output, okey, ooffset)
def status_text(self, max_depth=0, flags=None, color=False, exists=None):
count, existing_keys = self.count(keys=True)
exists = count >= self._abs_threshold()
if exists:
text = "existent"
_color = "green"
else:
text = "absent"
_color = "red" if not self.optional else "dark_grey"
text = colored(text, _color, style="bright") if color else text
text += " ({}/{})".format(count, len(self))
if flags and "missing" in flags and count != len(self):
missing_keys = [
str(key) for key in self.keys() if key not in existing_keys
]
text += ", missing: " + ",".join(missing_keys)
if max_depth > 0:
if isinstance(self.targets, (list, tuple)):
gen = enumerate(self.targets)
else: # dict
gen = six.iteritems(self.targets)
for key, item in gen:
text += "\n{}: ".format(key)
if isinstance(item, TargetCollection):
t = item.status_text(max_depth=max_depth - 1, color=color)
text += "\n ".join(t.split("\n"))
elif isinstance(item, Target):
t = item.status_text(color=color,
exists=key in existing_keys)
text += "{} ({})".format(t, item.repr(color=color))
else:
t = self.__class__(item).status_text(max_depth=max_depth -
1,
color=color)
text += "\n ".join(t.split("\n"))
return text
def _iter_output(output, offset):
lookup = _flatten_output(output, 0)
while lookup:
output, odepth, oprefix = lookup.pop(0)
ooffset = offset + odepth * ind
if isinstance(output, Target):
yield output, odepth, oprefix, ooffset, lookup
else:
# before updating the lookup list, but check if the output changes by this
_lookup = _flatten_output(output, odepth + 1)
if len(_lookup) > 0 and _lookup[0][0] == output:
print("{} {}{}".format(ooffset, oprefix, colored("not a target", color="red")))
else:
# print the key of the current structure
print("{} {}".format(ooffset, oprefix))
# update the lookup list
lookup[:0] = _lookup
def print_task_status(task, max_depth=0, target_depth=0, flags=None):
from law.workflow.base import BaseWorkflow
max_depth = int(max_depth)
target_depth = int(target_depth)
if flags:
flags = tuple(flags.lower().split("-"))
print("print task status with max_depth {} and target_depth {}".format(
max_depth, target_depth))
done = []
ind = "| "
for dep, _, depth in task.walk_deps(max_depth=max_depth, order="pre"):
offset = depth * ind
print(offset)
# when the dep is a workflow, preload its branch map which updates branch parameters
if isinstance(dep, BaseWorkflow):
dep.get_branch_map()
print("{}> check status of {}".format(offset, dep.repr(color=True)))
offset += ind
if dep in done:
print(offset + "- " + colored("outputs already checked", "yellow"))
continue
done.append(dep)
for outp in flatten(dep.output()):
print("{}- {}".format(offset, outp.repr(color=True)))
status_text = outp.status_text(max_depth=target_depth, flags=flags, color=True)
status_lines = status_text.split("\n")
status_text = status_lines[0]
for line in status_lines[1:]:
status_text += "\n{} {}".format(offset, line)
print("{} {}".format(offset, status_text))
def status_line(self,
counts,
last_counts=None,
sum_counts=None,
timestamp=True,
align=False,
color=False):
"""
Returns a job status line containing job counts per status. When *last_counts* is *True*,
the status line also contains the differences in job counts with respect to the counts from
the previous call to this method. When you pass a list or tuple, those values are used
intead to compute the differences. The status line starts with the sum of jobs which is
inferred from *counts*. When you want to use a custom value, set *sum_counts*. The length of
*counts* should match the length of *status_names* of this instance. When *timestamp* is
*True*, the status line begins with the current timestamp. When *timestamp* is a non-empty
string, it is used as the ``strftime`` format. *align* handles the alignment of the values
in the status line by using a maximum width. *True* will result in the default width of 4.
When *align* evaluates to *False*, no alignment is used. By default, some elements of the
status line are colored. Set *color* to *False* to disable this feature. Example:
.. code-block:: python
status_line((2, 0, 0, 0, 0))
# 12:45:18: all: 2, pending: 2, running: 0, finished: 0, retry: 0, failed: 0
status_line((0, 2, 0, 0), last_counts=(2, 0, 0, 0), skip=["retry"], timestamp=False)
# all: 2, pending: 0 (-2), running: 2 (+2), finished: 2 (+0), failed: 0 (+0)
"""
# check and or set last counts
use_last_counts = bool(last_counts)
if use_last_counts and not isinstance(last_counts, (list, tuple)):
last_counts = self.last_counts or ([0] * len(self.status_names))
if last_counts and len(last_counts) != len(self.status_names):
raise Exception("{} last status counts expected, got {}".format(
len(self.status_names), len(last_counts)))
# check current counts
if len(counts) != len(self.status_names):
raise Exception("{} status counts expected, got {}".format(
len(self.status_names), len(counts)))
# store current counts for next call
self.last_counts = counts
# calculate differences
if last_counts:
diffs = tuple(n - m for n, m in zip(counts, last_counts))
# number formatting
if isinstance(align, bool) or not isinstance(align, six.integer_types):
align = 4 if align else 0
count_fmt = "%d" if not align else "%{}d".format(align)
diff_fmt = "%+d" if not align else "%+{}d".format(align)
# build the status line
line = ""
if timestamp:
time_format = timestamp if isinstance(
timestamp, six.string_types) else "%H:%M:%S"
line += "{}: ".format(time.strftime(time_format))
if sum_counts is None:
sum_counts = sum(counts)
line += "all: " + count_fmt % (sum_counts, )
for i, (status, count) in enumerate(zip(self.status_names, counts)):
count = count_fmt % count
if color:
count = colored(count, style="bright")
line += ", {}: {}".format(status, count)
if last_counts:
diff = diff_fmt % diffs[i]
if color:
# 0 if negative, 1 if zero, 2 if positive
style_idx = (diffs[i] > 0) + (diffs[i] >= 0)
diff = colored(
diff, **self.status_diff_styles[status][style_idx])
line += " ({})".format(diff)
return line
def colored_repr(self):
tpl = (colored(self.__class__.__name__, "cyan"), colored(self.path, style="bright"),
hex(id(self)))
return "%s(path=%s, %s)" % tpl
def execute(args):
"""
Executes the *index* subprogram with parsed commandline *args*.
"""
cfg = Config.instance()
index_file = cfg.get_expanded("core", "index_file")
# just print the file location?
if args.location:
print(index_file)
return
# just show the file content?
if args.show:
if os.path.exists(index_file):
with open(index_file, "r") as f:
print(f.read())
return
else:
abort("index file {} does not exist".format(index_file))
# just remove the index file?
if args.remove:
if os.path.exists(index_file):
os.remove(index_file)
print("removed index file {}".format(index_file))
return
# get modules to lookup
lookup = [m.strip() for m in cfg.options("modules")]
if args.modules:
lookup += args.modules
print("indexing tasks in {} module(s)".format(len(lookup)))
# loop through modules, import everything to load tasks
for modid in lookup:
if not modid:
continue
if args.verbose:
sys.stdout.write("loading module '{}'".format(modid))
try:
import_module(modid)
except Exception as e:
if not args.verbose:
print("error in module '{}': {}".format(
colored(modid, "red"), str(e)))
else:
print("\n\nerror in module '{}':".format(colored(modid,
"red")))
traceback.print_exc()
continue
if args.verbose:
print(", {}".format(colored("done", style="bright")))
# determine tasks to write into the index file
seen_families = []
task_classes = []
lookup = [Task]
while lookup:
cls = lookup.pop(0)
lookup.extend(cls.__subclasses__())
# skip already seen task families
task_family = cls.get_task_family()
if task_family in seen_families:
continue
seen_families.append(task_family)
# skip when explicitly excluded
if cls.exclude_index:
continue
# skip external tasks
is_external_task = issubclass(cls, ExternalTask)
if args.no_externals and is_external_task:
continue
# skip non-external tasks without run implementation
run_is_callable = callable(getattr(cls, "run", None))
run_is_abstract = getattr(cls.run, "__isabstractmethod__", False)
if not is_external_task and (not run_is_callable or run_is_abstract):
continue
# show an error when there is a "-" in the task family as the luigi command line parser will
# automatically map it to "_", i.e., it will fail to lookup the actual task class
# skip the task
if "-" in task_family:
logger.critical(
"skipping task '{}' as its family '{}' contains a '-' which cannot be "
"interpreted by luigi's command line parser, please use '_' or alike"
.format(cls, task_family))
continue
# show an error when there is a "_" after a "." in the task family, i.e., when there is a
# "_" in the class name (which is bad python practice anyway), as the shell autocompletion
# is not able to decide whether it should complete the task family or a task-level parameter
# skip the task
if "_" in task_family.rsplit(".", 1)[-1]:
logger.error(
"skipping task '{}' as its family '{}' contains a '_' after the namespace "
"definition which would lead to ambiguities between task families and task-level "
"parameters in the law shell autocompletion".format(
cls, task_family))
continue
task_classes.append(cls)
def get_task_params(cls):
params = []
for attr in dir(cls):
member = getattr(cls, attr)
if isinstance(member, luigi.Parameter):
exclude = getattr(cls, "exclude_params_index", set())
if not multi_match(attr, exclude, any):
params.append(attr.replace("_", "-"))
return params
def index_line(cls, params):
# format: "module_id:task_family:param param ..."
return "{}:{}:{}".format(cls.__module__, cls.get_task_family(),
" ".join(params))
stats = OrderedDict()
# write the index file
if not os.path.exists(os.path.dirname(index_file)):
os.makedirs(os.path.dirname(index_file))
with open(index_file, "w") as f:
for cls in task_classes:
# get prams
params = get_task_params(cls)
# fill stats
if cls.__module__ not in stats:
stats[cls.__module__] = []
stats[cls.__module__].append((cls.get_task_family(), params))
f.write(index_line(cls, params) + "\n")
# print stats
if args.verbose:
for mod, data in six.iteritems(stats):
print("\nmodule '{}', {} task(s):".format(
colored(mod, style="bright"), len(data)))
for task_family, _ in data:
print(" - {}".format(colored(task_family, "green")))
print("")
print("written {} task(s) to index file '{}'".format(
len(task_classes), index_file))
def colored_repr(self):
tpl = (colored(self.__class__.__name__,
"cyan"), colored(len(self), style="bright"),
colored(self.threshold, style="bright"), hex(id(self)))
return "%s(len=%s, threshold=%s, %s)" % tpl
def _repr_class_name(cls, name, color=False):
return colored(name, "cyan") if color else name
def _repr_flag(cls, name, color=False):
return colored(name, color="magenta") if color else name
def _repr_pair(cls, key, value, color=False):
return "{}={}".format(
colored(key, color="blue", style="bright") if color else key,
value)
