def logger(body="", system_msg=False, head="", bad=False, timestamp=True):
if timestamp:
if timestamp is True:
dt = datetime.now()
else:
dt = timestamp
tstamp = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
click.secho(tstamp + " ", fg=LOGGER_TIMESTAMP, nl=False)
if head:
click.secho(head, fg=LOGGER_COLOR, nl=False)
click.secho(body, bold=system_msg, fg=LOGGER_BAD_COLOR if bad else None)
def echo_always(line, **kwargs):
kwargs["err"] = kwargs.get("err", True)
if kwargs.pop("indent", None):
line = "\n".join(INDENT + x for x in line.splitlines())
if "nl" not in kwargs or kwargs["nl"]:
line += ERASE_TO_EOL
top = kwargs.pop("padding_top", None)
bottom = kwargs.pop("padding_bottom", None)
highlight = kwargs.pop("highlight", HIGHLIGHT)
if top:
click.secho(ERASE_TO_EOL, **kwargs)
hl_bold = kwargs.pop("highlight_bold", True)
nl = kwargs.pop("nl", True)
fg = kwargs.pop("fg", None)
bold = kwargs.pop("bold", False)
kwargs["nl"] = False
hl = True
nobold = kwargs.pop("no_bold", False)
if nobold:
click.secho(line, **kwargs)
else:
for span in line.split("*"):
if hl:
hl = False
kwargs["fg"] = fg
kwargs["bold"] = bold
click.secho(span, **kwargs)
else:
hl = True
kwargs["fg"] = highlight
kwargs["bold"] = hl_bold
click.secho(span, **kwargs)
if nl:
kwargs["nl"] = True
click.secho("", **kwargs)
if bottom:
click.secho(ERASE_TO_EOL, **kwargs)
def echo_unicode(line, **kwargs):
click.secho(line.decode("UTF-8", errors="replace"), **kwargs)
def logs(obj, input_path, stdout=None, stderr=None, both=None, timestamps=False):
types = set()
if stdout:
types.add("stdout")
both = False
if stderr:
types.add("stderr")
both = False
if both:
types.update(("stdout", "stderr"))
streams = list(sorted(types, reverse=True))
# Pathspec can either be run_id/step_name or run_id/step_name/task_id.
parts = input_path.split("/")
if len(parts) == 2:
run_id, step_name = parts
task_id = None
elif len(parts) == 3:
run_id, step_name, task_id = parts
else:
raise CommandException(
"input_path should either be run_id/step_name "
"or run_id/step_name/task_id"
)
datastore_set = TaskDataStoreSet(
obj.flow_datastore, run_id, steps=[step_name], allow_not_done=True
)
if task_id:
ds_list = [
TaskDataStore(
obj.flow_datastore,
run_id=run_id,
step_name=step_name,
task_id=task_id,
mode="r",
allow_not_done=True,
)
]
else:
ds_list = list(datastore_set) # get all tasks
if ds_list:
def echo_unicode(line, **kwargs):
click.secho(line.decode("UTF-8", errors="replace"), **kwargs)
# old style logs are non mflog-style logs
maybe_old_style = True
for ds in ds_list:
echo(
"Dumping logs of run_id=*{run_id}* "
"step=*{step}* task_id=*{task_id}*".format(
run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
),
fg="magenta",
)
for stream in streams:
echo(stream, bold=True)
logs = ds.load_logs(LOG_SOURCES, stream)
if any(data for _, data in logs):
# attempt to read new, mflog-style logs
for line in mflog.merge_logs([blob for _, blob in logs]):
if timestamps:
ts = mflog.utc_to_local(line.utc_tstamp)
tstamp = ts.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
click.secho(tstamp + " ", fg=LOGGER_TIMESTAMP, nl=False)
echo_unicode(line.msg)
maybe_old_style = False
elif maybe_old_style:
# if they are not available, we may be looking at
# a legacy run (unless we have seen new-style data already
# for another stream). This return an empty string if
# nothing is found
log = ds.load_log_legacy(stream)
if log and timestamps:
raise CommandException(
"We can't show --timestamps for " "old runs. Sorry!"
)
echo_unicode(log, nl=False)
else:
raise CommandException(
"No Tasks found at the given path -- "
"either none exist or none have started yet"
)
def echo_always(line, **kwargs):
click.secho(line, **kwargs)
