def _add_last_error_info(self, exc_info=None):
ex = exc_info[1]
if show_exc_info(ex):
# if we are showing internal task - we don't know how to "minimize" stack.
# everything is databand.* ...
isolate = not self.task.task_definition.full_task_family.startswith(
"databand.")
traceback_str = self.settings.log.format_exception_as_str(
exc_info=exc_info, isolate=isolate)
self.banner.column(
colored("TRACEBACK", color="red", attrs=["bold"]),
traceback_str,
raw_name=True,
)
self.banner.column(
colored("ERROR MESSAGE", color="red", attrs=["bold"]),
str(ex),
raw_name=True,
)
self.banner.column(
colored("HELP", attrs=["bold"]),
get_help_msg(ex),
raw_name=True,
skip_if_empty=True,
)
self.banner.column(
colored("CAUSED BY", color="red", attrs=["bold"]),
nested_exceptions_str(ex, limit=3),
raw_name=True,
skip_if_empty=True,
)
def __init__(self, msg, color="white"):
StringIO.__init__(self)
self.color = color
self._banner_separator = colored("==================== \n", color)
self._banner_msg = colored("= %s\n" % msg, color)
self._section_separator = "-------\n"
# p = pprint.PrettyPrinter(indent=2, width=140)
self.write("\n")
self.write(self._banner_separator)
self.write(self._banner_msg)
def _describe_task(self, task, describe_format=None, msg=None, color=None):
describe_format = describe_format or self.describe_format
describe_config = self.config # type: DescribeConfig
msg = msg or ""
if color is None:
color = "white"
if not describe_config.no_checks:
color = "green" if self._get_task_complete(task) else "cyan"
if describe_format == DescribeFormat.short:
return colored(str(task.task_id), color)
if (describe_format == DescribeFormat.long
or describe_format == DescribeFormat.verbose):
title = "%s - %s" % (task.task_name, task.task_id)
if task.task_name != task.get_task_family():
title += "(%s)" % task.get_task_family()
if msg:
title += ": %s" % msg
return task.ctrl.visualiser.banner(
title,
color=color,
verbose=describe_format == DescribeFormat.verbose)
raise DatabandSystemError("Not supported format mode %s" %
self.describe_format)
def column(self, name, value, newline=True, raw_name=False, skip_if_empty=False):
# if separate:
# s.write(" \n")
if skip_if_empty and value is None:
return self
if isinstance(value, six.string_types):
if six.PY2:
value = value.encode("utf-8", errors="ignore")
else:
value = str(value)
else:
value = str(value)
if skip_if_empty and not value:
return self
if "\n" in value:
value = _TAB + _TAB.join(value.split("\n"))
# remove last \n, use new line or new section if you nee that
if value[-1:] == "\n":
value = value[:-1]
self.write(
" {name:<18} : {value}".format(
name=name if raw_name else colored(name, attrs=["bold"]), value=value
)
)
if newline:
self.write("\n")
return self
def read_dbnd_log_preview(log_path, spark_log_path=None):
"""
Reads the log_path using the current logging config
"""
logger_config = LoggingConfig.current()
max_head_size = logger_config.preview_head_bytes
max_tail_size = logger_config.preview_tail_bytes
# there is not need to run any of the logic if we don't need to read anything
if max_head_size == 0 and max_tail_size == 0:
return EMPTY_LOG_MSG
if spark_log_path:
# we need to read and than merge the content of dbnd log and spark long
# using half of the max_bytes for each of the logs to make sure we are not exceeding the max after merging
parts = merge_read_log_files(
log_path,
spark_log_path,
max_head_size // 2,
max_tail_size // 2,
)
else:
# safely reading a single log file
parts = read_head_and_tail(log_path, max_head_size, max_tail_size)
# check if all the parts contains information
if not any(parts):
# all the parts are truncated
return EMPTY_LOG_MSG
# join each part with new lines
joined_parts = [LINE_BRAKE.join(part) for part in parts]
# building the merge message between head and tail
merge_msg = colored(
MERGE_MSG.format(
head_size=max_head_size,
tail_size=max_tail_size,
file_size=os.path.getsize(log_path),
log_path=log_path,
),
MERGE_MSG_COLOR,
)
return merge_msg.join(joined_parts)
def _get_batch_progresss_banner(self, batch_response):
"""
{
'id': 6,
'state': 'success',
'appId': 'application_1534487568579_0008',
'appInfo': {
'driverLogUrl': None,
'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/'
},
'log': [
'\nYARN Diagnostics: '
]
}
:param batch_response:
:return:
"""
t = self.task
b = TextBanner("Spark Task %s is running at Livy:" % t.task_id,
color="yellow")
b.column("TASK", t.task_id)
b.column("JOB STATE", batch_response.get("state", None))
tracker_url = current_task_run().task_tracker_url
if tracker_url:
b.column("DATABAND LOG", tracker_url)
b.new_line()
b.column("LIVY ID", batch_response.get("id", None))
if "appId" in batch_response:
b.column("APP ID", batch_response["appId"])
app_info = batch_response["appInfo"]
b.column("DRIVER LOG", app_info["driverLogUrl"])
if "sparkUiUrl" in app_info:
spark_url = app_info["sparkUiUrl"]
b.column(
"SPARK",
colored(spark_url, on_color="on_blue", attrs=["bold"]))
b.new_section()
return b.getvalue()
def error_separator():
return colored("--------------------------------------", color="red")
def cyan(value):
return colored(value, color="cyan")
def red(value):
return colored(value, color="red")
def underline(value):
return colored(value, attrs=["underline"])
def bold(value):
return colored(value, attrs=["bold"])
def error_separator_small():
return colored("- - -", on_color="on_red")
def _important_value(value):
return colored(value, color="red")
def get_banner_str(self):
self.write(colored("=\n", self.color))
self.write(self._banner_separator)
return self.getvalue()
from dbnd._vendor.termcolor import colored
ERROR_SEPARATOR = colored("--------------------------------------",
color="red")
ERROR_SEPARATOR_SMALL = colored("- - -", on_color="on_red")
def bold(value):
return colored(value, attrs=["bold"])
