def _add_last_error_info(self, exc_info=None): ex = exc_info[1] if show_exc_info(ex): # if we are showing internal task - we don't know how to "minimize" stack. # everything is databand.* ... isolate = not self.task.task_definition.full_task_family.startswith( "databand.") traceback_str = self.settings.log.format_exception_as_str( exc_info=exc_info, isolate=isolate) self.banner.column( colored("TRACEBACK", color="red", attrs=["bold"]), traceback_str, raw_name=True, ) self.banner.column( colored("ERROR MESSAGE", color="red", attrs=["bold"]), str(ex), raw_name=True, ) self.banner.column( colored("HELP", attrs=["bold"]), get_help_msg(ex), raw_name=True, skip_if_empty=True, ) self.banner.column( colored("CAUSED BY", color="red", attrs=["bold"]), nested_exceptions_str(ex, limit=3), raw_name=True, skip_if_empty=True, )
def __init__(self, msg, color="white"): StringIO.__init__(self) self.color = color self._banner_separator = colored("==================== \n", color) self._banner_msg = colored("= %s\n" % msg, color) self._section_separator = "-------\n" # p = pprint.PrettyPrinter(indent=2, width=140) self.write("\n") self.write(self._banner_separator) self.write(self._banner_msg)
def _describe_task(self, task, describe_format=None, msg=None, color=None): describe_format = describe_format or self.describe_format describe_config = self.config # type: DescribeConfig msg = msg or "" if color is None: color = "white" if not describe_config.no_checks: color = "green" if self._get_task_complete(task) else "cyan" if describe_format == DescribeFormat.short: return colored(str(task.task_id), color) if (describe_format == DescribeFormat.long or describe_format == DescribeFormat.verbose): title = "%s - %s" % (task.task_name, task.task_id) if task.task_name != task.get_task_family(): title += "(%s)" % task.get_task_family() if msg: title += ": %s" % msg return task.ctrl.visualiser.banner( title, color=color, verbose=describe_format == DescribeFormat.verbose) raise DatabandSystemError("Not supported format mode %s" % self.describe_format)
def column(self, name, value, newline=True, raw_name=False, skip_if_empty=False): # if separate: # s.write(" \n") if skip_if_empty and value is None: return self if isinstance(value, six.string_types): if six.PY2: value = value.encode("utf-8", errors="ignore") else: value = str(value) else: value = str(value) if skip_if_empty and not value: return self if "\n" in value: value = _TAB + _TAB.join(value.split("\n")) # remove last \n, use new line or new section if you nee that if value[-1:] == "\n": value = value[:-1] self.write( " {name:<18} : {value}".format( name=name if raw_name else colored(name, attrs=["bold"]), value=value ) ) if newline: self.write("\n") return self
def read_dbnd_log_preview(log_path, spark_log_path=None): """ Reads the log_path using the current logging config """ logger_config = LoggingConfig.current() max_head_size = logger_config.preview_head_bytes max_tail_size = logger_config.preview_tail_bytes # there is not need to run any of the logic if we don't need to read anything if max_head_size == 0 and max_tail_size == 0: return EMPTY_LOG_MSG if spark_log_path: # we need to read and than merge the content of dbnd log and spark long # using half of the max_bytes for each of the logs to make sure we are not exceeding the max after merging parts = merge_read_log_files( log_path, spark_log_path, max_head_size // 2, max_tail_size // 2, ) else: # safely reading a single log file parts = read_head_and_tail(log_path, max_head_size, max_tail_size) # check if all the parts contains information if not any(parts): # all the parts are truncated return EMPTY_LOG_MSG # join each part with new lines joined_parts = [LINE_BRAKE.join(part) for part in parts] # building the merge message between head and tail merge_msg = colored( MERGE_MSG.format( head_size=max_head_size, tail_size=max_tail_size, file_size=os.path.getsize(log_path), log_path=log_path, ), MERGE_MSG_COLOR, ) return merge_msg.join(joined_parts)
def _get_batch_progresss_banner(self, batch_response): """ { 'id': 6, 'state': 'success', 'appId': 'application_1534487568579_0008', 'appInfo': { 'driverLogUrl': None, 'sparkUiUrl': 'http://ip-172-31-70-109.ec2.internal:20888/proxy/application_1534487568579_0008/' }, 'log': [ '\nYARN Diagnostics: ' ] } :param batch_response: :return: """ t = self.task b = TextBanner("Spark Task %s is running at Livy:" % t.task_id, color="yellow") b.column("TASK", t.task_id) b.column("JOB STATE", batch_response.get("state", None)) tracker_url = current_task_run().task_tracker_url if tracker_url: b.column("DATABAND LOG", tracker_url) b.new_line() b.column("LIVY ID", batch_response.get("id", None)) if "appId" in batch_response: b.column("APP ID", batch_response["appId"]) app_info = batch_response["appInfo"] b.column("DRIVER LOG", app_info["driverLogUrl"]) if "sparkUiUrl" in app_info: spark_url = app_info["sparkUiUrl"] b.column( "SPARK", colored(spark_url, on_color="on_blue", attrs=["bold"])) b.new_section() return b.getvalue()
def error_separator(): return colored("--------------------------------------", color="red")
def cyan(value): return colored(value, color="cyan")
def red(value): return colored(value, color="red")
def underline(value): return colored(value, attrs=["underline"])
def bold(value): return colored(value, attrs=["bold"])
def error_separator_small(): return colored("- - -", on_color="on_red")
def _important_value(value): return colored(value, color="red")
def get_banner_str(self): self.write(colored("=\n", self.color)) self.write(self._banner_separator) return self.getvalue()
from dbnd._vendor.termcolor import colored ERROR_SEPARATOR = colored("--------------------------------------", color="red") ERROR_SEPARATOR_SMALL = colored("- - -", on_color="on_red") def bold(value): return colored(value, attrs=["bold"])