def pformat_config_store_as_table(config_store, sections=None):
# type: (_ConfigStore, Optional[Iterable[str]]) -> str
header = ["Section", "Key", "Value", "Source", "Priority"]
data = []
if sections:
sections = [_lower_config_name(s) for s in sections]
else:
sections = config_store.keys()
for section in sections:
section_values = config_store.get(section)
if not section_values:
continue
for key, config_value in six.iteritems(section_values):
data.append((
section,
key,
safe_string(config_value.value, 300),
config_value.source,
config_value.priority,
))
if data:
return safe_tabulate(tabular_data=data, headers=header)
return ""
def build_instances_table(instances_data):
extract_keys = (
"name",
"is_sync_enabled",
"tracking_source_uid",
"base_url",
"external_url",
"api_mode",
)
headers = (
"Name",
"Active",
"Tracking source UID",
"Base url",
"External url",
"Api Mode",
"Instance Type",
)
table_data = []
for server_info in instances_data:
instance_row = [getattr(server_info, key, "") for key in extract_keys]
instance_row.append("Airflow" if getattr(server_info, "fetcher") ==
"web" else "GC Composer")
table_data.append(instance_row)
return safe_tabulate(table_data, headers)
def build_alerts_table(alerts_data):
extract_keys = (
"uid",
"job_name",
"task_name",
"task_repr",
"custom_name",
"severity",
"type",
"operator",
"value",
"user_metric",
)
headers = (
"uid",
"pipeline",
"task",
"task_repr",
"name",
"severity",
"type",
"op",
"value",
"metric",
)
table_data = []
for alert in alerts_data:
alert_row = [alert.get(key, "") for key in extract_keys]
table_data.append(alert_row)
return safe_tabulate(table_data, headers)
def add_params_table(
self,
all_params=False,
param_kind=True,
param_type=True,
param_format=False,
param_source=False,
param_section=False,
param_default=False,
):
# config
self.kind = param_kind
self.type = param_type
self.format = param_format
self.source = param_source
self.section = param_section
self.default = param_default
"""adds the param table to the banner"""
from dbnd._core.decorator.schemed_result import ResultProxyTarget
exclude = set(_TASK_FIELDS) # excluding those
# special treatment for result
t_result = getattr(self.task, "result", None)
if isinstance(t_result, ResultProxyTarget):
self.banner.column("RESULT", t_result)
exclude.add("result")
params_data = []
params_warnings = []
# iterating over the params and building the data for the table
for param_def, param_value, param_meta in self.task._params:
# filter switches
if not all_params:
if param_def.name in exclude:
continue
if param_def.system:
continue
# building a single row
param_row = self.build_record(param_def, param_meta, param_value)
params_data.append(param_row)
# extract param warnings
if param_meta and param_meta.warnings:
params_warnings.extend(param_meta.warnings)
if params_warnings:
self.banner.column("BUILD WARNINGS:", "")
self.banner.write("".join("\t%s\n" % pw for pw in params_warnings))
if params_data:
params_header = self.build_headers()
params_table = safe_tabulate(tabular_data=params_data,
headers=params_header)
self.banner.column("PARAMS:", "")
self.banner.write(params_table)
self.banner.new_line()
def two_columns_table(stats, split_by):
# type: (Dict, Callable[Tuple[Any,Any], bool]) -> str
"""
Split the items by a filter function "split_by" and merge
Example:
two_columns_table({1: "a", 2: "b", 3: "c", 4: "d"}, is_odd)
[ 2 , b , 1 , a ]
[ 3 , d , 3 , c ]
"""
splatted = partition(split_by, stats.items())
flatten_merged_rows = map(collapse, zip_longest(*splatted))
return safe_tabulate(flatten_merged_rows, headers=())
def _add_params_info(self):
b = self.banner
exclude = set(_TASK_FIELDS)
# special treatment for result
t_result = getattr(self.task, "result", None)
from dbnd._core.decorator.schemed_result import ResultProxyTarget
if isinstance(t_result, ResultProxyTarget):
b.column("RESULT", t_result)
exclude.add("result")
params_data = []
params_warnings = []
all_info = self.verbosity >= FormatterVerbosity.HIGH
relevant_params = []
for p in self.params.get_params():
if not all_info:
# we don't want to show all this switches
if p.name in exclude:
continue
if p.system:
continue
relevant_params.append(p)
for p in relevant_params:
value = self.params.get_value(p.name)
value_meta = self.params.get_value_meta(p.name)
target_config = p.target_config
if isinstance(value, DataTarget) and hasattr(value, "config"):
target_config = value.config
if p.is_output():
p_kind = "output"
value_str = b.f_io(value)
elif not p.load_on_build:
p_kind = "input"
value_str = b.f_io(value)
else:
p_kind = "param"
value_str = p.to_str(value)
value_str = safe_string(value_str, _MAX_VALUE_SIZE)
value_source = ""
if value_meta:
value_source = value_meta.source
if value_meta.warnings:
params_warnings.extend(value_meta.warnings)
type_handler = p.value_type_str
param_data = [p.name, p_kind, type_handler, target_config, value_source]
# add source task class of parameter
if all_info:
section = p.parameter_origin.get_task_family()
if section == self.task.get_task_family():
section = ""
default = str(p.default) # TODO
param_data += [default, section]
# add preview
if isinstance(value, Target) and value.target_meta:
preview_value = safe_string(
value.target_meta.value_preview, _MAX_VALUE_SIZE
)
# we should add minimal preview
if len(preview_value) < 100:
value_str += " :='%s'" % preview_value
if value_str and "\n" in value_str:
# some simple euristics around value
extra_padding = " " * len("\t".join(map(str, param_data)))
value_str = "".join(
"\n%s%s" % (extra_padding, l) for l in value_str.split("\n")
)
value_str = "-->\n" + value_str
param_data.append(value_str)
params_data.append(param_data)
# config_params = [
# (p.name, value)
# for p, value in c._params.get_param_values()
# if value is not None
# ]
#
# b.column(c.task_meta.task_family.upper(), b.f_params(config_params))
# b.new_line()
# 'Name' is missing, header is aligned to the last column
params_header = ["Name", "Kind", "Type", "Format", "Source"]
if all_info:
params_header.extend(["Default", "Class"])
params_header.append("-= Value =-")
if params_warnings:
b.column("BUILD WARNINGS:", "")
b.write("".join("\t%s\n" % pw for pw in params_warnings))
if params_data:
p = safe_tabulate(tabular_data=params_data, headers=params_header)
b.column("PARAMS:", "")
b.write(p)
# b.new_line()
# b.column("OUTPUTS", b.f_io(task_outputs_user))
b.new_line()
def add_params_table(
self,
all_params=False,
param_kind=True,
param_type=True,
param_format=False,
param_source=False,
param_section=False,
param_default=False,
):
# config
self.kind = param_kind
self.type = param_type
self.format = param_format
self.source = param_source
self.section = param_section
self.default = param_default
"""adds the param table to the banner"""
from dbnd._core.task_build.task_results import ResultProxyTarget
exclude = set(_TASK_FIELDS) # excluding those
# special treatment for result
t_result = getattr(self.task, "result", None)
if isinstance(t_result, ResultProxyTarget):
self.banner.column("RESULT", t_result)
exclude.add("result")
params_data = []
params_warnings = []
# iterating over the params and building the data for the table
for param_value in self.task._params:
try:
param_def = (param_value.parameter.
update_value_meta_conf_from_runtime_value(
param_value.value,
self.task.settings.tracking))
# filter switches
if not all_params:
if param_def.name in exclude:
continue
if param_def.system:
continue
# building a single row
if param_def.value_meta_conf.log_preview:
param_value_preview = param_value.value
else:
param_value_preview = ParameterValue.MASKED_VALUE_PREVIEW
param_row = self.build_record(param_def, param_value,
param_value_preview)
params_data.append(param_row)
# extract param warnings
if param_value and param_value.warnings:
params_warnings.extend(param_value.warnings)
except Exception as ex:
log_exception(
"Failed to calculate parameter value for %s.%s" %
(self.task.task_name, param_value.parameter.name),
ex,
non_critical=True,
)
if params_warnings:
self.banner.column("BUILD WARNINGS:", "")
self.banner.write("".join("\t%s\n" % pw for pw in params_warnings))
if params_data:
params_header = self.build_headers()
params_table = safe_tabulate(tabular_data=params_data,
headers=params_header)
self.banner.column("PARAMS:", "")
self.banner.write(params_table)
self.banner.new_line()
