def pformat_config_store_as_table(config_store, sections=None): # type: (_ConfigStore, Optional[Iterable[str]]) -> str header = ["Section", "Key", "Value", "Source", "Priority"] data = [] if sections: sections = [_lower_config_name(s) for s in sections] else: sections = config_store.keys() for section in sections: section_values = config_store.get(section) if not section_values: continue for key, config_value in six.iteritems(section_values): data.append(( section, key, safe_string(config_value.value, 300), config_value.source, config_value.priority, )) if data: return safe_tabulate(tabular_data=data, headers=header) return ""
def build_instances_table(instances_data): extract_keys = ( "name", "is_sync_enabled", "tracking_source_uid", "base_url", "external_url", "api_mode", ) headers = ( "Name", "Active", "Tracking source UID", "Base url", "External url", "Api Mode", "Instance Type", ) table_data = [] for server_info in instances_data: instance_row = [getattr(server_info, key, "") for key in extract_keys] instance_row.append("Airflow" if getattr(server_info, "fetcher") == "web" else "GC Composer") table_data.append(instance_row) return safe_tabulate(table_data, headers)
def build_alerts_table(alerts_data): extract_keys = ( "uid", "job_name", "task_name", "task_repr", "custom_name", "severity", "type", "operator", "value", "user_metric", ) headers = ( "uid", "pipeline", "task", "task_repr", "name", "severity", "type", "op", "value", "metric", ) table_data = [] for alert in alerts_data: alert_row = [alert.get(key, "") for key in extract_keys] table_data.append(alert_row) return safe_tabulate(table_data, headers)
def add_params_table( self, all_params=False, param_kind=True, param_type=True, param_format=False, param_source=False, param_section=False, param_default=False, ): # config self.kind = param_kind self.type = param_type self.format = param_format self.source = param_source self.section = param_section self.default = param_default """adds the param table to the banner""" from dbnd._core.decorator.schemed_result import ResultProxyTarget exclude = set(_TASK_FIELDS) # excluding those # special treatment for result t_result = getattr(self.task, "result", None) if isinstance(t_result, ResultProxyTarget): self.banner.column("RESULT", t_result) exclude.add("result") params_data = [] params_warnings = [] # iterating over the params and building the data for the table for param_def, param_value, param_meta in self.task._params: # filter switches if not all_params: if param_def.name in exclude: continue if param_def.system: continue # building a single row param_row = self.build_record(param_def, param_meta, param_value) params_data.append(param_row) # extract param warnings if param_meta and param_meta.warnings: params_warnings.extend(param_meta.warnings) if params_warnings: self.banner.column("BUILD WARNINGS:", "") self.banner.write("".join("\t%s\n" % pw for pw in params_warnings)) if params_data: params_header = self.build_headers() params_table = safe_tabulate(tabular_data=params_data, headers=params_header) self.banner.column("PARAMS:", "") self.banner.write(params_table) self.banner.new_line()
def two_columns_table(stats, split_by): # type: (Dict, Callable[Tuple[Any,Any], bool]) -> str """ Split the items by a filter function "split_by" and merge Example: two_columns_table({1: "a", 2: "b", 3: "c", 4: "d"}, is_odd) [ 2 , b , 1 , a ] [ 3 , d , 3 , c ] """ splatted = partition(split_by, stats.items()) flatten_merged_rows = map(collapse, zip_longest(*splatted)) return safe_tabulate(flatten_merged_rows, headers=())
def _add_params_info(self): b = self.banner exclude = set(_TASK_FIELDS) # special treatment for result t_result = getattr(self.task, "result", None) from dbnd._core.decorator.schemed_result import ResultProxyTarget if isinstance(t_result, ResultProxyTarget): b.column("RESULT", t_result) exclude.add("result") params_data = [] params_warnings = [] all_info = self.verbosity >= FormatterVerbosity.HIGH relevant_params = [] for p in self.params.get_params(): if not all_info: # we don't want to show all this switches if p.name in exclude: continue if p.system: continue relevant_params.append(p) for p in relevant_params: value = self.params.get_value(p.name) value_meta = self.params.get_value_meta(p.name) target_config = p.target_config if isinstance(value, DataTarget) and hasattr(value, "config"): target_config = value.config if p.is_output(): p_kind = "output" value_str = b.f_io(value) elif not p.load_on_build: p_kind = "input" value_str = b.f_io(value) else: p_kind = "param" value_str = p.to_str(value) value_str = safe_string(value_str, _MAX_VALUE_SIZE) value_source = "" if value_meta: value_source = value_meta.source if value_meta.warnings: params_warnings.extend(value_meta.warnings) type_handler = p.value_type_str param_data = [p.name, p_kind, type_handler, target_config, value_source] # add source task class of parameter if all_info: section = p.parameter_origin.get_task_family() if section == self.task.get_task_family(): section = "" default = str(p.default) # TODO param_data += [default, section] # add preview if isinstance(value, Target) and value.target_meta: preview_value = safe_string( value.target_meta.value_preview, _MAX_VALUE_SIZE ) # we should add minimal preview if len(preview_value) < 100: value_str += " :='%s'" % preview_value if value_str and "\n" in value_str: # some simple euristics around value extra_padding = " " * len("\t".join(map(str, param_data))) value_str = "".join( "\n%s%s" % (extra_padding, l) for l in value_str.split("\n") ) value_str = "-->\n" + value_str param_data.append(value_str) params_data.append(param_data) # config_params = [ # (p.name, value) # for p, value in c._params.get_param_values() # if value is not None # ] # # b.column(c.task_meta.task_family.upper(), b.f_params(config_params)) # b.new_line() # 'Name' is missing, header is aligned to the last column params_header = ["Name", "Kind", "Type", "Format", "Source"] if all_info: params_header.extend(["Default", "Class"]) params_header.append("-= Value =-") if params_warnings: b.column("BUILD WARNINGS:", "") b.write("".join("\t%s\n" % pw for pw in params_warnings)) if params_data: p = safe_tabulate(tabular_data=params_data, headers=params_header) b.column("PARAMS:", "") b.write(p) # b.new_line() # b.column("OUTPUTS", b.f_io(task_outputs_user)) b.new_line()
def add_params_table( self, all_params=False, param_kind=True, param_type=True, param_format=False, param_source=False, param_section=False, param_default=False, ): # config self.kind = param_kind self.type = param_type self.format = param_format self.source = param_source self.section = param_section self.default = param_default """adds the param table to the banner""" from dbnd._core.task_build.task_results import ResultProxyTarget exclude = set(_TASK_FIELDS) # excluding those # special treatment for result t_result = getattr(self.task, "result", None) if isinstance(t_result, ResultProxyTarget): self.banner.column("RESULT", t_result) exclude.add("result") params_data = [] params_warnings = [] # iterating over the params and building the data for the table for param_value in self.task._params: try: param_def = (param_value.parameter. update_value_meta_conf_from_runtime_value( param_value.value, self.task.settings.tracking)) # filter switches if not all_params: if param_def.name in exclude: continue if param_def.system: continue # building a single row if param_def.value_meta_conf.log_preview: param_value_preview = param_value.value else: param_value_preview = ParameterValue.MASKED_VALUE_PREVIEW param_row = self.build_record(param_def, param_value, param_value_preview) params_data.append(param_row) # extract param warnings if param_value and param_value.warnings: params_warnings.extend(param_value.warnings) except Exception as ex: log_exception( "Failed to calculate parameter value for %s.%s" % (self.task.task_name, param_value.parameter.name), ex, non_critical=True, ) if params_warnings: self.banner.column("BUILD WARNINGS:", "") self.banner.write("".join("\t%s\n" % pw for pw in params_warnings)) if params_data: params_header = self.build_headers() params_table = safe_tabulate(tabular_data=params_data, headers=params_header) self.banner.column("PARAMS:", "") self.banner.write(params_table) self.banner.new_line()