def _qubole_banner(self, status):
b = TextBanner(
"Spark task {} is submitted to Qubole cluster labeled: {}".format(
self.task.task_id, self.qubole_config.cluster_label),
color="magenta",
)
b.column("Status", status)
b.column("URL", self.qubole_job_url)
logger.info(b.get_banner_str())
def _handle_databricks_operator_execution(self, run_id, hook, task_id):
"""
Handles the Airflow + Databricks lifecycle logic for a Databricks operator
:param run_id: Databricks run_id
:param hook: Airflow databricks hook
:param task_id: Databand Task Id.
"""
b = TextBanner("Spark task %s is submitted to Databricks cluster:" %
task_id,
color="cyan")
url = hook.get_run_page_url(run_id)
self.task_run.set_external_resource_urls({"databricks url": url})
b.column("URL", url)
logger.info(b.get_banner_str())
while True:
b = TextBanner(
"Spark task %s is submitted to Databricks cluster:" % task_id,
color="cyan",
)
b.column("URL", url)
run_state = hook.get_run_state(run_id)
if run_state.is_terminal:
if run_state.is_successful:
b.column("Task completed successfully", task_id)
b.column("State:", run_state.life_cycle_state)
b.column("Message:", run_state.state_message)
break
else:
b.column("State", run_state.result_state)
b.column("Error Message:", run_state.state_message)
logger.info(b.get_banner_str())
raise failed_to_run_databricks_job(run_state.result_state,
run_state.state_message,
url)
else:
b.column("State:", run_state.life_cycle_state)
b.column("Message:", run_state.state_message)
time.sleep(
self.databricks_config.status_polling_interval_seconds)
logger.info(b.get_banner_str())
def pformat_all_layers(config, sections=None):
# type: (DbndConfig, Optional[Iterable[str]]) -> str
tb = TextBanner("Configs")
layers = config.config_layer.get_all_layers()
# start from the first one
for layer in reversed(layers):
layer_values = layer.layer_config.as_value_dict(sections=sections)
tb.column("LAYER {}".format(layer.config_layer_path),
tb.f_struct(layer_values))
return tb.get_banner_str()
def pformat_current_config(config, as_table=False, sections=None):
# type: (DbndConfig, bool, Optional[Iterable[str]]) -> str
config_layer = config.config_layer
tb = TextBanner("Config {}".format(config_layer.name))
tb.column("LAYERS", config_layer.config_layer_path)
if as_table:
view_str = pformat_config_store_as_table(
config_store=config_layer.config, sections=sections)
else:
value_dict = config_layer.config.as_value_dict(sections=sections)
view_str = tb.f_struct(value_dict)
tb.column("CONFIG", view_str)
return tb.get_banner_str()
class TaskVisualiser(object):
def __init__(self, task):
self.task = task
def banner(
self,
msg,
color=None,
verbose=False,
print_task_band=False,
task_run=None,
exc_info=None,
):
task_id = self.task.task_id
try:
# Saving banner for testability
self._banner = TextBanner(msg, color)
if verbose or is_verbose():
verbosity = FormatterVerbosity.HIGH
else:
verbosity = FormatterVerbosity.NORMAL
builder = _TaskBannerBuilder(
task=self.task,
banner=self._banner,
verbosity=verbosity,
print_task_band=print_task_band,
)
# different banners for tracking and orchestration
if TaskEssence.TRACKING.is_instance(self.task):
builder.build_tracking_banner(task_run=task_run,
exc_info=exc_info)
else:
if TaskEssence.CONFIG.is_instance(self.task):
builder.build_config_banner()
else:
builder.build_orchestration_banner(task_run=task_run,
exc_info=exc_info)
return self._banner.get_banner_str()
except Exception as ex:
log_exception("Failed to calculate banner for '%s'" % task_id,
ex,
non_critical=True)
return msg + (" ( task_id=%s)" % task_id)
class DoctorStatusReportBuilder(object):
def __init__(self, name):
self.name = name
msg = "Running %s Status Check:" % name
logger.info(msg)
self.text_banner = TextBanner(msg)
def log(self, key, value):
log_metric(key, value)
self.text_banner.column(key, value)
def add_sub_report(self, sub_report_str):
self.text_banner.write(sub_report_str)
self.text_banner.write("\n")
def get_status_str(self):
return self.text_banner.get_banner_str()
def banner(
self,
msg,
color=None,
verbose=False,
print_task_band=False,
task_run=None,
exc_info=None,
):
try:
banner = TextBanner(msg, color)
if verbose or is_verbose():
verbosity = FormatterVerbosity.HIGH
else:
verbosity = FormatterVerbosity.NORMAL
builder = _TaskBannerBuilder(
task=self.task,
banner=banner,
verbosity=verbosity,
print_task_band=print_task_band,
)
# different banners for tracking and orchestration
if TaskEssence.TRACKING.is_included(self.task):
builder.build_tracking_banner(task_run=task_run,
exc_info=exc_info)
else:
builder.build_orchestration_banner(task_run=task_run,
exc_info=exc_info)
return banner.get_banner_str()
except Exception as ex:
log_exception(
"Failed to calculate banner for '%s'" % self.task_id,
ex,
non_critical=True,
)
return msg + (" ( task_id=%s)" % self.task_id)
def print_table(header, instances_list):
banner = TextBanner(header)
banner.write(build_instances_table(instances_list))
logger.info(banner.get_banner_str())
def print_table(header, alerts_list):
banner = TextBanner(header)
banner.write(build_alerts_table(alerts_list))
logger.info(banner.get_banner_str())
