def show_table(self, result):
table = result.agate_table
rand_table = table.order_by(lambda x: random.random())
schema = result.node.schema
alias = result.node.alias
header = "Random sample of table: {}.{}".format(schema, alias)
with TextOnly():
logger.info("")
logger.info(header)
logger.info("-" * len(header))
rand_table.print_table(max_rows=10, max_columns=None)
with TextOnly():
logger.info("")
def run(self):
"""
Run dbt for the query, based on the graph.
"""
self._runtime_initialize()
if self._flattened_nodes is None:
raise InternalException(
'after _runtime_initialize, _flattened_nodes was still None'
)
if len(self._flattened_nodes) == 0:
logger.warning("WARNING: Nothing to do. Try checking your model "
"configs and model specification args")
return self.get_result(
results=[],
generated_at=datetime.utcnow(),
elapsed_time=0.0,
)
else:
with TextOnly():
logger.info("")
selected_uids = frozenset(n.unique_id for n in self._flattened_nodes)
result = self.execute_with_hooks(selected_uids)
if flags.WRITE_JSON:
self.write_manifest()
self.write_result(result)
self.task_end_messages(result.results)
return result
def execute_nodes(self):
num_threads = self.config.threads
target_name = self.config.target_name
text = "Concurrency: {} threads (target='{}')"
concurrency_line = text.format(num_threads, target_name)
with NodeCount(self.num_nodes):
print_timestamped_line(concurrency_line)
with TextOnly():
print_timestamped_line("")
pool = ThreadPool(num_threads)
try:
self.run_queue(pool)
except FailFastException as failure:
self._cancel_connections(pool)
print_run_result_error(failure.result)
raise
except KeyboardInterrupt:
self._cancel_connections(pool)
print_run_end_messages(self.node_results, keyboard_interrupt=True)
raise
pool.close()
pool.join()
return self.node_results
def run_hooks(self, adapter, hook_type: RunHookType, extra_context):
ordered_hooks = self.get_hooks_by_type(hook_type)
# on-run-* hooks should run outside of a transaction. This happens
# b/c psycopg2 automatically begins a transaction when a connection
# is created.
adapter.clear_transaction()
if not ordered_hooks:
return
num_hooks = len(ordered_hooks)
plural = 'hook' if num_hooks == 1 else 'hooks'
with TextOnly():
print_timestamped_line("")
print_timestamped_line('Running {} {} {}'.format(
num_hooks, hook_type, plural))
startctx = TimestampNamed('node_started_at')
finishctx = TimestampNamed('node_finished_at')
for idx, hook in enumerate(ordered_hooks, start=1):
sql = self.get_hook_sql(adapter, hook, idx, num_hooks,
extra_context)
hook_text = '{}.{}.{}'.format(hook.package_name, hook_type,
hook.index)
hook_meta_ctx = HookMetadata(hook, self.index_offset(idx))
with UniqueID(hook.unique_id):
with hook_meta_ctx, startctx:
print_hook_start_line(hook_text, idx, num_hooks)
status = 'OK'
with Timer() as timer:
if len(sql.strip()) > 0:
status, _ = adapter.execute(sql,
auto_begin=False,
fetch=False)
self.ran_hooks.append(hook)
with finishctx, DbtModelState({'node_status': 'passed'}):
print_hook_end_line(hook_text, status, idx, num_hooks,
timer.elapsed)
self._total_executed += len(ordered_hooks)
with TextOnly():
print_timestamped_line("")
def print_run_result_error(result,
newline: bool = True,
is_warning: bool = False) -> None:
if newline:
with TextOnly():
logger.info("")
if result.status == NodeStatus.Fail or (is_warning and result.status
== NodeStatus.Warn):
if is_warning:
color = ui.yellow
info = 'Warning'
logger_fn = logger.warning
else:
color = ui.red
info = 'Failure'
logger_fn = logger.error
logger_fn(
color("{} in {} {} ({})").format(info, result.node.resource_type,
result.node.name,
result.node.original_file_path))
try:
# if message is int, must be rows returned for a test
int(result.message)
except ValueError:
logger.error(" Status: {}".format(result.status))
else:
num_rows = utils.pluralize(result.message, 'result')
logger.error(" Got {}, expected 0.".format(num_rows))
if result.node.build_path is not None:
with TextOnly():
logger.info("")
logger.info(" compiled SQL at {}".format(result.node.build_path))
elif result.message is not None:
first = True
for line in result.message.split("\n"):
if first:
logger.error(ui.yellow(line))
first = False
else:
logger.error(line)
def print_run_result_error(result,
newline: bool = True,
is_warning: bool = False) -> None:
if newline:
with TextOnly():
logger.info("")
if result.fail or (is_warning and result.warn):
if is_warning:
color = yellow
info = 'Warning'
logger_fn = logger.warning
else:
color = red
info = 'Failure'
logger_fn = logger.error
logger_fn(
color("{} in {} {} ({})").format(info, result.node.resource_type,
result.node.name,
result.node.original_file_path))
try:
int(result.status)
except ValueError:
logger.error(" Status: {}".format(result.status))
else:
status = dbt.utils.pluralize(result.status, 'result')
logger.error(" Got {}, expected 0.".format(status))
if result.node.build_path is not None:
with TextOnly():
logger.info("")
logger.info(" compiled SQL at {}".format(result.node.build_path))
else:
first = True
for line in result.error.split("\n"):
if first:
logger.error(yellow(line))
first = False
else:
logger.error(line)
def print_results_line(self, results, execution_time):
nodes = [r.node for r in results] + self.ran_hooks
stat_line = get_counts(nodes)
execution = ""
if execution_time is not None:
execution = " in {execution_time:0.2f}s".format(
execution_time=execution_time)
with TextOnly():
print_timestamped_line("")
print_timestamped_line(
"Finished running {stat_line}{execution}.".format(
stat_line=stat_line, execution=execution))
def print_end_of_run_summary(num_errors: int,
num_warnings: int,
keyboard_interrupt: bool = False) -> None:
error_plural = utils.pluralize(num_errors, 'error')
warn_plural = utils.pluralize(num_warnings, 'warning')
if keyboard_interrupt:
message = ui.yellow('Exited because of keyboard interrupt.')
elif num_errors > 0:
message = ui.red("Completed with {} and {}:".format(
error_plural, warn_plural))
elif num_warnings > 0:
message = ui.yellow('Completed with {}:'.format(warn_plural))
else:
message = ui.green('Completed successfully')
with TextOnly():
logger.info('')
logger.info('{}'.format(message))
def execute_nodes(self):
num_threads = self.config.threads
target_name = self.config.target_name
text = "Concurrency: {} threads (target='{}')"
concurrency_line = text.format(num_threads, target_name)
with NodeCount(self.num_nodes):
dbt.ui.printer.print_timestamped_line(concurrency_line)
with TextOnly():
dbt.ui.printer.print_timestamped_line("")
pool = ThreadPool(num_threads)
try:
self.run_queue(pool)
except KeyboardInterrupt:
pool.close()
pool.terminate()
adapter = get_adapter(self.config)
if not adapter.is_cancelable():
msg = ("The {} adapter does not support query "
"cancellation. Some queries may still be "
"running!".format(adapter.type()))
yellow = dbt.ui.printer.COLOR_FG_YELLOW
dbt.ui.printer.print_timestamped_line(msg, yellow)
raise
for conn_name in adapter.cancel_open_connections():
dbt.ui.printer.print_cancel_line(conn_name)
pool.join()
dbt.ui.printer.print_run_end_messages(self.node_results,
early_exit=True)
raise
pool.close()
pool.join()
return self.node_results