def task_end_messages(self, results):
for result in results:
if result.status in (FreshnessStatus.Error,
FreshnessStatus.RuntimeErr):
print_run_result_error(result)
print_timestamped_line('Done.')
def _cancel_connections(self, pool):
"""Given a pool, cancel all adapter connections and wait until all
runners gentle terminates.
"""
pool.close()
pool.terminate()
adapter = get_adapter(self.config)
if not adapter.is_cancelable():
msg = ("The {} adapter does not support query "
"cancellation. Some queries may still be "
"running!".format(adapter.type()))
yellow = ui.COLOR_FG_YELLOW
print_timestamped_line(msg, yellow)
else:
with adapter.connection_named('master'):
for conn_name in adapter.cancel_open_connections():
if self.manifest is not None:
node = self.manifest.nodes.get(conn_name)
if node is not None and node.is_ephemeral_model:
continue
# if we don't have a manifest/don't have a node, print
# anyway.
print_cancel_line(conn_name)
pool.join()
def execute_nodes(self):
num_threads = self.config.threads
target_name = self.config.target_name
text = "Concurrency: {} threads (target='{}')"
concurrency_line = text.format(num_threads, target_name)
with NodeCount(self.num_nodes):
print_timestamped_line(concurrency_line)
with TextOnly():
print_timestamped_line("")
pool = ThreadPool(num_threads)
try:
self.run_queue(pool)
except FailFastException as failure:
self._cancel_connections(pool)
print_run_result_error(failure.result)
raise
except KeyboardInterrupt:
self._cancel_connections(pool)
print_run_end_messages(self.node_results, keyboard_interrupt=True)
raise
pool.close()
pool.join()
return self.node_results
def write_perf_info(self):
path = os.path.join(self.config.target_path, PERF_INFO_FILE_NAME)
write_file(
path,
json.dumps(self.loader._perf_info,
cls=dbt.utils.JSONEncoder,
indent=4))
print_timestamped_line(f"Performance info: {path}")
def print_results_line(self, results, execution_time):
nodes = [r.node for r in results] + self.ran_hooks
stat_line = get_counts(nodes)
execution = ""
if execution_time is not None:
execution = " in {execution_time:0.2f}s".format(
execution_time=execution_time)
with TextOnly():
print_timestamped_line("")
print_timestamped_line(
"Finished running {stat_line}{execution}.".format(
stat_line=stat_line, execution=execution))
def run(self):
print_timestamped_line('Start parsing.')
self.get_full_manifest()
if self.args.compile:
print_timestamped_line('Compiling.')
self.compile_manifest()
if self.args.write_manifest:
print_timestamped_line('Writing manifest.')
self.write_manifest()
self.write_perf_info()
print_timestamped_line('Done.')
def run_hooks(self, adapter, hook_type: RunHookType, extra_context):
ordered_hooks = self.get_hooks_by_type(hook_type)
# on-run-* hooks should run outside of a transaction. This happens
# b/c psycopg2 automatically begins a transaction when a connection
# is created.
adapter.clear_transaction()
if not ordered_hooks:
return
num_hooks = len(ordered_hooks)
plural = 'hook' if num_hooks == 1 else 'hooks'
with TextOnly():
print_timestamped_line("")
print_timestamped_line('Running {} {} {}'.format(
num_hooks, hook_type, plural))
startctx = TimestampNamed('node_started_at')
finishctx = TimestampNamed('node_finished_at')
for idx, hook in enumerate(ordered_hooks, start=1):
sql = self.get_hook_sql(adapter, hook, idx, num_hooks,
extra_context)
hook_text = '{}.{}.{}'.format(hook.package_name, hook_type,
hook.index)
hook_meta_ctx = HookMetadata(hook, self.index_offset(idx))
with UniqueID(hook.unique_id):
with hook_meta_ctx, startctx:
print_hook_start_line(hook_text, idx, num_hooks)
status = 'OK'
with Timer() as timer:
if len(sql.strip()) > 0:
status, _ = adapter.execute(sql,
auto_begin=False,
fetch=False)
self.ran_hooks.append(hook)
with finishctx, DbtModelState({'node_status': 'passed'}):
print_hook_end_line(hook_text, status, idx, num_hooks,
timer.elapsed)
self._total_executed += len(ordered_hooks)
with TextOnly():
print_timestamped_line("")
def get_full_manifest(self):
adapter = get_adapter(self.config) # type: ignore
root_config = self.config
macro_hook = adapter.connections.set_query_header
with PARSING_STATE:
start_load_all = time.perf_counter()
projects = root_config.load_dependencies()
print_timestamped_line("Dependencies loaded")
loader = ManifestLoader(root_config, projects, macro_hook)
print_timestamped_line("ManifestLoader created")
loader.load()
print_timestamped_line("Manifest loaded")
loader.write_manifest_for_partial_parse()
print_timestamped_line("Manifest for partial parse saved")
manifest = loader.update_manifest()
print_timestamped_line("Manifest updated")
_check_manifest(manifest, root_config)
print_timestamped_line("Manifest checked")
manifest.build_flat_graph()
print_timestamped_line("Flat graph built")
loader._perf_info.load_all_elapsed = (time.perf_counter() -
start_load_all)
self.loader = loader
self.manifest = manifest
print_timestamped_line("Manifest loaded")
def run(self) -> CatalogResults:
compile_results = None
if self.args.compile:
compile_results = CompileTask.run(self)
if any(r.error is not None for r in compile_results):
print_timestamped_line('compile failed, cannot generate docs')
return CatalogResults(nodes={},
sources={},
generated_at=datetime.utcnow(),
errors=None,
_compile_results=compile_results)
else:
self.manifest = get_full_manifest(self.config)
shutil.copyfile(DOCS_INDEX_FILE_PATH,
os.path.join(self.config.target_path, 'index.html'))
for asset_path in self.config.asset_paths:
to_asset_path = os.path.join(self.config.target_path, asset_path)
if os.path.exists(to_asset_path):
shutil.rmtree(to_asset_path)
if os.path.exists(asset_path):
shutil.copytree(asset_path, to_asset_path)
if self.manifest is None:
raise InternalException('self.manifest was None in run!')
adapter = get_adapter(self.config)
with adapter.connection_named('generate_catalog'):
print_timestamped_line("Building catalog")
catalog_table, exceptions = adapter.get_catalog(self.manifest)
catalog_data: List[PrimitiveDict] = [
dict(zip(catalog_table.column_names, map(_coerce_decimal, row)))
for row in catalog_table
]
catalog = Catalog(catalog_data)
errors: Optional[List[str]] = None
if exceptions:
errors = [str(e) for e in exceptions]
nodes, sources = catalog.make_unique_id_map(self.manifest)
results = self.get_catalog_results(
nodes=nodes,
sources=sources,
generated_at=datetime.utcnow(),
compile_results=compile_results,
errors=errors,
)
path = os.path.join(self.config.target_path, CATALOG_FILENAME)
results.write(path)
if self.args.compile:
self.write_manifest()
if exceptions:
logger.error(
'dbt encountered {} failure{} while writing the catalog'.
format(len(exceptions), (len(exceptions) != 1) * 's'))
print_timestamped_line('Catalog written to {}'.format(
os.path.abspath(path)))
return results
def task_end_messages(self, results):
for result in results:
if result.error is not None:
print_run_result_error(result)
print_timestamped_line('Done.')
def warning_on_hooks(hook_type):
msg = "{} is not supported in bigquery and will be ignored"
print_timestamped_line(msg.format(hook_type), ui.COLOR_FG_YELLOW)
def task_end_messages(self, results):
print_timestamped_line('Done.')
