def task_end_messages(self, results): for result in results: if result.status in (FreshnessStatus.Error, FreshnessStatus.RuntimeErr): print_run_result_error(result) print_timestamped_line('Done.')
def _cancel_connections(self, pool): """Given a pool, cancel all adapter connections and wait until all runners gentle terminates. """ pool.close() pool.terminate() adapter = get_adapter(self.config) if not adapter.is_cancelable(): msg = ("The {} adapter does not support query " "cancellation. Some queries may still be " "running!".format(adapter.type())) yellow = ui.COLOR_FG_YELLOW print_timestamped_line(msg, yellow) else: with adapter.connection_named('master'): for conn_name in adapter.cancel_open_connections(): if self.manifest is not None: node = self.manifest.nodes.get(conn_name) if node is not None and node.is_ephemeral_model: continue # if we don't have a manifest/don't have a node, print # anyway. print_cancel_line(conn_name) pool.join()
def execute_nodes(self): num_threads = self.config.threads target_name = self.config.target_name text = "Concurrency: {} threads (target='{}')" concurrency_line = text.format(num_threads, target_name) with NodeCount(self.num_nodes): print_timestamped_line(concurrency_line) with TextOnly(): print_timestamped_line("") pool = ThreadPool(num_threads) try: self.run_queue(pool) except FailFastException as failure: self._cancel_connections(pool) print_run_result_error(failure.result) raise except KeyboardInterrupt: self._cancel_connections(pool) print_run_end_messages(self.node_results, keyboard_interrupt=True) raise pool.close() pool.join() return self.node_results
def write_perf_info(self): path = os.path.join(self.config.target_path, PERF_INFO_FILE_NAME) write_file( path, json.dumps(self.loader._perf_info, cls=dbt.utils.JSONEncoder, indent=4)) print_timestamped_line(f"Performance info: {path}")
def print_results_line(self, results, execution_time): nodes = [r.node for r in results] + self.ran_hooks stat_line = get_counts(nodes) execution = "" if execution_time is not None: execution = " in {execution_time:0.2f}s".format( execution_time=execution_time) with TextOnly(): print_timestamped_line("") print_timestamped_line( "Finished running {stat_line}{execution}.".format( stat_line=stat_line, execution=execution))
def run(self): print_timestamped_line('Start parsing.') self.get_full_manifest() if self.args.compile: print_timestamped_line('Compiling.') self.compile_manifest() if self.args.write_manifest: print_timestamped_line('Writing manifest.') self.write_manifest() self.write_perf_info() print_timestamped_line('Done.')
def run_hooks(self, adapter, hook_type: RunHookType, extra_context): ordered_hooks = self.get_hooks_by_type(hook_type) # on-run-* hooks should run outside of a transaction. This happens # b/c psycopg2 automatically begins a transaction when a connection # is created. adapter.clear_transaction() if not ordered_hooks: return num_hooks = len(ordered_hooks) plural = 'hook' if num_hooks == 1 else 'hooks' with TextOnly(): print_timestamped_line("") print_timestamped_line('Running {} {} {}'.format( num_hooks, hook_type, plural)) startctx = TimestampNamed('node_started_at') finishctx = TimestampNamed('node_finished_at') for idx, hook in enumerate(ordered_hooks, start=1): sql = self.get_hook_sql(adapter, hook, idx, num_hooks, extra_context) hook_text = '{}.{}.{}'.format(hook.package_name, hook_type, hook.index) hook_meta_ctx = HookMetadata(hook, self.index_offset(idx)) with UniqueID(hook.unique_id): with hook_meta_ctx, startctx: print_hook_start_line(hook_text, idx, num_hooks) status = 'OK' with Timer() as timer: if len(sql.strip()) > 0: status, _ = adapter.execute(sql, auto_begin=False, fetch=False) self.ran_hooks.append(hook) with finishctx, DbtModelState({'node_status': 'passed'}): print_hook_end_line(hook_text, status, idx, num_hooks, timer.elapsed) self._total_executed += len(ordered_hooks) with TextOnly(): print_timestamped_line("")
def get_full_manifest(self): adapter = get_adapter(self.config) # type: ignore root_config = self.config macro_hook = adapter.connections.set_query_header with PARSING_STATE: start_load_all = time.perf_counter() projects = root_config.load_dependencies() print_timestamped_line("Dependencies loaded") loader = ManifestLoader(root_config, projects, macro_hook) print_timestamped_line("ManifestLoader created") loader.load() print_timestamped_line("Manifest loaded") loader.write_manifest_for_partial_parse() print_timestamped_line("Manifest for partial parse saved") manifest = loader.update_manifest() print_timestamped_line("Manifest updated") _check_manifest(manifest, root_config) print_timestamped_line("Manifest checked") manifest.build_flat_graph() print_timestamped_line("Flat graph built") loader._perf_info.load_all_elapsed = (time.perf_counter() - start_load_all) self.loader = loader self.manifest = manifest print_timestamped_line("Manifest loaded")
def run(self) -> CatalogResults: compile_results = None if self.args.compile: compile_results = CompileTask.run(self) if any(r.error is not None for r in compile_results): print_timestamped_line('compile failed, cannot generate docs') return CatalogResults(nodes={}, sources={}, generated_at=datetime.utcnow(), errors=None, _compile_results=compile_results) else: self.manifest = get_full_manifest(self.config) shutil.copyfile(DOCS_INDEX_FILE_PATH, os.path.join(self.config.target_path, 'index.html')) for asset_path in self.config.asset_paths: to_asset_path = os.path.join(self.config.target_path, asset_path) if os.path.exists(to_asset_path): shutil.rmtree(to_asset_path) if os.path.exists(asset_path): shutil.copytree(asset_path, to_asset_path) if self.manifest is None: raise InternalException('self.manifest was None in run!') adapter = get_adapter(self.config) with adapter.connection_named('generate_catalog'): print_timestamped_line("Building catalog") catalog_table, exceptions = adapter.get_catalog(self.manifest) catalog_data: List[PrimitiveDict] = [ dict(zip(catalog_table.column_names, map(_coerce_decimal, row))) for row in catalog_table ] catalog = Catalog(catalog_data) errors: Optional[List[str]] = None if exceptions: errors = [str(e) for e in exceptions] nodes, sources = catalog.make_unique_id_map(self.manifest) results = self.get_catalog_results( nodes=nodes, sources=sources, generated_at=datetime.utcnow(), compile_results=compile_results, errors=errors, ) path = os.path.join(self.config.target_path, CATALOG_FILENAME) results.write(path) if self.args.compile: self.write_manifest() if exceptions: logger.error( 'dbt encountered {} failure{} while writing the catalog'. format(len(exceptions), (len(exceptions) != 1) * 's')) print_timestamped_line('Catalog written to {}'.format( os.path.abspath(path))) return results
def task_end_messages(self, results): for result in results: if result.error is not None: print_run_result_error(result) print_timestamped_line('Done.')
def warning_on_hooks(hook_type): msg = "{} is not supported in bigquery and will be ignored" print_timestamped_line(msg.format(hook_type), ui.COLOR_FG_YELLOW)
def task_end_messages(self, results): print_timestamped_line('Done.')