def open(cls, connection): if connection.state == 'open': logger.debug('Connection is already open, skipping open.') return connection credentials = cls.get_credentials(connection.credentials) host = f'{credentials.host}:{credentials.port}/{credentials.database}' try: host = cls._build_host(credentials=credentials) handle = cx_Oracle.connect( credentials.user, credentials.password, host, encoding="UTF-8") connection.handle = handle connection.state = 'open' except cx_Oracle.DatabaseError as e: logger.debug("Got an error when attempting to open an connection to oracle\n" "using simple string method(host and port)\n" "Will try to connect using tnsanmes.ora file\n" "error: '{}'" .format(str(e))) try: handle = cx_Oracle.connect( credentials.user, credentials.password, credentials.database, encoding="UTF-8") connection.handle = handle connection.state = 'open' except cx_Oracle.DatabaseError as e: logger.error("Also Got an error when attempting to open an oracle connection\n" "using tnsnames.ora file, failing dbt job.\n" "error: '{}'" .format(str(e))) connection.handle = None connection.state = 'fail' raise dbt.exceptions.FailedToConnectException(str(e)) return connection
def main(args=None): if args is None: args = sys.argv[1:] with log_manager.applicationbound(): try: results, succeeded = handle_and_check(args) if succeeded: exit_code = ExitCodes.Success.value else: exit_code = ExitCodes.ModelError.value except KeyboardInterrupt: logger.info("ctrl-c") exit_code = ExitCodes.UnhandledError.value # This can be thrown by eg. argparse except SystemExit as e: exit_code = e.code except BaseException as e: logger.warning("Encountered an error:") logger.warning(str(e)) if log_manager.initialized: logger.debug(traceback.format_exc()) elif not isinstance(e, RuntimeException): # if it did not come from dbt proper and the logger is not # initialized (so there's no safe path to log to), log the # stack trace at error level. logger.error(traceback.format_exc()) exit_code = ExitCodes.UnhandledError.value sys.exit(exit_code)
def open(cls, connection): if connection.state == 'open': logger.debug('Connection is already open, skipping open.') return connection credentials = connection.credentials try: connection_args = {} if credentials.dsn: connection_args = {'DSN': credentials.dsn} else: connection_args = { 'DRIVER': credentials.driver, 'DBCNAME': credentials.host, 'PORT': credentials.port, 'DATABASE': credentials.database, 'SCHEMA': credentials.schema, #'SESSIONMODE': 'ANSI' } handle = pyodbc.connect(UID=credentials.username, PWD=credentials.password, autocommit=False, **connection_args) connection.state = 'open' connection.handle = handle except Exception as e: logger.error(f"Got an error when attempting to open a pyodbc " "connection '{e}'") connection.handle = None connection.state = 'fail' raise dbt.exceptions.FailedToConnectException(str(e)) return connection
def run(self) -> CatalogResults: compile_results = None if self.args.compile: compile_results = CompileTask.run(self) if any(r.error is not None for r in compile_results): dbt.ui.printer.print_timestamped_line( 'compile failed, cannot generate docs') return CatalogResults(nodes={}, sources={}, generated_at=datetime.utcnow(), errors=None, _compile_results=compile_results) else: self.manifest = get_full_manifest(self.config) shutil.copyfile(DOCS_INDEX_FILE_PATH, os.path.join(self.config.target_path, 'index.html')) if self.manifest is None: raise InternalException('self.manifest was None in run!') adapter = get_adapter(self.config) with adapter.connection_named('generate_catalog'): dbt.ui.printer.print_timestamped_line("Building catalog") catalog_table, exceptions = adapter.get_catalog(self.manifest) catalog_data: List[PrimitiveDict] = [ dict(zip(catalog_table.column_names, map(_coerce_decimal, row))) for row in catalog_table ] catalog = Catalog(catalog_data) errors: Optional[List[str]] = None if exceptions: errors = [str(e) for e in exceptions] nodes, sources = catalog.make_unique_id_map(self.manifest) results = self.get_catalog_results( nodes=nodes, sources=sources, generated_at=datetime.utcnow(), compile_results=compile_results, errors=errors, ) path = os.path.join(self.config.target_path, CATALOG_FILENAME) results.write(path) if self.args.compile: write_manifest(self.config, self.manifest) if exceptions: logger.error( 'dbt encountered {} failure{} while writing the catalog'. format(len(exceptions), (len(exceptions) != 1) * 's')) dbt.ui.printer.print_timestamped_line('Catalog written to {}'.format( os.path.abspath(path))) return results
def main(args=None): if args is None: args = sys.argv[1:] try: results, succeeded = handle_and_check(args) if succeeded: exit_code = ExitCodes.Success else: exit_code = ExitCodes.ModelError except KeyboardInterrupt as e: logger.info("ctrl-c") exit_code = ExitCodes.UnhandledError # This can be thrown by eg. argparse except SystemExit as e: exit_code = e.code except BaseException as e: logger.info("Encountered an error:") logger.info(str(e)) if logger_initialized: logger.debug(traceback.format_exc()) else: logger.error(traceback.format_exc()) exit_code = ExitCodes.UnhandledError sys.exit(exit_code)
def cancel(self, connection): connection_name = connection.name logger.info("Cancelling query '{}' ".format(connection_name)) try: connection.handle.close() except Exception as e: logger.error('Error closing connection for cancel request') raise Exception(str(e))
def _handle_generic_exception(self, e, ctx): node_description = self.node.get('build_path') if node_description is None: node_description = self.node.unique_id prefix = "Unhandled error while executing {}".format(node_description) error = "{prefix}\n{error}".format(prefix=dbt.ui.printer.red(prefix), error=str(e).strip()) logger.error(error) logger.debug('', exc_info=True) return dbt.compat.to_string(e)
def write_file(self): logger.debug(' - Writing snapshot to {}'.format(self.snapshot_path)) contents = self.build_archive_data() wrote = system.make_file(path=self.snapshot_path, contents=contents) if wrote: logger.info(' - Wrote new snapshot file to {}'.format( self.snapshot_path)) else: logger.error( ' - Error: Could not write new snapshot file to {}'.format( self.snapshot_path))
def cancel(cls, connection): connection_name = connection.name oracle_connection = connection.handle logger.info("Cancelling query '{}' ".format(connection_name)) try: Connection.close(oracle_connection) except Exception as e: logger.error('Error closing connection for cancel request') raise Exception(str(e)) logger.info("Canceled query '{}'".format(connection_name))
def from_args(cls, args): try: config = cls.ConfigType.from_args(args) except dbt.exceptions.DbtProjectError as exc: logger.error("Encountered an error while reading the project:") logger.error(" ERROR: {}".format(str(exc))) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException('Could not run dbt') except dbt.exceptions.DbtProfileError as exc: logger.error("Encountered an error while reading profiles:") logger.error(" ERROR {}".format(str(exc))) all_profiles = read_profiles(args.profiles_dir).keys() if len(all_profiles) > 0: logger.info("Defined profiles:") for profile in all_profiles: logger.info(" - {}".format(profile)) else: logger.info("There are no profiles defined in your " "profiles.yml file") logger.info(PROFILES_HELP_MESSAGE) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException('Could not run dbt') return cls(args, config)
def parse_cli_vars(var_string): try: cli_vars = yaml_helper.load_yaml_text(var_string) var_type = type(cli_vars) if var_type == dict: return cli_vars else: type_name = var_type.__name__ dbt.exceptions.raise_compiler_error( "The --vars argument must be a YAML dictionary, but was " "of type '{}'".format(type_name)) except dbt.exceptions.ValidationException as e: logger.error( "The YAML provided in the --vars argument is not valid.\n") raise
def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode: if not _is_writable(node): return node logger.debug(f'Writing injected SQL for node "{node.unique_id}"') if node.injected_sql is None: # this should not really happen, but it'd be a shame to crash # over it logger.error( f'Compiled node "{node.unique_id}" had no injected_sql, ' 'cannot write sql!') else: node.build_path = node.write_node(self.config.target_path, 'compiled', node.injected_sql) return node
def validate_with(schema, data): try: schema(data) except MultipleInvalid as e: logger.debug(schema) logger.debug(data) logger.error(str(e)) raise ValidationException(str(e)) except Invalid as e: logger.debug(schema) logger.debug(data) logger.error(str(e)) raise ValidationException(str(e))
def exception_handler(self, sql: str) -> ContextManager: try: yield except Exception as e: logger.error("Error running SQL: {}".format(sql)) logger.error("Rolling back transaction.") self.rollback_if_open() if isinstance(e, RuntimeException): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise raise RuntimeException(e) from e
def run(self): try: result = self._run_unsafe() except dbt.exceptions.Exception as exc: logger.error( 'Encountered an error while running operation: {}'.format(exc)) logger.debug('', exc_info=True) return False, None except Exception as exc: logger.error( 'Encountered an uncaught exception while running operation: {}' .format(exc)) logger.debug('', exc_info=True) return False, None else: return True, result
def _checkout(self): """Performs a shallow clone of the repository into the downloads directory. This function can be called repeatedly. If the project has already been checked out at this version, it will be a no-op. Returns the path to the checked out directory.""" try: dir_ = git.clone_and_checkout(self.git, get_downloads_path(), branch=self.revision, dirname=self._checkout_name) except ExecutableError as exc: if exc.cmd and exc.cmd[0] == 'git': logger.error( 'Make sure git is installed on your machine. More ' 'information: ' 'https://docs.getdbt.com/docs/package-management') raise return os.path.join(get_downloads_path(), dir_)
def track_run(task): dbt.tracking.track_invocation_start(config=task.config, args=task.args) try: yield dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="ok" ) except (dbt.exceptions.NotImplementedException, dbt.exceptions.FailedToConnectException) as e: logger.error('ERROR: {}'.format(e)) dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="error" ) except Exception: dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="error" ) raise finally: dbt.tracking.flush()
def task_exec(self) -> None: """task_exec runs first inside the child process""" signal.signal(signal.SIGTERM, sigterm_handler) # the first thing we do in a new process: push logging back over our # queue handler = QueueLogHandler(self.queue) with handler.applicationbound(): self._spawn_setup() # copy threads over into our credentials, if it exists and is set. # some commands, like 'debug', won't have a threads value at all. if getattr(self.task.args, 'threads', None) is not None: self.task.config.threads = self.task.args.threads rpc_exception = None result = None try: result = self.task.handle_request() except RPCException as exc: rpc_exception = exc except dbt.exceptions.RPCKilledException as exc: # do NOT log anything here, you risk triggering a deadlock on # the queue handler we inserted above rpc_exception = dbt_error(exc) except dbt.exceptions.Exception as exc: logger.debug('dbt runtime exception', exc_info=True) rpc_exception = dbt_error(exc) except Exception as exc: with OutputHandler(sys.stderr).applicationbound(): logger.error('uncaught python exception', exc_info=True) rpc_exception = server_error(exc) # put whatever result we got onto the queue as well. if rpc_exception is not None: handler.emit_error(rpc_exception.error) elif result is not None: handler.emit_result(result) else: error = dbt_error( dbt.exceptions.InternalException( 'after request handling, neither result nor error is None!' )) handler.emit_error(error.error)
def _checkout(self, project): """Performs a shallow clone of the repository into the downloads directory. This function can be called repeatedly. If the project has already been checked out at this version, it will be a no-op. Returns the path to the checked out directory.""" if len(self.version) != 1: dbt.exceptions.raise_dependency_error( 'Cannot checkout repository until the version is pinned.') try: dir_ = dbt.clients.git.clone_and_checkout( self.git, DOWNLOADS_PATH, branch=self.version[0], dirname=self._checkout_name) except dbt.exceptions.ExecutableError as exc: if exc.cmd and exc.cmd[0] == 'git': logger.error( 'Make sure git is installed on your machine. More ' 'information: ' 'https://docs.getdbt.com/docs/package-management' ) raise return os.path.join(DOWNLOADS_PATH, dir_)
def run(self) -> RunOperationResult: start = datetime.utcnow() self._runtime_initialize() try: self._run_unsafe() except dbt.exceptions.Exception as exc: logger.error( 'Encountered an error while running operation: {}'.format(exc)) logger.debug('', exc_info=True) success = False except Exception as exc: logger.error( 'Encountered an uncaught exception while running operation: {}' .format(exc)) logger.debug('', exc_info=True) success = False else: success = True end = datetime.utcnow() return RunOperationResult(results=[], generated_at=end, elapsed_time=(end - start).total_seconds(), success=success)
def print_run_result_error(result, newline: bool = True, is_warning: bool = False) -> None: if newline: with TextOnly(): logger.info("") if result.status == NodeStatus.Fail or (is_warning and result.status == NodeStatus.Warn): if is_warning: color = ui.yellow info = 'Warning' logger_fn = logger.warning else: color = ui.red info = 'Failure' logger_fn = logger.error logger_fn( color("{} in {} {} ({})").format(info, result.node.resource_type, result.node.name, result.node.original_file_path)) try: # if message is int, must be rows returned for a test int(result.message) except ValueError: logger.error(" Status: {}".format(result.status)) else: num_rows = utils.pluralize(result.message, 'result') logger.error(" Got {}, expected 0.".format(num_rows)) if result.node.build_path is not None: with TextOnly(): logger.info("") logger.info(" compiled SQL at {}".format(result.node.build_path)) elif result.message is not None: first = True for line in result.message.split("\n"): if first: logger.error(ui.yellow(line)) first = False else: logger.error(line)
def print_run_result_error(result, newline: bool = True, is_warning: bool = False) -> None: if newline: with TextOnly(): logger.info("") if result.fail or (is_warning and result.warn): if is_warning: color = yellow info = 'Warning' logger_fn = logger.warning else: color = red info = 'Failure' logger_fn = logger.error logger_fn( color("{} in {} {} ({})").format(info, result.node.resource_type, result.node.name, result.node.original_file_path)) try: int(result.status) except ValueError: logger.error(" Status: {}".format(result.status)) else: status = dbt.utils.pluralize(result.status, 'result') logger.error(" Got {}, expected 0.".format(status)) if result.node.build_path is not None: with TextOnly(): logger.info("") logger.info(" compiled SQL at {}".format(result.node.build_path)) else: first = True for line in result.error.split("\n"): if first: logger.error(yellow(line)) first = False else: logger.error(line)
def exception_handler(self, sql): try: yield except pyodbc.DatabaseError as e: logger.error(f'pyodbc error: {str(e)}') try: self.release() except pyodbc.DatabaseError: logger.error('Failed to release connection!') except Exception as e: logger.error("Error running SQL: {}".format(sql)) logger.error("Rolling back transaction.") self.release() if isinstance(e, dbt.exceptions.RuntimeException): # during a sql query, an internal to dbt exception was raised. # this sounds a lot like a signal handler and probably has # useful information, so raise it without modification. raise raise dbt.exceptions.RuntimeException(str(e))
def safe_run(self, manifest): catchable_errors = (CompilationException, RuntimeException) # result = self.DefaultResult(self.node) started = time.time() timing = [] error = None node = self.node result = None try: with collect_timing_info('compile') as timing_info: # if we fail here, we still have a compiled node to return # this has the benefit of showing a build path for the errant # model node = self.compile(manifest) timing.append(timing_info) # for ephemeral nodes, we only want to compile, not run if not node.is_ephemeral_model: with collect_timing_info('execute') as timing_info: result = self.run(node, manifest) node = result.node timing.append(timing_info) # result.extend(item.serialize() for item in timing) except catchable_errors as e: if e.node is None: e.node = node error = dbt.compat.to_string(e) except InternalException as e: build_path = self.node.build_path prefix = 'Internal error executing {}'.format(build_path) error = "{prefix}\n{error}\n\n{note}".format( prefix=dbt.ui.printer.red(prefix), error=str(e).strip(), note=INTERNAL_ERROR_STRING) logger.debug(error) error = dbt.compat.to_string(e) except Exception as e: node_description = self.node.get('build_path') if node_description is None: node_description = self.node.unique_id prefix = "Unhandled error while executing {description}".format( description=node_description) error = "{prefix}\n{error}".format( prefix=dbt.ui.printer.red(prefix), error=str(e).strip()) logger.error(error) logger.debug('', exc_info=True) error = dbt.compat.to_string(e) finally: exc_str = self._safe_release_connection() # if releasing failed and the result doesn't have an error yet, set # an error if exc_str is not None and result.error is None: error = exc_str if error is not None: # we could include compile time for runtime errors here result = self.error_result(node, error, started, []) elif result is not None: result = self.from_run_result(result, started, timing) else: result = self.ephemeral_result(node, started, timing) return result
def safe_run(self, manifest): catchable_errors = (dbt.exceptions.CompilationException, dbt.exceptions.RuntimeException) result = RunModelResult(self.node) started = time.time() exc_info = (None, None, None) try: # if we fail here, we still have a compiled node to return # this has the benefit of showing a build path for the errant model compiled_node = self.compile(manifest) result.node = compiled_node # for ephemeral nodes, we only want to compile, not run if not self.is_ephemeral_model(self.node): result = self.run(compiled_node, manifest) except catchable_errors as e: if e.node is None: e.node = result.node result.error = dbt.compat.to_string(e) result.status = 'ERROR' except dbt.exceptions.InternalException as e: build_path = self.node.build_path prefix = 'Internal error executing {}'.format(build_path) error = "{prefix}\n{error}\n\n{note}".format( prefix=dbt.ui.printer.red(prefix), error=str(e).strip(), note=INTERNAL_ERROR_STRING) logger.debug(error) result.error = dbt.compat.to_string(e) result.status = 'ERROR' except Exception as e: # set this here instead of finally, as python 2/3 exc_info() # behavior with re-raised exceptions are slightly different exc_info = sys.exc_info() prefix = "Unhandled error while executing {filepath}".format( filepath=self.node.build_path) error = "{prefix}\n{error}".format( prefix=dbt.ui.printer.red(prefix), error=str(e).strip()) logger.error(error) raise e finally: exc_str = self._safe_release_connection() # if we had an unhandled exception, re-raise it if exc_info and exc_info[1]: six.reraise(*exc_info) # if releasing failed and the result doesn't have an error yet, set # an error if exc_str is not None and result.error is None: result.error = exc_str result.status = 'ERROR' result.execution_time = time.time() - started return result