def open(cls, connection):
if connection.state == 'open':
logger.debug('Connection is already open, skipping open.')
return connection
credentials = cls.get_credentials(connection.credentials)
host = f'{credentials.host}:{credentials.port}/{credentials.database}'
try:
host = cls._build_host(credentials=credentials)
handle = cx_Oracle.connect( credentials.user, credentials.password, host, encoding="UTF-8")
connection.handle = handle
connection.state = 'open'
except cx_Oracle.DatabaseError as e:
logger.debug("Got an error when attempting to open an connection to oracle\n"
"using simple string method(host and port)\n"
"Will try to connect using tnsanmes.ora file\n"
"error: '{}'"
.format(str(e)))
try:
handle = cx_Oracle.connect( credentials.user, credentials.password, credentials.database, encoding="UTF-8")
connection.handle = handle
connection.state = 'open'
except cx_Oracle.DatabaseError as e:
logger.error("Also Got an error when attempting to open an oracle connection\n"
"using tnsnames.ora file, failing dbt job.\n"
"error: '{}'"
.format(str(e)))
connection.handle = None
connection.state = 'fail'
raise dbt.exceptions.FailedToConnectException(str(e))
return connection
def main(args=None):
if args is None:
args = sys.argv[1:]
with log_manager.applicationbound():
try:
results, succeeded = handle_and_check(args)
if succeeded:
exit_code = ExitCodes.Success.value
else:
exit_code = ExitCodes.ModelError.value
except KeyboardInterrupt:
logger.info("ctrl-c")
exit_code = ExitCodes.UnhandledError.value
# This can be thrown by eg. argparse
except SystemExit as e:
exit_code = e.code
except BaseException as e:
logger.warning("Encountered an error:")
logger.warning(str(e))
if log_manager.initialized:
logger.debug(traceback.format_exc())
elif not isinstance(e, RuntimeException):
# if it did not come from dbt proper and the logger is not
# initialized (so there's no safe path to log to), log the
# stack trace at error level.
logger.error(traceback.format_exc())
exit_code = ExitCodes.UnhandledError.value
sys.exit(exit_code)
def open(cls, connection):
if connection.state == 'open':
logger.debug('Connection is already open, skipping open.')
return connection
credentials = connection.credentials
try:
connection_args = {}
if credentials.dsn:
connection_args = {'DSN': credentials.dsn}
else:
connection_args = {
'DRIVER': credentials.driver,
'DBCNAME': credentials.host,
'PORT': credentials.port,
'DATABASE': credentials.database,
'SCHEMA': credentials.schema,
#'SESSIONMODE': 'ANSI'
}
handle = pyodbc.connect(UID=credentials.username,
PWD=credentials.password,
autocommit=False,
**connection_args)
connection.state = 'open'
connection.handle = handle
except Exception as e:
logger.error(f"Got an error when attempting to open a pyodbc "
"connection '{e}'")
connection.handle = None
connection.state = 'fail'
raise dbt.exceptions.FailedToConnectException(str(e))
return connection
def run(self) -> CatalogResults:
compile_results = None
if self.args.compile:
compile_results = CompileTask.run(self)
if any(r.error is not None for r in compile_results):
dbt.ui.printer.print_timestamped_line(
'compile failed, cannot generate docs')
return CatalogResults(nodes={},
sources={},
generated_at=datetime.utcnow(),
errors=None,
_compile_results=compile_results)
else:
self.manifest = get_full_manifest(self.config)
shutil.copyfile(DOCS_INDEX_FILE_PATH,
os.path.join(self.config.target_path, 'index.html'))
if self.manifest is None:
raise InternalException('self.manifest was None in run!')
adapter = get_adapter(self.config)
with adapter.connection_named('generate_catalog'):
dbt.ui.printer.print_timestamped_line("Building catalog")
catalog_table, exceptions = adapter.get_catalog(self.manifest)
catalog_data: List[PrimitiveDict] = [
dict(zip(catalog_table.column_names, map(_coerce_decimal, row)))
for row in catalog_table
]
catalog = Catalog(catalog_data)
errors: Optional[List[str]] = None
if exceptions:
errors = [str(e) for e in exceptions]
nodes, sources = catalog.make_unique_id_map(self.manifest)
results = self.get_catalog_results(
nodes=nodes,
sources=sources,
generated_at=datetime.utcnow(),
compile_results=compile_results,
errors=errors,
)
path = os.path.join(self.config.target_path, CATALOG_FILENAME)
results.write(path)
if self.args.compile:
write_manifest(self.config, self.manifest)
if exceptions:
logger.error(
'dbt encountered {} failure{} while writing the catalog'.
format(len(exceptions), (len(exceptions) != 1) * 's'))
dbt.ui.printer.print_timestamped_line('Catalog written to {}'.format(
os.path.abspath(path)))
return results
def main(args=None):
if args is None:
args = sys.argv[1:]
try:
results, succeeded = handle_and_check(args)
if succeeded:
exit_code = ExitCodes.Success
else:
exit_code = ExitCodes.ModelError
except KeyboardInterrupt as e:
logger.info("ctrl-c")
exit_code = ExitCodes.UnhandledError
# This can be thrown by eg. argparse
except SystemExit as e:
exit_code = e.code
except BaseException as e:
logger.info("Encountered an error:")
logger.info(str(e))
if logger_initialized:
logger.debug(traceback.format_exc())
else:
logger.error(traceback.format_exc())
exit_code = ExitCodes.UnhandledError
sys.exit(exit_code)
def cancel(self, connection):
connection_name = connection.name
logger.info("Cancelling query '{}' ".format(connection_name))
try:
connection.handle.close()
except Exception as e:
logger.error('Error closing connection for cancel request')
raise Exception(str(e))
def _handle_generic_exception(self, e, ctx):
node_description = self.node.get('build_path')
if node_description is None:
node_description = self.node.unique_id
prefix = "Unhandled error while executing {}".format(node_description)
error = "{prefix}\n{error}".format(prefix=dbt.ui.printer.red(prefix),
error=str(e).strip())
logger.error(error)
logger.debug('', exc_info=True)
return dbt.compat.to_string(e)
def write_file(self):
logger.debug(' - Writing snapshot to {}'.format(self.snapshot_path))
contents = self.build_archive_data()
wrote = system.make_file(path=self.snapshot_path, contents=contents)
if wrote:
logger.info(' - Wrote new snapshot file to {}'.format(
self.snapshot_path))
else:
logger.error(
' - Error: Could not write new snapshot file to {}'.format(
self.snapshot_path))
def cancel(cls, connection):
connection_name = connection.name
oracle_connection = connection.handle
logger.info("Cancelling query '{}' ".format(connection_name))
try:
Connection.close(oracle_connection)
except Exception as e:
logger.error('Error closing connection for cancel request')
raise Exception(str(e))
logger.info("Canceled query '{}'".format(connection_name))
def from_args(cls, args):
try:
config = cls.ConfigType.from_args(args)
except dbt.exceptions.DbtProjectError as exc:
logger.error("Encountered an error while reading the project:")
logger.error(" ERROR: {}".format(str(exc)))
tracking.track_invalid_invocation(args=args,
result_type=exc.result_type)
raise dbt.exceptions.RuntimeException('Could not run dbt')
except dbt.exceptions.DbtProfileError as exc:
logger.error("Encountered an error while reading profiles:")
logger.error(" ERROR {}".format(str(exc)))
all_profiles = read_profiles(args.profiles_dir).keys()
if len(all_profiles) > 0:
logger.info("Defined profiles:")
for profile in all_profiles:
logger.info(" - {}".format(profile))
else:
logger.info("There are no profiles defined in your "
"profiles.yml file")
logger.info(PROFILES_HELP_MESSAGE)
tracking.track_invalid_invocation(args=args,
result_type=exc.result_type)
raise dbt.exceptions.RuntimeException('Could not run dbt')
return cls(args, config)
def parse_cli_vars(var_string):
try:
cli_vars = yaml_helper.load_yaml_text(var_string)
var_type = type(cli_vars)
if var_type == dict:
return cli_vars
else:
type_name = var_type.__name__
dbt.exceptions.raise_compiler_error(
"The --vars argument must be a YAML dictionary, but was "
"of type '{}'".format(type_name))
except dbt.exceptions.ValidationException as e:
logger.error(
"The YAML provided in the --vars argument is not valid.\n")
raise
def _write_node(self, node: NonSourceCompiledNode) -> ManifestNode:
if not _is_writable(node):
return node
logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
if node.injected_sql is None:
# this should not really happen, but it'd be a shame to crash
# over it
logger.error(
f'Compiled node "{node.unique_id}" had no injected_sql, '
'cannot write sql!')
else:
node.build_path = node.write_node(self.config.target_path,
'compiled', node.injected_sql)
return node
def parse_cli_vars(var_string):
try:
cli_vars = yaml_helper.load_yaml_text(var_string)
var_type = type(cli_vars)
if var_type == dict:
return cli_vars
else:
type_name = var_type.__name__
dbt.exceptions.raise_compiler_error(
"The --vars argument must be a YAML dictionary, but was "
"of type '{}'".format(type_name))
except dbt.exceptions.ValidationException as e:
logger.error(
"The YAML provided in the --vars argument is not valid.\n")
raise
def validate_with(schema, data):
try:
schema(data)
except MultipleInvalid as e:
logger.debug(schema)
logger.debug(data)
logger.error(str(e))
raise ValidationException(str(e))
except Invalid as e:
logger.debug(schema)
logger.debug(data)
logger.error(str(e))
raise ValidationException(str(e))
def exception_handler(self, sql: str) -> ContextManager:
try:
yield
except Exception as e:
logger.error("Error running SQL: {}".format(sql))
logger.error("Rolling back transaction.")
self.rollback_if_open()
if isinstance(e, RuntimeException):
# during a sql query, an internal to dbt exception was raised.
# this sounds a lot like a signal handler and probably has
# useful information, so raise it without modification.
raise
raise RuntimeException(e) from e
def run(self):
try:
result = self._run_unsafe()
except dbt.exceptions.Exception as exc:
logger.error(
'Encountered an error while running operation: {}'.format(exc))
logger.debug('', exc_info=True)
return False, None
except Exception as exc:
logger.error(
'Encountered an uncaught exception while running operation: {}'
.format(exc))
logger.debug('', exc_info=True)
return False, None
else:
return True, result
def _checkout(self):
"""Performs a shallow clone of the repository into the downloads
directory. This function can be called repeatedly. If the project has
already been checked out at this version, it will be a no-op. Returns
the path to the checked out directory."""
try:
dir_ = git.clone_and_checkout(self.git,
get_downloads_path(),
branch=self.revision,
dirname=self._checkout_name)
except ExecutableError as exc:
if exc.cmd and exc.cmd[0] == 'git':
logger.error(
'Make sure git is installed on your machine. More '
'information: '
'https://docs.getdbt.com/docs/package-management')
raise
return os.path.join(get_downloads_path(), dir_)
def track_run(task):
dbt.tracking.track_invocation_start(config=task.config, args=task.args)
try:
yield
dbt.tracking.track_invocation_end(
config=task.config, args=task.args, result_type="ok"
)
except (dbt.exceptions.NotImplementedException,
dbt.exceptions.FailedToConnectException) as e:
logger.error('ERROR: {}'.format(e))
dbt.tracking.track_invocation_end(
config=task.config, args=task.args, result_type="error"
)
except Exception:
dbt.tracking.track_invocation_end(
config=task.config, args=task.args, result_type="error"
)
raise
finally:
dbt.tracking.flush()
def task_exec(self) -> None:
"""task_exec runs first inside the child process"""
signal.signal(signal.SIGTERM, sigterm_handler)
# the first thing we do in a new process: push logging back over our
# queue
handler = QueueLogHandler(self.queue)
with handler.applicationbound():
self._spawn_setup()
# copy threads over into our credentials, if it exists and is set.
# some commands, like 'debug', won't have a threads value at all.
if getattr(self.task.args, 'threads', None) is not None:
self.task.config.threads = self.task.args.threads
rpc_exception = None
result = None
try:
result = self.task.handle_request()
except RPCException as exc:
rpc_exception = exc
except dbt.exceptions.RPCKilledException as exc:
# do NOT log anything here, you risk triggering a deadlock on
# the queue handler we inserted above
rpc_exception = dbt_error(exc)
except dbt.exceptions.Exception as exc:
logger.debug('dbt runtime exception', exc_info=True)
rpc_exception = dbt_error(exc)
except Exception as exc:
with OutputHandler(sys.stderr).applicationbound():
logger.error('uncaught python exception', exc_info=True)
rpc_exception = server_error(exc)
# put whatever result we got onto the queue as well.
if rpc_exception is not None:
handler.emit_error(rpc_exception.error)
elif result is not None:
handler.emit_result(result)
else:
error = dbt_error(
dbt.exceptions.InternalException(
'after request handling, neither result nor error is None!'
))
handler.emit_error(error.error)
def _checkout(self, project):
"""Performs a shallow clone of the repository into the downloads
directory. This function can be called repeatedly. If the project has
already been checked out at this version, it will be a no-op. Returns
the path to the checked out directory."""
if len(self.version) != 1:
dbt.exceptions.raise_dependency_error(
'Cannot checkout repository until the version is pinned.')
try:
dir_ = dbt.clients.git.clone_and_checkout(
self.git, DOWNLOADS_PATH, branch=self.version[0],
dirname=self._checkout_name)
except dbt.exceptions.ExecutableError as exc:
if exc.cmd and exc.cmd[0] == 'git':
logger.error(
'Make sure git is installed on your machine. More '
'information: '
'https://docs.getdbt.com/docs/package-management'
)
raise
return os.path.join(DOWNLOADS_PATH, dir_)
def run(self) -> RunOperationResult:
start = datetime.utcnow()
self._runtime_initialize()
try:
self._run_unsafe()
except dbt.exceptions.Exception as exc:
logger.error(
'Encountered an error while running operation: {}'.format(exc))
logger.debug('', exc_info=True)
success = False
except Exception as exc:
logger.error(
'Encountered an uncaught exception while running operation: {}'
.format(exc))
logger.debug('', exc_info=True)
success = False
else:
success = True
end = datetime.utcnow()
return RunOperationResult(results=[],
generated_at=end,
elapsed_time=(end - start).total_seconds(),
success=success)
def print_run_result_error(result,
newline: bool = True,
is_warning: bool = False) -> None:
if newline:
with TextOnly():
logger.info("")
if result.status == NodeStatus.Fail or (is_warning and result.status
== NodeStatus.Warn):
if is_warning:
color = ui.yellow
info = 'Warning'
logger_fn = logger.warning
else:
color = ui.red
info = 'Failure'
logger_fn = logger.error
logger_fn(
color("{} in {} {} ({})").format(info, result.node.resource_type,
result.node.name,
result.node.original_file_path))
try:
# if message is int, must be rows returned for a test
int(result.message)
except ValueError:
logger.error(" Status: {}".format(result.status))
else:
num_rows = utils.pluralize(result.message, 'result')
logger.error(" Got {}, expected 0.".format(num_rows))
if result.node.build_path is not None:
with TextOnly():
logger.info("")
logger.info(" compiled SQL at {}".format(result.node.build_path))
elif result.message is not None:
first = True
for line in result.message.split("\n"):
if first:
logger.error(ui.yellow(line))
first = False
else:
logger.error(line)
def print_run_result_error(result,
newline: bool = True,
is_warning: bool = False) -> None:
if newline:
with TextOnly():
logger.info("")
if result.fail or (is_warning and result.warn):
if is_warning:
color = yellow
info = 'Warning'
logger_fn = logger.warning
else:
color = red
info = 'Failure'
logger_fn = logger.error
logger_fn(
color("{} in {} {} ({})").format(info, result.node.resource_type,
result.node.name,
result.node.original_file_path))
try:
int(result.status)
except ValueError:
logger.error(" Status: {}".format(result.status))
else:
status = dbt.utils.pluralize(result.status, 'result')
logger.error(" Got {}, expected 0.".format(status))
if result.node.build_path is not None:
with TextOnly():
logger.info("")
logger.info(" compiled SQL at {}".format(result.node.build_path))
else:
first = True
for line in result.error.split("\n"):
if first:
logger.error(yellow(line))
first = False
else:
logger.error(line)
def exception_handler(self, sql):
try:
yield
except pyodbc.DatabaseError as e:
logger.error(f'pyodbc error: {str(e)}')
try:
self.release()
except pyodbc.DatabaseError:
logger.error('Failed to release connection!')
except Exception as e:
logger.error("Error running SQL: {}".format(sql))
logger.error("Rolling back transaction.")
self.release()
if isinstance(e, dbt.exceptions.RuntimeException):
# during a sql query, an internal to dbt exception was raised.
# this sounds a lot like a signal handler and probably has
# useful information, so raise it without modification.
raise
raise dbt.exceptions.RuntimeException(str(e))
def safe_run(self, manifest):
catchable_errors = (CompilationException, RuntimeException)
# result = self.DefaultResult(self.node)
started = time.time()
timing = []
error = None
node = self.node
result = None
try:
with collect_timing_info('compile') as timing_info:
# if we fail here, we still have a compiled node to return
# this has the benefit of showing a build path for the errant
# model
node = self.compile(manifest)
timing.append(timing_info)
# for ephemeral nodes, we only want to compile, not run
if not node.is_ephemeral_model:
with collect_timing_info('execute') as timing_info:
result = self.run(node, manifest)
node = result.node
timing.append(timing_info)
# result.extend(item.serialize() for item in timing)
except catchable_errors as e:
if e.node is None:
e.node = node
error = dbt.compat.to_string(e)
except InternalException as e:
build_path = self.node.build_path
prefix = 'Internal error executing {}'.format(build_path)
error = "{prefix}\n{error}\n\n{note}".format(
prefix=dbt.ui.printer.red(prefix),
error=str(e).strip(),
note=INTERNAL_ERROR_STRING)
logger.debug(error)
error = dbt.compat.to_string(e)
except Exception as e:
node_description = self.node.get('build_path')
if node_description is None:
node_description = self.node.unique_id
prefix = "Unhandled error while executing {description}".format(
description=node_description)
error = "{prefix}\n{error}".format(
prefix=dbt.ui.printer.red(prefix), error=str(e).strip())
logger.error(error)
logger.debug('', exc_info=True)
error = dbt.compat.to_string(e)
finally:
exc_str = self._safe_release_connection()
# if releasing failed and the result doesn't have an error yet, set
# an error
if exc_str is not None and result.error is None:
error = exc_str
if error is not None:
# we could include compile time for runtime errors here
result = self.error_result(node, error, started, [])
elif result is not None:
result = self.from_run_result(result, started, timing)
else:
result = self.ephemeral_result(node, started, timing)
return result
def safe_run(self, manifest):
catchable_errors = (dbt.exceptions.CompilationException,
dbt.exceptions.RuntimeException)
result = RunModelResult(self.node)
started = time.time()
exc_info = (None, None, None)
try:
# if we fail here, we still have a compiled node to return
# this has the benefit of showing a build path for the errant model
compiled_node = self.compile(manifest)
result.node = compiled_node
# for ephemeral nodes, we only want to compile, not run
if not self.is_ephemeral_model(self.node):
result = self.run(compiled_node, manifest)
except catchable_errors as e:
if e.node is None:
e.node = result.node
result.error = dbt.compat.to_string(e)
result.status = 'ERROR'
except dbt.exceptions.InternalException as e:
build_path = self.node.build_path
prefix = 'Internal error executing {}'.format(build_path)
error = "{prefix}\n{error}\n\n{note}".format(
prefix=dbt.ui.printer.red(prefix),
error=str(e).strip(),
note=INTERNAL_ERROR_STRING)
logger.debug(error)
result.error = dbt.compat.to_string(e)
result.status = 'ERROR'
except Exception as e:
# set this here instead of finally, as python 2/3 exc_info()
# behavior with re-raised exceptions are slightly different
exc_info = sys.exc_info()
prefix = "Unhandled error while executing {filepath}".format(
filepath=self.node.build_path)
error = "{prefix}\n{error}".format(
prefix=dbt.ui.printer.red(prefix),
error=str(e).strip())
logger.error(error)
raise e
finally:
exc_str = self._safe_release_connection()
# if we had an unhandled exception, re-raise it
if exc_info and exc_info[1]:
six.reraise(*exc_info)
# if releasing failed and the result doesn't have an error yet, set
# an error
if exc_str is not None and result.error is None:
result.error = exc_str
result.status = 'ERROR'
result.execution_time = time.time() - started
return result
def safe_run(self, manifest):
catchable_errors = (CompilationException, RuntimeException)
# result = self.DefaultResult(self.node)
started = time.time()
timing = []
error = None
node = self.node
result = None
try:
with collect_timing_info('compile') as timing_info:
# if we fail here, we still have a compiled node to return
# this has the benefit of showing a build path for the errant
# model
node = self.compile(manifest)
timing.append(timing_info)
# for ephemeral nodes, we only want to compile, not run
if not node.is_ephemeral_model:
with collect_timing_info('execute') as timing_info:
result = self.run(node, manifest)
node = result.node
timing.append(timing_info)
# result.extend(item.serialize() for item in timing)
except catchable_errors as e:
if e.node is None:
e.node = node
error = dbt.compat.to_string(e)
except InternalException as e:
build_path = self.node.build_path
prefix = 'Internal error executing {}'.format(build_path)
error = "{prefix}\n{error}\n\n{note}".format(
prefix=dbt.ui.printer.red(prefix),
error=str(e).strip(),
note=INTERNAL_ERROR_STRING)
logger.debug(error)
error = dbt.compat.to_string(e)
except Exception as e:
node_description = self.node.get('build_path')
if node_description is None:
node_description = self.node.unique_id
prefix = "Unhandled error while executing {description}".format(
description=node_description)
error = "{prefix}\n{error}".format(
prefix=dbt.ui.printer.red(prefix),
error=str(e).strip())
logger.error(error)
logger.debug('', exc_info=True)
error = dbt.compat.to_string(e)
finally:
exc_str = self._safe_release_connection()
# if releasing failed and the result doesn't have an error yet, set
# an error
if exc_str is not None and result.error is None:
error = exc_str
if error is not None:
# we could include compile time for runtime errors here
result = self.error_result(node, error, started, [])
elif result is not None:
result = self.from_run_result(result, started, timing)
else:
result = self.ephemeral_result(node, started, timing)
return result
