def search(self, included_nodes: Set[UniqueId],
selector: str) -> Iterator[UniqueId]:
try:
resource_type = NodeType(selector)
except ValueError as exc:
raise RuntimeException(
f'Invalid resource_type selector "{selector}"') from exc
for node, real_node in self.parsed_nodes(included_nodes):
if real_node.resource_type == resource_type:
yield node
def _get_warehouse(self) -> str:
_, table = self.execute(
'select current_warehouse() as warehouse',
fetch=True
)
if len(table) == 0 or len(table[0]) == 0:
# can this happen?
raise RuntimeException(
'Could not get current warehouse: no results'
)
return str(table[0][0])
def __init__(self,
args,
config,
tasks: Optional[List[Type[RemoteMethod]]] = None) -> None:
if os.name == 'nt':
raise RuntimeException(
'The dbt RPC server is not supported on windows')
super().__init__(args, config)
self.task_manager = TaskManager(self.args, self.config,
TaskTypes(tasks))
signal.signal(signal.SIGHUP, self._sighup_handler)
def _match_to_int(match: Dict[str, str], key: str) -> Optional[int]:
raw = match.get(key)
# turn the empty string into None, too.
if not raw:
return None
try:
return int(raw)
except ValueError as exc:
raise RuntimeException(
f'Invalid node spec - could not handle parent depth {raw}'
) from exc
def prepend_ctes(self, prepended_ctes: List[InjectedCTE]):
self.extra_ctes_injected = True
self.extra_ctes = prepended_ctes
if self.compiled_sql is None:
raise RuntimeException('Cannot prepend ctes to an unparsed node',
self)
self.injected_sql = _inject_ctes_into_sql(
self.compiled_sql,
prepended_ctes,
)
self.validate(self.to_dict())
def __post_init__(self):
# some core things set database='', which we should ignore.
if self.database and self.database != self.schema:
raise RuntimeException(
f'Error while parsing relation {self.name}: \n'
f' identifier: {self.identifier} \n'
f' schema: {self.schema} \n'
f' database: {self.database} \n'
f'On Spark, database should not be set. Use the schema '
f'config to set a custom schema/database for this relation.'
)
def string_add_sql(
self, add_to: str, value: str, location='append',
) -> str:
if location == 'append':
return f"{add_to} || '{value}'"
elif location == 'prepend':
return f"'{value}' || {add_to}"
else:
raise RuntimeException(
f'Got an unexpected location value of "{location}"'
)
def exception_handler(self, sql: str):
try:
yield
except sqlite3.DatabaseError as e:
self.release()
logger.debug("sqlite3 error: {}".format(str(e)))
raise DatabaseException(str(e))
except Exception as e:
logger.debug("Error running SQL: {}".format(sql))
logger.debug("Rolling back transaction.")
self.release()
raise RuntimeException(str(e))
def list_schemas(self, database: str) -> List[str]:
try:
results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME,
kwargs={'database': database})
except DatabaseException as exc:
msg = (f'Database error while listing schemas in database '
f'"{database}"\n{exc}')
raise RuntimeException(msg)
# this uses 'show terse schemas in database', and the column name we
# want is 'name'
return [row['name'] for row in results]
def __init__(self, args, config):
if not args.from_archive:
raise RuntimeException('The --from-archive parameter is required!')
if args.apply:
args.write_files = True
args.migrate_database = True
super(MigrationTask, self).__init__(args, config)
self.snapshot_root = os.path.normpath(self.config.snapshot_paths[0])
system.make_directory(self.snapshot_root)
self.backups_made = []
self.snapshots_written = []
def create_profiles_file(self, profiles_file, sample_adapter):
# Line below raises an exception if the specified adapter is not found
load_plugin(sample_adapter)
adapter_path = get_include_paths(sample_adapter)[0]
sample_profiles_path = adapter_path / 'sample_profiles.yml'
if not sample_profiles_path.exists():
raise RuntimeException(f'No sample profile for {sample_adapter}')
if not os.path.exists(profiles_file):
shutil.copyfile(sample_profiles_path, profiles_file)
return True
return False
def exception_handler(self, sql: str) -> ContextManager:
try:
yield
except Exception as e:
logger.error("Error running SQL: {}".format(sql))
logger.error("Rolling back transaction.")
self.rollback_if_open()
if isinstance(e, RuntimeException):
# during a sql query, an internal to dbt exception was raised.
# this sounds a lot like a signal handler and probably has
# useful information, so raise it without modification.
raise
raise RuntimeException(e) from e
def run_from_args(parsed):
task = None
cfg = None
if parsed.which in ('init', 'debug'):
# bypass looking for a project file if we're running `dbt init` or
# `dbt debug`
task = parsed.cls(args=parsed)
else:
nearest_project_dir = get_nearest_project_dir()
if nearest_project_dir is None:
raise RuntimeException(
"fatal: Not a dbt project (or any of the parent directories). "
"Missing dbt_project.yml file")
os.chdir(nearest_project_dir)
res = invoke_dbt(parsed)
if res is None:
raise RuntimeException("Could not run dbt")
else:
task, cfg = res
log_path = None
if cfg is not None:
log_path = cfg.log_path
initialize_logger(parsed.debug, log_path)
logger.debug("Tracking: {}".format(dbt.tracking.active_user.state()))
dbt.tracking.track_invocation_start(config=cfg, args=parsed)
results = run_from_task(task, cfg, parsed)
return task, results
def search(self, included_nodes: Set[UniqueId],
selector: str) -> Iterator[UniqueId]:
search_types: Tuple[Type, ...]
if selector == 'schema':
search_types = (ParsedSchemaTestNode, CompiledSchemaTestNode)
elif selector == 'data':
search_types = (ParsedDataTestNode, CompiledDataTestNode)
else:
raise RuntimeException(
f'Invalid test type selector {selector}: expected "data" or '
'"schema"')
for node, real_node in self.parsed_nodes(included_nodes):
if isinstance(real_node, search_types):
yield node
def from_description(cls, name: str, raw_data_type: str) -> 'Column':
match = re.match(r'([^(]+)(\([^)]+\))?', raw_data_type)
if match is None:
raise RuntimeException(
f'Could not interpret data type "{raw_data_type}"')
data_type, size_info = match.groups()
char_size = None
numeric_precision = None
numeric_scale = None
if size_info is not None:
# strip out the parentheses
size_info = size_info[1:-1]
parts = size_info.split(',')
if len(parts) == 1:
try:
char_size = int(parts[0])
except ValueError:
raise RuntimeException(
f'Could not interpret data_type "{raw_data_type}": '
f'could not convert "{parts[0]}" to an integer')
elif len(parts) == 2:
try:
numeric_precision = int(parts[0])
except ValueError:
raise RuntimeException(
f'Could not interpret data_type "{raw_data_type}": '
f'could not convert "{parts[0]}" to an integer')
try:
numeric_scale = int(parts[1])
except ValueError:
raise RuntimeException(
f'Could not interpret data_type "{raw_data_type}": '
f'could not convert "{parts[1]}" to an integer')
return cls(name, data_type, char_size, numeric_precision,
numeric_scale)
def _model_prepend_ctes(
self, model: NonSourceCompiledNode,
prepended_ctes: List[InjectedCTE]) -> NonSourceCompiledNode:
if model.compiled_sql is None:
raise RuntimeException('Cannot prepend ctes to an unparsed node',
model)
injected_sql = self._inject_ctes_into_sql(
model.compiled_sql,
prepended_ctes,
)
model.extra_ctes_injected = True
model.extra_ctes = prepended_ctes
model.injected_sql = injected_sql
model.validate(model.to_dict())
return model
def __call__(self, *args, **kwargs):
if len(args) == 1 and len(kwargs) == 0:
opts = args[0]
elif len(args) == 0 and len(kwargs) > 0:
opts = kwargs
else:
raise_compiler_error("Invalid inline model config", self.model)
opts = self._transform_config(opts)
# it's ok to have a parse context with no context config, but you must
# not call it!
if self.context_config is None:
raise RuntimeException(
'At parse time, did not receive a context config')
self.context_config.update_in_model_config(opts)
return ''
def _iterate_selected_nodes(self):
nodes = sorted(self.select_nodes())
if not nodes:
logger.warning('No nodes selected!')
return
if self.manifest is None:
raise InternalException(
'manifest is None in _iterate_selected_nodes')
for node in nodes:
if node in self.manifest.nodes:
yield self.manifest.nodes[node]
elif node in self.manifest.sources:
yield self.manifest.sources[node]
else:
raise RuntimeException(
f'Got an unexpected result from node selection: "{node}"'
f'Expected a source or a node!')
def add_query(self,
sql,
auto_begin=True,
bindings=None,
abridge_sql_log=False):
connection = None
cursor = None
if bindings:
# The snowflake connector is more strict than, eg., psycopg2 -
# which allows any iterable thing to be passed as a binding.
bindings = tuple(bindings)
queries = self._split_queries(sql)
for individual_query in queries:
# hack -- after the last ';', remove comments and don't run
# empty queries. this avoids using exceptions as flow control,
# and also allows us to return the status of the last cursor
without_comments = re.sub(re.compile('^.*(--.*)$', re.MULTILINE),
'', individual_query).strip()
if without_comments == "":
continue
connection, cursor = super().add_query(
individual_query,
auto_begin,
bindings=bindings,
abridge_sql_log=abridge_sql_log)
if cursor is None:
conn = self.get_thread_connection()
if conn is None or conn.name is None:
conn_name = '<None>'
else:
conn_name = conn.name
raise RuntimeException(
"Tried to run an empty query on model '{}'. If you are "
"conditionally running\nsql, eg. in a model hook, make "
"sure your `else` clause contains valid sql!\n\n"
"Provided SQL:\n{}".format(conn_name, sql))
return connection, cursor
def __setitem__(self, key, value):
if key not in self:
super().__setitem__(key, value)
return
existing_type = self[key]
if isinstance(existing_type, _NullMarker):
# overwrite
super().__setitem__(key, value)
elif isinstance(value, _NullMarker):
# use the existing value
return
elif not isinstance(value, type(existing_type)):
# actual type mismatch!
raise RuntimeException(
f'Tables contain columns with the same names ({key}), '
f'but different types ({value} vs {existing_type})'
)
def __init__(self, node_spec: str):
self.raw = node_spec
self.select_children = False
self.select_children_max_depth = None
self.select_parents = False
self.select_parents_max_depth = None
self.select_childrens_parents = False
if node_spec.startswith(SELECTOR_CHILDREN_AND_ANCESTORS):
self.select_childrens_parents = True
node_spec = node_spec[1:]
matches = re.match(SELECTOR_PARENTS, node_spec)
if matches:
self.select_parents = True
if matches['depth']:
self.select_parents_max_depth = int(matches['depth'])
node_spec = matches['node']
matches = re.match(SELECTOR_CHILDREN, node_spec)
if matches:
self.select_children = True
if matches['depth']:
self.select_children_max_depth = int(matches['depth'])
node_spec = matches['node']
if self.select_children and self.select_childrens_parents:
raise RuntimeException(
'Invalid node spec {} - "@" prefix and "+" suffix are '
'incompatible'.format(self.raw)
)
if SELECTOR_DELIMITER in node_spec:
selector_parts = node_spec.split(SELECTOR_DELIMITER, 1)
selector_type, self.selector_value = selector_parts
self.selector_type = SELECTOR_FILTERS(selector_type)
else:
self.selector_value = node_spec
# if the selector type has an OS path separator in it, it can't
# really be a valid file name, so assume it's a path.
if _probably_path(node_spec):
self.selector_type = SELECTOR_FILTERS.PATH
else:
self.selector_type = SELECTOR_FILTERS.FQN
def from_dict(cls, raw: Any, dct: Dict[str, Any]) -> 'SelectionCriteria':
if 'value' not in dct:
raise RuntimeException(
f'Invalid node spec "{raw}" - no search value!')
method_name, method_arguments = cls.parse_method(dct)
parents_depth = _match_to_int(dct, 'parents_depth')
children_depth = _match_to_int(dct, 'children_depth')
return cls(
raw=raw,
method=method_name,
method_arguments=method_arguments,
value=dct['value'],
childrens_parents=bool(dct.get('childrens_parents')),
parents=bool(dct.get('parents')),
parents_depth=parents_depth,
children=bool(dct.get('children')),
children_depth=children_depth,
)
def search(self, included_nodes: Set[UniqueId],
selector: str) -> Iterator[UniqueId]:
parts = selector.split('.')
target_package = SELECTOR_GLOB
if len(parts) == 1:
target_name = parts[0]
elif len(parts) == 2:
target_package, target_name = parts
else:
msg = (
'Invalid exposure selector value "{}". Exposures must be of '
'the form ${{exposure_name}} or '
'${{exposure_package.exposure_name}}').format(selector)
raise RuntimeException(msg)
for node, real_node in self.exposure_nodes(included_nodes):
if target_package not in (real_node.package_name, SELECTOR_GLOB):
continue
if target_name not in (real_node.name, SELECTOR_GLOB):
continue
yield node
def add_query(self,
sql,
model_name=None,
auto_begin=True,
bindings=None,
abridge_sql_log=False):
connection = None
cursor = None
# TODO: is this sufficient? Largely copy+pasted from snowflake, so
# there's some common behavior here we can maybe factor out into the
# SQLAdapter?
queries = [q.rstrip(';') for q in sqlparse.split(sql)]
for individual_query in queries:
# hack -- after the last ';', remove comments and don't run
# empty queries. this avoids using exceptions as flow control,
# and also allows us to return the status of the last cursor
without_comments = re.sub(re.compile('^.*(--.*)$', re.MULTILINE),
'', individual_query).strip()
if without_comments == "":
continue
parent = super(HiveConnectionManager, self)
connection, cursor = parent.add_query(individual_query, model_name,
auto_begin, bindings,
abridge_sql_log)
if cursor is None:
raise RuntimeException(
"Tried to run an empty query on model '{}'. If you are "
"conditionally running\nsql, eg. in a model hook, make "
"sure your `else` clause contains valid sql!\n\n"
"Provided SQL:\n{}".format(model_name, sql))
return connection, cursor
def compile(self, manifest):
if self.node.resource_type != NodeType.Source:
# should be unreachable...
raise RuntimeException('fresnhess runner: got a non-Source')
# we don't do anything interesting when we compile a source node
return self.node
def __post_init__(self):
if self.children and self.childrens_parents:
raise RuntimeException(
f'Invalid node spec {self.raw} - "@" prefix and "+" suffix '
'are incompatible')
def render(self):
if self.include_policy.database and self.include_policy.schema:
raise RuntimeException(
'Got a spark relation with schema and database set to '
'include, but only one can be set')
return super().render()
def get_selector(self, name: str) -> SelectionSpec:
if name not in self.selectors:
raise RuntimeException(
f'Could not find selector named {name}, expected one of '
f'{list(self.selectors)}')
return self.selectors[name]
def on_skip(self):
raise RuntimeException('Freshness: nodes cannot be skipped!')
def __getattribute__(self, name):
if name in {'profile_name', 'target_name', 'threads'}:
raise RuntimeException(
f'Error: disallowed attribute "{name}" - no profile!')
return Profile.__getattribute__(self, name)
