def validate_args(self, source_name: str, table_name: str):
if not isinstance(source_name, str):
raise CompilationException(
f'The source name (first) argument to source() must be a '
f'string, got {type(source_name)}')
if not isinstance(table_name, str):
raise CompilationException(
f'The table name (second) argument to source() must be a '
f'string, got {type(table_name)}')
def validate_args(self, name: str, package: Optional[str]):
if not isinstance(name, str):
raise CompilationException(
f'The name argument to ref() must be a string, got '
f'{type(name)}')
if package is not None and not isinstance(package, str):
raise CompilationException(
f'The package argument to ref() must be a string or None, got '
f'{type(package)}')
def catch_jinja(node=None) -> Iterator[None]:
try:
yield
except jinja2.exceptions.TemplateSyntaxError as e:
e.translated = False
raise CompilationException(str(e), node) from e
except jinja2.exceptions.UndefinedError as e:
raise CompilationException(str(e), node) from e
except CompilationException as exc:
exc.add_node(node)
raise
def __post_init__(self):
# handle pesky jinja2.Undefined sneaking in here and messing up rende
if not isinstance(self.database, (type(None), str)):
raise CompilationException(
'Got an invalid path database: {}'.format(self.database))
if not isinstance(self.schema, (type(None), str)):
raise CompilationException('Got an invalid path schema: {}'.format(
self.schema))
if not isinstance(self.identifier, (type(None), str)):
raise CompilationException(
'Got an invalid path identifier: {}'.format(self.identifier))
def dispatch(
self, macro_name: str, packages: Optional[List[str]] = None
) -> MacroGenerator:
search_packages: List[Optional[str]]
if '.' in macro_name:
suggest_package, suggest_macro_name = macro_name.split('.', 1)
msg = (
f'In adapter.dispatch, got a macro name of "{macro_name}", '
f'but "." is not a valid macro name component. Did you mean '
f'`adapter.dispatch("{suggest_macro_name}", '
f'packages=["{suggest_package}"])`?'
)
raise CompilationException(msg)
if packages is None:
search_packages = [None]
elif isinstance(packages, str):
raise CompilationException(
f'In adapter.dispatch, got a string packages argument '
f'("{packages}"), but packages should be None or a list.'
)
else:
search_packages = packages
attempts = []
for package_name in search_packages:
for prefix in self._get_adapter_macro_prefixes():
search_name = f'{prefix}__{macro_name}'
try:
# this uses the namespace from the context
macro = self._namespace.get_from_package(
package_name, search_name
)
except CompilationException as exc:
raise CompilationException(
f'In dispatch: {exc.msg}',
) from exc
if package_name is None:
attempts.append(search_name)
else:
attempts.append(f'{package_name}.{search_name}')
if macro is not None:
return macro
searched = ', '.join(repr(a) for a in attempts)
msg = (
f"In dispatch: No macro named '{macro_name}' found\n"
f" Searched for: {searched}"
)
raise CompilationException(msg)
def get_key_dicts(self) -> Iterable[Dict[str, Any]]:
data = self.yaml.data.get(self.key, [])
if not isinstance(data, list):
raise CompilationException(
'{} must be a list, got {} instead: ({})'.format(
self.key, type(data), _trimmed(str(data))))
path = self.yaml.path.original_file_path
for entry in data:
if coerce_dict_str(entry) is not None:
yield entry
else:
msg = error_context(path, self.key, data,
'expected a dict with string keys')
raise CompilationException(msg)
def _expect_value(key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile,
name: str) -> V_T:
if key not in src:
raise CompilationException(
'Expected to find "{}" in cached "result.{}" based '
'on cached file information: {}!'.format(key, name, old_file))
return src[key]
def _process_node(
self,
node_id: str,
source_file: SourceFile,
old_file: SourceFile,
old_manifest: Any,
) -> None:
"""Nodes are a special kind of complicated - there can be multiple
with the same name, as long as all but one are disabled.
Only handle nodes where the matching node has the same resource type
as the current parser.
"""
source_path = source_file.path.original_file_path
found: bool = False
if node_id in old_manifest.nodes:
old_node = old_manifest.nodes[node_id]
if old_node.original_file_path == source_path:
self.add_node(source_file, old_node)
found = True
if node_id in old_manifest._disabled:
matches = old_manifest._get_disabled(node_id, source_file)
for match in matches:
self.add_disabled(source_file, match)
found = True
if not found:
raise CompilationException(
'Expected to find "{}" in cached "manifest.nodes" or '
'"manifest.disabled" based on cached file information: {}!'.
format(node_id, old_file))
def parse_file(self, block: FileBlock) -> None:
dct = self._yaml_from_file(block.file)
# mark the file as seen, even if there are no macros in it
self.results.get_file(block.file)
if dct:
try:
dct = self.raw_renderer.render_data(dct)
except CompilationException as exc:
raise CompilationException(
f'Failed to render {block.path.original_file_path} from '
f'project {self.project.project_name}: {exc}') from exc
yaml_block = YamlBlock.from_file_block(block, dct)
self._parse_format_version(yaml_block)
parser: YamlDocsReader
for key in NodeType.documentable():
plural = key.pluralize()
if key == NodeType.Source:
parser = SourceParser(self, yaml_block, plural)
elif key == NodeType.Macro:
parser = MacroPatchParser(self, yaml_block, plural)
elif key == NodeType.Analysis:
parser = AnalysisPatchParser(self, yaml_block, plural)
else:
parser = TestablePatchParser(self, yaml_block, plural)
for test_block in parser.parse():
self.parse_tests(test_block)
def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T:
path = self.yaml.path.original_file_path
try:
return cls.from_dict(data)
except (ValidationError, JSONValidationException) as exc:
msg = error_context(path, self.key, data, exc)
raise CompilationException(msg) from exc
def transform(self, node: IntermediateSnapshotNode) -> ParsedSnapshotNode:
try:
parsed_node = ParsedSnapshotNode.from_dict(node.to_dict())
self.set_snapshot_attributes(parsed_node)
return parsed_node
except ValidationError as exc:
raise CompilationException(validator_error_message(exc), node)
def __delitem__(self, key):
if hasattr(self, key):
msg = ('Error, tried to delete config key "{}": Cannot delete '
'built-in keys').format(key)
raise CompilationException(msg)
else:
del self._extra[key]
def adapter_macro(self, name: str, *args, **kwargs):
"""Find the most appropriate macro for the name, considering the
adapter type currently in use, and call that with the given arguments.
If the name has a `.` in it, the first section before the `.` is
interpreted as a package name, and the remainder as a macro name.
If no adapter is found, raise a compiler exception. If an invalid
package name is specified, raise a compiler exception.
Some examples:
{# dbt will call this macro by name, providing any arguments #}
{% macro create_table_as(temporary, relation, sql) -%}
{# dbt will dispatch the macro call to the relevant macro #}
{{ adapter_macro('create_table_as', temporary, relation, sql) }}
{%- endmacro %}
{#
If no macro matches the specified adapter, "default" will be
used
#}
{% macro default__create_table_as(temporary, relation, sql) -%}
...
{%- endmacro %}
{# Example which defines special logic for Redshift #}
{% macro redshift__create_table_as(temporary, relation, sql) -%}
...
{%- endmacro %}
{# Example which defines special logic for BigQuery #}
{% macro bigquery__create_table_as(temporary, relation, sql) -%}
...
{%- endmacro %}
"""
deprecations.warn('adapter-macro', macro_name=name)
original_name = name
package_names: Optional[List[str]] = None
if '.' in name:
package_name, name = name.split('.', 1)
package_names = [package_name]
try:
macro = self.db_wrapper.dispatch(macro_name=name,
packages=package_names)
except CompilationException as exc:
raise CompilationException(
f'In adapter_macro: {exc.msg}\n'
f" Original name: '{original_name}'",
node=self.model) from exc
return macro(*args, **kwargs)
def project_name_from_path(include_path: str) -> str:
# avoid an import cycle
from dbt.config.project import Project
partial = Project.partial_load(include_path)
if partial.project_name is None:
raise CompilationException(
f'Invalid project at {include_path}: name not set!')
return partial.project_name
def _get_dicts_for(self, yaml: YamlBlock,
key: str) -> Iterable[Dict[str, Any]]:
data = yaml.data.get(key, [])
if not isinstance(data, list):
raise CompilationException(
'{} must be a list, got {} instead: ({})'.format(
key, type(data), _trimmed(str(data))))
path = yaml.path.original_file_path
for entry in data:
str_keys = (isinstance(entry, dict)
and all(isinstance(k, str) for k in entry))
if str_keys:
yield entry
else:
msg = error_context(path, key, data,
'expected a dict with string keys')
raise CompilationException(msg)
def parse(self) -> Iterable[ParsedExposure]:
for data in self.get_key_dicts():
try:
unparsed = UnparsedExposure.from_dict(data)
except (ValidationError, JSONValidationException) as exc:
msg = error_context(self.yaml.path, self.key, data, exc)
raise CompilationException(msg) from exc
parsed = self.parse_exposure(unparsed)
yield parsed
def render_update(self, node: IntermediateNode,
config: ContextConfig) -> None:
try:
self.render_with_context(node, config)
self.update_parsed_node(node, config)
except ValidationError as exc:
# we got a ValidationError - probably bad types in config()
msg = validator_error_message(exc)
raise CompilationException(msg, node=node) from exc
def read_yaml_models(self, yaml: YamlBlock) -> Iterable[ModelTarget]:
path = yaml.path.original_file_path
yaml_key = 'models'
for data in self._get_dicts_for(yaml, yaml_key):
try:
model = UnparsedNodeUpdate.from_dict(data)
except (ValidationError, JSONValidationException) as exc:
msg = error_context(path, yaml_key, data, exc)
raise CompilationException(msg) from exc
else:
yield model
def _yaml_from_file(self,
source_file: SourceFile) -> Optional[Dict[str, Any]]:
"""If loading the yaml fails, raise an exception.
"""
path: str = source_file.path.relative_path
try:
return load_yaml_text(source_file.contents)
except ValidationException as e:
reason = validator_error_message(e)
raise CompilationException('Error reading {}: {} - {}'.format(
self.project.project_name, path, reason))
return None
def render_value(self,
value: Any,
keypath: Optional[Keypath] = None) -> Any:
# keypath is ignored.
# if it wasn't read as a string, ignore it
if not isinstance(value, str):
return value
try:
with catch_jinja():
return get_rendered(value, self.context, native=True)
except CompilationException as exc:
msg = f'Could not render {value}: {exc.msg}'
raise CompilationException(msg) from exc
def _materialization_relations(self, result: Any,
model) -> List[BaseRelation]:
if isinstance(result, str):
deprecations.warn('materialization-return',
materialization=model.get_materialization())
return [self.adapter.Relation.create_from(self.config, model)]
if isinstance(result, dict):
return _validate_materialization_relations_dict(result, model)
msg = ('Invalid return value from materialization, expected a dict '
'with key "relations", got: {}'.format(str(result)))
raise CompilationException(msg, node=model)
def read_yaml_sources(self, yaml: YamlBlock) -> Iterable[SourceTarget]:
path = yaml.path.original_file_path
yaml_key = 'sources'
for data in self._get_dicts_for(yaml, yaml_key):
try:
data = self._renderer.render_schema_source(data)
source = UnparsedSourceDefinition.from_dict(data)
except (ValidationError, JSONValidationException) as exc:
msg = error_context(path, yaml_key, data, exc)
raise CompilationException(msg) from exc
else:
for table in source.tables:
yield SourceTarget(source, table)
def sanitized_update(
self,
source_file: SourceFile,
old_result: 'ParseResult',
) -> bool:
"""Perform a santized update. If the file can't be updated, invalidate
it and return false.
"""
if isinstance(source_file.path, RemoteFile):
return False
old_file = old_result.get_file(source_file)
for doc_id in old_file.docs:
doc = _expect_value(doc_id, old_result.docs, old_file, "docs")
self.add_doc(source_file, doc)
for macro_id in old_file.macros:
macro = _expect_value(macro_id, old_result.macros, old_file,
"macros")
self.add_macro(source_file, macro)
for source_id in old_file.sources:
source = _expect_value(source_id, old_result.sources, old_file,
"sources")
self.add_source(source_file, source)
# because we know this is how we _parsed_ the node, we can safely
# assume if it's disabled it was done by the project or file, and
# we can keep our old data
for node_id in old_file.nodes:
if node_id in old_result.nodes:
node = old_result.nodes[node_id]
self.add_node(source_file, node)
elif node_id in old_result.disabled:
matches = old_result._get_disabled(node_id, source_file)
for match in matches:
self.add_disabled(source_file, match)
else:
raise CompilationException(
'Expected to find "{}" in cached "manifest.nodes" or '
'"manifest.disabled" based on cached file information: {}!'
.format(node_id, old_file))
for name in old_file.patches:
patch = _expect_value(name, old_result.patches, old_file,
"patches")
self.add_patch(source_file, patch)
return True
def _validate_materialization_relations_dict(inp: Dict[Any, Any],
model) -> List[BaseRelation]:
try:
relations_value = inp['relations']
except KeyError:
msg = ('Invalid return value from materialization, "relations" '
'not found, got keys: {}'.format(list(inp)))
raise CompilationException(msg, node=model) from None
if not isinstance(relations_value, list):
msg = ('Invalid return value from materialization, "relations" '
'not a list, got: {}'.format(relations_value))
raise CompilationException(msg, node=model) from None
relations: List[BaseRelation] = []
for relation in relations_value:
if not isinstance(relation, BaseRelation):
msg = ('Invalid return value from materialization, '
'"relations" contains non-Relation: {}'.format(relation))
raise CompilationException(msg, node=model)
assert isinstance(relation, BaseRelation)
relations.append(relation)
return relations
def get_unparsed_target(self) -> Iterable[NonSourceTarget]:
path = self.yaml.path.original_file_path
for data in self.get_key_dicts():
data.update({
'original_file_path': path,
'yaml_key': self.key,
'package_name': self.project.project_name,
})
try:
model = self._target_type().from_dict(data)
except (ValidationError, JSONValidationException) as exc:
msg = error_context(path, self.key, data, exc)
raise CompilationException(msg) from exc
else:
yield model
def create_test_node(
self,
target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate],
path: str,
config: ContextConfig,
tags: List[str],
fqn: List[str],
name: str,
raw_sql: str,
test_metadata: Dict[str, Any],
column_name: Optional[str],
) -> ParsedSchemaTestNode:
dct = {
'alias': name,
'schema': self.default_schema,
'database': self.default_database,
'fqn': fqn,
'name': name,
'root_path': self.project.project_root,
'resource_type': self.resource_type,
'tags': tags,
'path': path,
'original_file_path': target.original_file_path,
'package_name': self.project.project_name,
'raw_sql': raw_sql,
'unique_id': self.generate_unique_id(name),
'config': self.config_dict(config),
'test_metadata': test_metadata,
'column_name': column_name,
'checksum': FileHash.empty().to_dict(omit_none=True),
}
try:
ParsedSchemaTestNode.validate(dct)
return ParsedSchemaTestNode.from_dict(dct)
except ValidationError as exc:
msg = validator_error_message(exc)
# this is a bit silly, but build an UnparsedNode just for error
# message reasons
node = self._create_error_node(
name=target.name,
path=path,
original_file_path=target.original_file_path,
raw_sql=raw_sql,
)
raise CompilationException(msg, node=node) from exc
def parse_test(self, target_block: TargetBlock, test: TestDef,
column_name: Optional[str]) -> None:
if isinstance(test, str):
test = {test: {}}
block = SchemaTestBlock.from_target_block(src=target_block,
test=test,
column_name=column_name)
try:
self.parse_node(block)
except CompilationException as exc:
context = _trimmed(str(block.target))
msg = ('Invalid test config given in {}:'
'\n\t{}\n\t@: {}'.format(block.path.original_file_path,
exc.msg, context))
raise CompilationException(msg) from exc
def _create_parsetime_node(
self,
block: ConfiguredBlockType,
path: str,
config: ContextConfig,
fqn: List[str],
name=None,
**kwargs,
) -> IntermediateNode:
"""Create the node that will be passed in to the parser context for
"rendering". Some information may be partial, as it'll be updated by
config() and any ref()/source() calls discovered during rendering.
"""
if name is None:
name = block.name
dct = {
'alias': name,
'schema': self.default_schema,
'database': self.default_database,
'fqn': fqn,
'name': name,
'root_path': self.project.project_root,
'resource_type': self.resource_type,
'path': path,
'original_file_path': block.path.original_file_path,
'package_name': self.project.project_name,
'raw_sql': block.contents,
'unique_id': self.generate_unique_id(name),
'config': self.config_dict(config),
'checksum': block.file.checksum.to_dict(omit_none=True),
}
dct.update(kwargs)
try:
return self.parse_from_dict(dct, validate=True)
except ValidationError as exc:
msg = validator_error_message(exc)
# this is a bit silly, but build an UnparsedNode just for error
# message reasons
node = self._create_error_node(
name=block.name,
path=path,
original_file_path=block.path.original_file_path,
raw_sql=block.contents,
)
raise CompilationException(msg, node=node)
def render_test_update(self, node, config, builder):
macro_unique_id = self.macro_resolver.get_macro_id(
node.package_name, 'test_' + builder.name)
# Add the depends_on here so we can limit the macros added
# to the context in rendering processing
node.depends_on.add_macro(macro_unique_id)
if (macro_unique_id
in ['macro.dbt.test_not_null', 'macro.dbt.test_unique']):
self.update_parsed_node(node, config)
if builder.severity() is not None:
node.unrendered_config['severity'] = builder.severity()
node.config['severity'] = builder.severity()
if builder.enabled() is not None:
node.config['enabled'] = builder.enabled()
# source node tests are processed at patch_source time
if isinstance(builder.target, UnpatchedSourceDefinition):
sources = [builder.target.fqn[-2], builder.target.fqn[-1]]
node.sources.append(sources)
else: # all other nodes
node.refs.append([builder.target.name])
else:
try:
# make a base context that doesn't have the magic kwargs field
context = generate_test_context(
node,
self.root_project,
self.manifest,
config,
self.macro_resolver,
)
# update with rendered test kwargs (which collects any refs)
add_rendered_test_kwargs(context, node, capture_macros=True)
# the parsed node is not rendered in the native context.
get_rendered(node.raw_sql, context, node, capture_macros=True)
self.update_parsed_node(node, config)
except ValidationError as exc:
# we got a ValidationError - probably bad types in config()
msg = validator_error_message(exc)
raise CompilationException(msg, node=node) from exc
def parse_unparsed_macros(
self, base_node: UnparsedMacro) -> Iterable[ParsedMacro]:
try:
blocks: List[jinja.BlockTag] = [
t for t in jinja.extract_toplevel_blocks(
base_node.raw_sql,
allowed_blocks={'macro', 'materialization'},
collect_raw_data=False,
) if isinstance(t, jinja.BlockTag)
]
except CompilationException as exc:
exc.add_node(base_node)
raise
for block in blocks:
try:
ast = jinja.parse(block.full_block)
except CompilationException as e:
e.add_node(base_node)
raise
macro_nodes = list(ast.find_all(jinja2.nodes.Macro))
if len(macro_nodes) != 1:
# things have gone disastrously wrong, we thought we only
# parsed one block!
raise CompilationException(
f'Found multiple macros in {block.full_block}, expected 1',
node=base_node)
macro_name = macro_nodes[0].name
if not macro_name.startswith(MACRO_PREFIX):
continue
name: str = macro_name.replace(MACRO_PREFIX, '')
node = self.parse_macro(block, base_node, name)
yield node
