def parse_models_entry(self, model_dict, path, package_name, root_dir):
model_name = model_dict['name']
refs = ParserRef()
for column in model_dict.get('columns', []):
column_tests = self._parse_column(model_dict, column, package_name,
root_dir, path, refs)
for node in column_tests:
yield 'test', node
for test in model_dict.get('tests', []):
try:
node = self.build_test_node(model_dict, package_name, test,
root_dir, path)
except dbt.exceptions.CompilationException as exc:
dbt.exceptions.warn_or_error(
'in {}: {}'.format(path, exc.msg), test
)
continue
yield 'test', node
context = {'doc': dbt.context.parser.docs(model_dict, refs.docrefs)}
description = model_dict.get('description', '')
get_rendered(description, context)
patch = ParsedNodePatch(
name=model_name,
original_file_path=path,
description=description,
columns=refs.column_info,
docrefs=refs.docrefs
)
yield 'patch', patch
def update_parsed_node(self, parsed_node: IntermediateNode,
config: ContextConfig) -> None:
"""Given the ContextConfig used for parsing and the parsed node,
generate and set the true values to use, overriding the temporary parse
values set in _build_intermediate_parsed_node.
"""
config_dict = config.build_config_dict()
# Set tags on node provided in config blocks
model_tags = config_dict.get('tags', [])
parsed_node.tags.extend(model_tags)
parsed_node.unrendered_config = config.build_config_dict(
rendered=False)
# do this once before we parse the node database/schema/alias, so
# parsed_node.config is what it would be if they did nothing
self.update_parsed_node_config(parsed_node, config_dict)
self.update_parsed_node_name(parsed_node, config_dict)
# at this point, we've collected our hooks. Use the node context to
# render each hook and collect refs/sources
hooks = list(
itertools.chain(parsed_node.config.pre_hook,
parsed_node.config.post_hook))
# skip context rebuilding if there aren't any hooks
if not hooks:
return
# we could cache the original context from parsing this node. Is that
# worth the cost in memory/complexity?
context = self._context_for(parsed_node, config)
for hook in hooks:
get_rendered(hook.sql, context, parsed_node, capture_macros=True)
def _process_docs_for_node(
context: Dict[str, Any],
node: ManifestNode,
):
node.description = get_rendered(node.description, context)
for column_name, column in node.columns.items():
column.description = get_rendered(column.description, context)
def parse_models_entry(self, model_dict, path, package_name, root_dir):
model_name = model_dict['name']
refs = ParserRef()
for column in model_dict.get('columns', []):
column_tests = self._parse_column(model_dict, column, package_name,
root_dir, path, refs)
for node in column_tests:
yield 'test', node
for test in model_dict.get('tests', []):
try:
node = self.build_test_node(model_dict, package_name, test,
root_dir, path)
except dbt.exceptions.CompilationException as exc:
dbt.exceptions.warn_or_error('in {}: {}'.format(path, exc.msg),
test)
continue
yield 'test', node
context = {'doc': dbt.context.parser.docs(model_dict, refs.docrefs)}
description = model_dict.get('description', '')
get_rendered(description, context)
patch = ParsedNodePatch(name=model_name,
original_file_path=path,
description=description,
columns=refs.column_info,
docrefs=refs.docrefs)
yield 'patch', patch
def parse_exposure(self, unparsed: UnparsedExposure) -> ParsedExposure:
package_name = self.project.project_name
unique_id = f'{NodeType.Exposure}.{package_name}.{unparsed.name}'
path = self.yaml.path.relative_path
fqn = self.schema_parser.get_fqn_prefix(path)
fqn.append(unparsed.name)
parsed = ParsedExposure(
package_name=package_name,
root_path=self.project.project_root,
path=path,
original_file_path=self.yaml.path.original_file_path,
unique_id=unique_id,
fqn=fqn,
name=unparsed.name,
type=unparsed.type,
url=unparsed.url,
description=unparsed.description,
owner=unparsed.owner,
maturity=unparsed.maturity,
)
ctx = generate_parse_exposure(
parsed,
self.root_project,
self.schema_parser.macro_manifest,
package_name,
)
depends_on_jinja = '\n'.join('{{ ' + line + '}}'
for line in unparsed.depends_on)
get_rendered(depends_on_jinja, ctx, parsed, capture_macros=True)
# parsed now has a populated refs/sources
return parsed
def test_jinja_rendering(value, text_expectation, native_expectation):
foo_value = yaml.safe_load(value)['foo']
ctx = {'a_str': '100', 'a_int': 100, 'b_str': 'hello'}
with text_expectation as text_result:
assert text_result == get_rendered(foo_value, ctx, native=False)
with native_expectation as native_result:
assert native_result == get_rendered(foo_value, ctx, native=True)
def collect_docrefs(
target: UnparsedSchemaYaml,
refs: ParserRef,
column_name: Optional[str],
*descriptions: str,
) -> None:
context = {'doc': docs(target, refs.docrefs, column_name)}
for description in descriptions:
get_rendered(description, context)
def test_regular_render(self):
s = '{{ "some_value" | as_native }}'
value = get_rendered(s, {}, native=False)
assert value == 'some_value'
s = '{{ 1991 | as_native }}'
value = get_rendered(s, {}, native=False)
assert value == '1991'
s = '{{ "some_value" | as_text }}'
value = get_rendered(s, {}, native=False)
assert value == 'some_value'
s = '{{ 1991 | as_text }}'
value = get_rendered(s, {}, native=False)
assert value == '1991'
def render_with_context(
self, parsed_node: IntermediateNode, config: SourceConfig
) -> None:
"""Given the parsed node and a SourceConfig to use during parsing,
render the node's sql wtih macro capture enabled.
Note: this mutates the config object when config() calls are rendered.
"""
context = dbt.context.parser.generate(
parsed_node, self.root_project, self.macro_manifest, config
)
get_rendered(parsed_node.raw_sql, context, parsed_node,
capture_macros=True)
def test_native_render(self):
s = '{{ "some_value" }}'
value = get_rendered(s, {}, native=True)
assert value == 'some_value'
s = '{{ 1991 }}'
value = get_rendered(s, {}, native=True)
assert value == 1991
s = '{{ "some_value" | as_text }}'
value = get_rendered(s, {}, native=True)
assert value == 'some_value'
s = '{{ 1991 | as_text }}'
value = get_rendered(s, {}, native=True)
assert value == '1991'
def _process_docs_for_source(
context: Dict[str, Any],
source: ParsedSourceDefinition,
):
table_description = source.description
source_description = source.source_description
table_description = get_rendered(table_description, context)
source_description = get_rendered(source_description, context)
source.description = table_description
source.source_description = source_description
for column in source.columns.values():
column_desc = column.description
column_desc = get_rendered(column_desc, context)
column.description = column_desc
def _compile_node(
self,
node: ManifestNode,
manifest: Manifest,
extra_context: Optional[Dict[str, Any]] = None,
) -> NonSourceCompiledNode:
if extra_context is None:
extra_context = {}
logger.debug("Compiling {}".format(node.unique_id))
data = node.to_dict(omit_none=True)
data.update({
'compiled': False,
'compiled_sql': None,
'extra_ctes_injected': False,
'extra_ctes': [],
})
compiled_node = _compiled_type_for(node).from_dict(data)
context = self._create_node_context(compiled_node, manifest,
extra_context)
compiled_node.compiled_sql = jinja.get_rendered(
node.raw_sql,
context,
node,
)
compiled_node.relation_name = self._get_relation_name(node)
compiled_node.compiled = True
return compiled_node
def render_with_context(self, parsed_node: IntermediateNode,
config: ContextConfigType) -> None:
"""Given the parsed node and a ContextConfigType to use during parsing,
render the node's sql wtih macro capture enabled.
Note: this mutates the config object when config() calls are rendered.
"""
# during parsing, we don't have a connection, but we might need one, so
# we have to acquire it.
with get_adapter(self.root_project).connection_for(parsed_node):
context = self._context_for(parsed_node, config)
get_rendered(parsed_node.raw_sql,
context,
parsed_node,
capture_macros=True)
def __init__(
self,
test: Dict[str, Any],
target: Target,
package_name: str,
render_ctx: Dict[str, Any],
column_name: str = None,
) -> None:
test_name, test_args = self.extract_test_args(test, column_name)
self.args: Dict[str, Any] = test_args
self.package_name: str = package_name
self.target: Target = target
match = self.TEST_NAME_PATTERN.match(test_name)
if match is None:
raise_compiler_error(
'Test name string did not match expected pattern: {}'.format(
test_name))
groups = match.groupdict()
self.name: str = groups['test_name']
self.namespace: str = groups['test_namespace']
self.modifiers: Dict[str, Any] = {}
for key, default in self.MODIFIER_ARGS.items():
value = self.args.pop(key, default)
if isinstance(value, str):
value = get_rendered(value, render_ctx)
self.modifiers[key] = value
if self.namespace is not None:
self.package_name = self.namespace
compiled_name, fqn_name = self.get_test_name()
self.compiled_name: str = compiled_name
self.fqn_name: str = fqn_name
def render_with_context(
self,
node: ParsedSchemaTestNode,
config: ContextConfigType,
) -> None:
"""Given the parsed node and a ContextConfigType to use during
parsing, collect all the refs that might be squirreled away in the test
arguments. This includes the implicit "model" argument.
"""
# make a base context that doesn't have the magic kwargs field
context = self._context_for(node, config)
# update it with the rendered test kwargs (which collects any refs)
add_rendered_test_kwargs(context, node, capture_macros=True)
# the parsed node is not rendered in the native context.
get_rendered(node.raw_sql, context, node, capture_macros=True)
def get_rendered_var(self, var_name):
raw = self.merged[var_name]
# if bool/int/float/etc are passed in, don't compile anything
if not isinstance(raw, str):
return raw
return get_rendered(raw, self.context)
def render_value(self, value, keypath=None):
# keypath is ignored.
# if it wasn't read as a string, ignore it
if not isinstance(value, compat.basestring):
return value
# force the result of rendering into this python version's native
# string type
return compat.to_native_string(get_rendered(value, self.context))
def render_value(self, value, keypath=None):
# keypath is ignored.
# if it wasn't read as a string, ignore it
if not isinstance(value, compat.basestring):
return value
# force the result of rendering into this python version's native
# string type
return compat.to_native_string(get_rendered(value, self.context))
def render_with_context(self, parsed_node: IntermediateNode,
config: ContextConfig) -> None:
# Given the parsed node and a ContextConfig to use during parsing,
# render the node's sql wtih macro capture enabled.
# Note: this mutates the config object when config calls are rendered.
# during parsing, we don't have a connection, but we might need one, so
# we have to acquire it.
with get_adapter(self.root_project).connection_for(parsed_node):
context = self._context_for(parsed_node, config)
# this goes through the process of rendering, but just throws away
# the rendered result. The "macro capture" is the point?
get_rendered(parsed_node.raw_sql,
context,
parsed_node,
capture_macros=True)
def generate_source_node(self, source, table, path, package_name, root_dir,
refs):
unique_id = self.get_path(NodeType.Source, package_name, source.name,
table.name)
context = {'doc': dbt.context.parser.docs(source, refs.docrefs)}
description = table.get('description', '')
source_description = source.get('description', '')
get_rendered(description, context)
get_rendered(source_description, context)
freshness = dbt.utils.deep_merge(source.get('freshness', {}),
table.get('freshness', {}))
loaded_at_field = table.get('loaded_at_field',
source.get('loaded_at_field'))
# use 'or {}' to allow quoting: null
source_quoting = source.get('quoting') or {}
table_quoting = table.get('quoting') or {}
quoting = dbt.utils.deep_merge(source_quoting, table_quoting)
default_database = self.root_project_config.credentials.database
return ParsedSourceDefinition(
package_name=package_name,
database=source.get('database', default_database),
schema=source.get('schema', source.name),
identifier=table.get('identifier', table.name),
root_path=root_dir,
path=path,
original_file_path=path,
columns=refs.column_info,
unique_id=unique_id,
name=table.name,
description=description,
source_name=source.name,
source_description=source_description,
loader=source.get('loader', ''),
docrefs=refs.docrefs,
loaded_at_field=loaded_at_field,
freshness=freshness,
quoting=quoting,
resource_type=NodeType.Source,
fqn=[package_name, source.name, table.name])
def generate_source_node(self, source, table, path, package_name, root_dir,
refs):
unique_id = self.get_path(NodeType.Source, package_name,
source.name, table.name)
context = {'doc': dbt.context.parser.docs(source, refs.docrefs)}
description = table.get('description', '')
source_description = source.get('description', '')
get_rendered(description, context)
get_rendered(source_description, context)
freshness = dbt.utils.deep_merge(source.get('freshness', {}),
table.get('freshness', {}))
loaded_at_field = table.get('loaded_at_field',
source.get('loaded_at_field'))
# use 'or {}' to allow quoting: null
source_quoting = source.get('quoting') or {}
table_quoting = table.get('quoting') or {}
quoting = dbt.utils.deep_merge(source_quoting, table_quoting)
default_database = self.root_project_config.credentials.database
return ParsedSourceDefinition(
package_name=package_name,
database=source.get('database', default_database),
schema=source.get('schema', source.name),
identifier=table.get('identifier', table.name),
root_path=root_dir,
path=path,
original_file_path=path,
columns=refs.column_info,
unique_id=unique_id,
name=table.name,
description=description,
source_name=source.name,
source_description=source_description,
loader=source.get('loader', ''),
docrefs=refs.docrefs,
loaded_at_field=loaded_at_field,
freshness=freshness,
quoting=quoting,
resource_type=NodeType.Source
)
def _parse_column(self, target, column, package_name, root_dir, path,
refs):
# this should yield ParsedNodes where resource_type == NodeType.Test
column_name = column['name']
description = column.get('description', '')
refs.add(column_name, description)
context = {
'doc': dbt.context.parser.docs(target, refs.docrefs, column_name)
}
get_rendered(description, context)
for test in column.get('tests', []):
try:
yield self.build_test_node(target, package_name, test,
root_dir, path, column_name)
except dbt.exceptions.CompilationException as exc:
dbt.exceptions.warn_or_error('in {}: {}'.format(path, exc.msg),
None)
continue
def render_value(self,
value: Any,
keypath: Optional[Keypath] = None) -> Any:
# keypath is ignored.
# if it wasn't read as a string, ignore it
if not isinstance(value, str):
return value
try:
with catch_jinja():
return get_rendered(value, self.context, native=True)
except CompilationException as exc:
msg = f'Could not render {value}: {exc.msg}'
raise CompilationException(msg) from exc
def render_test_update(self, node, config, builder):
macro_unique_id = self.macro_resolver.get_macro_id(
node.package_name, 'test_' + builder.name)
# Add the depends_on here so we can limit the macros added
# to the context in rendering processing
node.depends_on.add_macro(macro_unique_id)
if (macro_unique_id
in ['macro.dbt.test_not_null', 'macro.dbt.test_unique']):
self.update_parsed_node(node, config)
if builder.severity() is not None:
node.unrendered_config['severity'] = builder.severity()
node.config['severity'] = builder.severity()
if builder.enabled() is not None:
node.config['enabled'] = builder.enabled()
# source node tests are processed at patch_source time
if isinstance(builder.target, UnpatchedSourceDefinition):
sources = [builder.target.fqn[-2], builder.target.fqn[-1]]
node.sources.append(sources)
else: # all other nodes
node.refs.append([builder.target.name])
else:
try:
# make a base context that doesn't have the magic kwargs field
context = generate_test_context(
node,
self.root_project,
self.manifest,
config,
self.macro_resolver,
)
# update with rendered test kwargs (which collects any refs)
add_rendered_test_kwargs(context, node, capture_macros=True)
# the parsed node is not rendered in the native context.
get_rendered(node.raw_sql, context, node, capture_macros=True)
self.update_parsed_node(node, config)
except ValidationError as exc:
# we got a ValidationError - probably bad types in config()
msg = validator_error_message(exc)
raise CompilationException(msg, node=node) from exc
def _parse_column(self, target, column, package_name, root_dir, path,
refs):
# this should yield ParsedNodes where resource_type == NodeType.Test
column_name = column['name']
description = column.get('description', '')
refs.add(column_name, description)
context = {
'doc': dbt.context.parser.docs(target, refs.docrefs, column_name)
}
get_rendered(description, context)
for test in column.get('tests', []):
try:
yield self.build_test_node(
target, package_name, test, root_dir,
path, column_name
)
except dbt.exceptions.CompilationException as exc:
dbt.exceptions.warn_or_error(
'in {}: {}'.format(path, exc.msg), None
)
continue
def parse_block(self,
block: BlockContents) -> Iterable[ParsedDocumentation]:
unique_id = self.generate_unique_id(block.name)
contents = get_rendered(block.contents, {}).strip()
doc = ParsedDocumentation(
root_path=self.project.project_root,
path=block.file.path.relative_path,
original_file_path=block.path.original_file_path,
package_name=self.project.project_name,
unique_id=unique_id,
name=block.name,
block_contents=contents,
)
return [doc]
def __init__(
self,
test: Dict[str, Any],
target: Testable,
package_name: str,
render_ctx: Dict[str, Any],
column_name: str = None,
) -> None:
test_name, test_args = self.extract_test_args(test, column_name)
self.args: Dict[str, Any] = test_args
if 'model' in self.args:
raise_compiler_error(
'Test arguments include "model", which is a reserved argument',
)
self.package_name: str = package_name
self.target: Testable = target
self.args['model'] = self.build_model_str()
match = self.TEST_NAME_PATTERN.match(test_name)
if match is None:
raise_compiler_error(
'Test name string did not match expected pattern: {}'.format(
test_name))
groups = match.groupdict()
self.name: str = groups['test_name']
self.namespace: str = groups['test_namespace']
self.modifiers: Dict[str, Any] = {}
for key in self.MODIFIER_ARGS:
value = self.args.pop(key, None)
if isinstance(value, str):
value = get_rendered(value, render_ctx, native=True)
if value is not None:
self.modifiers[key] = value
if self.namespace is not None:
self.package_name = self.namespace
compiled_name, fqn_name = self.get_test_name()
self.compiled_name: str = compiled_name
self.fqn_name: str = fqn_name
def __init__(self, test, target, column_name, package_name, render_ctx):
test_name, test_args = self.extract_test_args(test, column_name)
self.args = test_args
self.package_name = package_name
self.target = target
match = self.TEST_NAME_PATTERN.match(test_name)
if match is None:
dbt.exceptions.raise_compiler_error(
'Test name string did not match expected pattern: {}'.format(
test_name))
groups = match.groupdict()
self.name = groups['test_name']
self.namespace = groups['test_namespace']
self.modifiers = {}
for key, default in self.MODIFIER_ARGS.items():
value = self.args.pop(key, default)
if isinstance(value, basestring):
value = get_rendered(value, render_ctx)
self.modifiers[key] = value
if self.namespace is not None:
self.package_name = self.namespace
def _process_docs_for_exposure(context: Dict[str, Any],
exposure: ParsedExposure) -> None:
exposure.description = get_rendered(exposure.description, context)
def _process_docs_for_macro(context: Dict[str, Any],
macro: ParsedMacro) -> None:
macro.description = get_rendered(macro.description, context)
for arg in macro.arguments:
arg.description = get_rendered(arg.description, context)
def fn(string):
return get_rendered(string, context, node)
def render_value(self, value, keypath=None):
# keypath is ignored.
# if it wasn't read as a string, ignore it
if not isinstance(value, str):
return value
return str(get_rendered(value, self.context))
def render(self, string: str) -> str:
return get_rendered(string, self._ctx, self.model)