def build_unparsed_node(cls, model_name, package_name, test_type,
test_args, test_namespace, root_dir,
original_file_path):
"""Given a model name (for the model under test), a pacakge name,
a test type (identifying the test macro to use), arguments dictionary,
the root directory of the search, and the original file path to the
schema.yml file that specified the test, build an UnparsedNode
representing the test.
"""
test_path = os.path.basename(original_file_path)
raw_sql = build_test_raw_sql(test_namespace, model_name, test_type,
test_args)
hashed_name, full_name = get_nice_schema_test_name(
test_type, model_name, test_args)
hashed_path = get_pseudo_test_path(hashed_name, test_path,
'schema_test')
full_path = get_pseudo_test_path(full_name, test_path, 'schema_test')
return UnparsedNode(name=full_name,
resource_type=NodeType.Test,
package_name=package_name,
root_path=root_dir,
path=hashed_path,
original_file_path=original_file_path,
raw_sql=raw_sql)
def parse_schema_test(test_base,
model_name,
test_config,
test_type,
root_project_config,
package_project_config,
all_projects,
macros=None):
if isinstance(test_config, (basestring, int, float, bool)):
test_args = {'arg': test_config}
else:
test_args = test_config
# sort the dict so the keys are rendered deterministically (for tests)
kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)]
macro_name = "test_{}".format(test_type)
if package_project_config.get('name') != \
root_project_config.get('name'):
macro_name = "{}.{}".format(package_project_config.get('name'),
macro_name)
raw_sql = "{{{{ {macro}(model=ref('{model}'), {kwargs}) }}}}".format(
**{
'model': model_name,
'macro': macro_name,
'kwargs': ", ".join(kwargs)
})
base_path = test_base.get('path')
hashed_name, full_name = get_nice_schema_test_name(test_type, model_name,
test_args)
hashed_path = get_pseudo_test_path(hashed_name, base_path, 'schema_test')
full_path = get_pseudo_test_path(full_name, base_path, 'schema_test')
# supply our own fqn which overrides the hashed version from the path
fqn_override = get_fqn(full_path, package_project_config)
to_return = {
'name': full_name,
'resource_type': test_base.get('resource_type'),
'package_name': test_base.get('package_name'),
'root_path': test_base.get('root_path'),
'path': hashed_path,
'original_file_path': test_base.get('original_file_path'),
'raw_sql': raw_sql
}
return parse_node(to_return,
get_test_path(test_base.get('package_name'), full_name),
root_project_config,
package_project_config,
all_projects,
tags={'schema'},
fqn_extra=None,
fqn=fqn_override,
macros=macros)
def parse_schema_test(cls, test_base, model_name, test_config,
test_namespace, test_type, root_project_config,
package_project_config, all_projects, macros=None):
if isinstance(test_config, (basestring, int, float, bool)):
test_args = {'arg': test_config}
else:
test_args = test_config
# sort the dict so the keys are rendered deterministically (for tests)
kwargs = [as_kwarg(key, test_args[key]) for key in sorted(test_args)]
if test_namespace is None:
macro_name = "test_{}".format(test_type)
else:
macro_name = "{}.test_{}".format(test_namespace, test_type)
raw_sql = "{{{{ {macro}(model=ref('{model}'), {kwargs}) }}}}".format(
**{
'model': model_name,
'macro': macro_name,
'kwargs': ", ".join(kwargs)
}
)
base_path = test_base.get('path')
hashed_name, full_name = get_nice_schema_test_name(test_type,
model_name,
test_args)
hashed_path = get_pseudo_test_path(hashed_name, base_path,
'schema_test')
full_path = get_pseudo_test_path(full_name, base_path,
'schema_test')
# supply our own fqn which overrides the hashed version from the path
fqn_override = cls.get_fqn(full_path, package_project_config)
package_name = test_base.get('package_name')
node_path = cls.get_path(NodeType.Test, package_name, full_name)
to_return = UnparsedNode(
name=full_name,
resource_type=test_base.get('resource_type'),
package_name=package_name,
root_path=test_base.get('root_path'),
path=hashed_path,
original_file_path=test_base.get('original_file_path'),
raw_sql=raw_sql
)
return cls.parse_node(to_return,
node_path,
root_project_config,
package_project_config,
all_projects,
tags=['schema'],
fqn_extra=None,
fqn=fqn_override,
macros=macros)
def build_test_node(self, test_target, package_name, test, root_dir, path,
column_name=None):
"""Build a test node against the given target (a model or a source).
:param test_target: An unparsed form of the target.
"""
test_type, test_args = _build_test_args(test, column_name)
test_namespace, test_type, package_name = calculate_test_namespace(
test_type, package_name
)
source_package = self.all_projects.get(package_name)
if source_package is None:
desc = '"{}" test on {}'.format(
test_type, self._describe_test_target(test_target)
)
dbt.exceptions.raise_dep_not_found(None, desc, test_namespace)
test_path = os.path.basename(path)
hashed_name, full_name = self._generate_test_name(test_target,
test_type,
test_args)
hashed_path = get_pseudo_test_path(hashed_name, test_path,
'schema_test')
full_path = get_pseudo_test_path(full_name, test_path, 'schema_test')
raw_sql = self._build_raw_sql(test_namespace, test_target, test_type,
test_args)
unparsed = UnparsedNode(
name=full_name,
resource_type=NodeType.Test,
package_name=package_name,
root_path=root_dir,
path=hashed_path,
original_file_path=path,
raw_sql=raw_sql
)
# supply our own fqn which overrides the hashed version from the path
# TODO: is this necessary even a little bit for tests?
fqn_override = self.get_fqn(full_path, source_package)
node_path = self.get_path(NodeType.Test, unparsed.package_name,
unparsed.name)
return self.parse_node(unparsed,
node_path,
source_package,
tags=['schema'],
fqn_extra=None,
fqn=fqn_override,
column_name=column_name)
def build_test_node(self,
test_target,
package_name,
test,
root_dir,
path,
column_name=None):
"""Build a test node against the given target (a model or a source).
:param test_target: An unparsed form of the target.
"""
test_type, test_args = _build_test_args(test, column_name)
test_namespace, test_type, package_name = calculate_test_namespace(
test_type, package_name)
source_package = self.all_projects.get(package_name)
if source_package is None:
desc = '"{}" test on {}'.format(
test_type, self._describe_test_target(test_target))
dbt.exceptions.raise_dep_not_found(None, desc, test_namespace)
test_path = os.path.basename(path)
hashed_name, full_name = self._generate_test_name(
test_target, test_type, test_args)
hashed_path = get_pseudo_test_path(hashed_name, test_path,
'schema_test')
full_path = get_pseudo_test_path(full_name, test_path, 'schema_test')
raw_sql = self._build_raw_sql(test_namespace, test_target, test_type,
test_args)
unparsed = UnparsedNode(name=full_name,
resource_type=NodeType.Test,
package_name=package_name,
root_path=root_dir,
path=hashed_path,
original_file_path=path,
raw_sql=raw_sql)
# supply our own fqn which overrides the hashed version from the path
# TODO: is this necessary even a little bit for tests?
fqn_override = self.get_fqn(full_path, source_package)
node_path = self.get_path(NodeType.Test, unparsed.package_name,
unparsed.name)
return self.parse_node(unparsed,
node_path,
source_package,
tags=['schema'],
fqn_extra=None,
fqn=fqn_override,
column_name=column_name)
def parse_node(self, block: SchemaTestBlock) -> ParsedTestNode:
"""In schema parsing, we rewrite most of the part of parse_node that
builds the initial node to be parsed, but rendering is basically the
same
"""
render_ctx = ConfigRenderContext(self.root_project.cli_vars).to_dict()
builder = TestBuilder[Target](
test=block.test,
target=block.target,
column_name=block.column_name,
package_name=self.project.project_name,
render_ctx=render_ctx,
)
original_name = os.path.basename(block.path.original_file_path)
compiled_path = get_pseudo_test_path(
builder.compiled_name,
original_name,
'schema_test',
)
fqn_path = get_pseudo_test_path(
builder.fqn_name,
original_name,
'schema_test',
)
# the fqn for tests actually happens in the test target's name, which
# is not necessarily this package's name
fqn = self.get_fqn(fqn_path, builder.fqn_name)
config = self.initial_config(fqn)
metadata = {
'namespace': builder.namespace,
'name': builder.name,
'kwargs': builder.args,
}
node = self._create_parsetime_node(
block=block,
path=compiled_path,
config=config,
tags=['schema'],
name=builder.fqn_name,
raw_sql=builder.build_raw_sql(),
column_name=block.column_name,
test_metadata=metadata,
)
self.render_update(node, config)
self.add_result_node(block, node)
return node
def build_parsed_node(self, unparsed, model_name, test_namespace,
test_type, column_name):
"""Given an UnparsedNode with a node type of Test and some extra
information, build a ParsedNode representing the test.
"""
test_path = os.path.basename(unparsed.original_file_path)
source_package = self.all_projects.get(unparsed.package_name)
if source_package is None:
desc = '"{}" test on model "{}"'.format(test_type, model_name)
dbt.exceptions.raise_dep_not_found(None, desc, test_namespace)
# supply our own fqn which overrides the hashed version from the path
full_path = get_pseudo_test_path(unparsed.name, test_path,
'schema_test')
fqn_override = self.get_fqn(full_path, source_package)
node_path = self.get_path(NodeType.Test, unparsed.package_name,
unparsed.name)
return self.parse_node(unparsed,
node_path,
source_package,
tags=['schema'],
fqn_extra=None,
fqn=fqn_override,
column_name=column_name)
def _parse_generic_test(
self,
target: Testable,
test: Dict[str, Any],
tags: List[str],
column_name: Optional[str],
) -> ParsedSchemaTestNode:
render_ctx = generate_target_context(self.root_project,
self.root_project.cli_vars)
try:
builder = TestBuilder(
test=test,
target=target,
column_name=column_name,
package_name=target.package_name,
render_ctx=render_ctx,
)
except CompilationException as exc:
context = _trimmed(str(target))
msg = ('Invalid test config given in {}:'
'\n\t{}\n\t@: {}'.format(target.original_file_path, exc.msg,
context))
raise CompilationException(msg) from exc
original_name = os.path.basename(target.original_file_path)
compiled_path = get_pseudo_test_path(
builder.compiled_name,
original_name,
'schema_test',
)
fqn_path = get_pseudo_test_path(
builder.fqn_name,
original_name,
'schema_test',
)
# the fqn for tests actually happens in the test target's name, which
# is not necessarily this package's name
fqn = self.get_fqn(fqn_path, builder.fqn_name)
config = self.initial_config(fqn)
metadata = {
'namespace': builder.namespace,
'name': builder.name,
'kwargs': builder.args,
}
tags = sorted(set(itertools.chain(tags, builder.tags())))
if 'schema' not in tags:
tags.append('schema')
node = self.create_test_node(
target=target,
path=compiled_path,
config=config,
fqn=fqn,
tags=tags,
name=builder.fqn_name,
raw_sql=builder.build_raw_sql(),
column_name=column_name,
test_metadata=metadata,
)
self.render_update(node, config)
return node
def get_compiled_path(cls, block: FileBlock):
return get_pseudo_test_path(block.name, block.path.relative_path,
'data_test')
def build_test_node(self,
test_target,
package_name,
test,
root_dir,
path,
column_name=None):
"""Build a test node against the given target (a model or a source).
:param test_target: An unparsed form of the target.
"""
if isinstance(test, basestring):
test = {test: {}}
ctx = generate_config_context(self.root_project_config.cli_vars)
test_info = self.Builder(test, test_target, column_name, package_name,
ctx)
source_package = self.all_projects.get(test_info.package_name)
if source_package is None:
desc = '"{}" test on {}'.format(test_info.name,
test_info.describe_test_target())
dbt.exceptions.raise_dep_not_found(None, desc, test_info.namespace)
test_path = os.path.basename(path)
hashed_name, full_name = test_info.get_test_name()
hashed_path = get_pseudo_test_path(hashed_name, test_path,
'schema_test')
full_path = get_pseudo_test_path(full_name, test_path, 'schema_test')
raw_sql = test_info.build_raw_sql()
unparsed = UnparsedNode(name=full_name,
resource_type=NodeType.Test,
package_name=test_info.package_name,
root_path=root_dir,
path=hashed_path,
original_file_path=path,
raw_sql=raw_sql)
# supply our own fqn which overrides the hashed version from the path
# TODO: is this necessary even a little bit for tests?
fqn_override = self.get_fqn(unparsed.incorporate(path=full_path),
source_package)
node_path = self.get_path(NodeType.Test, unparsed.package_name,
unparsed.name)
result = self.parse_node(unparsed,
node_path,
source_package,
tags=['schema'],
fqn_extra=None,
fqn=fqn_override,
column_name=column_name)
parse_ok = self.check_block_parsing(full_name, test_path, raw_sql)
if not parse_ok:
# if we had a parse error in parse_node, we would not get here. So
# this means we rejected a good file :(
raise dbt.exceptions.InternalException(
'the block parser rejected a good node: {} was marked invalid '
'but is actually valid!'.format(test_path))
return result