def get_model_doc_patches(cls, nodes, docs):
patches = {}
for doc_name, docfile in docs.items():
parts = doc_name.split('__')
if len(parts) == 2:
maybe_model, maybe_column = parts
else:
maybe_model, maybe_column = doc_name, None
# Look for docs whose name matches that of a model node.
# TODO: Should this work for other non-model types as well?
model_name = 'model.' + maybe_model
node = nodes.get(model_name)
if not node:
continue
column_info = deepcopy(node['columns'])
description = node['description']
doc_ref = "{{ doc('%s') }}" % docfile.name
docrefs = []
updated = False
if maybe_column:
# Replace the column's description if it's not already set.
column = node['columns'].get(maybe_column)
if not column or not column.get('description'):
info = column_info.setdefault(
maybe_column, {'name': maybe_column})
info['description'] = doc_ref
context = {'doc': dbt.context.parser.docs(node, docrefs, maybe_column)}
updated = True
elif not description:
# Replace the node's description if it's not already set.
description = doc_ref
context = {'doc': dbt.context.parser.docs(node, docrefs)}
updated = True
if not updated:
continue
dbt.clients.jinja.get_rendered(doc_ref, context)
patch = ParsedNodePatch(
name=node.name,
original_file_path=node['original_file_path'],
description=description,
columns=column_info,
docrefs=docrefs
)
if node.name in patches:
patches[node.name].incorporate(**patch.serialize())
else:
patches[node.name] = patch
return patches
def parse_models_entry(self, model_dict, path, package_name, root_dir):
model_name = model_dict['name']
refs = ParserRef()
for column in model_dict.get('columns', []):
column_tests = self._parse_column(model_dict, column, package_name,
root_dir, path, refs)
for node in column_tests:
yield 'test', node
for test in model_dict.get('tests', []):
try:
node = self.build_test_node(model_dict, package_name, test,
root_dir, path)
except dbt.exceptions.CompilationException as exc:
dbt.exceptions.warn_or_error('in {}: {}'.format(path, exc.msg),
test)
continue
yield 'test', node
context = {'doc': dbt.context.parser.docs(model_dict, refs.docrefs)}
description = model_dict.get('description', '')
get_rendered(description, context)
patch = ParsedNodePatch(name=model_name,
original_file_path=path,
description=description,
columns=refs.column_info,
docrefs=refs.docrefs)
yield 'patch', patch
def patch_invalid(self):
initial = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
)
# invalid patch: description can't be None
patch = ParsedNodePatch(
name='foo',
yaml_key='models',
package_name='test',
description=None,
original_file_path='/path/to/schema.yml',
columns={},
docs=Docs(),
)
with self.assertRaises(ValidationError):
initial.patch(patch)
def parse_model(cls, model, package_name, root_dir, path, root_project,
all_projects, macros):
"""Given an UnparsedNodeUpdate, return column info about the model
- column info (name and maybe description) as a dict
- a list of ParsedNodes repreenting tests
This is only used in parsing the v2 schema.
"""
model_name = model['name']
docrefs = []
column_info = {}
for column in model.get('columns', []):
column_name = column['name']
description = column.get('description', '')
column_info[column_name] = {
'name': column_name,
'description': description,
}
context = {
'doc': dbt.context.parser.docs(model, docrefs, column_name)
}
dbt.clients.jinja.get_rendered(description, context)
for test in column.get('tests', []):
test_type, test_args = cls._build_v2_test_args(
test, column_name
)
node = cls.build_node(
model_name, package_name, test_type, test_args, root_dir,
path, root_project, all_projects, macros, column_name
)
yield 'test', node
for test in model.get('tests', []):
# table tests don't inject any extra values, model name is
# available via `model.name`
test_type, test_args = cls._build_v2_test_args(test, None)
node = cls.build_node(model_name, package_name, test_type,
test_args, root_dir, path, root_project,
all_projects, macros)
yield 'test', node
context = {'doc': dbt.context.parser.docs(model, docrefs)}
description = model.get('description', '')
dbt.clients.jinja.get_rendered(description, context)
patch = ParsedNodePatch(
name=model_name,
original_file_path=path,
description=description,
columns=column_info,
docrefs=docrefs
)
yield 'patch', patch
def generate_node_patch(self, block: TargetBlock,
refs: ParserRef) -> ParsedNodePatch:
assert isinstance(block.target, UnparsedNodeUpdate)
description = block.target.description
collect_docrefs(block.target, refs, None, description)
return ParsedNodePatch(
name=block.target.name,
original_file_path=block.path.original_file_path,
description=description,
columns=refs.column_info,
docrefs=refs.docrefs)
def parse_patch(self, block: TargetBlock[NodeTarget],
refs: ParserRef) -> None:
result = ParsedNodePatch(
name=block.target.name,
original_file_path=block.target.original_file_path,
yaml_key=block.target.yaml_key,
package_name=block.target.package_name,
description=block.target.description,
columns=refs.column_info,
meta=block.target.meta,
docs=block.target.docs,
)
self.results.add_patch(self.yaml.file, result)
def test__parse_basic_model_tests(self):
block = self.file_block_for(SINGLE_TABLE_MODEL_TESTS, 'test_one.yml')
self.parser.parse_file(block)
self.assert_has_results_length(self.parser.results, patches=1, nodes=3)
patch = list(self.parser.results.patches.values())[0]
self.assertEqual(len(patch.columns), 1)
self.assertEqual(patch.name, 'my_model')
self.assertEqual(patch.description, 'A description of my model')
expected_patch = ParsedNodePatch(
name='my_model',
description='A description of my model',
columns={'color': ColumnInfo(name='color', description='The color value')},
docrefs=[],
original_file_path=normalize('models/test_one.yml'),
)
self.assertEqual(patch, expected_patch)
tests = sorted(self.parser.results.nodes.values(), key=lambda n: n.unique_id)
self.assertEqual(tests[0].config.severity, 'ERROR')
self.assertEqual(tests[0].tags, ['schema'])
self.assertEqual(tests[0].refs, [['my_model']])
self.assertEqual(tests[0].column_name, 'color')
self.assertEqual(tests[0].package_name, 'snowplow')
self.assertTrue(tests[0].name.startswith('accepted_values_'))
self.assertEqual(tests[0].fqn, ['snowplow', 'schema_test', tests[0].name])
self.assertEqual(tests[0].unique_id.split('.'), ['test', 'snowplow', tests[0].name])
self.assertEqual(tests[0].test_metadata.name, 'accepted_values')
self.assertIsNone(tests[0].test_metadata.namespace)
self.assertEqual(
tests[0].test_metadata.kwargs,
{
'column_name': 'color',
'values': ['red', 'blue', 'green'],
}
)
# foreign packages are a bit weird, they include the macro package
# name in the test name
self.assertEqual(tests[1].config.severity, 'ERROR')
self.assertEqual(tests[1].tags, ['schema'])
self.assertEqual(tests[1].refs, [['my_model']])
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].fqn, ['snowplow', 'schema_test', tests[1].name])
self.assertTrue(tests[1].name.startswith('foreign_package_test_case_'))
self.assertEqual(tests[1].package_name, 'snowplow')
self.assertEqual(tests[1].unique_id.split('.'), ['test', 'snowplow', tests[1].name])
self.assertEqual(tests[1].test_metadata.name, 'test_case')
self.assertEqual(tests[1].test_metadata.namespace, 'foreign_package')
self.assertEqual(
tests[1].test_metadata.kwargs,
{
'column_name': 'color',
'arg': 100,
},
)
self.assertEqual(tests[2].config.severity, 'WARN')
self.assertEqual(tests[2].tags, ['schema'])
self.assertEqual(tests[2].refs, [['my_model']])
self.assertEqual(tests[2].column_name, 'color')
self.assertEqual(tests[2].package_name, 'snowplow')
self.assertTrue(tests[2].name.startswith('not_null_'))
self.assertEqual(tests[2].fqn, ['snowplow', 'schema_test', tests[2].name])
self.assertEqual(tests[2].unique_id.split('.'), ['test', 'snowplow', tests[2].name])
self.assertEqual(tests[2].test_metadata.name, 'not_null')
self.assertIsNone(tests[2].test_metadata.namespace)
self.assertEqual(
tests[2].test_metadata.kwargs,
{
'column_name': 'color',
},
)
path = get_abs_os_path('./dbt_modules/snowplow/models/test_one.yml')
self.assertIn(path, self.parser.results.files)
self.assertEqual(sorted(self.parser.results.files[path].nodes),
[t.unique_id for t in tests])
self.assertIn(path, self.parser.results.files)
self.assertEqual(self.parser.results.files[path].patches, ['my_model'])
def test_patch_ok(self):
initial = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
)
patch = ParsedNodePatch(
name='foo',
yaml_key='models',
package_name='test',
description='The foo model',
original_file_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
meta={},
)
initial.patch(patch)
expected_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': []
},
'database': 'test_db',
'description': 'The foo model',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'meta': {},
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'patch_path': '/path/to/schema.yml',
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'docs': {
'show': True
},
}
expected = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='The foo model',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
patch_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
)
self.assert_symmetric(expected, expected_dict) # sanity check
self.assertEqual(initial, expected)
self.assert_symmetric(initial, expected_dict)