def test_complex(self):
node_dict = self._complex()
cfg = TestConfig(column_types={'a': 'text'},
materialized='table',
severity='WARN')
cfg._extra.update({'extra_key': 'extra value'})
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("bar") }}',
name='foo',
resource_type=NodeType.Test,
unique_id='test.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.test.bar']),
description='My parsed node',
database='test_db',
schema='test_schema',
alias='bar',
tags=['tag'],
meta={},
config=cfg,
columns={'a': ColumnInfo('a', 'a text field', {})},
column_name='id',
docs=Docs(show=False),
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
def _get_node_column(node, column_name):
"""Given a ParsedNode, add some fields that might be missing. Return a
reference to the dict that refers to the given column, creating it if
it doesn't yet exist.
"""
if column_name in node.columns:
column = node.columns[column_name]
else:
node.columns[column_name] = ColumnInfo(name=column_name)
node.columns[column_name] = column
return column
def test_select_state_changed_seed_columns_documented_withdocs(
manifest, previous_state, seed):
seed_doc_columns = replace_config(seed, persist_docs={'columns': True})
seed_doc_columns_documented_columns = seed_doc_columns.replace(
columns={'a': ColumnInfo(name='a', description='a description')}, )
change_node(manifest, seed_doc_columns)
change_node(previous_state.manifest, seed_doc_columns_documented_columns)
method = statemethod(manifest, previous_state)
assert search_manifest_using_method(manifest, method,
'modified') == {'seed'}
assert not search_manifest_using_method(manifest, method, 'new')
def add(
self,
column: Union[HasDocs, UnparsedColumn],
description: str,
data_type: Optional[str],
meta: Dict[str, Any],
):
tags: List[str] = []
tags.extend(getattr(column, 'tags', ()))
self.column_info[column.name] = ColumnInfo(
name=column.name,
description=description,
data_type=data_type,
meta=meta,
tags=tags,
)
def add(
self,
column: Union[HasDocs, UnparsedColumn],
description: str,
data_type: Optional[str],
meta: Dict[str, Any],
):
tags: List[str] = []
tags.extend(getattr(column, 'tags', ()))
quote: Optional[bool]
if isinstance(column, UnparsedColumn):
quote = column.quote
else:
quote = None
self.column_info[column.name] = ColumnInfo(name=column.name,
description=description,
data_type=data_type,
meta=meta,
tags=tags,
quote=quote,
_extra=column.extra)
def test_populated(self):
dct = {
'name': 'foo',
'description': 'The foo model',
'original_file_path': '/path/to/schema.yml',
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'docs': {
'show': False
},
'meta': {
'key': ['value']
},
'yaml_key': 'models',
'package_name': 'test',
}
patch = self.ContractType(
name='foo',
description='The foo model',
original_file_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
meta={'key': ['value']},
yaml_key='models',
package_name='test',
docs=Docs(show=False),
)
self.assert_symmetric(patch, dct)
pickle.loads(pickle.dumps(patch))
def add(self, column_name, description, data_type):
self.column_info[column_name] = ColumnInfo(name=column_name,
description=description,
data_type=data_type)
lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': False})), u.replace(
config=u.config.replace(persist_docs={'relation': False}))),
lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': False})), u.replace(
config=u.config.replace(persist_docs={'columns': False}))),
# True -> True
lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace(
config=u.config.replace(persist_docs={'relation': True}))),
lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace(
config=u.config.replace(persist_docs={'columns': True}))),
# only columns docs enabled, but description changed
lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace(
config=u.config.replace(persist_docs={'columns': True}), description='a model description')),
# only relation docs eanbled, but columns changed
lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace(config=u.config.replace(
persist_docs={'relation': True}), columns={'a': ColumnInfo(name='a', description='a column description')}))
]
changed_compiled_models = [
lambda u: (u, None),
lambda u: (u, u.replace(raw_sql='select * from wherever')),
lambda u: (u, u.replace(fqn=['test', 'models', 'subdir', 'foo'],
original_file_path='models/subdir/foo.sql', path='/root/models/subdir/foo.sql')),
lambda u: (u, replace_config(u, full_refresh=True)),
lambda u: (u, replace_config(u, post_hook=['select 1 as id'])),
lambda u: (u, replace_config(u, pre_hook=['select 1 as id'])),
lambda u: (u, replace_config(
u, quoting={'database': True, 'schema': False, 'identifier': False})),
# we changed persist_docs values
lambda u: (u, replace_config(u, persist_docs={'relation': True})),
def test__parse_basic_model_tests(self):
block = self.file_block_for(SINGLE_TABLE_MODEL_TESTS, 'test_one.yml')
self.parser.parse_file(block)
self.assert_has_results_length(self.parser.results, patches=1, nodes=3)
patch = list(self.parser.results.patches.values())[0]
self.assertEqual(len(patch.columns), 1)
self.assertEqual(patch.name, 'my_model')
self.assertEqual(patch.description, 'A description of my model')
expected_patch = ParsedNodePatch(
name='my_model',
description='A description of my model',
columns={'color': ColumnInfo(name='color', description='The color value')},
docrefs=[],
original_file_path=normalize('models/test_one.yml'),
)
self.assertEqual(patch, expected_patch)
tests = sorted(self.parser.results.nodes.values(), key=lambda n: n.unique_id)
self.assertEqual(tests[0].config.severity, 'ERROR')
self.assertEqual(tests[0].tags, ['schema'])
self.assertEqual(tests[0].refs, [['my_model']])
self.assertEqual(tests[0].column_name, 'color')
self.assertEqual(tests[0].package_name, 'snowplow')
self.assertTrue(tests[0].name.startswith('accepted_values_'))
self.assertEqual(tests[0].fqn, ['snowplow', 'schema_test', tests[0].name])
self.assertEqual(tests[0].unique_id.split('.'), ['test', 'snowplow', tests[0].name])
self.assertEqual(tests[0].test_metadata.name, 'accepted_values')
self.assertIsNone(tests[0].test_metadata.namespace)
self.assertEqual(
tests[0].test_metadata.kwargs,
{
'column_name': 'color',
'values': ['red', 'blue', 'green'],
}
)
# foreign packages are a bit weird, they include the macro package
# name in the test name
self.assertEqual(tests[1].config.severity, 'ERROR')
self.assertEqual(tests[1].tags, ['schema'])
self.assertEqual(tests[1].refs, [['my_model']])
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].fqn, ['snowplow', 'schema_test', tests[1].name])
self.assertTrue(tests[1].name.startswith('foreign_package_test_case_'))
self.assertEqual(tests[1].package_name, 'snowplow')
self.assertEqual(tests[1].unique_id.split('.'), ['test', 'snowplow', tests[1].name])
self.assertEqual(tests[1].test_metadata.name, 'test_case')
self.assertEqual(tests[1].test_metadata.namespace, 'foreign_package')
self.assertEqual(
tests[1].test_metadata.kwargs,
{
'column_name': 'color',
'arg': 100,
},
)
self.assertEqual(tests[2].config.severity, 'WARN')
self.assertEqual(tests[2].tags, ['schema'])
self.assertEqual(tests[2].refs, [['my_model']])
self.assertEqual(tests[2].column_name, 'color')
self.assertEqual(tests[2].package_name, 'snowplow')
self.assertTrue(tests[2].name.startswith('not_null_'))
self.assertEqual(tests[2].fqn, ['snowplow', 'schema_test', tests[2].name])
self.assertEqual(tests[2].unique_id.split('.'), ['test', 'snowplow', tests[2].name])
self.assertEqual(tests[2].test_metadata.name, 'not_null')
self.assertIsNone(tests[2].test_metadata.namespace)
self.assertEqual(
tests[2].test_metadata.kwargs,
{
'column_name': 'color',
},
)
path = get_abs_os_path('./dbt_modules/snowplow/models/test_one.yml')
self.assertIn(path, self.parser.results.files)
self.assertEqual(sorted(self.parser.results.files[path].nodes),
[t.unique_id for t in tests])
self.assertIn(path, self.parser.results.files)
self.assertEqual(self.parser.results.files[path].patches, ['my_model'])
def test_complex(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Operation),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from {{ ref("bar") }}',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': ['model.test.bar']
},
'database': 'test_db',
'description': 'My parsed node',
'schema': 'test_schema',
'alias': 'bar',
'tags': ['tag'],
'meta': {},
'config': {
'column_types': {
'a': 'text'
},
'enabled': True,
'materialized': 'table',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'docs': {
'show': True
},
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'index': 13,
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("bar") }}',
name='foo',
resource_type=NodeType.Operation,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.test.bar']),
description='My parsed node',
database='test_db',
schema='test_schema',
alias='bar',
tags=['tag'],
meta={},
config=NodeConfig(column_types={'a': 'text'},
materialized='table',
post_hook=[]),
columns={'a': ColumnInfo('a', 'a text field', {})},
index=13,
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertFalse(node.is_refable)
self.assertEqual(node.get_materialization(), 'table')
def test_patch_ok(self):
initial = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
)
patch = ParsedNodePatch(
name='foo',
yaml_key='models',
package_name='test',
description='The foo model',
original_file_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
meta={},
)
initial.patch(patch)
expected_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': []
},
'database': 'test_db',
'description': 'The foo model',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'meta': {},
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'patch_path': '/path/to/schema.yml',
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'docs': {
'show': True
},
}
expected = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='The foo model',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
patch_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
)
self.assert_symmetric(expected, expected_dict) # sanity check
self.assertEqual(initial, expected)
self.assert_symmetric(initial, expected_dict)