def basic_compiled_model(): return CompiledModelNode( package_name='test', root_path='/root/', path='/root/models/foo.sql', original_file_path='models/foo.sql', raw_sql='select * from {{ ref("other") }}', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), deferred=True, description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), meta={}, compiled=True, extra_ctes=[InjectedCTE('whatever', 'select * from other')], extra_ctes_injected=True, compiled_sql='with whatever as (select * from other) select * from whatever', checksum=FileHash.from_contents(''), unrendered_config={} )
def test_complex(self): node_dict = self._complex() cfg = TestConfig(column_types={'a': 'text'}, materialized='table', severity='WARN') cfg._extra.update({'extra_key': 'extra value'}) node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from {{ ref("bar") }}', name='foo', resource_type=NodeType.Test, unique_id='test.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(nodes=['model.test.bar']), description='My parsed node', database='test_db', schema='test_schema', alias='bar', tags=['tag'], meta={}, config=cfg, columns={'a': ColumnInfo('a', 'a text field', {})}, column_name='id', docs=Docs(show=False), test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}), ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty)
def mock_model(): return mock.MagicMock( __class__=ParsedModelNode, alias='model_one', name='model_one', database='dbt', schema='analytics', resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], package_name='root', original_file_path='model_one.sql', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }), tags=[], path='model_one.sql', raw_sql='', description='', columns={}, )
def test_basic(self): raw_sql = 'select 1 as id' block = self.file_block_for(raw_sql, 'nested/analysis_1.sql') self.parser.parse_file(block) self.assert_has_results_length(self.parser.results, nodes=1) node = list(self.parser.results.nodes.values())[0] expected = ParsedAnalysisNode( alias='analysis_1', name='analysis_1', database='test', schema='analytics', resource_type=NodeType.Analysis, unique_id='analysis.snowplow.analysis_1', fqn=['snowplow', 'analysis', 'nested', 'analysis_1'], package_name='snowplow', original_file_path=normalize('analyses/nested/analysis_1.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), depends_on=DependsOn(), config=NodeConfig(), path=normalize('analysis/nested/analysis_1.sql'), raw_sql=raw_sql, ) self.assertEqual(node, expected) path = get_abs_os_path('./dbt_modules/snowplow/analyses/nested/analysis_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['analysis.snowplow.analysis_1'])
def patch_invalid(self): initial = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), ) # invalid patch: description can't be None patch = ParsedNodePatch( name='foo', yaml_key='models', package_name='test', description=None, original_file_path='/path/to/schema.yml', columns={}, docs=Docs(), ) with self.assertRaises(ValidationError): initial.patch(patch)
def basic_compiled_schema_test_node(): return CompiledSchemaTestNode( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from {{ ref("other") }}', name='foo', resource_type=NodeType.Test, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), deferred=False, description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=TestConfig(severity='warn'), meta={}, compiled=True, extra_ctes=[InjectedCTE('whatever', 'select * from other')], extra_ctes_injected=True, compiled_sql='with whatever as (select * from other) select * from whatever', column_name='id', test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}), checksum=FileHash.from_contents(''), unrendered_config={ 'severity': 'warn', } )
def make_model(pkg, name, sql, refs=None, sources=None, tags=None, path=None, alias=None, config_kwargs=None, fqn_extras=None): if refs is None: refs = [] if sources is None: sources = [] if tags is None: tags = [] if path is None: path = f'{name}.sql' if alias is None: alias = name if config_kwargs is None: config_kwargs = {} if fqn_extras is None: fqn_extras = [] fqn = [pkg] + fqn_extras + [name] depends_on_nodes = [] source_values = [] ref_values = [] for ref in refs: ref_values.append([ref.name]) depends_on_nodes.append(ref.unique_id) for src in sources: source_values.append([src.source_name, src.name]) depends_on_nodes.append(src.unique_id) return ParsedModelNode( raw_sql=sql, database='dbt', schema='dbt_schema', alias=alias, name=name, fqn=fqn, unique_id=f'model.{pkg}.{name}', package_name=pkg, root_path='/usr/dbt/some-project', path=path, original_file_path=f'models/{path}', config=NodeConfig(**config_kwargs), tags=tags, refs=ref_values, sources=source_values, depends_on=DependsOn(nodes=depends_on_nodes), resource_type=NodeType.Model, checksum=FileHash.from_contents(''), )
def make_data_test(pkg, name, sql, refs=None, sources=None, tags=None, path=None, config_kwargs=None): if refs is None: refs = [] if sources is None: sources = [] if tags is None: tags = ['data'] if path is None: path = f'{name}.sql' if config_kwargs is None: config_kwargs = {} fqn = ['minimal', 'data_test', name] depends_on_nodes = [] source_values = [] ref_values = [] for ref in refs: ref_values.append([ref.name]) depends_on_nodes.append(ref.unique_id) for src in sources: source_values.append([src.source_name, src.name]) depends_on_nodes.append(src.unique_id) return ParsedDataTestNode( raw_sql=sql, database='dbt', schema='dbt_schema', name=name, alias=name, fqn=fqn, unique_id=f'test.{pkg}.{name}', package_name=pkg, root_path='/usr/dbt/some-project', path=path, original_file_path=f'tests/{path}', config=TestConfig(**config_kwargs), tags=tags, refs=ref_values, sources=source_values, depends_on=DependsOn(nodes=depends_on_nodes), resource_type=NodeType.Test, checksum=FileHash.from_contents(''), )
def test_ok(self): node_dict = self._hook_ok() node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Operation, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), index=10, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertFalse(node.is_refable) self.assertEqual(node.get_materialization(), 'view') node.index = None minimum = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Operation), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'database': 'test_db', 'schema': 'test_schema', 'alias': 'bar', } self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node))
def test_get_resource_fqns(self): nodes = copy.copy(self.nested_nodes) nodes['seed.root.seed'] = ParsedSeedNode( name='seed', database='dbt', schema='analytics', alias='seed', resource_type='seed', unique_id='seed.root.seed', fqn=['root', 'seed'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='seed.csv', original_file_path='seed.csv', root_path='', raw_sql='-- csv --', seed_file_path='data/seed.csv') manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], files={}) expect = { 'models': frozenset([ ('snowplow', 'events'), ('root', 'events'), ('root', 'dep'), ('root', 'nested'), ('root', 'sibling'), ('root', 'multi'), ]), 'seeds': frozenset([('root', 'seed')]), } resource_fqns = manifest.get_resource_fqns() self.assertEqual(resource_fqns, expect)
def setUp(self): self.model = ParsedModelNode( alias='model_one', name='model_one', database='dbt', schema='analytics', resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], package_name='root', original_file_path='model_one.sql', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }), tags=[], path='model_one.sql', raw_sql='', description='', columns={}, checksum=FileHash.from_contents(''), ) self.context = mock.MagicMock() self.provider = VarProvider({}) self.config = mock.MagicMock(config_version=2, vars=self.provider, cli_vars={}, project_name='root')
def test_complex(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Operation), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from {{ ref("bar") }}', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': { 'macros': [], 'nodes': ['model.test.bar'] }, 'database': 'test_db', 'description': 'My parsed node', 'schema': 'test_schema', 'alias': 'bar', 'tags': ['tag'], 'meta': {}, 'config': { 'column_types': { 'a': 'text' }, 'enabled': True, 'materialized': 'table', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, }, 'docs': { 'show': True }, 'columns': { 'a': { 'name': 'a', 'description': 'a text field', 'meta': {}, 'tags': [], }, }, 'index': 13, } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from {{ ref("bar") }}', name='foo', resource_type=NodeType.Operation, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(nodes=['model.test.bar']), description='My parsed node', database='test_db', schema='test_schema', alias='bar', tags=['tag'], meta={}, config=NodeConfig(column_types={'a': 'text'}, materialized='table', post_hook=[]), columns={'a': ColumnInfo('a', 'a text field', {})}, index=13, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertFalse(node.is_refable) self.assertEqual(node.get_materialization(), 'table')
def setUp(self): dbt.flags.STRICT_MODE = True self.maxDiff = None self.model_config = NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }) self.nested_nodes = { 'model.snowplow.events': ParsedModelNode(name='events', database='dbt', schema='analytics', alias='events', resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], package_name='snowplow', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter'), 'model.root.events': ParsedModelNode(name='events', database='dbt', schema='analytics', alias='events', resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], package_name='root', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter'), 'model.root.dep': ParsedModelNode(name='dep', database='dbt', schema='analytics', alias='dep', resource_type=NodeType.Model, unique_id='model.root.dep', fqn=['root', 'dep'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.events']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.nested': ParsedModelNode(name='nested', database='dbt', schema='analytics', alias='nested', resource_type=NodeType.Model, unique_id='model.root.nested', fqn=['root', 'nested'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.dep']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.sibling': ParsedModelNode(name='sibling', database='dbt', schema='analytics', alias='sibling', resource_type=NodeType.Model, unique_id='model.root.sibling', fqn=['root', 'sibling'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.events']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.multi': ParsedModelNode( name='multi', database='dbt', schema='analytics', alias='multi', resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn( nodes=['model.root.nested', 'model.root.sibling']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), } for node in self.nested_nodes.values(): node.validate(node.to_dict())
def test_timestamp_ok(self): node_dict = self._ts_ok() node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Snapshot, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=TimestampSnapshotConfig( strategy=SnapshotStrategy.Timestamp, unique_key='id', updated_at='last_update', target_database='some_snapshot_db', target_schema='some_snapshot_schema', ), ) cfg = EmptySnapshotConfig() cfg._extra.update({ 'strategy': 'timestamp', 'unique_key': 'id', 'updated_at': 'last_update', 'target_database': 'some_snapshot_db', 'target_schema': 'some_snapshot_schema', }) inter = IntermediateSnapshotNode( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Snapshot, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=cfg, ) self.assert_symmetric(node, node_dict) self.assert_symmetric(inter, node_dict, cls=IntermediateSnapshotNode) self.assertEqual(self.ContractType.from_dict(inter.to_dict()), node) self.assertTrue(node.is_refable) self.assertFalse(node.is_ephemeral) pickle.loads(pickle.dumps(node))
def test_basic_uncompiled(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'database': 'test_db', 'description': '', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, }, 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': False, 'extra_ctes': [], 'extra_ctes_injected': False, } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertTrue(node.is_refable) self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {}) minimum = self._minimum() self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node))
def test_basic_compiled(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Test), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from {{ ref("other") }}', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'database': 'test_db', 'description': '', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, 'severity': 'warn', }, 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': True, 'compiled_sql': 'select * from whatever', 'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}], 'extra_ctes_injected': True, 'injected_sql': 'with whatever as (select * from other) select * from whatever', 'column_name': 'id', 'test_metadata': { 'name': 'foo', 'kwargs': {}, }, } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from {{ ref("other") }}', name='foo', resource_type=NodeType.Test, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=TestConfig(severity='warn'), meta={}, compiled=True, compiled_sql='select * from whatever', extra_ctes=[InjectedCTE('whatever', 'select * from other')], extra_ctes_injected=True, injected_sql='with whatever as (select * from other) select * from whatever', column_name='id', test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}), ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertFalse(node.is_refable) self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {})
def test_check_ok(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Snapshot), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': { 'macros': [], 'nodes': [] }, 'database': 'test_db', 'description': '', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'snapshot', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, 'target_database': 'some_snapshot_db', 'target_schema': 'some_snapshot_schema', 'unique_key': 'id', 'strategy': 'check', 'check_cols': 'all', }, 'docs': { 'show': True }, 'columns': {}, 'meta': {}, } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Snapshot, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=CheckSnapshotConfig( strategy=SnapshotStrategy.Check, unique_key='id', check_cols=All.All, target_database='some_snapshot_db', target_schema='some_snapshot_schema', ), ) cfg = EmptySnapshotConfig() cfg._extra.update({ 'unique_key': 'id', 'strategy': 'check', 'check_cols': 'all', 'target_database': 'some_snapshot_db', 'target_schema': 'some_snapshot_schema', }) inter = IntermediateSnapshotNode( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Snapshot, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=cfg, ) self.assert_symmetric(node, node_dict) self.assert_symmetric(inter, node_dict, cls=IntermediateSnapshotNode) self.assertEqual(self.ContractType.from_dict(inter.to_dict()), node) self.assertTrue(node.is_refable) self.assertFalse(node.is_ephemeral)
def test__prepend_ctes__already_has_cte(self): ephemeral_config = self.model_config.replace(materialized='ephemeral') input_graph = Manifest( macros={}, nodes={ 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(nodes=['model.root.ephemeral']), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from {{ref("ephemeral")}}', compiled=True, extra_ctes_injected=False, extra_ctes=[ InjectedCTE(id='model.root.ephemeral', sql='select * from source_table') ], injected_sql='', compiled_sql=('with cte as (select * from something_else) ' 'select * from __dbt__CTE__ephemeral')), 'model.root.ephemeral': CompiledModelNode(name='ephemeral', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root_project', 'ephemeral'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=ephemeral_config, tags=[], path='ephemeral.sql', original_file_path='ephemeral.sql', raw_sql='select * from source_table', compiled=True, compiled_sql='select * from source_table', extra_ctes_injected=False, extra_ctes=[], injected_sql=''), }, docs={}, # '2018-02-14T09:15:13Z' generated_at=datetime(2018, 2, 14, 9, 15, 13), disabled=[], files={}, ) result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes['model.root.view'], input_graph) self.assertEqual(result, output_graph.nodes['model.root.view']) self.assertEqual(result.extra_ctes_injected, True) self.assertEqualIgnoreWhitespace( result.injected_sql, ('with __dbt__CTE__ephemeral as (' 'select * from source_table' '), cte as (select * from something_else) ' 'select * from __dbt__CTE__ephemeral')) self.assertEqual( input_graph.nodes['model.root.ephemeral'].extra_ctes_injected, True)
def test__prepend_ctes__no_ctes(self): input_graph = Manifest( macros={}, nodes={ 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root_project', 'view'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql=('with cte as (select * from something_else) ' 'select * from source_table'), compiled=True, extra_ctes_injected=False, extra_ctes=[], injected_sql='', compiled_sql=('with cte as (select * from something_else) ' 'select * from source_table')), 'model.root.view_no_cte': CompiledModelNode(name='view_no_cte', database='dbt', schema='analytics', alias='view_no_cte', resource_type=NodeType.Model, unique_id='model.root.view_no_cte', fqn=['root_project', 'view_no_cte'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from source_table', compiled=True, extra_ctes_injected=False, extra_ctes=[], injected_sql='', compiled_sql=('select * from source_table')), }, docs={}, generated_at='2018-02-14T09:15:13Z', disabled=[], files={}, ) result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes.get('model.root.view'), input_graph) self.assertEqual(result, output_graph.nodes.get('model.root.view')) self.assertTrue(result.extra_ctes_injected) self.assertEqualIgnoreWhitespace( result.injected_sql, output_graph.nodes.get('model.root.view').compiled_sql) result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes.get('model.root.view_no_cte'), input_graph) self.assertEqual(result, output_graph.nodes.get('model.root.view_no_cte')) self.assertTrue(result.extra_ctes_injected) self.assertEqualIgnoreWhitespace( result.injected_sql, output_graph.nodes.get('model.root.view_no_cte').compiled_sql)
def test__prepend_ctes__no_ctes(self): manifest = Manifest( macros={}, nodes={ 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root', 'view'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql=('with cte as (select * from something_else) ' 'select * from source_table'), compiled=True, extra_ctes_injected=False, extra_ctes=[], compiled_sql=('with cte as (select * from something_else) ' 'select * from source_table'), checksum=FileHash.from_contents(''), ), 'model.root.view_no_cte': CompiledModelNode( name='view_no_cte', database='dbt', schema='analytics', alias='view_no_cte', resource_type=NodeType.Model, unique_id='model.root.view_no_cte', fqn=['root', 'view_no_cte'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from source_table', compiled=True, extra_ctes_injected=False, extra_ctes=[], compiled_sql=('select * from source_table'), checksum=FileHash.from_contents(''), ), }, sources={}, docs={}, disabled=[], files={}, exposures={}, selectors={}, ) compiler = dbt.compilation.Compiler(self.config) result, _ = compiler._recursively_prepend_ctes( manifest.nodes['model.root.view'], manifest, {} ) self.assertEqual( result, manifest.nodes.get('model.root.view')) self.assertTrue(result.extra_ctes_injected) self.assertEqualIgnoreWhitespace( result.compiled_sql, manifest.nodes.get('model.root.view').compiled_sql) compiler = dbt.compilation.Compiler(self.config) result, _ = compiler._recursively_prepend_ctes( manifest.nodes.get('model.root.view_no_cte'), manifest, {}) self.assertEqual( result, manifest.nodes.get('model.root.view_no_cte')) self.assertTrue(result.extra_ctes_injected) self.assertEqualIgnoreWhitespace( result.compiled_sql, manifest.nodes.get('model.root.view_no_cte').compiled_sql)
def test_patch_ok(self): initial = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], meta={}, config=NodeConfig(), ) patch = ParsedNodePatch( name='foo', yaml_key='models', package_name='test', description='The foo model', original_file_path='/path/to/schema.yml', columns={ 'a': ColumnInfo(name='a', description='a text field', meta={}) }, docs=Docs(), meta={}, ) initial.patch(patch) expected_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': { 'macros': [], 'nodes': [] }, 'database': 'test_db', 'description': 'The foo model', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'meta': {}, 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, }, 'patch_path': '/path/to/schema.yml', 'columns': { 'a': { 'name': 'a', 'description': 'a text field', 'meta': {}, 'tags': [], }, }, 'docs': { 'show': True }, } expected = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='The foo model', database='test_db', schema='test_schema', alias='bar', tags=[], meta={}, config=NodeConfig(), patch_path='/path/to/schema.yml', columns={ 'a': ColumnInfo(name='a', description='a text field', meta={}) }, docs=Docs(), ) self.assert_symmetric(expected, expected_dict) # sanity check self.assertEqual(initial, expected) self.assert_symmetric(initial, expected_dict)
def test__prepend_ctes__already_has_cte(self): ephemeral_config = self.model_config.replace(materialized='ephemeral') manifest = Manifest( macros={}, nodes={ 'model.root.view': CompiledModelNode( name='view', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.view', fqn=['root', 'view'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(nodes=['model.root.ephemeral']), config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from {{ref("ephemeral")}}', compiled=True, extra_ctes_injected=False, extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')], compiled_sql=( 'with cte as (select * from something_else) ' 'select * from __dbt__cte__ephemeral'), checksum=FileHash.from_contents(''), ), 'model.root.ephemeral': CompiledModelNode( name='ephemeral', database='dbt', schema='analytics', alias='view', resource_type=NodeType.Model, unique_id='model.root.ephemeral', fqn=['root', 'ephemeral'], package_name='root', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=ephemeral_config, tags=[], path='ephemeral.sql', original_file_path='ephemeral.sql', raw_sql='select * from source_table', compiled=True, compiled_sql='select * from source_table', extra_ctes_injected=False, extra_ctes=[], checksum=FileHash.from_contents(''), ), }, sources={}, docs={}, disabled=[], files={}, exposures={}, selectors={}, ) compiler = dbt.compilation.Compiler(self.config) result, _ = compiler._recursively_prepend_ctes( manifest.nodes['model.root.view'], manifest, {} ) self.assertEqual(result, manifest.nodes['model.root.view']) self.assertEqual(result.extra_ctes_injected, True) self.assertEqualIgnoreWhitespace( result.compiled_sql, ('with __dbt__cte__ephemeral as (' 'select * from source_table' '), cte as (select * from something_else) ' 'select * from __dbt__cte__ephemeral')) self.assertEqual( manifest.nodes['model.root.ephemeral'].extra_ctes_injected, False)
def make_schema_test(pkg, test_name, test_model, test_kwargs, path=None, refs=None, sources=None, tags=None, column_name=None): kwargs = test_kwargs.copy() ref_values = [] source_values = [] # this doesn't really have to be correct if isinstance(test_model, ParsedSourceDefinition): kwargs['model'] = "{{ source('" + test_model.source_name + \ "', '" + test_model.name + "') }}" source_values.append([test_model.source_name, test_model.name]) else: kwargs['model'] = "{{ ref('" + test_model.name + "')}}" ref_values.append([test_model.name]) if column_name is not None: kwargs['column_name'] = column_name # whatever args_name = test_model.search_name.replace(".", "_") if column_name is not None: args_name += '_' + column_name node_name = f'{test_name}_{args_name}' raw_sql = '{{ config(severity="ERROR") }}{{ test_' + \ test_name + '(**dbt_schema_test_kwargs) }}' name_parts = test_name.split('.') if len(name_parts) == 2: namespace, test_name = name_parts macro_depends = f'model.{namespace}.{test_name}' elif len(name_parts) == 1: namespace = None macro_depends = f'model.dbt.{test_name}' else: assert False, f'invalid test name: {test_name}' if path is None: path = 'schema.yml' if tags is None: tags = ['schema'] if refs is None: refs = [] if sources is None: sources = [] depends_on_nodes = [] for ref in refs: ref_values.append([ref.name]) depends_on_nodes.append(ref.unique_id) for source in sources: source_values.append([source.source_name, source.name]) depends_on_nodes.append(source.unique_id) return ParsedSchemaTestNode( raw_sql=raw_sql, test_metadata=TestMetadata( namespace=namespace, name=test_name, kwargs=kwargs, ), database='dbt', schema='dbt_postgres', name=node_name, alias=node_name, fqn=['minimal', 'schema_test', node_name], unique_id=f'test.{pkg}.{node_name}', package_name=pkg, root_path='/usr/dbt/some-project', path=f'schema_test/{node_name}.sql', original_file_path=f'models/{path}', resource_type=NodeType.Test, tags=tags, refs=ref_values, sources=[], depends_on=DependsOn(macros=[macro_depends], nodes=depends_on_nodes), column_name=column_name, checksum=FileHash.from_contents(''), )