def basic_compiled_model(): return CompiledModelNode( package_name='test', root_path='/root/', path='/root/models/foo.sql', original_file_path='models/foo.sql', raw_sql='select * from {{ ref("other") }}', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), deferred=True, description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), meta={}, compiled=True, extra_ctes=[InjectedCTE('whatever', 'select * from other')], extra_ctes_injected=True, compiled_sql='with whatever as (select * from other) select * from whatever', checksum=FileHash.from_contents(''), unrendered_config={} )
def test_basic(self): raw_sql = '{{ config(materialized="table") }}select 1 as id' block = self.file_block_for(raw_sql, 'nested/model_1.sql') self.parser.parse_file(block) self.assert_has_results_length(self.parser.results, nodes=1) node = list(self.parser.results.nodes.values())[0] expected = ParsedModelNode( alias='model_1', name='model_1', database='test', schema='analytics', resource_type=NodeType.Model, unique_id='model.snowplow.model_1', fqn=['snowplow', 'nested', 'model_1'], package_name='snowplow', original_file_path=normalize('models/nested/model_1.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), config=NodeConfig(materialized='table'), path=normalize('nested/model_1.sql'), raw_sql=raw_sql, ) self.assertEqual(node, expected) path = get_abs_os_path('./dbt_modules/snowplow/models/nested/model_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['model.snowplow.model_1'])
def test_basic(self): raw_sql = 'select 1 as id' block = self.file_block_for(raw_sql, 'nested/analysis_1.sql') self.parser.parse_file(block) self.assert_has_results_length(self.parser.results, nodes=1) node = list(self.parser.results.nodes.values())[0] expected = ParsedAnalysisNode( alias='analysis_1', name='analysis_1', database='test', schema='analytics', resource_type=NodeType.Analysis, unique_id='analysis.snowplow.analysis_1', fqn=['snowplow', 'analysis', 'nested', 'analysis_1'], package_name='snowplow', original_file_path=normalize('analyses/nested/analysis_1.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), depends_on=DependsOn(), config=NodeConfig(), path=normalize('analysis/nested/analysis_1.sql'), raw_sql=raw_sql, ) self.assertEqual(node, expected) path = get_abs_os_path('./dbt_modules/snowplow/analyses/nested/analysis_1.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].nodes, ['analysis.snowplow.analysis_1'])
def mock_model(): return mock.MagicMock( __class__=ParsedModelNode, alias='model_one', name='model_one', database='dbt', schema='analytics', resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], package_name='root', original_file_path='model_one.sql', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }), tags=[], path='model_one.sql', raw_sql='', description='', columns={}, )
def setUp(self): dbt.flags.STRICT_MODE = True self.maxDiff = None self.root_project_config = { 'name': 'root_project', 'version': '0.1', 'profile': 'test', 'project-root': os.path.abspath('.'), } self.snowplow_project_config = { 'name': 'snowplow', 'version': '0.1', 'project-root': os.path.abspath('./dbt_modules/snowplow'), } self.model_config = NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], })
def make_model(pkg, name, sql, refs=None, sources=None, tags=None, path=None, alias=None, config_kwargs=None, fqn_extras=None): if refs is None: refs = [] if sources is None: sources = [] if tags is None: tags = [] if path is None: path = f'{name}.sql' if alias is None: alias = name if config_kwargs is None: config_kwargs = {} if fqn_extras is None: fqn_extras = [] fqn = [pkg] + fqn_extras + [name] depends_on_nodes = [] source_values = [] ref_values = [] for ref in refs: ref_values.append([ref.name]) depends_on_nodes.append(ref.unique_id) for src in sources: source_values.append([src.source_name, src.name]) depends_on_nodes.append(src.unique_id) return ParsedModelNode( raw_sql=sql, database='dbt', schema='dbt_schema', alias=alias, name=name, fqn=fqn, unique_id=f'model.{pkg}.{name}', package_name=pkg, root_path='/usr/dbt/some-project', path=path, original_file_path=f'models/{path}', config=NodeConfig(**config_kwargs), tags=tags, refs=ref_values, sources=source_values, depends_on=DependsOn(nodes=depends_on_nodes), resource_type=NodeType.Model, checksum=FileHash.from_contents(''), )
def setUp(self): self.model = ParsedModelNode( alias='model_one', name='model_one', database='dbt', schema='analytics', resource_type=NodeType.Model, unique_id='model.root.model_one', fqn=['root', 'model_one'], package_name='root', original_file_path='model_one.sql', root_path='/usr/src/app', refs=[], sources=[], depends_on=DependsOn(), config=NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }), tags=[], path='model_one.sql', raw_sql='', description='', columns={}, checksum=FileHash.from_contents(''), ) self.context = mock.MagicMock() self.provider = VarProvider({}) self.config = mock.MagicMock(config_version=2, vars=self.provider, cli_vars={}, project_name='root')
def setUp(self): dbt.flags.STRICT_MODE = True self.maxDiff = None self.model_config = NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }) project_cfg = { 'name': 'X', 'version': '0.1', 'profile': 'test', 'project-root': '/tmp/dbt/does-not-exist', 'config-version': 2, } profile_cfg = { 'outputs': { 'test': { 'type': 'postgres', 'dbname': 'postgres', 'user': '******', 'host': 'thishostshouldnotexist', 'pass': '******', 'port': 5432, 'schema': 'public' } }, 'target': 'test' } self.config = config_from_parts_or_dicts(project_cfg, profile_cfg) self._generate_runtime_model_patch = patch.object(dbt.compilation, 'generate_runtime_model') self.mock_generate_runtime_model = self._generate_runtime_model_patch.start() inject_adapter(Plugin.adapter(self.config), Plugin) # self.mock_adapter = PostgresAdapter MagicMock(type=MagicMock(return_value='postgres')) # self.mock_adapter.Relation = # self.mock_adapter.get_compiler.return_value = dbt.compilation.Compiler # self.mock_plugin = MagicMock( # adapter=MagicMock( # credentials=MagicMock(return_value='postgres') # ) # ) # inject_adapter(self.mock_adapter, self.mock_plugin) # so we can make an adapter def mock_generate_runtime_model_context(model, config, manifest): def ref(name): result = f'__dbt__cte__{name}' unique_id = f'model.root.{name}' model.extra_ctes.append(InjectedCTE(id=unique_id, sql=None)) return result return {'ref': ref} self.mock_generate_runtime_model.side_effect = mock_generate_runtime_model_context
def test_basic_uncompiled(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from wherever', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': {'macros': [], 'nodes': []}, 'database': 'test_db', 'description': '', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, }, 'docs': {'show': True}, 'columns': {}, 'meta': {}, 'compiled': False, 'extra_ctes': [], 'extra_ctes_injected': False, } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from wherever', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), meta={}, compiled=False, extra_ctes=[], extra_ctes_injected=False, ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertTrue(node.is_refable) self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {}) minimum = self._minimum() self.assert_from_dict(node, minimum) pickle.loads(pickle.dumps(node))
def test_basic_compiled(self): node_dict = { 'name': 'foo', 'root_path': '/root/', 'resource_type': str(NodeType.Model), 'path': '/root/x/path.sql', 'original_file_path': '/root/path.sql', 'package_name': 'test', 'raw_sql': 'select * from {{ ref("other") }}', 'unique_id': 'model.test.foo', 'fqn': ['test', 'models', 'foo'], 'refs': [], 'sources': [], 'depends_on': { 'macros': [], 'nodes': [] }, 'database': 'test_db', 'description': '', 'schema': 'test_schema', 'alias': 'bar', 'tags': [], 'config': { 'column_types': {}, 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'quoting': {}, 'tags': [], 'vars': {}, }, 'docrefs': [], 'columns': {}, 'compiled': True, 'compiled_sql': 'select * from whatever', 'extra_ctes': [{ 'id': 'whatever', 'sql': 'select * from other' }], 'extra_ctes_injected': True, 'injected_sql': 'with whatever as (select * from other) select * from whatever', 'wrapped_sql': 'None', } node = self.ContractType( package_name='test', root_path='/root/', path='/root/x/path.sql', original_file_path='/root/path.sql', raw_sql='select * from {{ ref("other") }}', name='foo', resource_type=NodeType.Model, unique_id='model.test.foo', fqn=['test', 'models', 'foo'], refs=[], sources=[], depends_on=DependsOn(), description='', database='test_db', schema='test_schema', alias='bar', tags=[], config=NodeConfig(), compiled=True, compiled_sql='select * from whatever', extra_ctes=[InjectedCTE('whatever', 'select * from other')], extra_ctes_injected=True, injected_sql= 'with whatever as (select * from other) select * from whatever', wrapped_sql='None', ) self.assert_symmetric(node, node_dict) self.assertFalse(node.empty) self.assertTrue(node.is_refable) self.assertFalse(node.is_ephemeral) self.assertEqual(node.local_vars(), {})
def setUp(self): dbt.flags.STRICT_MODE = True self.maxDiff = None self.model_config = NodeConfig.from_dict({ 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], }) self.nested_nodes = { 'model.snowplow.events': ParsedModelNode(name='events', database='dbt', schema='analytics', alias='events', resource_type=NodeType.Model, unique_id='model.snowplow.events', fqn=['snowplow', 'events'], package_name='snowplow', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter'), 'model.root.events': ParsedModelNode(name='events', database='dbt', schema='analytics', alias='events', resource_type=NodeType.Model, unique_id='model.root.events', fqn=['root', 'events'], package_name='root', refs=[], sources=[], depends_on=DependsOn(), config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter'), 'model.root.dep': ParsedModelNode(name='dep', database='dbt', schema='analytics', alias='dep', resource_type=NodeType.Model, unique_id='model.root.dep', fqn=['root', 'dep'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.events']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.nested': ParsedModelNode(name='nested', database='dbt', schema='analytics', alias='nested', resource_type=NodeType.Model, unique_id='model.root.nested', fqn=['root', 'nested'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.dep']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.sibling': ParsedModelNode(name='sibling', database='dbt', schema='analytics', alias='sibling', resource_type=NodeType.Model, unique_id='model.root.sibling', fqn=['root', 'sibling'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn(nodes=['model.root.events']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.multi': ParsedModelNode( name='multi', database='dbt', schema='analytics', alias='multi', resource_type=NodeType.Model, unique_id='model.root.multi', fqn=['root', 'multi'], package_name='root', refs=[['events']], sources=[], depends_on=DependsOn( nodes=['model.root.nested', 'model.root.sibling']), config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), } for node in self.nested_nodes.values(): node.validate(node.to_dict())