def recursively_prepend_ctes(model, manifest): if model.extra_ctes_injected: return (model, model.extra_ctes, manifest) if dbt.flags.STRICT_MODE: # ensure that the cte we're adding to is compiled CompiledNode(**model.serialize()) prepended_ctes = [] for cte in model.extra_ctes: cte_id = cte['id'] cte_to_add = manifest.nodes.get(cte_id) cte_to_add, new_prepended_ctes, manifest = recursively_prepend_ctes( cte_to_add, manifest) _extend_prepended_ctes(prepended_ctes, new_prepended_ctes) new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name')) sql = ' {} as (\n{}\n)'.format(new_cte_name, cte_to_add.compiled_sql) _add_prepended_cte(prepended_ctes, {'id': cte_id, 'sql': sql}) model.prepend_ctes(prepended_ctes) manifest.nodes[model.unique_id] = model return (model, prepended_ctes, manifest)
def compile_node(self, node, manifest, extra_context=None): if extra_context is None: extra_context = {} logger.debug("Compiling {}".format(node.get('unique_id'))) data = node.to_dict() data.update({ 'compiled': False, 'compiled_sql': None, 'extra_ctes_injected': False, 'extra_ctes': [], 'injected_sql': None, }) compiled_node = CompiledNode(**data) context = dbt.context.runtime.generate( compiled_node, self.config, manifest) context.update(extra_context) compiled_node.compiled_sql = dbt.clients.jinja.get_rendered( node.get('raw_sql'), context, node) compiled_node.compiled = True injected_node, _ = prepend_ctes(compiled_node, manifest) should_wrap = {NodeType.Test, NodeType.Operation} if injected_node.resource_type in should_wrap: # data tests get wrapped in count(*) # TODO : move this somewhere more reasonable if 'data' in injected_node.tags and \ is_type(injected_node, NodeType.Test): injected_node.wrapped_sql = ( "select count(*) from (\n{test_sql}\n) sbq").format( test_sql=injected_node.injected_sql) else: # don't wrap schema tests or analyses. injected_node.wrapped_sql = injected_node.injected_sql elif is_type(injected_node, NodeType.Archive): # unfortunately we do everything automagically for # archives. in the future it'd be nice to generate # the SQL at the parser level. pass elif(is_type(injected_node, NodeType.Model) and get_materialization(injected_node) == 'ephemeral'): pass else: injected_node.wrapped_sql = None return injected_node
def test__prepend_ctes__already_has_cte(self): ephemeral_config = self.model_config.copy() ephemeral_config['materialized'] = 'ephemeral' input_graph = Manifest( macros={}, nodes={ 'model.root.view': CompiledNode( name='view', schema='analytics', alias='view', resource_type='model', unique_id='model.root.view', fqn=['root_project', 'view'], empty=False, package_name='root', root_path='/usr/src/app', refs=[], depends_on={ 'nodes': ['model.root.ephemeral'], 'macros': [] }, config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from {{ref("ephemeral")}}', compiled=True, extra_ctes_injected=False, extra_ctes=[{ 'id': 'model.root.ephemeral', 'sql': None }], injected_sql='', compiled_sql=('with cte as (select * from something_else) ' 'select * from __dbt__CTE__ephemeral')), 'model.root.ephemeral': CompiledNode(name='ephemeral', schema='analytics', alias='view', resource_type='model', unique_id='model.root.ephemeral', fqn=['root_project', 'ephemeral'], empty=False, package_name='root', root_path='/usr/src/app', refs=[], depends_on={ 'nodes': [], 'macros': [] }, config=ephemeral_config, tags=[], path='ephemeral.sql', original_file_path='ephemeral.sql', raw_sql='select * from source_table', compiled=True, compiled_sql='select * from source_table', extra_ctes_injected=False, extra_ctes=[], injected_sql=''), }, docs={}, generated_at='2018-02-14T09:15:13Z') result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes['model.root.view'], input_graph) self.assertEqual(result, output_graph.nodes['model.root.view']) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), ('with __dbt__CTE__ephemeral as (' 'select * from source_table' '), cte as (select * from something_else) ' 'select * from __dbt__CTE__ephemeral')) self.assertEqual((input_graph.nodes.get( 'model.root.ephemeral', {}).get('extra_ctes_injected')), True)
def test__prepend_ctes__no_ctes(self): input_graph = Manifest( macros={}, nodes={ 'model.root.view': CompiledNode( name='view', schema='analytics', alias='view', resource_type='model', unique_id='model.root.view', fqn=['root_project', 'view'], empty=False, package_name='root', root_path='/usr/src/app', refs=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql=('with cte as (select * from something_else) ' 'select * from source_table'), compiled=True, extra_ctes_injected=False, extra_ctes=[], injected_sql='', compiled_sql=('with cte as (select * from something_else) ' 'select * from source_table')), 'model.root.view_no_cte': CompiledNode(name='view_no_cte', schema='analytics', alias='view_no_cte', resource_type='model', unique_id='model.root.view_no_cte', fqn=['root_project', 'view_no_cte'], empty=False, package_name='root', root_path='/usr/src/app', refs=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='view.sql', original_file_path='view.sql', raw_sql='select * from source_table', compiled=True, extra_ctes_injected=False, extra_ctes=[], injected_sql='', compiled_sql=('select * from source_table')), }, docs={}, generated_at='2018-02-14T09:15:13Z') result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes.get('model.root.view'), input_graph) self.assertEqual(result, output_graph.nodes.get('model.root.view')) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), (output_graph.nodes.get('model.root.view').get('compiled_sql'))) result, output_graph = dbt.compilation.prepend_ctes( input_graph.nodes.get('model.root.view_no_cte'), input_graph) self.assertEqual(result, output_graph.nodes.get('model.root.view_no_cte')) self.assertEqual(result.get('extra_ctes_injected'), True) self.assertEqualIgnoreWhitespace( result.get('injected_sql'), (output_graph.nodes.get( 'model.root.view_no_cte').get('compiled_sql')))
def setUp(self): dbt.flags.STRICT_MODE = True self.maxDiff = None self.model_config = { 'enabled': True, 'materialized': 'view', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], 'vars': {}, 'quoting': {}, 'column_types': {}, 'tags': [], } self.nested_nodes = { 'model.snowplow.events': CompiledNode(name='events', database='dbt', schema='analytics', alias='events', resource_type='model', unique_id='model.snowplow.events', fqn=['snowplow', 'events'], empty=False, package_name='snowplow', refs=[], sources=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter', compiled=True, compiled_sql='also does not matter', extra_ctes_injected=True, injected_sql=None, extra_ctes=[]), 'model.root.events': CompiledNode(name='events', database='dbt', schema='analytics', alias='events', resource_type='model', unique_id='model.root.events', fqn=['root', 'events'], empty=False, package_name='root', refs=[], sources=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='events.sql', original_file_path='events.sql', root_path='', raw_sql='does not matter', compiled=True, compiled_sql='also does not matter', extra_ctes_injected=True, injected_sql='and this also does not matter', extra_ctes=[]), 'model.root.dep': ParsedNode(name='dep', database='dbt', schema='analytics', alias='dep', resource_type='model', unique_id='model.root.dep', fqn=['root', 'dep'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] }, config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.nested': ParsedNode(name='nested', database='dbt', schema='analytics', alias='nested', resource_type='model', unique_id='model.root.nested', fqn=['root', 'nested'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': ['model.root.dep'], 'macros': [] }, config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.sibling': ParsedNode(name='sibling', database='dbt', schema='analytics', alias='sibling', resource_type='model', unique_id='model.root.sibling', fqn=['root', 'sibling'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': ['model.root.events'], 'macros': [] }, config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), 'model.root.multi': ParsedNode(name='multi', database='dbt', schema='analytics', alias='multi', resource_type='model', unique_id='model.root.multi', fqn=['root', 'multi'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': ['model.root.nested', 'model.root.sibling'], 'macros': [] }, config=self.model_config, tags=[], path='multi.sql', original_file_path='multi.sql', root_path='', raw_sql='does not matter'), }