def compile_node(self, node, manifest, extra_context=None):
if extra_context is None:
extra_context = {}
logger.debug("Compiling {}".format(node.get('unique_id')))
data = node.to_dict()
data.update({
'compiled': False,
'compiled_sql': None,
'extra_ctes_injected': False,
'extra_ctes': [],
'injected_sql': None,
})
compiled_node = CompiledNode(**data)
context = dbt.context.runtime.generate(
compiled_node, self.config, manifest)
context.update(extra_context)
compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
node.get('raw_sql'),
context,
node)
compiled_node.compiled = True
injected_node, _ = prepend_ctes(compiled_node, manifest)
should_wrap = {NodeType.Test, NodeType.Operation}
if injected_node.resource_type in should_wrap:
# data tests get wrapped in count(*)
# TODO : move this somewhere more reasonable
if 'data' in injected_node.tags and \
is_type(injected_node, NodeType.Test):
injected_node.wrapped_sql = (
"select count(*) from (\n{test_sql}\n) sbq").format(
test_sql=injected_node.injected_sql)
else:
# don't wrap schema tests or analyses.
injected_node.wrapped_sql = injected_node.injected_sql
elif is_type(injected_node, NodeType.Archive):
# unfortunately we do everything automagically for
# archives. in the future it'd be nice to generate
# the SQL at the parser level.
pass
elif(is_type(injected_node, NodeType.Model) and
get_materialization(injected_node) == 'ephemeral'):
pass
else:
injected_node.wrapped_sql = None
return injected_node
def recursively_prepend_ctes(model, manifest):
if model.extra_ctes_injected:
return (model, model.extra_ctes, manifest)
if dbt.flags.STRICT_MODE:
# ensure that the cte we're adding to is compiled
CompiledNode(**model.serialize())
prepended_ctes = []
for cte in model.extra_ctes:
cte_id = cte['id']
cte_to_add = manifest.nodes.get(cte_id)
cte_to_add, new_prepended_ctes, manifest = recursively_prepend_ctes(
cte_to_add, manifest)
_extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name'))
sql = ' {} as (\n{}\n)'.format(new_cte_name, cte_to_add.compiled_sql)
_add_prepended_cte(prepended_ctes, {'id': cte_id, 'sql': sql})
model.prepend_ctes(prepended_ctes)
manifest.nodes[model.unique_id] = model
return (model, prepended_ctes, manifest)
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.copy()
ephemeral_config['materialized'] = 'ephemeral'
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledNode(
name='view',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.view',
fqn=['root_project', 'view'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': ['model.root.ephemeral'],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[{
'id': 'model.root.ephemeral',
'sql': None
}],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral')),
'model.root.ephemeral':
CompiledNode(name='ephemeral',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.ephemeral',
fqn=['root_project', 'ephemeral'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql=''),
},
docs={},
generated_at='2018-02-14T09:15:13Z')
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes['model.root.view'], input_graph)
self.assertEqual(result, output_graph.nodes['model.root.view'])
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'),
('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'))
self.assertEqual((input_graph.nodes.get(
'model.root.ephemeral', {}).get('extra_ctes_injected')), True)
def test__prepend_ctes__no_ctes(self):
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledNode(
name='view',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.view',
fqn=['root_project', 'view'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table')),
'model.root.view_no_cte':
CompiledNode(name='view_no_cte',
schema='analytics',
alias='view_no_cte',
resource_type='model',
unique_id='model.root.view_no_cte',
fqn=['root_project', 'view_no_cte'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('select * from source_table')),
},
docs={},
generated_at='2018-02-14T09:15:13Z')
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view'), input_graph)
self.assertEqual(result, output_graph.nodes.get('model.root.view'))
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'),
(output_graph.nodes.get('model.root.view').get('compiled_sql')))
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view_no_cte'), input_graph)
self.assertEqual(result,
output_graph.nodes.get('model.root.view_no_cte'))
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'), (output_graph.nodes.get(
'model.root.view_no_cte').get('compiled_sql')))
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = {
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
}
self.nested_nodes = {
'model.snowplow.events':
CompiledNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type='model',
unique_id='model.snowplow.events',
fqn=['snowplow', 'events'],
empty=False,
package_name='snowplow',
refs=[],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter',
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql=None,
extra_ctes=[]),
'model.root.events':
CompiledNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type='model',
unique_id='model.root.events',
fqn=['root', 'events'],
empty=False,
package_name='root',
refs=[],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter',
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql='and this also does not matter',
extra_ctes=[]),
'model.root.dep':
ParsedNode(name='dep',
database='dbt',
schema='analytics',
alias='dep',
resource_type='model',
unique_id='model.root.dep',
fqn=['root', 'dep'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': ['model.root.events'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.nested':
ParsedNode(name='nested',
database='dbt',
schema='analytics',
alias='nested',
resource_type='model',
unique_id='model.root.nested',
fqn=['root', 'nested'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': ['model.root.dep'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.sibling':
ParsedNode(name='sibling',
database='dbt',
schema='analytics',
alias='sibling',
resource_type='model',
unique_id='model.root.sibling',
fqn=['root', 'sibling'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': ['model.root.events'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.multi':
ParsedNode(name='multi',
database='dbt',
schema='analytics',
alias='multi',
resource_type='model',
unique_id='model.root.multi',
fqn=['root', 'multi'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes':
['model.root.nested', 'model.root.sibling'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
}
