def basic_compiled_model():
return CompiledModelNode(
package_name='test',
root_path='/root/',
path='/root/models/foo.sql',
original_file_path='models/foo.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
deferred=True,
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
meta={},
compiled=True,
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
compiled_sql='with whatever as (select * from other) select * from whatever',
checksum=FileHash.from_contents(''),
unrendered_config={}
)
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
compiled_sql=(
'with cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'),
checksum=FileHash.from_contents(''),
),
'model.root.ephemeral': CompiledModelNode(
name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(result, manifest.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
('with __dbt__cte__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'))
self.assertEqual(
manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
False)
def test__prepend_ctes__no_ctes(self):
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table'),
checksum=FileHash.from_contents(''),
),
'model.root.view_no_cte': CompiledModelNode(
name='view_no_cte',
database='dbt',
schema='analytics',
alias='view_no_cte',
resource_type=NodeType.Model,
unique_id='model.root.view_no_cte',
fqn=['root', 'view_no_cte'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('select * from source_table'),
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(
result,
manifest.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view').compiled_sql)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes.get('model.root.view_no_cte'),
manifest,
{})
self.assertEqual(
result,
manifest.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view_no_cte').compiled_sql)
def parse_manifest(
manifest: dict[str, Any]
) -> tuple[dict[str, ParsedModelNode | CompiledModelNode] | None,
dict[str, ParsedSeedNode | CompiledSeedNode] | None,
dict[str, ParsedGenericTestNode | CompiledGenericTestNode] | None,
dict[str, ParsedSourceDefinition] | None, ]:
"""
Parse the manifest.
Only V4 manifest is supported.
Parameters
----------
manifest : dict[str, Any]
The raw manifest.
Returns
-------
dbt manifest parsed into a tuple of dicts containing: models_nodes, seed_nodes, test_nodes and source_nodes.
Any of them may be `None` if they were not in the manifest.
Raises
------
NotImplementedError :
If the dbt schema is not equal to the V4 manifest
Source
------
https://docs.getdbt.com/reference/artifacts/manifest-json
"""
if manifest.get('nodes') is not None:
model_nodes = {
node_name: CompiledModelNode(**node)
if "compiled" in node.keys() else ParsedModelNode(**node)
for node_name, node in manifest["nodes"].items()
if node["resource_type"] == NodeType.Model
}
seed_nodes = {
node_name: CompiledSeedNode(**node)
if "compiled" in node.keys() else ParsedSeedNode(**node)
for node_name, node in manifest["nodes"].items()
if node["resource_type"] == NodeType.Seed
}
test_nodes = {}
for node_name, node in manifest["nodes"].items():
if node["resource_type"] == NodeType.Test:
if "test_metadata" in node.keys():
if "compiled" in node.keys():
node = CompiledGenericTestNode(**node)
else:
node = ParsedGenericTestNode(**node)
test_nodes[node_name] = node
else:
logger.info(f"Ignoring unsupported {node_name}")
else:
model_nodes = None
seed_nodes = None
test_nodes = None
if manifest.get('sources') is not None:
source_nodes: Optional[dict] = {
source_name: ParsedSourceDefinition(**source)
for source_name, source in manifest["sources"].items()
if source['resource_type'] == NodeType.Source
}
else:
source_nodes = None
return model_nodes, seed_nodes, test_nodes, source_nodes
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root_project', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[
InjectedCTE(id='model.root.ephemeral',
sql='select * from source_table')
],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral')),
'model.root.ephemeral':
CompiledModelNode(name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root_project', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql=''),
},
docs={},
# '2018-02-14T09:15:13Z'
generated_at=datetime(2018, 2, 14, 9, 15, 13),
disabled=[],
files={},
)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes['model.root.view'], input_graph)
self.assertEqual(result, output_graph.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.injected_sql, ('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'))
self.assertEqual(
input_graph.nodes['model.root.ephemeral'].extra_ctes_injected,
True)
def test__prepend_ctes__no_ctes(self):
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root_project', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table')),
'model.root.view_no_cte':
CompiledModelNode(name='view_no_cte',
database='dbt',
schema='analytics',
alias='view_no_cte',
resource_type=NodeType.Model,
unique_id='model.root.view_no_cte',
fqn=['root_project', 'view_no_cte'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('select * from source_table')),
},
docs={},
generated_at='2018-02-14T09:15:13Z',
disabled=[],
files={},
)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view'), input_graph)
self.assertEqual(result, output_graph.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
output_graph.nodes.get('model.root.view').compiled_sql)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view_no_cte'), input_graph)
self.assertEqual(result,
output_graph.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
output_graph.nodes.get('model.root.view_no_cte').compiled_sql)
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
})
self.nested_nodes = {
'model.snowplow.events':
CompiledModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.snowplow.events',
fqn=['snowplow', 'events'],
package_name='snowplow',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter',
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql=None,
extra_ctes=[]),
'model.root.events':
CompiledModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.root.events',
fqn=['root', 'events'],
package_name='root',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter',
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql='and this also does not matter',
extra_ctes=[]),
'model.root.dep':
ParsedModelNode(name='dep',
database='dbt',
schema='analytics',
alias='dep',
resource_type=NodeType.Model,
unique_id='model.root.dep',
fqn=['root', 'dep'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.nested':
ParsedModelNode(name='nested',
database='dbt',
schema='analytics',
alias='nested',
resource_type=NodeType.Model,
unique_id='model.root.nested',
fqn=['root', 'nested'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.dep']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.sibling':
ParsedModelNode(name='sibling',
database='dbt',
schema='analytics',
alias='sibling',
resource_type=NodeType.Model,
unique_id='model.root.sibling',
fqn=['root', 'sibling'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.multi':
ParsedModelNode(
name='multi',
database='dbt',
schema='analytics',
alias='multi',
resource_type=NodeType.Model,
unique_id='model.root.multi',
fqn=['root', 'multi'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(
nodes=['model.root.nested', 'model.root.sibling']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
}