def __init__(
self,
root_project: RuntimeConfig,
all_projects: Mapping[str, Project],
macro_hook: Optional[Callable[[Manifest], Any]] = None,
) -> None:
self.root_project: RuntimeConfig = root_project
self.all_projects: Mapping[str, Project] = all_projects
self.manifest: Manifest = Manifest({}, {}, {}, {}, {}, {}, [], {})
self.manifest.metadata = root_project.get_metadata()
# This is a MacroQueryStringSetter callable, which is called
# later after we set the MacroManifest in the adapter. It sets
# up the query headers.
self.macro_hook: Callable[[Manifest], Any]
if macro_hook is None:
self.macro_hook = lambda m: None
else:
self.macro_hook = macro_hook
self._perf_info = ManifestLoaderInfo(
is_partial_parse_enabled=self._partial_parse_enabled()
)
# State check determines whether the old_manifest and the current
# manifest match well enough to do partial parsing
self.manifest.state_check = self.build_manifest_state_check()
# This is a saved manifest from a previous run that's used for partial parsing
self.old_manifest: Optional[Manifest] = self.read_saved_manifest()
def manifest(seed, source, ephemeral_model, view_model, table_model,
ext_source, ext_model, union_model, ext_source_2,
ext_source_other, ext_source_other_2, table_id_unique,
table_id_not_null, view_id_unique, ext_source_id_unique,
view_test_nothing, namespaced_seed, namespace_model,
namespaced_union_model):
nodes = [
seed, ephemeral_model, view_model, table_model, union_model, ext_model,
table_id_unique, table_id_not_null, view_id_unique,
ext_source_id_unique, view_test_nothing, namespaced_seed,
namespace_model, namespaced_union_model
]
sources = [
source, ext_source, ext_source_2, ext_source_other, ext_source_other_2
]
manifest = Manifest(
nodes={n.unique_id: n
for n in nodes},
sources={s.unique_id: s
for s in sources},
macros={},
docs={},
files={},
exposures={},
disabled=[],
selectors={},
)
return manifest
def load_all(cls, project_config, all_projects):
root_project = project_config
macros = MacroLoader.load_all(root_project, all_projects)
macros.update(OperationLoader.load_all(root_project, all_projects))
nodes = {}
for loader in cls._LOADERS:
nodes.update(loader.load_all(root_project, all_projects, macros))
docs = DocumentationLoader.load_all(root_project, all_projects)
tests, patches = SchemaTestLoader.load_all(root_project, all_projects)
manifest = Manifest(nodes=nodes, macros=macros, docs=docs,
generated_at=timestring(), config=project_config)
manifest.add_nodes(tests)
manifest.patch_nodes(patches)
patches = DocumentationLoader.get_model_doc_patches(nodes, docs)
manifest.patch_nodes(patches)
manifest = dbt.parser.ParserUtils.process_refs(
manifest,
root_project.project_name
)
manifest = dbt.parser.ParserUtils.process_docs(manifest, root_project)
return manifest
def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
metadata = ManifestMetadata(
project_id='098f6bcd4621d373cade4e832627b4f6')
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
metadata=metadata,
files={})
self.assertEqual(
manifest.writable_manifest().to_dict(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
'files': {},
})
def create_manifest(self) -> Manifest:
# before we do anything else, patch the sources. This mutates
# results.disabled, so it needs to come before the final 'disabled'
# list is created
sources = patch_sources(self.results, self.root_project)
disabled = []
for value in self.results.disabled.values():
disabled.extend(value)
nodes: MutableMapping[str, ManifestNode] = {
k: v for k, v in self.results.nodes.items()
}
manifest = Manifest(
nodes=nodes,
sources=sources,
macros=self.results.macros,
docs=self.results.docs,
exposures=self.results.exposures,
generated_at=datetime.utcnow(),
metadata=self.root_project.get_metadata(),
disabled=disabled,
files=self.results.files,
)
manifest.patch_nodes(self.results.patches)
manifest.patch_macros(self.results.macro_patches)
self.process_manifest(manifest)
return manifest
def setUp(self):
x_depends_on = mock.MagicMock()
y_depends_on = mock.MagicMock()
x_uid = 'model.project.x'
y_uid = 'model.otherproject.y'
src_uid = 'source.thirdproject.src.tbl'
remote_docref = mock.MagicMock(documentation_package='otherproject', documentation_name='my_doc', column_name=None)
docref = mock.MagicMock(documentation_package='', documentation_name='my_doc', column_name=None)
self.x_node = mock.MagicMock(
refs=[], sources=[['src', 'tbl']], docrefs=[remote_docref], unique_id=x_uid,
resource_type=NodeType.Model, depends_on=x_depends_on,
description='other_project: {{ doc("otherproject", "my_doc") }}',
)
self.y_node = mock.MagicMock(
refs=[['x']], sources=[], docrefs=[docref], unique_id=y_uid,
resource_type=NodeType.Model, depends_on=y_depends_on,
description='{{ doc("my_doc") }}',
)
self.src_node = mock.MagicMock(
resource_type=NodeType.Source, unique_id=src_uid,
)
nodes = {
x_uid: self.x_node,
y_uid: self.y_node,
src_uid: self.src_node,
}
docs = {
'otherproject.my_doc': mock.MagicMock(block_contents='some docs')
}
self.manifest = Manifest(
nodes=nodes, macros={}, docs=docs, disabled=[], files={}, generated_at=mock.MagicMock()
)
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring(),
disabled=[])
self.assertEqual(manifest.get_resource_fqns(), {})
def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
config = mock.MagicMock()
# md5 of 'test'
config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6'
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring(),
disabled=[],
config=config)
metadata = {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
}
self.assertEqual(
manifest.serialize(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
})
def create_manifest(self) -> Manifest:
nodes: Dict[str, CompileResultNode] = {}
nodes.update(self.results.nodes)
nodes.update(self.results.sources)
disabled = []
for value in self.results.disabled.values():
disabled.extend(value)
manifest = Manifest(
nodes=nodes,
macros=self.results.macros,
docs=self.results.docs,
generated_at=datetime.utcnow(),
metadata=self.root_project.get_metadata(),
disabled=disabled,
files=self.results.files,
)
manifest.patch_nodes(self.results.patches)
manifest = ParserUtils.process_sources(
manifest, self.root_project.project_name
)
manifest = ParserUtils.process_refs(
manifest, self.root_project.project_name
)
manifest = ParserUtils.process_docs(
manifest, self.root_project.project_name
)
return manifest
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
self.assertEqual(manifest.get_resource_fqns(), {})
def load(self, internal_manifest=None):
self._load_macros(internal_manifest=internal_manifest)
# make a manifest with just the macros to get the context
self.macro_manifest = Manifest(macros=self.macros,
nodes={},
docs={},
generated_at=timestring(),
disabled=[])
self._load_nodes()
self._load_docs()
self._load_schema_tests()
def setUp(self):
super().setUp()
x_depends_on = mock.MagicMock()
y_depends_on = mock.MagicMock()
self.x_node = MockNode(
package='project',
name='x',
config=mock.MagicMock(enabled=True),
refs=[],
sources=[['src', 'tbl']],
depends_on=x_depends_on,
description='other_project: {{ doc("otherproject", "my_doc") }}',
)
self.y_node = MockNode(
package='otherproject',
name='y',
config=mock.MagicMock(enabled=True),
refs=[['x']],
sources=[],
depends_on=y_depends_on,
description='{{ doc("my_doc") }}',
)
self.src_node = MockSource(
package='thirdproject',
source_name='src',
name='tbl',
config=mock.MagicMock(enabled=True),
)
self.doc = MockDocumentation(
package='otherproject',
name='my_doc',
block_contents='some docs',
)
nodes = {
self.x_node.unique_id: self.x_node,
self.y_node.unique_id: self.y_node,
}
sources = {
self.src_node.unique_id: self.src_node,
}
docs = {
self.doc.unique_id: self.doc,
}
self.manifest = Manifest(
nodes=nodes,
sources=sources,
macros={},
docs=docs,
disabled=[],
files={},
exposures={},
selectors={},
)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
for node in flat_nodes.values():
self.assertEqual(set(node), expected_keys)
def create_manifest(self):
manifest = Manifest(nodes=self.nodes,
macros=self.macros,
docs=self.docs,
generated_at=timestring(),
config=self.root_project,
disabled=self.disabled)
manifest.add_nodes(self.tests)
manifest.patch_nodes(self.patches)
manifest = ParserUtils.process_sources(manifest, self.root_project)
manifest = ParserUtils.process_refs(manifest,
self.root_project.project_name)
manifest = ParserUtils.process_docs(manifest, self.root_project)
return manifest
def test__build_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
manifest.build_flat_graph()
flat_graph = manifest.flat_graph
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes']))
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
for node in flat_nodes.values():
self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS)
def test__nested_nodes(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
serialized = manifest.writable_manifest().to_dict()
self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
self.assertEqual(serialized['docs'], {})
self.assertEqual(serialized['disabled'], [])
self.assertEqual(serialized['files'], {})
parent_map = serialized['parent_map']
child_map = serialized['child_map']
# make sure there aren't any extra/missing keys.
self.assertEqual(set(parent_map), set(nodes))
self.assertEqual(set(child_map), set(nodes))
self.assertEqual(parent_map['model.root.sibling'],
['model.root.events'])
self.assertEqual(parent_map['model.root.nested'], ['model.root.dep'])
self.assertEqual(parent_map['model.root.dep'], ['model.root.events'])
# order doesn't matter.
self.assertEqual(set(parent_map['model.root.multi']),
set(['model.root.nested', 'model.root.sibling']))
self.assertEqual(
parent_map['model.root.events'],
[],
)
self.assertEqual(
parent_map['model.snowplow.events'],
[],
)
self.assertEqual(
child_map['model.root.sibling'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.nested'],
['model.root.multi'],
)
self.assertEqual(child_map['model.root.dep'], ['model.root.nested'])
self.assertEqual(child_map['model.root.multi'], [])
self.assertEqual(set(child_map['model.root.events']),
set(['model.root.dep', 'model.root.sibling']))
self.assertEqual(child_map['model.snowplow.events'], [])
def _load_macro_nodes(self, resource_type):
parser = MacroParser(self.root_project, self.all_projects)
for project_name, project in self.all_projects.items():
self.macros.update(
parser.load_and_parse(
package_name=project_name,
root_dir=project.project_root,
relative_dirs=project.macro_paths,
resource_type=resource_type,
))
# make a manifest with just the macros to get the context
self.macro_manifest = Manifest(macros=self.macros,
nodes={},
docs={},
generated_at=timestring(),
disabled=[])
def test__no_nodes(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
self.assertEqual(
manifest.writable_manifest().to_dict(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {},
'disabled': [],
'files': {},
})
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedNode(name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --')
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=timestring(),
disabled=[])
expect = {
'models':
frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds':
frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'}
compiled_count = 0
for node in flat_nodes.values():
if node.get('compiled'):
self.assertEqual(set(node), compiled_keys)
compiled_count += 1
else:
self.assertEqual(set(node), parsed_keys)
self.assertEqual(compiled_count, 2)
def test__no_nodes(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring())
self.assertEqual(
manifest.serialize(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': None,
'user_id': None,
'send_anonymous_usage_stats': None,
},
})
def create_manifest(self) -> Manifest:
# before we do anything else, patch the sources. This mutates
# results.disabled, so it needs to come before the final 'disabled'
# list is created
start_patch = time.perf_counter()
sources = patch_sources(self.results, self.root_project)
self._perf_info.patch_sources_elapsed = (time.perf_counter() -
start_patch)
disabled = []
for value in self.results.disabled.values():
disabled.extend(value)
nodes: MutableMapping[str, ManifestNode] = {
k: v
for k, v in self.results.nodes.items()
}
manifest = Manifest(
nodes=nodes,
sources=sources,
macros=self.results.macros,
docs=self.results.docs,
exposures=self.results.exposures,
metadata=self.root_project.get_metadata(),
disabled=disabled,
files=self.results.files,
selectors=self.root_project.manifest_selectors,
)
manifest.patch_nodes(self.results.patches)
manifest.patch_macros(self.results.macro_patches)
start_process = time.perf_counter()
self.process_manifest(manifest)
self._perf_info.process_manifest_elapsed = (time.perf_counter() -
start_process)
return manifest
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.copy()
ephemeral_config['materialized'] = 'ephemeral'
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledNode(
name='view',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.view',
fqn=['root_project', 'view'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': ['model.root.ephemeral'],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[{
'id': 'model.root.ephemeral',
'sql': None
}],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral')),
'model.root.ephemeral':
CompiledNode(name='ephemeral',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.ephemeral',
fqn=['root_project', 'ephemeral'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql=''),
},
docs={},
generated_at='2018-02-14T09:15:13Z')
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes['model.root.view'], input_graph)
self.assertEqual(result, output_graph.nodes['model.root.view'])
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'),
('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'))
self.assertEqual((input_graph.nodes.get(
'model.root.ephemeral', {}).get('extra_ctes_injected')), True)
def test__prepend_ctes__no_ctes(self):
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledNode(
name='view',
schema='analytics',
alias='view',
resource_type='model',
unique_id='model.root.view',
fqn=['root_project', 'view'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table')),
'model.root.view_no_cte':
CompiledNode(name='view_no_cte',
schema='analytics',
alias='view_no_cte',
resource_type='model',
unique_id='model.root.view_no_cte',
fqn=['root_project', 'view_no_cte'],
empty=False,
package_name='root',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('select * from source_table')),
},
docs={},
generated_at='2018-02-14T09:15:13Z')
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view'), input_graph)
self.assertEqual(result, output_graph.nodes.get('model.root.view'))
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'),
(output_graph.nodes.get('model.root.view').get('compiled_sql')))
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view_no_cte'), input_graph)
self.assertEqual(result,
output_graph.nodes.get('model.root.view_no_cte'))
self.assertEqual(result.get('extra_ctes_injected'), True)
self.assertEqualIgnoreWhitespace(
result.get('injected_sql'), (output_graph.nodes.get(
'model.root.view_no_cte').get('compiled_sql')))
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
compiled_sql=(
'with cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'),
checksum=FileHash.from_contents(''),
),
'model.root.ephemeral': CompiledModelNode(
name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(result, manifest.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
('with __dbt__cte__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'))
self.assertEqual(
manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
False)
def test__prepend_ctes__no_ctes(self):
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table'),
checksum=FileHash.from_contents(''),
),
'model.root.view_no_cte': CompiledModelNode(
name='view_no_cte',
database='dbt',
schema='analytics',
alias='view_no_cte',
resource_type=NodeType.Model,
unique_id='model.root.view_no_cte',
fqn=['root', 'view_no_cte'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('select * from source_table'),
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(
result,
manifest.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view').compiled_sql)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes.get('model.root.view_no_cte'),
manifest,
{})
self.assertEqual(
result,
manifest.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view_no_cte').compiled_sql)
def setUp(self):
super().setUp()
self.parser = MacroParser(project=self.snowplow_project_config,
manifest=Manifest({}, {}, {}, {}, {}, {}, {},
{}))
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root_project', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[
InjectedCTE(id='model.root.ephemeral',
sql='select * from source_table')
],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral')),
'model.root.ephemeral':
CompiledModelNode(name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root_project', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql=''),
},
docs={},
# '2018-02-14T09:15:13Z'
generated_at=datetime(2018, 2, 14, 9, 15, 13),
disabled=[],
files={},
)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes['model.root.view'], input_graph)
self.assertEqual(result, output_graph.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.injected_sql, ('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'))
self.assertEqual(
input_graph.nodes['model.root.ephemeral'].extra_ctes_injected,
True)
def _new_manifest(self):
state_check = ManifestStateCheck(MatchingHash(), MatchingHash, [])
manifest = Manifest({}, {}, {}, {}, {}, {}, [], {})
manifest.state_check = state_check
return manifest
def parse_node(cls,
node,
node_path,
root_project_config,
package_project_config,
all_projects,
tags=None,
fqn_extra=None,
fqn=None,
macros=None,
agate_table=None,
archive_config=None,
column_name=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
archive_config should be set if the node is an Archive node.
column_name should be set if the node is a Test node associated with a
particular column.
"""
logger.debug("Parsing {}".format(node_path))
node = node.serialize()
if agate_table is not None:
node['agate_table'] = agate_table
tags = coalesce(tags, [])
fqn_extra = coalesce(fqn_extra, [])
macros = coalesce(macros, {})
node.update({
'refs': [],
'depends_on': {
'nodes': [],
'macros': [],
}
})
if fqn is None:
fqn = cls.get_fqn(node.get('path'), package_project_config,
fqn_extra)
config = dbt.model.SourceConfig(root_project_config,
package_project_config, fqn,
node['resource_type'])
node['unique_id'] = node_path
node['empty'] = ('raw_sql' in node
and len(node['raw_sql'].strip()) == 0)
node['fqn'] = fqn
node['tags'] = tags
# Set this temporarily. Not the full config yet (as config() hasn't
# been called from jinja yet). But the Var() call below needs info
# about project level configs b/c they might contain refs.
# TODO: Restructure this?
config_dict = coalesce(archive_config, {})
config_dict.update(config.config)
node['config'] = config_dict
# Set this temporarily so get_rendered() has access to a schema & alias
profile = dbt.utils.get_profile_from_project(root_project_config)
default_schema = profile.get('schema', 'public')
node['schema'] = default_schema
default_alias = node.get('name')
node['alias'] = default_alias
# if there's a column, it should end up part of the ParsedNode
if column_name is not None:
node['column_name'] = column_name
# make a manifest with just the macros to get the context
manifest = Manifest(macros=macros,
nodes={},
docs={},
generated_at=dbt.utils.timestring())
parsed_node = ParsedNode(**node)
context = dbt.context.parser.generate(parsed_node, root_project_config,
manifest, config)
dbt.clients.jinja.get_rendered(parsed_node.raw_sql,
context,
parsed_node.to_shallow_dict(),
capture_macros=True)
# Clean up any open conns opened by adapter functions that hit the db
db_wrapper = context['adapter']
adapter = db_wrapper.adapter
profile = db_wrapper.profile
adapter.release_connection(profile, parsed_node.name)
# Special macro defined in the global project
schema_override = config.config.get('schema')
get_schema = context.get('generate_schema_name',
lambda x: default_schema)
parsed_node.schema = get_schema(schema_override)
parsed_node.alias = config.config.get('alias', default_alias)
# Overwrite node config
config_dict = parsed_node.get('config', {})
config_dict.update(config.config)
parsed_node.config = config_dict
for hook_type in dbt.hooks.ModelHookType.Both:
parsed_node.config[hook_type] = dbt.hooks.get_hooks(
node, hook_type)
parsed_node.validate()
return parsed_node
