def basic_compiled_model():
return CompiledModelNode(
package_name='test',
root_path='/root/',
path='/root/models/foo.sql',
original_file_path='models/foo.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
deferred=True,
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
meta={},
compiled=True,
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
compiled_sql='with whatever as (select * from other) select * from whatever',
checksum=FileHash.from_contents(''),
unrendered_config={}
)
def test_basic(self):
raw_sql = '{{ config(materialized="table") }}select 1 as id'
block = self.file_block_for(raw_sql, 'nested/model_1.sql')
self.parser.parse_file(block)
self.assert_has_results_length(self.parser.results, nodes=1)
node = list(self.parser.results.nodes.values())[0]
expected = ParsedModelNode(
alias='model_1',
name='model_1',
database='test',
schema='analytics',
resource_type=NodeType.Model,
unique_id='model.snowplow.model_1',
fqn=['snowplow', 'nested', 'model_1'],
package_name='snowplow',
original_file_path=normalize('models/nested/model_1.sql'),
root_path=get_abs_os_path('./dbt_modules/snowplow'),
config=NodeConfig(materialized='table'),
path=normalize('nested/model_1.sql'),
raw_sql=raw_sql,
)
self.assertEqual(node, expected)
path = get_abs_os_path('./dbt_modules/snowplow/models/nested/model_1.sql')
self.assertIn(path, self.parser.results.files)
self.assertEqual(self.parser.results.files[path].nodes, ['model.snowplow.model_1'])
def test_basic(self):
raw_sql = 'select 1 as id'
block = self.file_block_for(raw_sql, 'nested/analysis_1.sql')
self.parser.parse_file(block)
self.assert_has_results_length(self.parser.results, nodes=1)
node = list(self.parser.results.nodes.values())[0]
expected = ParsedAnalysisNode(
alias='analysis_1',
name='analysis_1',
database='test',
schema='analytics',
resource_type=NodeType.Analysis,
unique_id='analysis.snowplow.analysis_1',
fqn=['snowplow', 'analysis', 'nested', 'analysis_1'],
package_name='snowplow',
original_file_path=normalize('analyses/nested/analysis_1.sql'),
root_path=get_abs_os_path('./dbt_modules/snowplow'),
depends_on=DependsOn(),
config=NodeConfig(),
path=normalize('analysis/nested/analysis_1.sql'),
raw_sql=raw_sql,
)
self.assertEqual(node, expected)
path = get_abs_os_path('./dbt_modules/snowplow/analyses/nested/analysis_1.sql')
self.assertIn(path, self.parser.results.files)
self.assertEqual(self.parser.results.files[path].nodes, ['analysis.snowplow.analysis_1'])
def mock_model():
return mock.MagicMock(
__class__=ParsedModelNode,
alias='model_one',
name='model_one',
database='dbt',
schema='analytics',
resource_type=NodeType.Model,
unique_id='model.root.model_one',
fqn=['root', 'model_one'],
package_name='root',
original_file_path='model_one.sql',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
}),
tags=[],
path='model_one.sql',
raw_sql='',
description='',
columns={},
)
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.root_project_config = {
'name': 'root_project',
'version': '0.1',
'profile': 'test',
'project-root': os.path.abspath('.'),
}
self.snowplow_project_config = {
'name': 'snowplow',
'version': '0.1',
'project-root': os.path.abspath('./dbt_modules/snowplow'),
}
self.model_config = NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
})
def make_model(pkg,
name,
sql,
refs=None,
sources=None,
tags=None,
path=None,
alias=None,
config_kwargs=None,
fqn_extras=None):
if refs is None:
refs = []
if sources is None:
sources = []
if tags is None:
tags = []
if path is None:
path = f'{name}.sql'
if alias is None:
alias = name
if config_kwargs is None:
config_kwargs = {}
if fqn_extras is None:
fqn_extras = []
fqn = [pkg] + fqn_extras + [name]
depends_on_nodes = []
source_values = []
ref_values = []
for ref in refs:
ref_values.append([ref.name])
depends_on_nodes.append(ref.unique_id)
for src in sources:
source_values.append([src.source_name, src.name])
depends_on_nodes.append(src.unique_id)
return ParsedModelNode(
raw_sql=sql,
database='dbt',
schema='dbt_schema',
alias=alias,
name=name,
fqn=fqn,
unique_id=f'model.{pkg}.{name}',
package_name=pkg,
root_path='/usr/dbt/some-project',
path=path,
original_file_path=f'models/{path}',
config=NodeConfig(**config_kwargs),
tags=tags,
refs=ref_values,
sources=source_values,
depends_on=DependsOn(nodes=depends_on_nodes),
resource_type=NodeType.Model,
checksum=FileHash.from_contents(''),
)
def setUp(self):
self.model = ParsedModelNode(
alias='model_one',
name='model_one',
database='dbt',
schema='analytics',
resource_type=NodeType.Model,
unique_id='model.root.model_one',
fqn=['root', 'model_one'],
package_name='root',
original_file_path='model_one.sql',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
}),
tags=[],
path='model_one.sql',
raw_sql='',
description='',
columns={},
checksum=FileHash.from_contents(''),
)
self.context = mock.MagicMock()
self.provider = VarProvider({})
self.config = mock.MagicMock(config_version=2,
vars=self.provider,
cli_vars={},
project_name='root')
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
})
project_cfg = {
'name': 'X',
'version': '0.1',
'profile': 'test',
'project-root': '/tmp/dbt/does-not-exist',
'config-version': 2,
}
profile_cfg = {
'outputs': {
'test': {
'type': 'postgres',
'dbname': 'postgres',
'user': '******',
'host': 'thishostshouldnotexist',
'pass': '******',
'port': 5432,
'schema': 'public'
}
},
'target': 'test'
}
self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
self._generate_runtime_model_patch = patch.object(dbt.compilation, 'generate_runtime_model')
self.mock_generate_runtime_model = self._generate_runtime_model_patch.start()
inject_adapter(Plugin.adapter(self.config), Plugin)
# self.mock_adapter = PostgresAdapter MagicMock(type=MagicMock(return_value='postgres'))
# self.mock_adapter.Relation =
# self.mock_adapter.get_compiler.return_value = dbt.compilation.Compiler
# self.mock_plugin = MagicMock(
# adapter=MagicMock(
# credentials=MagicMock(return_value='postgres')
# )
# )
# inject_adapter(self.mock_adapter, self.mock_plugin)
# so we can make an adapter
def mock_generate_runtime_model_context(model, config, manifest):
def ref(name):
result = f'__dbt__cte__{name}'
unique_id = f'model.root.{name}'
model.extra_ctes.append(InjectedCTE(id=unique_id, sql=None))
return result
return {'ref': ref}
self.mock_generate_runtime_model.side_effect = mock_generate_runtime_model_context
def test_basic_uncompiled(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {'macros': [], 'nodes': []},
'database': 'test_db',
'description': '',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'docs': {'show': True},
'columns': {},
'meta': {},
'compiled': False,
'extra_ctes': [],
'extra_ctes_injected': False,
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
meta={},
compiled=False,
extra_ctes=[],
extra_ctes_injected=False,
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertTrue(node.is_refable)
self.assertFalse(node.is_ephemeral)
self.assertEqual(node.local_vars(), {})
minimum = self._minimum()
self.assert_from_dict(node, minimum)
pickle.loads(pickle.dumps(node))
def test_basic_compiled(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from {{ ref("other") }}',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': []
},
'database': 'test_db',
'description': '',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'docrefs': [],
'columns': {},
'compiled': True,
'compiled_sql': 'select * from whatever',
'extra_ctes': [{
'id': 'whatever',
'sql': 'select * from other'
}],
'extra_ctes_injected': True,
'injected_sql':
'with whatever as (select * from other) select * from whatever',
'wrapped_sql': 'None',
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
compiled=True,
compiled_sql='select * from whatever',
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
injected_sql=
'with whatever as (select * from other) select * from whatever',
wrapped_sql='None',
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertTrue(node.is_refable)
self.assertFalse(node.is_ephemeral)
self.assertEqual(node.local_vars(), {})
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
})
self.nested_nodes = {
'model.snowplow.events':
ParsedModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.snowplow.events',
fqn=['snowplow', 'events'],
package_name='snowplow',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'),
'model.root.events':
ParsedModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.root.events',
fqn=['root', 'events'],
package_name='root',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'),
'model.root.dep':
ParsedModelNode(name='dep',
database='dbt',
schema='analytics',
alias='dep',
resource_type=NodeType.Model,
unique_id='model.root.dep',
fqn=['root', 'dep'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.nested':
ParsedModelNode(name='nested',
database='dbt',
schema='analytics',
alias='nested',
resource_type=NodeType.Model,
unique_id='model.root.nested',
fqn=['root', 'nested'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.dep']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.sibling':
ParsedModelNode(name='sibling',
database='dbt',
schema='analytics',
alias='sibling',
resource_type=NodeType.Model,
unique_id='model.root.sibling',
fqn=['root', 'sibling'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.multi':
ParsedModelNode(
name='multi',
database='dbt',
schema='analytics',
alias='multi',
resource_type=NodeType.Model,
unique_id='model.root.multi',
fqn=['root', 'multi'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(
nodes=['model.root.nested', 'model.root.sibling']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
}
for node in self.nested_nodes.values():
node.validate(node.to_dict())