def basic_compiled_model():
return CompiledModelNode(
package_name='test',
root_path='/root/',
path='/root/models/foo.sql',
original_file_path='models/foo.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
deferred=True,
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
meta={},
compiled=True,
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
compiled_sql='with whatever as (select * from other) select * from whatever',
checksum=FileHash.from_contents(''),
unrendered_config={}
)
def test_complex(self):
node_dict = self._complex()
cfg = TestConfig(column_types={'a': 'text'},
materialized='table',
severity='WARN')
cfg._extra.update({'extra_key': 'extra value'})
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("bar") }}',
name='foo',
resource_type=NodeType.Test,
unique_id='test.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.test.bar']),
description='My parsed node',
database='test_db',
schema='test_schema',
alias='bar',
tags=['tag'],
meta={},
config=cfg,
columns={'a': ColumnInfo('a', 'a text field', {})},
column_name='id',
docs=Docs(show=False),
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
def mock_model():
return mock.MagicMock(
__class__=ParsedModelNode,
alias='model_one',
name='model_one',
database='dbt',
schema='analytics',
resource_type=NodeType.Model,
unique_id='model.root.model_one',
fqn=['root', 'model_one'],
package_name='root',
original_file_path='model_one.sql',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
}),
tags=[],
path='model_one.sql',
raw_sql='',
description='',
columns={},
)
def test_basic(self):
raw_sql = 'select 1 as id'
block = self.file_block_for(raw_sql, 'nested/analysis_1.sql')
self.parser.parse_file(block)
self.assert_has_results_length(self.parser.results, nodes=1)
node = list(self.parser.results.nodes.values())[0]
expected = ParsedAnalysisNode(
alias='analysis_1',
name='analysis_1',
database='test',
schema='analytics',
resource_type=NodeType.Analysis,
unique_id='analysis.snowplow.analysis_1',
fqn=['snowplow', 'analysis', 'nested', 'analysis_1'],
package_name='snowplow',
original_file_path=normalize('analyses/nested/analysis_1.sql'),
root_path=get_abs_os_path('./dbt_modules/snowplow'),
depends_on=DependsOn(),
config=NodeConfig(),
path=normalize('analysis/nested/analysis_1.sql'),
raw_sql=raw_sql,
)
self.assertEqual(node, expected)
path = get_abs_os_path('./dbt_modules/snowplow/analyses/nested/analysis_1.sql')
self.assertIn(path, self.parser.results.files)
self.assertEqual(self.parser.results.files[path].nodes, ['analysis.snowplow.analysis_1'])
def patch_invalid(self):
initial = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
)
# invalid patch: description can't be None
patch = ParsedNodePatch(
name='foo',
yaml_key='models',
package_name='test',
description=None,
original_file_path='/path/to/schema.yml',
columns={},
docs=Docs(),
)
with self.assertRaises(ValidationError):
initial.patch(patch)
def basic_compiled_schema_test_node():
return CompiledSchemaTestNode(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Test,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
deferred=False,
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=TestConfig(severity='warn'),
meta={},
compiled=True,
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
compiled_sql='with whatever as (select * from other) select * from whatever',
column_name='id',
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
checksum=FileHash.from_contents(''),
unrendered_config={
'severity': 'warn',
}
)
def make_model(pkg,
name,
sql,
refs=None,
sources=None,
tags=None,
path=None,
alias=None,
config_kwargs=None,
fqn_extras=None):
if refs is None:
refs = []
if sources is None:
sources = []
if tags is None:
tags = []
if path is None:
path = f'{name}.sql'
if alias is None:
alias = name
if config_kwargs is None:
config_kwargs = {}
if fqn_extras is None:
fqn_extras = []
fqn = [pkg] + fqn_extras + [name]
depends_on_nodes = []
source_values = []
ref_values = []
for ref in refs:
ref_values.append([ref.name])
depends_on_nodes.append(ref.unique_id)
for src in sources:
source_values.append([src.source_name, src.name])
depends_on_nodes.append(src.unique_id)
return ParsedModelNode(
raw_sql=sql,
database='dbt',
schema='dbt_schema',
alias=alias,
name=name,
fqn=fqn,
unique_id=f'model.{pkg}.{name}',
package_name=pkg,
root_path='/usr/dbt/some-project',
path=path,
original_file_path=f'models/{path}',
config=NodeConfig(**config_kwargs),
tags=tags,
refs=ref_values,
sources=source_values,
depends_on=DependsOn(nodes=depends_on_nodes),
resource_type=NodeType.Model,
checksum=FileHash.from_contents(''),
)
def make_data_test(pkg,
name,
sql,
refs=None,
sources=None,
tags=None,
path=None,
config_kwargs=None):
if refs is None:
refs = []
if sources is None:
sources = []
if tags is None:
tags = ['data']
if path is None:
path = f'{name}.sql'
if config_kwargs is None:
config_kwargs = {}
fqn = ['minimal', 'data_test', name]
depends_on_nodes = []
source_values = []
ref_values = []
for ref in refs:
ref_values.append([ref.name])
depends_on_nodes.append(ref.unique_id)
for src in sources:
source_values.append([src.source_name, src.name])
depends_on_nodes.append(src.unique_id)
return ParsedDataTestNode(
raw_sql=sql,
database='dbt',
schema='dbt_schema',
name=name,
alias=name,
fqn=fqn,
unique_id=f'test.{pkg}.{name}',
package_name=pkg,
root_path='/usr/dbt/some-project',
path=path,
original_file_path=f'tests/{path}',
config=TestConfig(**config_kwargs),
tags=tags,
refs=ref_values,
sources=source_values,
depends_on=DependsOn(nodes=depends_on_nodes),
resource_type=NodeType.Test,
checksum=FileHash.from_contents(''),
)
def test_ok(self):
node_dict = self._hook_ok()
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Operation,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
index=10,
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertFalse(node.is_refable)
self.assertEqual(node.get_materialization(), 'view')
node.index = None
minimum = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Operation),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'database': 'test_db',
'schema': 'test_schema',
'alias': 'bar',
}
self.assert_from_dict(node, minimum)
pickle.loads(pickle.dumps(node))
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedSeedNode(
name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --',
seed_file_path='data/seed.csv')
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
expect = {
'models':
frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds':
frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def setUp(self):
self.model = ParsedModelNode(
alias='model_one',
name='model_one',
database='dbt',
schema='analytics',
resource_type=NodeType.Model,
unique_id='model.root.model_one',
fqn=['root', 'model_one'],
package_name='root',
original_file_path='model_one.sql',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
}),
tags=[],
path='model_one.sql',
raw_sql='',
description='',
columns={},
checksum=FileHash.from_contents(''),
)
self.context = mock.MagicMock()
self.provider = VarProvider({})
self.config = mock.MagicMock(config_version=2,
vars=self.provider,
cli_vars={},
project_name='root')
def test_complex(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Operation),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from {{ ref("bar") }}',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': ['model.test.bar']
},
'database': 'test_db',
'description': 'My parsed node',
'schema': 'test_schema',
'alias': 'bar',
'tags': ['tag'],
'meta': {},
'config': {
'column_types': {
'a': 'text'
},
'enabled': True,
'materialized': 'table',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'docs': {
'show': True
},
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'index': 13,
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("bar") }}',
name='foo',
resource_type=NodeType.Operation,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.test.bar']),
description='My parsed node',
database='test_db',
schema='test_schema',
alias='bar',
tags=['tag'],
meta={},
config=NodeConfig(column_types={'a': 'text'},
materialized='table',
post_hook=[]),
columns={'a': ColumnInfo('a', 'a text field', {})},
index=13,
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertFalse(node.is_refable)
self.assertEqual(node.get_materialization(), 'table')
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = NodeConfig.from_dict({
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
})
self.nested_nodes = {
'model.snowplow.events':
ParsedModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.snowplow.events',
fqn=['snowplow', 'events'],
package_name='snowplow',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'),
'model.root.events':
ParsedModelNode(name='events',
database='dbt',
schema='analytics',
alias='events',
resource_type=NodeType.Model,
unique_id='model.root.events',
fqn=['root', 'events'],
package_name='root',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'),
'model.root.dep':
ParsedModelNode(name='dep',
database='dbt',
schema='analytics',
alias='dep',
resource_type=NodeType.Model,
unique_id='model.root.dep',
fqn=['root', 'dep'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.nested':
ParsedModelNode(name='nested',
database='dbt',
schema='analytics',
alias='nested',
resource_type=NodeType.Model,
unique_id='model.root.nested',
fqn=['root', 'nested'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.dep']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.sibling':
ParsedModelNode(name='sibling',
database='dbt',
schema='analytics',
alias='sibling',
resource_type=NodeType.Model,
unique_id='model.root.sibling',
fqn=['root', 'sibling'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(nodes=['model.root.events']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
'model.root.multi':
ParsedModelNode(
name='multi',
database='dbt',
schema='analytics',
alias='multi',
resource_type=NodeType.Model,
unique_id='model.root.multi',
fqn=['root', 'multi'],
package_name='root',
refs=[['events']],
sources=[],
depends_on=DependsOn(
nodes=['model.root.nested', 'model.root.sibling']),
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'),
}
for node in self.nested_nodes.values():
node.validate(node.to_dict())
def test_timestamp_ok(self):
node_dict = self._ts_ok()
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Snapshot,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=TimestampSnapshotConfig(
strategy=SnapshotStrategy.Timestamp,
unique_key='id',
updated_at='last_update',
target_database='some_snapshot_db',
target_schema='some_snapshot_schema',
),
)
cfg = EmptySnapshotConfig()
cfg._extra.update({
'strategy': 'timestamp',
'unique_key': 'id',
'updated_at': 'last_update',
'target_database': 'some_snapshot_db',
'target_schema': 'some_snapshot_schema',
})
inter = IntermediateSnapshotNode(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Snapshot,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=cfg,
)
self.assert_symmetric(node, node_dict)
self.assert_symmetric(inter, node_dict, cls=IntermediateSnapshotNode)
self.assertEqual(self.ContractType.from_dict(inter.to_dict()), node)
self.assertTrue(node.is_refable)
self.assertFalse(node.is_ephemeral)
pickle.loads(pickle.dumps(node))
def test_basic_uncompiled(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {'macros': [], 'nodes': []},
'database': 'test_db',
'description': '',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'docs': {'show': True},
'columns': {},
'meta': {},
'compiled': False,
'extra_ctes': [],
'extra_ctes_injected': False,
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=NodeConfig(),
meta={},
compiled=False,
extra_ctes=[],
extra_ctes_injected=False,
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertTrue(node.is_refable)
self.assertFalse(node.is_ephemeral)
self.assertEqual(node.local_vars(), {})
minimum = self._minimum()
self.assert_from_dict(node, minimum)
pickle.loads(pickle.dumps(node))
def test_basic_compiled(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Test),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from {{ ref("other") }}',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {'macros': [], 'nodes': []},
'database': 'test_db',
'description': '',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
'severity': 'warn',
},
'docs': {'show': True},
'columns': {},
'meta': {},
'compiled': True,
'compiled_sql': 'select * from whatever',
'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
'extra_ctes_injected': True,
'injected_sql': 'with whatever as (select * from other) select * from whatever',
'column_name': 'id',
'test_metadata': {
'name': 'foo',
'kwargs': {},
},
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from {{ ref("other") }}',
name='foo',
resource_type=NodeType.Test,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=TestConfig(severity='warn'),
meta={},
compiled=True,
compiled_sql='select * from whatever',
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
injected_sql='with whatever as (select * from other) select * from whatever',
column_name='id',
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
)
self.assert_symmetric(node, node_dict)
self.assertFalse(node.empty)
self.assertFalse(node.is_refable)
self.assertFalse(node.is_ephemeral)
self.assertEqual(node.local_vars(), {})
def test_check_ok(self):
node_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Snapshot),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': []
},
'database': 'test_db',
'description': '',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'config': {
'column_types': {},
'enabled': True,
'materialized': 'snapshot',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
'target_database': 'some_snapshot_db',
'target_schema': 'some_snapshot_schema',
'unique_key': 'id',
'strategy': 'check',
'check_cols': 'all',
},
'docs': {
'show': True
},
'columns': {},
'meta': {},
}
node = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Snapshot,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=CheckSnapshotConfig(
strategy=SnapshotStrategy.Check,
unique_key='id',
check_cols=All.All,
target_database='some_snapshot_db',
target_schema='some_snapshot_schema',
),
)
cfg = EmptySnapshotConfig()
cfg._extra.update({
'unique_key': 'id',
'strategy': 'check',
'check_cols': 'all',
'target_database': 'some_snapshot_db',
'target_schema': 'some_snapshot_schema',
})
inter = IntermediateSnapshotNode(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Snapshot,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
config=cfg,
)
self.assert_symmetric(node, node_dict)
self.assert_symmetric(inter, node_dict, cls=IntermediateSnapshotNode)
self.assertEqual(self.ContractType.from_dict(inter.to_dict()), node)
self.assertTrue(node.is_refable)
self.assertFalse(node.is_ephemeral)
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root_project', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[
InjectedCTE(id='model.root.ephemeral',
sql='select * from source_table')
],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral')),
'model.root.ephemeral':
CompiledModelNode(name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root_project', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql=''),
},
docs={},
# '2018-02-14T09:15:13Z'
generated_at=datetime(2018, 2, 14, 9, 15, 13),
disabled=[],
files={},
)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes['model.root.view'], input_graph)
self.assertEqual(result, output_graph.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.injected_sql, ('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'))
self.assertEqual(
input_graph.nodes['model.root.ephemeral'].extra_ctes_injected,
True)
def test__prepend_ctes__no_ctes(self):
input_graph = Manifest(
macros={},
nodes={
'model.root.view':
CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root_project', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table')),
'model.root.view_no_cte':
CompiledModelNode(name='view_no_cte',
database='dbt',
schema='analytics',
alias='view_no_cte',
resource_type=NodeType.Model,
unique_id='model.root.view_no_cte',
fqn=['root_project', 'view_no_cte'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('select * from source_table')),
},
docs={},
generated_at='2018-02-14T09:15:13Z',
disabled=[],
files={},
)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view'), input_graph)
self.assertEqual(result, output_graph.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
output_graph.nodes.get('model.root.view').compiled_sql)
result, output_graph = dbt.compilation.prepend_ctes(
input_graph.nodes.get('model.root.view_no_cte'), input_graph)
self.assertEqual(result,
output_graph.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
output_graph.nodes.get('model.root.view_no_cte').compiled_sql)
def test__prepend_ctes__no_ctes(self):
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql=('with cte as (select * from something_else) '
'select * from source_table'),
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table'),
checksum=FileHash.from_contents(''),
),
'model.root.view_no_cte': CompiledModelNode(
name='view_no_cte',
database='dbt',
schema='analytics',
alias='view_no_cte',
resource_type=NodeType.Model,
unique_id='model.root.view_no_cte',
fqn=['root', 'view_no_cte'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from source_table',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
compiled_sql=('select * from source_table'),
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(
result,
manifest.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view').compiled_sql)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes.get('model.root.view_no_cte'),
manifest,
{})
self.assertEqual(
result,
manifest.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
manifest.nodes.get('model.root.view_no_cte').compiled_sql)
def test_patch_ok(self):
initial = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
)
patch = ParsedNodePatch(
name='foo',
yaml_key='models',
package_name='test',
description='The foo model',
original_file_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
meta={},
)
initial.patch(patch)
expected_dict = {
'name': 'foo',
'root_path': '/root/',
'resource_type': str(NodeType.Model),
'path': '/root/x/path.sql',
'original_file_path': '/root/path.sql',
'package_name': 'test',
'raw_sql': 'select * from wherever',
'unique_id': 'model.test.foo',
'fqn': ['test', 'models', 'foo'],
'refs': [],
'sources': [],
'depends_on': {
'macros': [],
'nodes': []
},
'database': 'test_db',
'description': 'The foo model',
'schema': 'test_schema',
'alias': 'bar',
'tags': [],
'meta': {},
'config': {
'column_types': {},
'enabled': True,
'materialized': 'view',
'persist_docs': {},
'post-hook': [],
'pre-hook': [],
'quoting': {},
'tags': [],
'vars': {},
},
'patch_path': '/path/to/schema.yml',
'columns': {
'a': {
'name': 'a',
'description': 'a text field',
'meta': {},
'tags': [],
},
},
'docs': {
'show': True
},
}
expected = self.ContractType(
package_name='test',
root_path='/root/',
path='/root/x/path.sql',
original_file_path='/root/path.sql',
raw_sql='select * from wherever',
name='foo',
resource_type=NodeType.Model,
unique_id='model.test.foo',
fqn=['test', 'models', 'foo'],
refs=[],
sources=[],
depends_on=DependsOn(),
description='The foo model',
database='test_db',
schema='test_schema',
alias='bar',
tags=[],
meta={},
config=NodeConfig(),
patch_path='/path/to/schema.yml',
columns={
'a': ColumnInfo(name='a', description='a text field', meta={})
},
docs=Docs(),
)
self.assert_symmetric(expected, expected_dict) # sanity check
self.assertEqual(initial, expected)
self.assert_symmetric(initial, expected_dict)
def test__prepend_ctes__already_has_cte(self):
ephemeral_config = self.model_config.replace(materialized='ephemeral')
manifest = Manifest(
macros={},
nodes={
'model.root.view': CompiledModelNode(
name='view',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.view',
fqn=['root', 'view'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(nodes=['model.root.ephemeral']),
config=self.model_config,
tags=[],
path='view.sql',
original_file_path='view.sql',
raw_sql='select * from {{ref("ephemeral")}}',
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
compiled_sql=(
'with cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'),
checksum=FileHash.from_contents(''),
),
'model.root.ephemeral': CompiledModelNode(
name='ephemeral',
database='dbt',
schema='analytics',
alias='view',
resource_type=NodeType.Model,
unique_id='model.root.ephemeral',
fqn=['root', 'ephemeral'],
package_name='root',
root_path='/usr/src/app',
refs=[],
sources=[],
depends_on=DependsOn(),
config=ephemeral_config,
tags=[],
path='ephemeral.sql',
original_file_path='ephemeral.sql',
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
checksum=FileHash.from_contents(''),
),
},
sources={},
docs={},
disabled=[],
files={},
exposures={},
selectors={},
)
compiler = dbt.compilation.Compiler(self.config)
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
self.assertEqual(result, manifest.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.compiled_sql,
('with __dbt__cte__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
'select * from __dbt__cte__ephemeral'))
self.assertEqual(
manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
False)
def make_schema_test(pkg,
test_name,
test_model,
test_kwargs,
path=None,
refs=None,
sources=None,
tags=None,
column_name=None):
kwargs = test_kwargs.copy()
ref_values = []
source_values = []
# this doesn't really have to be correct
if isinstance(test_model, ParsedSourceDefinition):
kwargs['model'] = "{{ source('" + test_model.source_name + \
"', '" + test_model.name + "') }}"
source_values.append([test_model.source_name, test_model.name])
else:
kwargs['model'] = "{{ ref('" + test_model.name + "')}}"
ref_values.append([test_model.name])
if column_name is not None:
kwargs['column_name'] = column_name
# whatever
args_name = test_model.search_name.replace(".", "_")
if column_name is not None:
args_name += '_' + column_name
node_name = f'{test_name}_{args_name}'
raw_sql = '{{ config(severity="ERROR") }}{{ test_' + \
test_name + '(**dbt_schema_test_kwargs) }}'
name_parts = test_name.split('.')
if len(name_parts) == 2:
namespace, test_name = name_parts
macro_depends = f'model.{namespace}.{test_name}'
elif len(name_parts) == 1:
namespace = None
macro_depends = f'model.dbt.{test_name}'
else:
assert False, f'invalid test name: {test_name}'
if path is None:
path = 'schema.yml'
if tags is None:
tags = ['schema']
if refs is None:
refs = []
if sources is None:
sources = []
depends_on_nodes = []
for ref in refs:
ref_values.append([ref.name])
depends_on_nodes.append(ref.unique_id)
for source in sources:
source_values.append([source.source_name, source.name])
depends_on_nodes.append(source.unique_id)
return ParsedSchemaTestNode(
raw_sql=raw_sql,
test_metadata=TestMetadata(
namespace=namespace,
name=test_name,
kwargs=kwargs,
),
database='dbt',
schema='dbt_postgres',
name=node_name,
alias=node_name,
fqn=['minimal', 'schema_test', node_name],
unique_id=f'test.{pkg}.{node_name}',
package_name=pkg,
root_path='/usr/dbt/some-project',
path=f'schema_test/{node_name}.sql',
original_file_path=f'models/{path}',
resource_type=NodeType.Test,
tags=tags,
refs=ref_values,
sources=[],
depends_on=DependsOn(macros=[macro_depends], nodes=depends_on_nodes),
column_name=column_name,
checksum=FileHash.from_contents(''),
)
