def setUp(self):
self.model = ParsedNode(alias='model_one',
name='model_one',
schema='analytics',
resource_type='model',
unique_id='model.root.model_one',
fqn=['root', 'model_one'],
empty=False,
package_name='root',
original_file_path='model_one.sql',
root_path='/usr/src/app',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config={
'enabled': True,
'materialized': 'view',
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
'tags': [],
},
tags=[],
path='model_one.sql',
raw_sql='',
description='',
columns={})
self.context = mock.MagicMock()
def as_node_list(self, selected_nodes, ephemeral_only=False):
dependency_list = self.linker.as_dependency_list(
selected_nodes, ephemeral_only=ephemeral_only)
concurrent_dependency_list = []
for level in dependency_list:
node_level = [
ParsedNode(**self.linker.get_node(node)) for node in level
]
concurrent_dependency_list.append(node_level)
return concurrent_dependency_list
def parse_node(self,
node,
node_path,
package_project_config,
tags=None,
fqn_extra=None,
fqn=None,
agate_table=None,
archive_config=None,
column_name=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
archive_config should be set if the node is an Archive node.
column_name should be set if the node is a Test node associated with a
particular column.
"""
logger.debug("Parsing {}".format(node_path))
tags = coalesce(tags, [])
fqn_extra = coalesce(fqn_extra, [])
if fqn is None:
fqn = self.get_fqn(node.path, package_project_config, fqn_extra)
config = SourceConfig(self.root_project_config, package_project_config,
fqn, node.resource_type)
parsed_dict = self._build_intermediate_node_dict(
config, node.serialize(), node_path, config, tags, fqn,
agate_table, archive_config, column_name)
parsed_node = ParsedNode(**parsed_dict)
self._render_with_context(parsed_node, config)
self._update_parsed_node_info(parsed_node, config)
parsed_node.validate()
return parsed_node
def parse_node(self, node, node_path, package_project_config, tags=None,
fqn_extra=None, fqn=None, agate_table=None,
archive_config=None, column_name=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
archive_config should be set if the node is an Archive node.
column_name should be set if the node is a Test node associated with a
particular column.
"""
logger.debug("Parsing {}".format(node_path))
tags = coalesce(tags, [])
fqn_extra = coalesce(fqn_extra, [])
if fqn is None:
fqn = self.get_fqn(node.path, package_project_config, fqn_extra)
config = SourceConfig(
self.root_project_config,
package_project_config,
fqn,
node.resource_type)
parsed_dict = self._build_intermediate_node_dict(
config, node.serialize(), node_path, config, tags, fqn,
agate_table, archive_config, column_name
)
parsed_node = ParsedNode(**parsed_dict)
self._render_with_context(parsed_node, config)
self._update_parsed_node_info(parsed_node, config)
parsed_node.validate()
return parsed_node
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedNode(name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --')
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=timestring(),
disabled=[])
expect = {
'models':
frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds':
frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def setUp(self):
dbt.flags.STRICT_MODE = True
self.maxDiff = None
self.model_config = {
'enabled': True,
'materialized': 'view',
'post-hook': [],
'pre-hook': [],
'vars': {},
'quoting': {},
'column_types': {},
}
self.nested_nodes = {
'model.snowplow.events': ParsedNode(
name='events',
schema='analytics',
alias='events',
resource_type='model',
unique_id='model.snowplow.events',
fqn=['snowplow', 'events'],
empty=False,
package_name='snowplow',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'
),
'model.root.events': ParsedNode(
name='events',
schema='analytics',
alias='events',
resource_type='model',
unique_id='model.root.events',
fqn=['root', 'events'],
empty=False,
package_name='root',
refs=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='events.sql',
original_file_path='events.sql',
root_path='',
raw_sql='does not matter'
),
'model.root.dep': ParsedNode(
name='dep',
schema='analytics',
alias='dep',
resource_type='model',
unique_id='model.root.dep',
fqn=['root', 'dep'],
empty=False,
package_name='root',
refs=[['events']],
depends_on={
'nodes': ['model.root.events'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'
),
'model.root.nested': ParsedNode(
name='nested',
schema='analytics',
alias='nested',
resource_type='model',
unique_id='model.root.nested',
fqn=['root', 'nested'],
empty=False,
package_name='root',
refs=[['events']],
depends_on={
'nodes': ['model.root.dep'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'
),
'model.root.sibling': ParsedNode(
name='sibling',
schema='analytics',
alias='sibling',
resource_type='model',
unique_id='model.root.sibling',
fqn=['root', 'sibling'],
empty=False,
package_name='root',
refs=[['events']],
depends_on={
'nodes': ['model.root.events'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'
),
'model.root.multi': ParsedNode(
name='multi',
schema='analytics',
alias='multi',
resource_type='model',
unique_id='model.root.multi',
fqn=['root', 'multi'],
empty=False,
package_name='root',
refs=[['events']],
depends_on={
'nodes': ['model.root.nested', 'model.root.sibling'],
'macros': []
},
config=self.model_config,
tags=[],
path='multi.sql',
original_file_path='multi.sql',
root_path='',
raw_sql='does not matter'
),
}
def parse_node(cls,
node,
node_path,
root_project_config,
package_project_config,
all_projects,
tags=None,
fqn_extra=None,
fqn=None,
macros=None,
agate_table=None,
archive_config=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
archive_config should be set if the node is an Archive node.
"""
logger.debug("Parsing {}".format(node_path))
node = node.serialize()
if agate_table is not None:
node['agate_table'] = agate_table
tags = coalesce(tags, [])
fqn_extra = coalesce(fqn_extra, [])
macros = coalesce(macros, {})
node.update({
'refs': [],
'depends_on': {
'nodes': [],
'macros': [],
}
})
if fqn is None:
fqn = cls.get_fqn(node.get('path'), package_project_config,
fqn_extra)
config = dbt.model.SourceConfig(root_project_config,
package_project_config, fqn,
node['resource_type'])
node['unique_id'] = node_path
node['empty'] = ('raw_sql' in node
and len(node['raw_sql'].strip()) == 0)
node['fqn'] = fqn
node['tags'] = tags
node['config_reference'] = config
# Set this temporarily. Not the full config yet (as config() hasn't
# been called from jinja yet). But the Var() call below needs info
# about project level configs b/c they might contain refs.
# TODO: Restructure this?
config_dict = coalesce(archive_config, {})
config_dict.update(config.config)
node['config'] = config_dict
# Set this temporarily so get_rendered() has access to a schema & alias
profile = dbt.utils.get_profile_from_project(root_project_config)
default_schema = profile.get('schema', 'public')
node['schema'] = default_schema
default_alias = node.get('name')
node['alias'] = default_alias
context = dbt.context.parser.generate(node, root_project_config,
{"macros": macros})
dbt.clients.jinja.get_rendered(node.get('raw_sql'),
context,
node,
capture_macros=True)
# Clean up any open conns opened by adapter functions that hit the db
db_wrapper = context['adapter']
adapter = db_wrapper.adapter
profile = db_wrapper.profile
adapter.release_connection(profile, node.get('name'))
# Special macro defined in the global project
schema_override = config.config.get('schema')
get_schema = context.get('generate_schema_name',
lambda x: default_schema)
node['schema'] = get_schema(schema_override)
node['alias'] = config.config.get('alias', default_alias)
# Overwrite node config
config_dict = node.get('config', {})
config_dict.update(config.config)
node['config'] = config_dict
for hook_type in dbt.hooks.ModelHookType.Both:
node['config'][hook_type] = dbt.hooks.get_hooks(node, hook_type)
del node['config_reference']
return ParsedNode(**node)
def parse_node(cls,
node,
node_path,
root_project_config,
package_project_config,
all_projects,
tags=None,
fqn_extra=None,
fqn=None,
macros=None,
agate_table=None,
archive_config=None,
column_name=None):
"""Parse a node, given an UnparsedNode and any other required information.
agate_table should be set if the node came from a seed file.
archive_config should be set if the node is an Archive node.
column_name should be set if the node is a Test node associated with a
particular column.
"""
logger.debug("Parsing {}".format(node_path))
node = node.serialize()
if agate_table is not None:
node['agate_table'] = agate_table
tags = coalesce(tags, [])
fqn_extra = coalesce(fqn_extra, [])
macros = coalesce(macros, {})
node.update({
'refs': [],
'depends_on': {
'nodes': [],
'macros': [],
}
})
if fqn is None:
fqn = cls.get_fqn(node.get('path'), package_project_config,
fqn_extra)
config = dbt.model.SourceConfig(root_project_config,
package_project_config, fqn,
node['resource_type'])
node['unique_id'] = node_path
node['empty'] = ('raw_sql' in node
and len(node['raw_sql'].strip()) == 0)
node['fqn'] = fqn
node['tags'] = tags
# Set this temporarily. Not the full config yet (as config() hasn't
# been called from jinja yet). But the Var() call below needs info
# about project level configs b/c they might contain refs.
# TODO: Restructure this?
config_dict = coalesce(archive_config, {})
config_dict.update(config.config)
node['config'] = config_dict
# Set this temporarily so get_rendered() has access to a schema & alias
default_schema = getattr(root_project_config.credentials, 'schema',
'public')
node['schema'] = default_schema
default_alias = node.get('name')
node['alias'] = default_alias
# if there's a column, it should end up part of the ParsedNode
if column_name is not None:
node['column_name'] = column_name
# make a manifest with just the macros to get the context
manifest = Manifest(macros=macros,
nodes={},
docs={},
generated_at=dbt.utils.timestring())
parsed_node = ParsedNode(**node)
context = dbt.context.parser.generate(parsed_node, root_project_config,
manifest, config)
dbt.clients.jinja.get_rendered(parsed_node.raw_sql,
context,
parsed_node.to_shallow_dict(),
capture_macros=True)
# Clean up any open conns opened by adapter functions that hit the db
db_wrapper = context['adapter']
adapter = db_wrapper.adapter
runtime_config = db_wrapper.config
adapter.release_connection(parsed_node.name)
# Special macro defined in the global project
schema_override = config.config.get('schema')
get_schema = context.get('generate_schema_name',
lambda x: default_schema)
parsed_node.schema = get_schema(schema_override)
parsed_node.alias = config.config.get('alias', default_alias)
# Overwrite node config
config_dict = parsed_node.get('config', {})
config_dict.update(config.config)
parsed_node.config = config_dict
for hook_type in dbt.hooks.ModelHookType.Both:
parsed_node.config[hook_type] = dbt.hooks.get_hooks(
parsed_node, hook_type)
parsed_node.validate()
return parsed_node