def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
config = mock.MagicMock()
# md5 of 'test'
config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6'
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[],
config=config)
metadata = {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
}
self.assertEqual(
manifest.serialize(),
{
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
}
)
def load(self, internal_manifest=None):
self._load_macros(internal_manifest=internal_manifest)
# make a manifest with just the macros to get the context
self.macro_manifest = Manifest(macros=self.macros, nodes={}, docs={},
generated_at=timestring(), disabled=[])
self._load_nodes()
self._load_docs()
self._load_schema_tests()
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
for node in flat_nodes.values():
self.assertEqual(set(node), expected_keys)
def create_manifest(self):
manifest = Manifest(
nodes=self.nodes,
macros=self.macros,
docs=self.docs,
generated_at=timestring(),
config=self.root_project,
disabled=self.disabled
)
manifest.add_nodes(self.tests)
manifest.patch_nodes(self.patches)
manifest = ParserUtils.process_sources(manifest, self.root_project)
manifest = ParserUtils.process_refs(manifest,
self.root_project.project_name)
manifest = ParserUtils.process_docs(manifest, self.root_project)
return manifest
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'}
compiled_count = 0
for node in flat_nodes.values():
if node.get('compiled'):
self.assertEqual(set(node), compiled_keys)
compiled_count += 1
else:
self.assertEqual(set(node), parsed_keys)
self.assertEqual(compiled_count, 2)
def test__nested_nodes(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=timestring())
serialized = manifest.serialize()
self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
parent_map = serialized['parent_map']
child_map = serialized['child_map']
# make sure there aren't any extra/missing keys.
self.assertEqual(set(parent_map), set(nodes))
self.assertEqual(set(child_map), set(nodes))
self.assertEqual(parent_map['model.root.sibling'],
['model.root.events'])
self.assertEqual(parent_map['model.root.nested'], ['model.root.dep'])
self.assertEqual(parent_map['model.root.dep'], ['model.root.events'])
# order doesn't matter.
self.assertEqual(set(parent_map['model.root.multi']),
set(['model.root.nested', 'model.root.sibling']))
self.assertEqual(
parent_map['model.root.events'],
[],
)
self.assertEqual(
parent_map['model.snowplow.events'],
[],
)
self.assertEqual(
child_map['model.root.sibling'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.nested'],
['model.root.multi'],
)
self.assertEqual(child_map['model.root.dep'], ['model.root.nested'])
self.assertEqual(child_map['model.root.multi'], [])
self.assertEqual(set(child_map['model.root.events']),
set(['model.root.dep', 'model.root.sibling']))
self.assertEqual(child_map['model.snowplow.events'], [])
def test__no_nodes(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(
manifest.serialize(),
{
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': None,
'user_id': None,
'send_anonymous_usage_stats': None,
},
'disabled': [],
}
)
def test__no_nodes(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(
manifest.serialize(),
{
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': None,
'user_id': None,
'send_anonymous_usage_stats': None,
},
'disabled': [],
}
)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=timestring())
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'}
compiled_count = 0
for node in flat_nodes.values():
if node.get('compiled'):
self.assertEqual(set(node), compiled_keys)
compiled_count += 1
else:
self.assertEqual(set(node), parsed_keys)
self.assertEqual(compiled_count, 2)
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedNode(
name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --'
)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
expect = {
'models': frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds': frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def load_all(cls, project_obj, all_projects):
root_project = project_obj.cfg
macros = MacroLoader.load_all(root_project, all_projects)
macros.update(OperationLoader.load_all(root_project, all_projects))
nodes = {}
for loader in cls._LOADERS:
nodes.update(loader.load_all(root_project, all_projects, macros))
docs = DocumentationLoader.load_all(root_project, all_projects)
tests, patches = SchemaTestLoader.load_all(root_project, all_projects)
manifest = Manifest(nodes=nodes,
macros=macros,
docs=docs,
generated_at=timestring(),
project=project_obj)
manifest.add_nodes(tests)
manifest.patch_nodes(patches)
manifest = dbt.parser.ParserUtils.process_refs(
manifest, root_project.get('name'))
manifest = dbt.parser.ParserUtils.process_docs(manifest, root_project)
return manifest
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(manifest.get_resource_fqns(), {})
def begin(self):
self.set('started_at', timestring())
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(manifest.get_resource_fqns(), {})
def end(self):
self.set('completed_at', timestring())
def begin(self):
self.set('started_at', timestring())
def test__nested_nodes(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
serialized = manifest.serialize()
self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
self.assertEqual(serialized['disabled'], [])
parent_map = serialized['parent_map']
child_map = serialized['child_map']
# make sure there aren't any extra/missing keys.
self.assertEqual(set(parent_map), set(nodes))
self.assertEqual(set(child_map), set(nodes))
self.assertEqual(
parent_map['model.root.sibling'],
['model.root.events']
)
self.assertEqual(
parent_map['model.root.nested'],
['model.root.dep']
)
self.assertEqual(
parent_map['model.root.dep'],
['model.root.events']
)
# order doesn't matter.
self.assertEqual(
set(parent_map['model.root.multi']),
set(['model.root.nested', 'model.root.sibling'])
)
self.assertEqual(
parent_map['model.root.events'],
[],
)
self.assertEqual(
parent_map['model.snowplow.events'],
[],
)
self.assertEqual(
child_map['model.root.sibling'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.nested'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.dep'],
['model.root.nested']
)
self.assertEqual(
child_map['model.root.multi'],
[]
)
self.assertEqual(
set(child_map['model.root.events']),
set(['model.root.dep', 'model.root.sibling'])
)
self.assertEqual(
child_map['model.snowplow.events'],
[]
)
def end(self):
self.set('completed_at', timestring())
