def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
config = mock.MagicMock()
# md5 of 'test'
config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6'
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring(),
disabled=[],
config=config)
metadata = {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
}
self.assertEqual(
manifest.serialize(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
})
def _process_sources_for_node(
manifest: Manifest, current_project: str, node: ManifestNode
):
target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None
for source_name, table_name in node.sources:
target_source = manifest.resolve_source(
source_name,
table_name,
current_project,
node.package_name,
)
if target_source is None or isinstance(target_source, Disabled):
# this folows the same pattern as refs
node.config.enabled = False
invalid_source_fail_unless_test(
node,
source_name,
table_name,
disabled=(isinstance(target_source, Disabled))
)
continue
target_source_id = target_source.unique_id
node.depends_on.nodes.append(target_source_id)
manifest.update_node(node)
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring(),
disabled=[])
self.assertEqual(manifest.get_resource_fqns(), {})
def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
config = mock.MagicMock()
# md5 of 'test'
config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6'
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[],
config=config)
metadata = {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
}
self.assertEqual(
manifest.serialize(),
{
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
}
)
def _get_compiled_model(
self,
manifest: Manifest,
cte_id: str,
extra_context: Dict[str, Any],
) -> NonSourceCompiledNode:
if cte_id not in manifest.nodes:
raise InternalException(
f'During compilation, found a cte reference that could not be '
f'resolved: {cte_id}')
cte_model = manifest.nodes[cte_id]
if getattr(cte_model, 'compiled', False):
assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
return cast(NonSourceCompiledNode, cte_model)
elif cte_model.is_ephemeral_model:
# this must be some kind of parsed node that we can compile.
# we know it's not a parsed source definition
assert isinstance(cte_model, tuple(COMPILED_TYPES))
# update the node so
node = self.compile_node(cte_model, manifest, extra_context)
manifest.sync_update_node(node)
return node
else:
raise InternalException(
f'During compilation, found an uncompiled cte that '
f'was not an ephemeral model: {cte_id}')
def test_no_nodes_with_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
metadata = ManifestMetadata(
project_id='098f6bcd4621d373cade4e832627b4f6')
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
metadata=metadata,
files={})
self.assertEqual(
manifest.writable_manifest().to_dict(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
},
'disabled': [],
'files': {},
})
def create_manifest(self) -> Manifest:
nodes: Dict[str, CompileResultNode] = {}
nodes.update(self.results.nodes)
nodes.update(self.results.sources)
disabled = []
for value in self.results.disabled.values():
disabled.extend(value)
manifest = Manifest(
nodes=nodes,
macros=self.results.macros,
docs=self.results.docs,
generated_at=datetime.utcnow(),
metadata=self.root_project.get_metadata(),
disabled=disabled,
files=self.results.files,
)
manifest.patch_nodes(self.results.patches)
manifest = ParserUtils.process_sources(
manifest, self.root_project.project_name
)
manifest = ParserUtils.process_refs(
manifest, self.root_project.project_name
)
manifest = ParserUtils.process_docs(
manifest, self.root_project.project_name
)
return manifest
def _runtime_initialize():
with open(graph_path, "rb") as f:
task.graph = Graph(graph=pickle.load(f))
with open(manifest_path) as f:
loaded_manifest = json.load(f)
# If I'm taking something from this experience, it's this Mashumaro
# package. I spent a long time trying to build a manifest, when I only
# had to call from_dict. Amazing stuff.
Manifest.from_dict(loaded_manifest)
task.manifest = Manifest.from_dict(loaded_manifest)
# What follows is the remaining _runtime_initialize method of
# GraphRunnableTask.
task.job_queue = task.get_graph_queue()
task._flattened_nodes = []
for uid in task.job_queue.get_selected_nodes():
if uid in task.manifest.nodes:
task._flattened_nodes.append(task.manifest.nodes[uid])
elif uid in task.manifest.sources:
task._flattened_nodes.append(task.manifest.sources[uid])
else:
raise InternalException(
f"Node selection returned {uid}, expected a node or a "
f"source"
)
task.num_nodes = len(
[n for n in task._flattened_nodes if not n.is_ephemeral_model]
)
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
self.assertEqual(manifest.get_resource_fqns(), {})
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
for node in flat_nodes.values():
self.assertEqual(set(node), expected_keys)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
for node in flat_nodes.values():
self.assertEqual(set(node), expected_keys)
def load_all(cls, project_config, all_projects):
root_project = project_config
macros = MacroLoader.load_all(root_project, all_projects)
macros.update(OperationLoader.load_all(root_project, all_projects))
nodes = {}
for loader in cls._LOADERS:
nodes.update(loader.load_all(root_project, all_projects, macros))
docs = DocumentationLoader.load_all(root_project, all_projects)
tests, patches = SchemaTestLoader.load_all(root_project, all_projects)
manifest = Manifest(nodes=nodes, macros=macros, docs=docs,
generated_at=timestring(), config=project_config)
manifest.add_nodes(tests)
manifest.patch_nodes(patches)
patches = DocumentationLoader.get_model_doc_patches(nodes, docs)
manifest.patch_nodes(patches)
manifest = dbt.parser.ParserUtils.process_refs(
manifest,
root_project.project_name
)
manifest = dbt.parser.ParserUtils.process_docs(manifest, root_project)
return manifest
def test__build_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
manifest.build_flat_graph()
flat_graph = manifest.flat_graph
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes']))
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
for node in flat_nodes.values():
self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS)
def add_new_refs(manifest: Manifest, config: RuntimeConfig,
node: ParsedRPCNode, macros: Dict[str, Any]) -> None:
"""Given a new node that is not in the manifest, insert the new node
into it as if it were part of regular ref processing.
"""
if config.args.single_threaded or flags.SINGLE_THREADED_HANDLER:
manifest = manifest.deepcopy()
# it's ok for macros to silently override a local project macro name
manifest.macros.update(macros)
for macro in macros.values():
process_macro(config, manifest, macro)
manifest.add_nodes({node.unique_id: node})
process_node(config, manifest, node)
def setUp(self):
dbt.flags.STRICT_MODE = True
dbt.flags.WARN_ERROR = True
self.maxDiff = None
profile_data = {
'target': 'test',
'quoting': {},
'outputs': {
'test': {
'type': 'redshift',
'host': 'localhost',
'schema': 'analytics',
'user': '******',
'pass': '******',
'dbname': 'test',
'port': 1,
}
}
}
root_project = {
'name': 'root',
'version': '0.1',
'profile': 'test',
'project-root': normalize('/usr/src/app'),
}
self.root_project_config = config_from_parts_or_dicts(
project=root_project,
profile=profile_data,
cli_vars='{"test_schema_name": "foo"}')
snowplow_project = {
'name': 'snowplow',
'version': '0.1',
'profile': 'test',
'project-root': get_abs_os_path('./dbt_modules/snowplow'),
}
self.snowplow_project_config = config_from_parts_or_dicts(
project=snowplow_project, profile=profile_data)
self.all_projects = {
'root': self.root_project_config,
'snowplow': self.snowplow_project_config
}
self.root_project_config.dependencies = self.all_projects
self.snowplow_project_config.dependencies = self.all_projects
self.patcher = mock.patch('dbt.context.providers.get_adapter')
self.factory = self.patcher.start()
self.parser_patcher = mock.patch('dbt.parser.base.get_adapter')
self.factory_parser = self.parser_patcher.start()
self.macro_manifest = Manifest.from_macros(
macros={m.unique_id: m
for m in generate_name_macros('root')})
def create_manifest(self):
manifest = Manifest(
nodes=self.nodes,
macros=self.macros,
docs=self.docs,
generated_at=timestring(),
config=self.root_project,
disabled=self.disabled
)
manifest.add_nodes(self.tests)
manifest.patch_nodes(self.patches)
manifest = ParserUtils.process_sources(manifest, self.root_project)
manifest = ParserUtils.process_refs(manifest,
self.root_project.project_name)
manifest = ParserUtils.process_docs(manifest, self.root_project)
return manifest
def test_multiple_raw_blocks(self):
parser = docs.DocumentationParser(
results=ParseResult.rpc(),
root_project=self.root_project_config,
project=self.subdir_project_config,
macro_manifest=Manifest.from_macros())
file_block = self._build_file(MULTIPLE_RAW_BLOCKS, 'test_file.md')
parser.parse_file(file_block)
results = sorted(parser.results.docs.values(), key=lambda n: n.name)
self.assertEqual(len(results), 2)
for result in results:
self.assertIsInstance(result, ParsedDocumentation)
self.assertEqual(result.package_name, 'some_package')
self.assertEqual(result.original_file_path, self.testfile_path)
self.assertEqual(result.root_path, self.subdir_path)
self.assertEqual(result.resource_type, NodeType.Documentation)
self.assertEqual(result.path, 'test_file.md')
self.assertEqual(results[0].name, 'other_doc')
self.assertEqual(results[0].block_contents,
'```\n {% docs %}other doc{% enddocs %}\n ```')
self.assertEqual(results[1].name, 'some_doc')
self.assertEqual(
results[1].block_contents,
'```\n {% docs %}some doc{% enddocs %}\n ```',
)
def __init__(
self,
root_project: RuntimeConfig,
all_projects: Mapping[str, Project],
macro_hook: Optional[Callable[[Manifest], Any]] = None,
) -> None:
self.root_project: RuntimeConfig = root_project
self.all_projects: Mapping[str, Project] = all_projects
self.manifest: Manifest = Manifest({}, {}, {}, {}, {}, {}, [], {})
self.manifest.metadata = root_project.get_metadata()
# This is a MacroQueryStringSetter callable, which is called
# later after we set the MacroManifest in the adapter. It sets
# up the query headers.
self.macro_hook: Callable[[Manifest], Any]
if macro_hook is None:
self.macro_hook = lambda m: None
else:
self.macro_hook = macro_hook
self._perf_info = ManifestLoaderInfo(
is_partial_parse_enabled=self._partial_parse_enabled()
)
# State check determines whether the old_manifest and the current
# manifest match well enough to do partial parsing
self.manifest.state_check = self.build_manifest_state_check()
# This is a saved manifest from a previous run that's used for partial parsing
self.old_manifest: Optional[Manifest] = self.read_saved_manifest()
def test_load_file(self):
parser = docs.DocumentationParser(
results=ParseResult.rpc(),
root_project=self.root_project_config,
project=self.subdir_project_config,
macro_manifest=Manifest.from_macros())
file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md')
parser.parse_file(file_block)
results = sorted(parser.results.docs.values(), key=lambda n: n.name)
self.assertEqual(len(results), 2)
for result in results:
self.assertIsInstance(result, ParsedDocumentation)
self.assertEqual(result.package_name, 'some_package')
self.assertNotEqual(result.file_contents, TEST_DOCUMENTATION_FILE)
self.assertEqual(result.original_file_path, self.testfile_path)
self.assertEqual(result.root_path, self.subdir_path)
self.assertEqual(result.resource_type, NodeType.Documentation)
self.assertEqual(result.path, 'test_file.md')
self.assertEqual(results[0].name, 'snowplow_sessions')
self.assertEqual(results[0].file_contents, SNOWPLOW_SESSIONS_BLOCK)
self.assertEqual(results[1].name, 'snowplow_sessions__session_id')
self.assertEqual(results[1].file_contents,
SNOWPLOW_SESSIONS_SESSION_ID_BLOCK)
def load_only_macros(self) -> Manifest:
old_results = self.read_parse_results()
self._load_macros(old_results, internal_manifest=None)
# make a manifest with just the macros to get the context
macro_manifest = Manifest.from_macros(macros=self.results.macros,
files=self.results.files)
return macro_manifest
def manifest(seed, source, ephemeral_model, view_model, table_model,
ext_source, ext_model, union_model, ext_source_2,
ext_source_other, ext_source_other_2, table_id_unique,
table_id_not_null, view_id_unique, ext_source_id_unique,
view_test_nothing, namespaced_seed, namespace_model,
namespaced_union_model):
nodes = [
seed, ephemeral_model, view_model, table_model, union_model, ext_model,
table_id_unique, table_id_not_null, view_id_unique,
ext_source_id_unique, view_test_nothing, namespaced_seed,
namespace_model, namespaced_union_model
]
sources = [
source, ext_source, ext_source_2, ext_source_other, ext_source_other_2
]
manifest = Manifest(
nodes={n.unique_id: n
for n in nodes},
sources={s.unique_id: s
for s in sources},
macros={},
docs={},
files={},
exposures={},
disabled=[],
selectors={},
)
return manifest
def setUp(self):
x_depends_on = mock.MagicMock()
y_depends_on = mock.MagicMock()
x_uid = 'model.project.x'
y_uid = 'model.otherproject.y'
src_uid = 'source.thirdproject.src.tbl'
remote_docref = mock.MagicMock(documentation_package='otherproject', documentation_name='my_doc', column_name=None)
docref = mock.MagicMock(documentation_package='', documentation_name='my_doc', column_name=None)
self.x_node = mock.MagicMock(
refs=[], sources=[['src', 'tbl']], docrefs=[remote_docref], unique_id=x_uid,
resource_type=NodeType.Model, depends_on=x_depends_on,
description='other_project: {{ doc("otherproject", "my_doc") }}',
)
self.y_node = mock.MagicMock(
refs=[['x']], sources=[], docrefs=[docref], unique_id=y_uid,
resource_type=NodeType.Model, depends_on=y_depends_on,
description='{{ doc("my_doc") }}',
)
self.src_node = mock.MagicMock(
resource_type=NodeType.Source, unique_id=src_uid,
)
nodes = {
x_uid: self.x_node,
y_uid: self.y_node,
src_uid: self.src_node,
}
docs = {
'otherproject.my_doc': mock.MagicMock(block_contents='some docs')
}
self.manifest = Manifest(
nodes=nodes, macros={}, docs=docs, disabled=[], files={}, generated_at=mock.MagicMock()
)
def sorted_ephemeral_ancestors(
self, manifest: Manifest, unique_id: str
) -> Iterable[str]:
"""Get the ephemeral ancestors of unique_id, stopping at the first
non-ephemeral node in each chain, in graph-topological order.
"""
to_check: Set[str] = {unique_id}
ephemerals: Set[str] = set()
visited: Set[str] = set()
while to_check:
# note that this avoids collecting unique_id itself
nextval = to_check.pop()
for pred in self.graph.predecessors(nextval):
if pred in visited:
continue
visited.add(pred)
node = manifest.expect(pred)
if node.resource_type != NodeType.Model:
continue
if node.get_materialization() != 'ephemeral': # type: ignore
continue
# this is an ephemeral model! We have to find everything it
# refs and do it all over again until we exhaust them all
ephemerals.add(pred)
to_check.add(pred)
ephemeral_graph = self.build_subset_graph(ephemerals)
# we can just topo sort this because we know there are no cycles.
return nx.topological_sort(ephemeral_graph)
def test__nested_nodes(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
serialized = manifest.writable_manifest().to_dict()
self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
self.assertEqual(serialized['docs'], {})
self.assertEqual(serialized['disabled'], [])
self.assertEqual(serialized['files'], {})
parent_map = serialized['parent_map']
child_map = serialized['child_map']
# make sure there aren't any extra/missing keys.
self.assertEqual(set(parent_map), set(nodes))
self.assertEqual(set(child_map), set(nodes))
self.assertEqual(parent_map['model.root.sibling'],
['model.root.events'])
self.assertEqual(parent_map['model.root.nested'], ['model.root.dep'])
self.assertEqual(parent_map['model.root.dep'], ['model.root.events'])
# order doesn't matter.
self.assertEqual(set(parent_map['model.root.multi']),
set(['model.root.nested', 'model.root.sibling']))
self.assertEqual(
parent_map['model.root.events'],
[],
)
self.assertEqual(
parent_map['model.snowplow.events'],
[],
)
self.assertEqual(
child_map['model.root.sibling'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.nested'],
['model.root.multi'],
)
self.assertEqual(child_map['model.root.dep'], ['model.root.nested'])
self.assertEqual(child_map['model.root.multi'], [])
self.assertEqual(set(child_map['model.root.events']),
set(['model.root.dep', 'model.root.sibling']))
self.assertEqual(child_map['model.snowplow.events'], [])
def write_graph(self, outfile: str, manifest: Manifest):
"""Write the graph to a gpickle file. Before doing so, serialize and
include all nodes in their corresponding graph entries.
"""
out_graph = self.graph.copy()
for node_id in self.graph.nodes():
data = manifest.expect(node_id).to_dict(omit_none=True)
out_graph.add_node(node_id, **data)
nx.write_gpickle(out_graph, outfile)
def get_manifest():
path = './target/partial_parse.msgpack'
if os.path.exists(path):
with open(path, 'rb') as fp:
manifest_mp = fp.read()
manifest: Manifest = Manifest.from_msgpack(manifest_mp)
return manifest
else:
return None
def _process_refs_for_node(
manifest: Manifest, current_project: str, node: ManifestNode
):
"""Given a manifest and a node in that manifest, process its refs"""
for ref in node.refs:
target_model: Optional[Union[Disabled, ManifestNode]] = None
target_model_name: str
target_model_package: Optional[str] = None
if len(ref) == 1:
target_model_name = ref[0]
elif len(ref) == 2:
target_model_package, target_model_name = ref
else:
raise dbt.exceptions.InternalException(
f'Refs should always be 1 or 2 arguments - got {len(ref)}'
)
target_model = manifest.resolve_ref(
target_model_name,
target_model_package,
current_project,
node.package_name,
)
if target_model is None or isinstance(target_model, Disabled):
# This may raise. Even if it doesn't, we don't want to add
# this node to the graph b/c there is no destination node
node.config.enabled = False
invalid_ref_fail_unless_test(
node, target_model_name, target_model_package,
disabled=(isinstance(target_model, Disabled))
)
continue
target_model_id = target_model.unique_id
node.depends_on.nodes.append(target_model_id)
# TODO: I think this is extraneous, node should already be the same
# as manifest.nodes[node.unique_id] (we're mutating node here, not
# making a new one)
# Q: could we stop doing this?
manifest.update_node(node)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'}
compiled_count = 0
for node in flat_nodes.values():
if node.get('compiled'):
self.assertEqual(set(node), compiled_keys)
compiled_count += 1
else:
self.assertEqual(set(node), parsed_keys)
self.assertEqual(compiled_count, 2)
def test__no_nodes(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=timestring())
self.assertEqual(
manifest.serialize(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': None,
'user_id': None,
'send_anonymous_usage_stats': None,
},
})
def test__no_nodes(self):
manifest = Manifest(nodes={},
macros={},
docs={},
generated_at=datetime.utcnow(),
disabled=[],
files={})
self.assertEqual(
manifest.writable_manifest().to_dict(), {
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {},
'disabled': [],
'files': {},
})
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedNode(name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --')
manifest = Manifest(nodes=nodes,
macros={},
docs={},
generated_at=timestring(),
disabled=[])
expect = {
'models':
frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds':
frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def test__to_flat_graph(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
flat_graph = manifest.to_flat_graph()
flat_nodes = flat_graph['nodes']
self.assertEqual(set(flat_graph), set(['nodes', 'macros']))
self.assertEqual(flat_graph['macros'], {})
self.assertEqual(set(flat_nodes), set(self.nested_nodes))
parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'}
compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'}
compiled_count = 0
for node in flat_nodes.values():
if node.get('compiled'):
self.assertEqual(set(node), compiled_keys)
compiled_count += 1
else:
self.assertEqual(set(node), parsed_keys)
self.assertEqual(compiled_count, 2)
def _process_sources_for_exposure(manifest: Manifest, current_project: str,
exposure: ParsedExposure):
target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None
for source_name, table_name in exposure.sources:
target_source = manifest.resolve_source(
source_name,
table_name,
current_project,
exposure.package_name,
)
if target_source is None or isinstance(target_source, Disabled):
invalid_source_fail_unless_test(exposure,
source_name,
table_name,
disabled=(isinstance(
target_source, Disabled)))
continue
target_source_id = target_source.unique_id
exposure.depends_on.nodes.append(target_source_id)
manifest.update_exposure(exposure)
def load(self, internal_manifest=None):
self._load_macros(internal_manifest=internal_manifest)
# make a manifest with just the macros to get the context
self.macro_manifest = Manifest(macros=self.macros,
nodes={},
docs={},
generated_at=timestring(),
disabled=[])
self._load_nodes()
self._load_docs()
self._load_schema_tests()
def test__no_nodes(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(
manifest.serialize(),
{
'nodes': {},
'macros': {},
'parent_map': {},
'child_map': {},
'generated_at': '2018-02-14T09:15:13Z',
'docs': {},
'metadata': {
'project_id': None,
'user_id': None,
'send_anonymous_usage_stats': None,
},
'disabled': [],
}
)
def test_get_resource_fqns(self):
nodes = copy.copy(self.nested_nodes)
nodes['seed.root.seed'] = ParsedNode(
name='seed',
database='dbt',
schema='analytics',
alias='seed',
resource_type='seed',
unique_id='seed.root.seed',
fqn=['root', 'seed'],
empty=False,
package_name='root',
refs=[['events']],
sources=[],
depends_on={
'nodes': [],
'macros': []
},
config=self.model_config,
tags=[],
path='seed.csv',
original_file_path='seed.csv',
root_path='',
raw_sql='-- csv --'
)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
expect = {
'models': frozenset([
('snowplow', 'events'),
('root', 'events'),
('root', 'dep'),
('root', 'nested'),
('root', 'sibling'),
('root', 'multi'),
]),
'seeds': frozenset([('root', 'seed')]),
}
resource_fqns = manifest.get_resource_fqns()
self.assertEqual(resource_fqns, expect)
def test_get_metadata(self, mock_user):
mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf'
mock_user.do_not_track = True
config = mock.MagicMock()
# md5 of 'test'
config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6'
self.assertEqual(
Manifest.get_metadata(config),
{
'project_id': '098f6bcd4621d373cade4e832627b4f6',
'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf',
'send_anonymous_usage_stats': False,
}
)
def test_get_resource_fqns_empty(self):
manifest = Manifest(nodes={}, macros={}, docs={},
generated_at=timestring(), disabled=[])
self.assertEqual(manifest.get_resource_fqns(), {})
def test__nested_nodes(self):
nodes = copy.copy(self.nested_nodes)
manifest = Manifest(nodes=nodes, macros={}, docs={},
generated_at=timestring(), disabled=[])
serialized = manifest.serialize()
self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z')
self.assertEqual(serialized['disabled'], [])
parent_map = serialized['parent_map']
child_map = serialized['child_map']
# make sure there aren't any extra/missing keys.
self.assertEqual(set(parent_map), set(nodes))
self.assertEqual(set(child_map), set(nodes))
self.assertEqual(
parent_map['model.root.sibling'],
['model.root.events']
)
self.assertEqual(
parent_map['model.root.nested'],
['model.root.dep']
)
self.assertEqual(
parent_map['model.root.dep'],
['model.root.events']
)
# order doesn't matter.
self.assertEqual(
set(parent_map['model.root.multi']),
set(['model.root.nested', 'model.root.sibling'])
)
self.assertEqual(
parent_map['model.root.events'],
[],
)
self.assertEqual(
parent_map['model.snowplow.events'],
[],
)
self.assertEqual(
child_map['model.root.sibling'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.nested'],
['model.root.multi'],
)
self.assertEqual(
child_map['model.root.dep'],
['model.root.nested']
)
self.assertEqual(
child_map['model.root.multi'],
[]
)
self.assertEqual(
set(child_map['model.root.events']),
set(['model.root.dep', 'model.root.sibling'])
)
self.assertEqual(
child_map['model.snowplow.events'],
[]
)
