def test_no_nodes_with_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' mock_user.do_not_track = True config = mock.MagicMock() # md5 of 'test' config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6' manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[], config=config) metadata = { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, } self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, }, 'disabled': [], })
def _process_sources_for_node( manifest: Manifest, current_project: str, node: ManifestNode ): target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None for source_name, table_name in node.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, node.package_name, ) if target_source is None or isinstance(target_source, Disabled): # this folows the same pattern as refs node.config.enabled = False invalid_source_fail_unless_test( node, source_name, table_name, disabled=(isinstance(target_source, Disabled)) ) continue target_source_id = target_source.unique_id node.depends_on.nodes.append(target_source_id) manifest.update_node(node)
def test_get_resource_fqns_empty(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[]) self.assertEqual(manifest.get_resource_fqns(), {})
def test_no_nodes_with_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' mock_user.do_not_track = True config = mock.MagicMock() # md5 of 'test' config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6' manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[], config=config) metadata = { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, } self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, }, 'disabled': [], } )
def _get_compiled_model( self, manifest: Manifest, cte_id: str, extra_context: Dict[str, Any], ) -> NonSourceCompiledNode: if cte_id not in manifest.nodes: raise InternalException( f'During compilation, found a cte reference that could not be ' f'resolved: {cte_id}') cte_model = manifest.nodes[cte_id] if getattr(cte_model, 'compiled', False): assert isinstance(cte_model, tuple(COMPILED_TYPES.values())) return cast(NonSourceCompiledNode, cte_model) elif cte_model.is_ephemeral_model: # this must be some kind of parsed node that we can compile. # we know it's not a parsed source definition assert isinstance(cte_model, tuple(COMPILED_TYPES)) # update the node so node = self.compile_node(cte_model, manifest, extra_context) manifest.sync_update_node(node) return node else: raise InternalException( f'During compilation, found an uncompiled cte that ' f'was not an ephemeral model: {cte_id}')
def test_no_nodes_with_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' mock_user.do_not_track = True metadata = ManifestMetadata( project_id='098f6bcd4621d373cade4e832627b4f6') manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], metadata=metadata, files={}) self.assertEqual( manifest.writable_manifest().to_dict(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, }, 'disabled': [], 'files': {}, })
def create_manifest(self) -> Manifest: nodes: Dict[str, CompileResultNode] = {} nodes.update(self.results.nodes) nodes.update(self.results.sources) disabled = [] for value in self.results.disabled.values(): disabled.extend(value) manifest = Manifest( nodes=nodes, macros=self.results.macros, docs=self.results.docs, generated_at=datetime.utcnow(), metadata=self.root_project.get_metadata(), disabled=disabled, files=self.results.files, ) manifest.patch_nodes(self.results.patches) manifest = ParserUtils.process_sources( manifest, self.root_project.project_name ) manifest = ParserUtils.process_refs( manifest, self.root_project.project_name ) manifest = ParserUtils.process_docs( manifest, self.root_project.project_name ) return manifest
def _runtime_initialize(): with open(graph_path, "rb") as f: task.graph = Graph(graph=pickle.load(f)) with open(manifest_path) as f: loaded_manifest = json.load(f) # If I'm taking something from this experience, it's this Mashumaro # package. I spent a long time trying to build a manifest, when I only # had to call from_dict. Amazing stuff. Manifest.from_dict(loaded_manifest) task.manifest = Manifest.from_dict(loaded_manifest) # What follows is the remaining _runtime_initialize method of # GraphRunnableTask. task.job_queue = task.get_graph_queue() task._flattened_nodes = [] for uid in task.job_queue.get_selected_nodes(): if uid in task.manifest.nodes: task._flattened_nodes.append(task.manifest.nodes[uid]) elif uid in task.manifest.sources: task._flattened_nodes.append(task.manifest.sources[uid]) else: raise InternalException( f"Node selection returned {uid}, expected a node or a " f"source" ) task.num_nodes = len( [n for n in task._flattened_nodes if not n.is_ephemeral_model] )
def test_get_resource_fqns_empty(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], files={}) self.assertEqual(manifest.get_resource_fqns(), {})
def test__to_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) flat_graph = manifest.to_flat_graph() flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes', 'macros'])) self.assertEqual(flat_graph['macros'], {}) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'} for node in flat_nodes.values(): self.assertEqual(set(node), expected_keys)
def load_all(cls, project_config, all_projects): root_project = project_config macros = MacroLoader.load_all(root_project, all_projects) macros.update(OperationLoader.load_all(root_project, all_projects)) nodes = {} for loader in cls._LOADERS: nodes.update(loader.load_all(root_project, all_projects, macros)) docs = DocumentationLoader.load_all(root_project, all_projects) tests, patches = SchemaTestLoader.load_all(root_project, all_projects) manifest = Manifest(nodes=nodes, macros=macros, docs=docs, generated_at=timestring(), config=project_config) manifest.add_nodes(tests) manifest.patch_nodes(patches) patches = DocumentationLoader.get_model_doc_patches(nodes, docs) manifest.patch_nodes(patches) manifest = dbt.parser.ParserUtils.process_refs( manifest, root_project.project_name ) manifest = dbt.parser.ParserUtils.process_docs(manifest, root_project) return manifest
def test__build_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], files={}) manifest.build_flat_graph() flat_graph = manifest.flat_graph flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes'])) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) for node in flat_nodes.values(): self.assertEqual(frozenset(node), REQUIRED_PARSED_NODE_KEYS)
def add_new_refs(manifest: Manifest, config: RuntimeConfig, node: ParsedRPCNode, macros: Dict[str, Any]) -> None: """Given a new node that is not in the manifest, insert the new node into it as if it were part of regular ref processing. """ if config.args.single_threaded or flags.SINGLE_THREADED_HANDLER: manifest = manifest.deepcopy() # it's ok for macros to silently override a local project macro name manifest.macros.update(macros) for macro in macros.values(): process_macro(config, manifest, macro) manifest.add_nodes({node.unique_id: node}) process_node(config, manifest, node)
def setUp(self): dbt.flags.STRICT_MODE = True dbt.flags.WARN_ERROR = True self.maxDiff = None profile_data = { 'target': 'test', 'quoting': {}, 'outputs': { 'test': { 'type': 'redshift', 'host': 'localhost', 'schema': 'analytics', 'user': '******', 'pass': '******', 'dbname': 'test', 'port': 1, } } } root_project = { 'name': 'root', 'version': '0.1', 'profile': 'test', 'project-root': normalize('/usr/src/app'), } self.root_project_config = config_from_parts_or_dicts( project=root_project, profile=profile_data, cli_vars='{"test_schema_name": "foo"}') snowplow_project = { 'name': 'snowplow', 'version': '0.1', 'profile': 'test', 'project-root': get_abs_os_path('./dbt_modules/snowplow'), } self.snowplow_project_config = config_from_parts_or_dicts( project=snowplow_project, profile=profile_data) self.all_projects = { 'root': self.root_project_config, 'snowplow': self.snowplow_project_config } self.root_project_config.dependencies = self.all_projects self.snowplow_project_config.dependencies = self.all_projects self.patcher = mock.patch('dbt.context.providers.get_adapter') self.factory = self.patcher.start() self.parser_patcher = mock.patch('dbt.parser.base.get_adapter') self.factory_parser = self.parser_patcher.start() self.macro_manifest = Manifest.from_macros( macros={m.unique_id: m for m in generate_name_macros('root')})
def create_manifest(self): manifest = Manifest( nodes=self.nodes, macros=self.macros, docs=self.docs, generated_at=timestring(), config=self.root_project, disabled=self.disabled ) manifest.add_nodes(self.tests) manifest.patch_nodes(self.patches) manifest = ParserUtils.process_sources(manifest, self.root_project) manifest = ParserUtils.process_refs(manifest, self.root_project.project_name) manifest = ParserUtils.process_docs(manifest, self.root_project) return manifest
def test_multiple_raw_blocks(self): parser = docs.DocumentationParser( results=ParseResult.rpc(), root_project=self.root_project_config, project=self.subdir_project_config, macro_manifest=Manifest.from_macros()) file_block = self._build_file(MULTIPLE_RAW_BLOCKS, 'test_file.md') parser.parse_file(file_block) results = sorted(parser.results.docs.values(), key=lambda n: n.name) self.assertEqual(len(results), 2) for result in results: self.assertIsInstance(result, ParsedDocumentation) self.assertEqual(result.package_name, 'some_package') self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.root_path, self.subdir_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, 'test_file.md') self.assertEqual(results[0].name, 'other_doc') self.assertEqual(results[0].block_contents, '```\n {% docs %}other doc{% enddocs %}\n ```') self.assertEqual(results[1].name, 'some_doc') self.assertEqual( results[1].block_contents, '```\n {% docs %}some doc{% enddocs %}\n ```', )
def __init__( self, root_project: RuntimeConfig, all_projects: Mapping[str, Project], macro_hook: Optional[Callable[[Manifest], Any]] = None, ) -> None: self.root_project: RuntimeConfig = root_project self.all_projects: Mapping[str, Project] = all_projects self.manifest: Manifest = Manifest({}, {}, {}, {}, {}, {}, [], {}) self.manifest.metadata = root_project.get_metadata() # This is a MacroQueryStringSetter callable, which is called # later after we set the MacroManifest in the adapter. It sets # up the query headers. self.macro_hook: Callable[[Manifest], Any] if macro_hook is None: self.macro_hook = lambda m: None else: self.macro_hook = macro_hook self._perf_info = ManifestLoaderInfo( is_partial_parse_enabled=self._partial_parse_enabled() ) # State check determines whether the old_manifest and the current # manifest match well enough to do partial parsing self.manifest.state_check = self.build_manifest_state_check() # This is a saved manifest from a previous run that's used for partial parsing self.old_manifest: Optional[Manifest] = self.read_saved_manifest()
def test_load_file(self): parser = docs.DocumentationParser( results=ParseResult.rpc(), root_project=self.root_project_config, project=self.subdir_project_config, macro_manifest=Manifest.from_macros()) file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md') parser.parse_file(file_block) results = sorted(parser.results.docs.values(), key=lambda n: n.name) self.assertEqual(len(results), 2) for result in results: self.assertIsInstance(result, ParsedDocumentation) self.assertEqual(result.package_name, 'some_package') self.assertNotEqual(result.file_contents, TEST_DOCUMENTATION_FILE) self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.root_path, self.subdir_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, 'test_file.md') self.assertEqual(results[0].name, 'snowplow_sessions') self.assertEqual(results[0].file_contents, SNOWPLOW_SESSIONS_BLOCK) self.assertEqual(results[1].name, 'snowplow_sessions__session_id') self.assertEqual(results[1].file_contents, SNOWPLOW_SESSIONS_SESSION_ID_BLOCK)
def load_only_macros(self) -> Manifest: old_results = self.read_parse_results() self._load_macros(old_results, internal_manifest=None) # make a manifest with just the macros to get the context macro_manifest = Manifest.from_macros(macros=self.results.macros, files=self.results.files) return macro_manifest
def manifest(seed, source, ephemeral_model, view_model, table_model, ext_source, ext_model, union_model, ext_source_2, ext_source_other, ext_source_other_2, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing, namespaced_seed, namespace_model, namespaced_union_model): nodes = [ seed, ephemeral_model, view_model, table_model, union_model, ext_model, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing, namespaced_seed, namespace_model, namespaced_union_model ] sources = [ source, ext_source, ext_source_2, ext_source_other, ext_source_other_2 ] manifest = Manifest( nodes={n.unique_id: n for n in nodes}, sources={s.unique_id: s for s in sources}, macros={}, docs={}, files={}, exposures={}, disabled=[], selectors={}, ) return manifest
def setUp(self): x_depends_on = mock.MagicMock() y_depends_on = mock.MagicMock() x_uid = 'model.project.x' y_uid = 'model.otherproject.y' src_uid = 'source.thirdproject.src.tbl' remote_docref = mock.MagicMock(documentation_package='otherproject', documentation_name='my_doc', column_name=None) docref = mock.MagicMock(documentation_package='', documentation_name='my_doc', column_name=None) self.x_node = mock.MagicMock( refs=[], sources=[['src', 'tbl']], docrefs=[remote_docref], unique_id=x_uid, resource_type=NodeType.Model, depends_on=x_depends_on, description='other_project: {{ doc("otherproject", "my_doc") }}', ) self.y_node = mock.MagicMock( refs=[['x']], sources=[], docrefs=[docref], unique_id=y_uid, resource_type=NodeType.Model, depends_on=y_depends_on, description='{{ doc("my_doc") }}', ) self.src_node = mock.MagicMock( resource_type=NodeType.Source, unique_id=src_uid, ) nodes = { x_uid: self.x_node, y_uid: self.y_node, src_uid: self.src_node, } docs = { 'otherproject.my_doc': mock.MagicMock(block_contents='some docs') } self.manifest = Manifest( nodes=nodes, macros={}, docs=docs, disabled=[], files={}, generated_at=mock.MagicMock() )
def sorted_ephemeral_ancestors( self, manifest: Manifest, unique_id: str ) -> Iterable[str]: """Get the ephemeral ancestors of unique_id, stopping at the first non-ephemeral node in each chain, in graph-topological order. """ to_check: Set[str] = {unique_id} ephemerals: Set[str] = set() visited: Set[str] = set() while to_check: # note that this avoids collecting unique_id itself nextval = to_check.pop() for pred in self.graph.predecessors(nextval): if pred in visited: continue visited.add(pred) node = manifest.expect(pred) if node.resource_type != NodeType.Model: continue if node.get_materialization() != 'ephemeral': # type: ignore continue # this is an ephemeral model! We have to find everything it # refs and do it all over again until we exhaust them all ephemerals.add(pred) to_check.add(pred) ephemeral_graph = self.build_subset_graph(ephemerals) # we can just topo sort this because we know there are no cycles. return nx.topological_sort(ephemeral_graph)
def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], files={}) serialized = manifest.writable_manifest().to_dict() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') self.assertEqual(serialized['docs'], {}) self.assertEqual(serialized['disabled'], []) self.assertEqual(serialized['files'], {}) parent_map = serialized['parent_map'] child_map = serialized['child_map'] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual(parent_map['model.root.sibling'], ['model.root.events']) self.assertEqual(parent_map['model.root.nested'], ['model.root.dep']) self.assertEqual(parent_map['model.root.dep'], ['model.root.events']) # order doesn't matter. self.assertEqual(set(parent_map['model.root.multi']), set(['model.root.nested', 'model.root.sibling'])) self.assertEqual( parent_map['model.root.events'], [], ) self.assertEqual( parent_map['model.snowplow.events'], [], ) self.assertEqual( child_map['model.root.sibling'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.nested'], ['model.root.multi'], ) self.assertEqual(child_map['model.root.dep'], ['model.root.nested']) self.assertEqual(child_map['model.root.multi'], []) self.assertEqual(set(child_map['model.root.events']), set(['model.root.dep', 'model.root.sibling'])) self.assertEqual(child_map['model.snowplow.events'], [])
def write_graph(self, outfile: str, manifest: Manifest): """Write the graph to a gpickle file. Before doing so, serialize and include all nodes in their corresponding graph entries. """ out_graph = self.graph.copy() for node_id in self.graph.nodes(): data = manifest.expect(node_id).to_dict(omit_none=True) out_graph.add_node(node_id, **data) nx.write_gpickle(out_graph, outfile)
def get_manifest(): path = './target/partial_parse.msgpack' if os.path.exists(path): with open(path, 'rb') as fp: manifest_mp = fp.read() manifest: Manifest = Manifest.from_msgpack(manifest_mp) return manifest else: return None
def _process_refs_for_node( manifest: Manifest, current_project: str, node: ManifestNode ): """Given a manifest and a node in that manifest, process its refs""" for ref in node.refs: target_model: Optional[Union[Disabled, ManifestNode]] = None target_model_name: str target_model_package: Optional[str] = None if len(ref) == 1: target_model_name = ref[0] elif len(ref) == 2: target_model_package, target_model_name = ref else: raise dbt.exceptions.InternalException( f'Refs should always be 1 or 2 arguments - got {len(ref)}' ) target_model = manifest.resolve_ref( target_model_name, target_model_package, current_project, node.package_name, ) if target_model is None or isinstance(target_model, Disabled): # This may raise. Even if it doesn't, we don't want to add # this node to the graph b/c there is no destination node node.config.enabled = False invalid_ref_fail_unless_test( node, target_model_name, target_model_package, disabled=(isinstance(target_model, Disabled)) ) continue target_model_id = target_model.unique_id node.depends_on.nodes.append(target_model_id) # TODO: I think this is extraneous, node should already be the same # as manifest.nodes[node.unique_id] (we're mutating node here, not # making a new one) # Q: could we stop doing this? manifest.update_node(node)
def test__to_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) flat_graph = manifest.to_flat_graph() flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes', 'macros'])) self.assertEqual(flat_graph['macros'], {}) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'} compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'} compiled_count = 0 for node in flat_nodes.values(): if node.get('compiled'): self.assertEqual(set(node), compiled_keys) compiled_count += 1 else: self.assertEqual(set(node), parsed_keys) self.assertEqual(compiled_count, 2)
def test__no_nodes(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring()) self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': None, 'user_id': None, 'send_anonymous_usage_stats': None, }, })
def test__no_nodes(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=datetime.utcnow(), disabled=[], files={}) self.assertEqual( manifest.writable_manifest().to_dict(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': {}, 'disabled': [], 'files': {}, })
def test_get_resource_fqns(self): nodes = copy.copy(self.nested_nodes) nodes['seed.root.seed'] = ParsedNode(name='seed', database='dbt', schema='analytics', alias='seed', resource_type='seed', unique_id='seed.root.seed', fqn=['root', 'seed'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='seed.csv', original_file_path='seed.csv', root_path='', raw_sql='-- csv --') manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) expect = { 'models': frozenset([ ('snowplow', 'events'), ('root', 'events'), ('root', 'dep'), ('root', 'nested'), ('root', 'sibling'), ('root', 'multi'), ]), 'seeds': frozenset([('root', 'seed')]), } resource_fqns = manifest.get_resource_fqns() self.assertEqual(resource_fqns, expect)
def _process_sources_for_exposure(manifest: Manifest, current_project: str, exposure: ParsedExposure): target_source: Optional[Union[Disabled, ParsedSourceDefinition]] = None for source_name, table_name in exposure.sources: target_source = manifest.resolve_source( source_name, table_name, current_project, exposure.package_name, ) if target_source is None or isinstance(target_source, Disabled): invalid_source_fail_unless_test(exposure, source_name, table_name, disabled=(isinstance( target_source, Disabled))) continue target_source_id = target_source.unique_id exposure.depends_on.nodes.append(target_source_id) manifest.update_exposure(exposure)
def load(self, internal_manifest=None): self._load_macros(internal_manifest=internal_manifest) # make a manifest with just the macros to get the context self.macro_manifest = Manifest(macros=self.macros, nodes={}, docs={}, generated_at=timestring(), disabled=[]) self._load_nodes() self._load_docs() self._load_schema_tests()
def test__no_nodes(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[]) self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': None, 'user_id': None, 'send_anonymous_usage_stats': None, }, 'disabled': [], } )
def test_get_resource_fqns(self): nodes = copy.copy(self.nested_nodes) nodes['seed.root.seed'] = ParsedNode( name='seed', database='dbt', schema='analytics', alias='seed', resource_type='seed', unique_id='seed.root.seed', fqn=['root', 'seed'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='seed.csv', original_file_path='seed.csv', root_path='', raw_sql='-- csv --' ) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) expect = { 'models': frozenset([ ('snowplow', 'events'), ('root', 'events'), ('root', 'dep'), ('root', 'nested'), ('root', 'sibling'), ('root', 'multi'), ]), 'seeds': frozenset([('root', 'seed')]), } resource_fqns = manifest.get_resource_fqns() self.assertEqual(resource_fqns, expect)
def test_get_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' mock_user.do_not_track = True config = mock.MagicMock() # md5 of 'test' config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6' self.assertEqual( Manifest.get_metadata(config), { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, } )
def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) serialized = manifest.serialize() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') self.assertEqual(serialized['disabled'], []) parent_map = serialized['parent_map'] child_map = serialized['child_map'] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual( parent_map['model.root.sibling'], ['model.root.events'] ) self.assertEqual( parent_map['model.root.nested'], ['model.root.dep'] ) self.assertEqual( parent_map['model.root.dep'], ['model.root.events'] ) # order doesn't matter. self.assertEqual( set(parent_map['model.root.multi']), set(['model.root.nested', 'model.root.sibling']) ) self.assertEqual( parent_map['model.root.events'], [], ) self.assertEqual( parent_map['model.snowplow.events'], [], ) self.assertEqual( child_map['model.root.sibling'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.nested'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.dep'], ['model.root.nested'] ) self.assertEqual( child_map['model.root.multi'], [] ) self.assertEqual( set(child_map['model.root.events']), set(['model.root.dep', 'model.root.sibling']) ) self.assertEqual( child_map['model.snowplow.events'], [] )