def test_no_nodes_with_metadata(self, mock_user): mock_user.id = 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf' mock_user.do_not_track = True config = mock.MagicMock() # md5 of 'test' config.hashed_name.return_value = '098f6bcd4621d373cade4e832627b4f6' manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[], config=config) metadata = { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, } self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': '098f6bcd4621d373cade4e832627b4f6', 'user_id': 'cfc9500f-dc7f-4c83-9ea7-2c581c1b38cf', 'send_anonymous_usage_stats': False, }, 'disabled': [], } )
def load(self, internal_manifest=None): self._load_macros(internal_manifest=internal_manifest) # make a manifest with just the macros to get the context self.macro_manifest = Manifest(macros=self.macros, nodes={}, docs={}, generated_at=timestring(), disabled=[]) self._load_nodes() self._load_docs() self._load_schema_tests()
def test__to_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) flat_graph = manifest.to_flat_graph() flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes', 'macros'])) self.assertEqual(flat_graph['macros'], {}) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) expected_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'} for node in flat_nodes.values(): self.assertEqual(set(node), expected_keys)
def create_manifest(self): manifest = Manifest( nodes=self.nodes, macros=self.macros, docs=self.docs, generated_at=timestring(), config=self.root_project, disabled=self.disabled ) manifest.add_nodes(self.tests) manifest.patch_nodes(self.patches) manifest = ParserUtils.process_sources(manifest, self.root_project) manifest = ParserUtils.process_refs(manifest, self.root_project.project_name) manifest = ParserUtils.process_docs(manifest, self.root_project) return manifest
def test__to_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) flat_graph = manifest.to_flat_graph() flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes', 'macros'])) self.assertEqual(flat_graph['macros'], {}) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'} compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'} compiled_count = 0 for node in flat_nodes.values(): if node.get('compiled'): self.assertEqual(set(node), compiled_keys) compiled_count += 1 else: self.assertEqual(set(node), parsed_keys) self.assertEqual(compiled_count, 2)
def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring()) serialized = manifest.serialize() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') parent_map = serialized['parent_map'] child_map = serialized['child_map'] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual(parent_map['model.root.sibling'], ['model.root.events']) self.assertEqual(parent_map['model.root.nested'], ['model.root.dep']) self.assertEqual(parent_map['model.root.dep'], ['model.root.events']) # order doesn't matter. self.assertEqual(set(parent_map['model.root.multi']), set(['model.root.nested', 'model.root.sibling'])) self.assertEqual( parent_map['model.root.events'], [], ) self.assertEqual( parent_map['model.snowplow.events'], [], ) self.assertEqual( child_map['model.root.sibling'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.nested'], ['model.root.multi'], ) self.assertEqual(child_map['model.root.dep'], ['model.root.nested']) self.assertEqual(child_map['model.root.multi'], []) self.assertEqual(set(child_map['model.root.events']), set(['model.root.dep', 'model.root.sibling'])) self.assertEqual(child_map['model.snowplow.events'], [])
def test__no_nodes(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[]) self.assertEqual( manifest.serialize(), { 'nodes': {}, 'macros': {}, 'parent_map': {}, 'child_map': {}, 'generated_at': '2018-02-14T09:15:13Z', 'docs': {}, 'metadata': { 'project_id': None, 'user_id': None, 'send_anonymous_usage_stats': None, }, 'disabled': [], } )
def test__to_flat_graph(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring()) flat_graph = manifest.to_flat_graph() flat_nodes = flat_graph['nodes'] self.assertEqual(set(flat_graph), set(['nodes', 'macros'])) self.assertEqual(flat_graph['macros'], {}) self.assertEqual(set(flat_nodes), set(self.nested_nodes)) parsed_keys = set(ParsedNode.SCHEMA['required']) | {'agate_table'} compiled_keys = set(CompiledNode.SCHEMA['required']) | {'agate_table'} compiled_count = 0 for node in flat_nodes.values(): if node.get('compiled'): self.assertEqual(set(node), compiled_keys) compiled_count += 1 else: self.assertEqual(set(node), parsed_keys) self.assertEqual(compiled_count, 2)
def test_get_resource_fqns(self): nodes = copy.copy(self.nested_nodes) nodes['seed.root.seed'] = ParsedNode( name='seed', database='dbt', schema='analytics', alias='seed', resource_type='seed', unique_id='seed.root.seed', fqn=['root', 'seed'], empty=False, package_name='root', refs=[['events']], sources=[], depends_on={ 'nodes': [], 'macros': [] }, config=self.model_config, tags=[], path='seed.csv', original_file_path='seed.csv', root_path='', raw_sql='-- csv --' ) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) expect = { 'models': frozenset([ ('snowplow', 'events'), ('root', 'events'), ('root', 'dep'), ('root', 'nested'), ('root', 'sibling'), ('root', 'multi'), ]), 'seeds': frozenset([('root', 'seed')]), } resource_fqns = manifest.get_resource_fqns() self.assertEqual(resource_fqns, expect)
def load_all(cls, project_obj, all_projects): root_project = project_obj.cfg macros = MacroLoader.load_all(root_project, all_projects) macros.update(OperationLoader.load_all(root_project, all_projects)) nodes = {} for loader in cls._LOADERS: nodes.update(loader.load_all(root_project, all_projects, macros)) docs = DocumentationLoader.load_all(root_project, all_projects) tests, patches = SchemaTestLoader.load_all(root_project, all_projects) manifest = Manifest(nodes=nodes, macros=macros, docs=docs, generated_at=timestring(), project=project_obj) manifest.add_nodes(tests) manifest.patch_nodes(patches) manifest = dbt.parser.ParserUtils.process_refs( manifest, root_project.get('name')) manifest = dbt.parser.ParserUtils.process_docs(manifest, root_project) return manifest
def test_get_resource_fqns_empty(self): manifest = Manifest(nodes={}, macros={}, docs={}, generated_at=timestring(), disabled=[]) self.assertEqual(manifest.get_resource_fqns(), {})
def begin(self): self.set('started_at', timestring())
def end(self): self.set('completed_at', timestring())
def test__nested_nodes(self): nodes = copy.copy(self.nested_nodes) manifest = Manifest(nodes=nodes, macros={}, docs={}, generated_at=timestring(), disabled=[]) serialized = manifest.serialize() self.assertEqual(serialized['generated_at'], '2018-02-14T09:15:13Z') self.assertEqual(serialized['disabled'], []) parent_map = serialized['parent_map'] child_map = serialized['child_map'] # make sure there aren't any extra/missing keys. self.assertEqual(set(parent_map), set(nodes)) self.assertEqual(set(child_map), set(nodes)) self.assertEqual( parent_map['model.root.sibling'], ['model.root.events'] ) self.assertEqual( parent_map['model.root.nested'], ['model.root.dep'] ) self.assertEqual( parent_map['model.root.dep'], ['model.root.events'] ) # order doesn't matter. self.assertEqual( set(parent_map['model.root.multi']), set(['model.root.nested', 'model.root.sibling']) ) self.assertEqual( parent_map['model.root.events'], [], ) self.assertEqual( parent_map['model.snowplow.events'], [], ) self.assertEqual( child_map['model.root.sibling'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.nested'], ['model.root.multi'], ) self.assertEqual( child_map['model.root.dep'], ['model.root.nested'] ) self.assertEqual( child_map['model.root.multi'], [] ) self.assertEqual( set(child_map['model.root.events']), set(['model.root.dep', 'model.root.sibling']) ) self.assertEqual( child_map['model.snowplow.events'], [] )