def test_multiple_raw_blocks(self): parser = docs.DocumentationParser( results=ParseResult.rpc(), root_project=self.root_project_config, project=self.subdir_project_config, macro_manifest=Manifest.from_macros()) file_block = self._build_file(MULTIPLE_RAW_BLOCKS, 'test_file.md') parser.parse_file(file_block) results = sorted(parser.results.docs.values(), key=lambda n: n.name) self.assertEqual(len(results), 2) for result in results: self.assertIsInstance(result, ParsedDocumentation) self.assertEqual(result.package_name, 'some_package') self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.root_path, self.subdir_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, 'test_file.md') self.assertEqual(results[0].name, 'other_doc') self.assertEqual(results[0].block_contents, '```\n {% docs %}other doc{% enddocs %}\n ```') self.assertEqual(results[1].name, 'some_doc') self.assertEqual( results[1].block_contents, '```\n {% docs %}some doc{% enddocs %}\n ```', )
def test_load_file(self): parser = docs.DocumentationParser( results=ParseResult.rpc(), root_project=self.root_project_config, project=self.subdir_project_config, macro_manifest=Manifest.from_macros()) file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md') parser.parse_file(file_block) results = sorted(parser.results.docs.values(), key=lambda n: n.name) self.assertEqual(len(results), 2) for result in results: self.assertIsInstance(result, ParsedDocumentation) self.assertEqual(result.package_name, 'some_package') self.assertNotEqual(result.file_contents, TEST_DOCUMENTATION_FILE) self.assertEqual(result.original_file_path, self.testfile_path) self.assertEqual(result.root_path, self.subdir_path) self.assertEqual(result.resource_type, NodeType.Documentation) self.assertEqual(result.path, 'test_file.md') self.assertEqual(results[0].name, 'snowplow_sessions') self.assertEqual(results[0].file_contents, SNOWPLOW_SESSIONS_BLOCK) self.assertEqual(results[1].name, 'snowplow_sessions__session_id') self.assertEqual(results[1].file_contents, SNOWPLOW_SESSIONS_SESSION_ID_BLOCK)
def load_only_macros(self) -> Manifest: old_results = self.read_parse_results() self._load_macros(old_results, internal_manifest=None) # make a manifest with just the macros to get the context macro_manifest = Manifest.from_macros(macros=self.results.macros, files=self.results.files) return macro_manifest
def setUp(self): dbt.flags.STRICT_MODE = True dbt.flags.WARN_ERROR = True self.maxDiff = None profile_data = { 'target': 'test', 'quoting': {}, 'outputs': { 'test': { 'type': 'redshift', 'host': 'localhost', 'schema': 'analytics', 'user': '******', 'pass': '******', 'dbname': 'test', 'port': 1, } } } root_project = { 'name': 'root', 'version': '0.1', 'profile': 'test', 'project-root': normalize('/usr/src/app'), } self.root_project_config = config_from_parts_or_dicts( project=root_project, profile=profile_data, cli_vars='{"test_schema_name": "foo"}') snowplow_project = { 'name': 'snowplow', 'version': '0.1', 'profile': 'test', 'project-root': get_abs_os_path('./dbt_modules/snowplow'), } self.snowplow_project_config = config_from_parts_or_dicts( project=snowplow_project, profile=profile_data) self.all_projects = { 'root': self.root_project_config, 'snowplow': self.snowplow_project_config } self.root_project_config.dependencies = self.all_projects self.snowplow_project_config.dependencies = self.all_projects self.patcher = mock.patch('dbt.context.providers.get_adapter') self.factory = self.patcher.start() self.parser_patcher = mock.patch('dbt.parser.base.get_adapter') self.factory_parser = self.parser_patcher.start() self.macro_manifest = Manifest.from_macros( macros={m.unique_id: m for m in generate_name_macros('root')})
def __init__( self, results: ParseResult, root_project: RuntimeConfig, ) -> None: self.results = results self.root_project = root_project self.macro_manifest = Manifest.from_macros(macros=self.results.macros, files=self.results.files) self.schema_parsers: Dict[str, SchemaParser] = {} self.patches_used: Dict[SourceKey, Set[str]] = {} self.sources: Dict[str, ParsedSourceDefinition] = {}
def load(self, internal_manifest: Optional[Manifest] = None): old_results = self.read_parse_results() if old_results is not None: logger.debug('Got an acceptable cached parse result') self._load_macros(old_results, internal_manifest=internal_manifest) # make a manifest with just the macros to get the context macro_manifest = Manifest.from_macros(macros=self.results.macros, files=self.results.files) self.macro_hook(macro_manifest) for project in self.all_projects.values(): # parse a single project self.parse_project(project, macro_manifest, old_results)
def load_only_macros(self) -> Manifest: old_results = self.read_parse_results() for project in self.all_projects.values(): parser = MacroParser(self.results, project) for path in parser.search(): self.parse_with_cache(path, parser, old_results) # make a manifest with just the macros to get the context macro_manifest = Manifest.from_macros( macros=self.results.macros, files=self.results.files ) self.macro_hook(macro_manifest) return macro_manifest
def test_load_file_extras(self): TEST_DOCUMENTATION_FILE + '{% model foo %}select 1 as id{% endmodel %}' parser = docs.DocumentationParser( results=ParseResult.rpc(), root_project=self.root_project_config, project=self.subdir_project_config, macro_manifest=Manifest.from_macros()) file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md') parser.parse_file(file_block) results = sorted(parser.results.docs.values(), key=lambda n: n.name) self.assertEqual(len(results), 2) for result in results: self.assertIsInstance(result, ParsedDocumentation) self.assertEqual(results[0].name, 'snowplow_sessions') self.assertEqual(results[1].name, 'snowplow_sessions__session_id')
def setUp(self): dbt.flags.STRICT_MODE = True self.graph_result = None self.write_gpickle_patcher = patch('networkx.write_gpickle') self.load_projects_patcher = patch( 'dbt.parser.manifest._load_projects') self.file_system_patcher = patch.object( dbt.parser.search.FilesystemSearcher, '__new__') self.hook_patcher = patch.object(dbt.parser.hooks.HookParser, '__new__') self.get_adapter_patcher = patch('dbt.context.providers.get_adapter') self.factory = self.get_adapter_patcher.start() # also patch this one self.get_adapter_patcher_parser = patch('dbt.parser.base.get_adapter') self.factory_cmn = self.get_adapter_patcher_parser.start() def mock_write_gpickle(graph, outfile): self.graph_result = graph self.mock_write_gpickle = self.write_gpickle_patcher.start() self.mock_write_gpickle.side_effect = mock_write_gpickle self.profile = { 'outputs': { 'test': { 'type': 'postgres', 'threads': 4, 'host': 'thishostshouldnotexist', 'port': 5432, 'user': '******', 'pass': '******', 'dbname': 'dbt', 'schema': 'dbt_test' } }, 'target': 'test' } self.mock_load_projects = self.load_projects_patcher.start() def _load_projects(config, paths): yield config.project_name, config self.mock_load_projects.side_effect = _load_projects self.mock_models = [] def _mock_parse_result(config, all_projects): return ParseResult( vars_hash=FileHash.from_contents('vars'), project_hashes={ name: FileHash.from_contents(name) for name in all_projects }, profile_hash=FileHash.from_contents('profile'), ) self.load_patch = patch('dbt.parser.manifest.make_parse_result') self.mock_parse_result = self.load_patch.start() self.mock_parse_result.side_effect = _mock_parse_result self.load_source_file_patcher = patch.object(BaseParser, 'load_file') self.mock_source_file = self.load_source_file_patcher.start() self.mock_source_file.side_effect = lambda path: [ n for n in self.mock_models if n.path == path ][0] # self.relation_update_patcher = patch.object(RelationUpdate, '_relation_components', lambda: []) # self.mock_relation_update = self.relation_update_patcher.start() self.internal_manifest = Manifest.from_macros(macros={ n.unique_id: n for n in generate_name_macros('test_models_compile') }) def filesystem_iter(iter_self): if 'sql' not in iter_self.extension: return [] if 'models' not in iter_self.relative_dirs: return [] return [model.path for model in self.mock_models] def create_filesystem_searcher(cls, project, relative_dirs, extension): result = MagicMock(project=project, relative_dirs=relative_dirs, extension=extension) result.__iter__.side_effect = lambda: iter(filesystem_iter(result)) return result def create_hook_patcher(cls, results, project, relative_dirs, extension): result = MagicMock(results=results, project=project, relative_dirs=relative_dirs, extension=extension) result.__iter__.side_effect = lambda: iter([]) return result self.mock_filesystem_constructor = self.file_system_patcher.start() self.mock_filesystem_constructor.side_effect = create_filesystem_searcher self.mock_hook_constructor = self.hook_patcher.start() self.mock_hook_constructor.side_effect = create_hook_patcher