def make_parse_result(
config: RuntimeConfig, all_projects: Mapping[str, Project]
) -> ParseResult:
"""Make a ParseResult from the project configuration and the profile."""
# if any of these change, we need to reject the parser
vars_hash = FileHash.from_contents(
'\0'.join([
getattr(config.args, 'vars', '{}') or '{}',
getattr(config.args, 'profile', '') or '',
getattr(config.args, 'target', '') or '',
__version__
])
)
profile_path = os.path.join(config.args.profiles_dir, 'profiles.yml')
with open(profile_path) as fp:
profile_hash = FileHash.from_contents(fp.read())
project_hashes = {}
for name, project in all_projects.items():
path = os.path.join(project.project_root, 'dbt_project.yml')
with open(path) as fp:
project_hashes[name] = FileHash.from_contents(fp.read())
return ParseResult(
vars_hash=vars_hash,
profile_hash=profile_hash,
project_hashes=project_hashes,
)
def test_multiple_raw_blocks(self):
parser = docs.DocumentationParser(
results=ParseResult.rpc(),
root_project=self.root_project_config,
project=self.subdir_project_config,
macro_manifest=Manifest.from_macros())
file_block = self._build_file(MULTIPLE_RAW_BLOCKS, 'test_file.md')
parser.parse_file(file_block)
results = sorted(parser.results.docs.values(), key=lambda n: n.name)
self.assertEqual(len(results), 2)
for result in results:
self.assertIsInstance(result, ParsedDocumentation)
self.assertEqual(result.package_name, 'some_package')
self.assertEqual(result.original_file_path, self.testfile_path)
self.assertEqual(result.root_path, self.subdir_path)
self.assertEqual(result.resource_type, NodeType.Documentation)
self.assertEqual(result.path, 'test_file.md')
self.assertEqual(results[0].name, 'other_doc')
self.assertEqual(results[0].block_contents,
'```\n {% docs %}other doc{% enddocs %}\n ```')
self.assertEqual(results[1].name, 'some_doc')
self.assertEqual(
results[1].block_contents,
'```\n {% docs %}some doc{% enddocs %}\n ```',
)
def test_load_file(self):
parser = docs.DocumentationParser(
results=ParseResult.rpc(),
root_project=self.root_project_config,
project=self.subdir_project_config,
macro_manifest=Manifest.from_macros())
file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md')
parser.parse_file(file_block)
results = sorted(parser.results.docs.values(), key=lambda n: n.name)
self.assertEqual(len(results), 2)
for result in results:
self.assertIsInstance(result, ParsedDocumentation)
self.assertEqual(result.package_name, 'some_package')
self.assertNotEqual(result.file_contents, TEST_DOCUMENTATION_FILE)
self.assertEqual(result.original_file_path, self.testfile_path)
self.assertEqual(result.root_path, self.subdir_path)
self.assertEqual(result.resource_type, NodeType.Documentation)
self.assertEqual(result.path, 'test_file.md')
self.assertEqual(results[0].name, 'snowplow_sessions')
self.assertEqual(results[0].file_contents, SNOWPLOW_SESSIONS_BLOCK)
self.assertEqual(results[1].name, 'snowplow_sessions__session_id')
self.assertEqual(results[1].file_contents,
SNOWPLOW_SESSIONS_SESSION_ID_BLOCK)
def _get_exec_node(self):
if self.manifest is None:
raise InternalException('manifest not set in _get_exec_node')
results = ParseResult.rpc()
macro_overrides = {}
macros = self.args.macros
sql, macros = self._extract_request_data(self.args.sql)
if macros:
macro_parser = RPCMacroParser(results, self.config)
for node in macro_parser.parse_remote(macros):
macro_overrides[node.unique_id] = node
self.manifest.macros.update(macro_overrides)
rpc_parser = RPCCallParser(
results=results,
project=self.config,
root_project=self.config,
macro_manifest=self.manifest,
)
rpc_node = rpc_parser.parse_remote(sql, self.args.name)
add_new_refs(manifest=self.manifest,
config=self.config,
node=rpc_node,
macros=macro_overrides)
# don't write our new, weird manifest!
self.linker = compile_manifest(self.config, self.manifest, write=False)
self._compile_ancestors(rpc_node.unique_id)
return rpc_node
def _get_exec_node(self):
if self.manifest is None:
raise InternalException('manifest not set in _get_exec_node')
results = ParseResult.rpc()
macro_overrides = {}
macros = self.args.macros
sql, macros = self._extract_request_data(self.args.sql)
if macros:
macro_parser = RPCMacroParser(results, self.config)
for node in macro_parser.parse_remote(macros):
macro_overrides[node.unique_id] = node
self.manifest.macros.update(macro_overrides)
rpc_parser = RPCCallParser(
results=results,
project=self.config,
root_project=self.config,
macro_manifest=self.manifest,
)
rpc_node = rpc_parser.parse_remote(sql, self.args.name)
add_new_refs(manifest=self.manifest,
config=self.config,
node=rpc_node,
macros=macro_overrides)
# don't write our new, weird manifest!
adapter = get_adapter(self.config)
compiler = adapter.get_compiler()
self.graph = compiler.compile(self.manifest, write=False)
# previously, this compiled the ancestors, but they are compiled at
# runtime now.
return rpc_node
def _mock_parse_result(config, all_projects):
return ParseResult(
vars_hash=FileHash.from_contents('vars'),
project_hashes={
name: FileHash.from_contents(name)
for name in all_projects
},
profile_hash=FileHash.from_contents('profile'),
)
def setUp(self):
flags.STRICT_MODE = False
profile_cfg = {
'outputs': {
'test': {
'type': 'snowflake',
'account': 'test_account',
'user': '******',
'database': 'test_database',
'warehouse': 'test_warehouse',
'schema': 'public',
},
},
'target': 'test',
}
project_cfg = {
'name': 'X',
'version': '0.1',
'profile': 'test',
'project-root': '/tmp/dbt/does-not-exist',
'quoting': {
'identifier': False,
'schema': True,
},
'query-comment': 'dbt',
}
self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
self.assertEqual(self.config.query_comment.comment, 'dbt')
self.assertEqual(self.config.query_comment.append, False)
self.handle = mock.MagicMock(
spec=snowflake_connector.SnowflakeConnection)
self.cursor = self.handle.cursor.return_value
self.mock_execute = self.cursor.execute
self.patcher = mock.patch(
'dbt.adapters.snowflake.connections.snowflake.connector.connect'
)
self.snowflake = self.patcher.start()
self.load_patch = mock.patch('dbt.parser.manifest.make_parse_result')
self.mock_parse_result = self.load_patch.start()
self.mock_parse_result.return_value = ParseResult.rpc()
self.snowflake.return_value = self.handle
self.adapter = SnowflakeAdapter(self.config)
self.adapter.connections.query_header = MacroQueryStringSetter(self.config, mock.MagicMock(macros={}))
self.qh_patch = mock.patch.object(self.adapter.connections.query_header, 'add')
self.mock_query_header_add = self.qh_patch.start()
self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(q)
self.adapter.acquire_connection()
inject_adapter(self.adapter)
def setUp(self):
flags.STRICT_MODE = False
self.target_dict = {
'type': 'postgres',
'dbname': 'postgres',
'user': '******',
'host': 'thishostshouldnotexist',
'pass': '******',
'port': 5432,
'schema': 'public'
}
profile_cfg = {
'outputs': {
'test': self.target_dict,
},
'target': 'test'
}
project_cfg = {
'name': 'X',
'version': '0.1',
'profile': 'test',
'project-root': '/tmp/dbt/does-not-exist',
'quoting': {
'identifier': False,
'schema': True,
},
}
self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
self.handle = mock.MagicMock(spec=psycopg2_extensions.connection)
self.cursor = self.handle.cursor.return_value
self.mock_execute = self.cursor.execute
self.patcher = mock.patch('dbt.adapters.postgres.connections.psycopg2')
self.psycopg2 = self.patcher.start()
self.psycopg2.connect.return_value = self.handle
self.adapter = PostgresAdapter(self.config)
self.adapter.connections.query_header = MacroQueryStringSetter(
self.config, mock.MagicMock(macros={}))
self.qh_patch = mock.patch.object(
self.adapter.connections.query_header, 'add')
self.mock_query_header_add = self.qh_patch.start()
self.mock_query_header_add.side_effect = lambda q: '/* dbt */\n{}'.format(
q)
self.adapter.acquire_connection()
inject_adapter(self.adapter)
self.load_patch = mock.patch('dbt.parser.manifest.make_parse_result')
self.mock_parse_result = self.load_patch.start()
self.mock_parse_result.return_value = ParseResult.rpc()
def test_load_file_extras(self):
TEST_DOCUMENTATION_FILE + '{% model foo %}select 1 as id{% endmodel %}'
parser = docs.DocumentationParser(
results=ParseResult.rpc(),
root_project=self.root_project_config,
project=self.subdir_project_config,
macro_manifest=Manifest.from_macros())
file_block = self._build_file(TEST_DOCUMENTATION_FILE, 'test_file.md')
parser.parse_file(file_block)
results = sorted(parser.results.docs.values(), key=lambda n: n.name)
self.assertEqual(len(results), 2)
for result in results:
self.assertIsInstance(result, ParsedDocumentation)
self.assertEqual(results[0].name, 'snowplow_sessions')
self.assertEqual(results[1].name, 'snowplow_sessions__session_id')