def test_multiple_blocks(self): raw_sql = '{% macro foo(a, b) %}a ~ b{% endmacro %}\n{% macro bar(c, d) %}c + d{% endmacro %}' block = self.file_block_for(raw_sql, 'macro.sql') self.parser.parse_file(block) self.assert_has_results_length(self.parser.results, macros=2) macros = sorted(self.parser.results.macros.values(), key=lambda m: m.name) expected_bar = ParsedMacro( name='bar', resource_type=NodeType.Macro, unique_id='macro.snowplow.bar', package_name='snowplow', original_file_path=normalize('macros/macro.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), path=normalize('macros/macro.sql'), macro_sql='{% macro bar(c, d) %}c + d{% endmacro %}', ) expected_foo = ParsedMacro( name='foo', resource_type=NodeType.Macro, unique_id='macro.snowplow.foo', package_name='snowplow', original_file_path=normalize('macros/macro.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), path=normalize('macros/macro.sql'), macro_sql='{% macro foo(a, b) %}a ~ b{% endmacro %}', ) self.assertEqual(macros, [expected_bar, expected_foo]) path = get_abs_os_path('./dbt_modules/snowplow/macros/macro.sql') self.assertIn(path, self.parser.results.files) self.assertEqual( sorted(self.parser.results.files[path].macros), ['macro.snowplow.bar', 'macro.snowplow.foo'], )
def generate_name_macros(package): from dbt.contracts.graph.parsed import ParsedMacro from dbt.node_types import NodeType name_sql = {} for component in ('database', 'schema', 'alias'): if component == 'alias': source = 'node.name' else: source = f'target.{component}' name = f'generate_{component}_name' sql = f'{{% macro {name}(value, node) %}} {{% if value %}} {{{{ value }}}} {{% else %}} {{{{ {source} }}}} {{% endif %}} {{% endmacro %}}' name_sql[name] = sql all_sql = '\n'.join(name_sql.values()) for name, sql in name_sql.items(): pm = ParsedMacro( name=name, resource_type=NodeType.Macro, unique_id=f'macro.{package}.{name}', package_name=package, original_file_path=normalize('macros/macro.sql'), root_path='./dbt_modules/root', path=normalize('macros/macro.sql'), raw_sql=all_sql, macro_sql=sql, ) yield pm
def parse_macro_file(self, macro_file_path, macro_file_contents, root_path, package_name, resource_type, tags=None, context=None): logger.debug("Parsing {}".format(macro_file_path)) to_return = {} if tags is None: tags = [] # change these to actual kwargs base_node = UnparsedMacro( path=macro_file_path, original_file_path=macro_file_path, package_name=package_name, raw_sql=macro_file_contents, root_path=root_path, ) try: ast = dbt.clients.jinja.parse(macro_file_contents) except dbt.exceptions.CompilationException as e: e.node = base_node raise e for macro_node in ast.find_all(jinja2.nodes.Macro): macro_name = macro_node.name node_type = None if macro_name.startswith(dbt.utils.MACRO_PREFIX): node_type = NodeType.Macro name = macro_name.replace(dbt.utils.MACRO_PREFIX, '') if node_type != resource_type: continue unique_id = self.get_path(resource_type, package_name, name) merged = dbt.utils.deep_merge( base_node.serialize(), { 'name': name, 'unique_id': unique_id, 'tags': tags, 'resource_type': resource_type, 'depends_on': { 'macros': [] }, }) new_node = ParsedMacro(**merged) to_return[unique_id] = new_node return to_return
def parse_macro(self, base_node: UnparsedMacro, name: str) -> ParsedMacro: unique_id = self.generate_unique_id(name) return ParsedMacro( path=base_node.path, original_file_path=base_node.original_file_path, package_name=base_node.package_name, raw_sql=base_node.raw_sql, root_path=base_node.root_path, resource_type=base_node.resource_type, name=name, unique_id=unique_id, )
def parse_macro(self, block: jinja.BlockTag, base_node: UnparsedMacro, name: str) -> ParsedMacro: unique_id = self.generate_unique_id(name) return ParsedMacro( path=base_node.path, macro_sql=block.full_block, original_file_path=base_node.original_file_path, package_name=base_node.package_name, root_path=base_node.root_path, resource_type=base_node.resource_type, name=name, unique_id=unique_id, )
def test_single_block(self): raw_sql = '{% macro foo(a, b) %}a ~ b{% endmacro %}' block = self.file_block_for(raw_sql, 'macro.sql') self.parser.parse_file(block) self.assert_has_results_length(self.parser.results, macros=1) macro = list(self.parser.results.macros.values())[0] expected = ParsedMacro( name='foo', resource_type=NodeType.Macro, unique_id='macro.snowplow.foo', package_name='snowplow', original_file_path=normalize('macros/macro.sql'), root_path=get_abs_os_path('./dbt_modules/snowplow'), path=normalize('macros/macro.sql'), raw_sql=raw_sql ) self.assertEqual(macro, expected) path = get_abs_os_path('./dbt_modules/snowplow/macros/macro.sql') self.assertIn(path, self.parser.results.files) self.assertEqual(self.parser.results.files[path].macros, ['macro.snowplow.foo'])
def _generate_macros(self): name_sql = {} for component in ('database', 'schema', 'alias'): if component == 'alias': source = 'node.name' else: source = f'target.{component}' name = f'generate_{component}_name' sql = f'{{% macro {name}(value, node) %}} {{% if value %}} {{{{ value }}}} {{% else %}} {{{{ {source} }}}} {{% endif %}} {{% endmacro %}}' name_sql[name] = sql for name, sql in name_sql.items(): pm = ParsedMacro( name=name, resource_type=NodeType.Macro, unique_id=f'macro.root.{name}', package_name='root', original_file_path=normalize('macros/macro.sql'), root_path=get_abs_os_path('./dbt_modules/root'), path=normalize('macros/macro.sql'), macro_sql=sql, ) yield pm
def _process_docs_for_macro(context: Dict[str, Any], macro: ParsedMacro) -> None: macro.description = get_rendered(macro.description, context) for arg in macro.arguments: arg.description = get_rendered(arg.description, context)
def parse_macro_file(cls, macro_file_path, macro_file_contents, root_path, package_name, resource_type, tags=None, context=None): logger.debug("Parsing {}".format(macro_file_path)) to_return = {} if tags is None: tags = [] context = {} # change these to actual kwargs base_node = UnparsedMacro( path=macro_file_path, original_file_path=macro_file_path, package_name=package_name, raw_sql=macro_file_contents, root_path=root_path, ) try: template = dbt.clients.jinja.get_template(macro_file_contents, context, node=base_node) except dbt.exceptions.CompilationException as e: e.node = base_node raise e for key, item in template.module.__dict__.items(): if type(item) != jinja2.runtime.Macro: continue node_type = None if key.startswith(dbt.utils.MACRO_PREFIX): node_type = NodeType.Macro name = key.replace(dbt.utils.MACRO_PREFIX, '') elif key.startswith(dbt.utils.OPERATION_PREFIX): node_type = NodeType.Operation name = key.replace(dbt.utils.OPERATION_PREFIX, '') if node_type != resource_type: continue unique_id = cls.get_path(resource_type, package_name, name) merged = dbt.utils.deep_merge( base_node.serialize(), { 'name': name, 'unique_id': unique_id, 'tags': tags, 'resource_type': resource_type, 'depends_on': { 'macros': [] }, }) new_node = ParsedMacro(template=template, **merged) to_return[unique_id] = new_node return to_return