def test_endif(self): body = '{% snapshot foo %}select * from thing{% endsnapshot%}{% endif %}' with self.assertRaises(CompilationException) as err: extract_toplevel_blocks(body) self.assertIn( 'Got an unexpected control flow end tag, got endif but never saw a preceeding if (@ 1:53)', str(err.exception))
def test_if_endfor_newlines(self): body = '{% if x %}\n ...\n {% endfor %}\n{% endif %}' with self.assertRaises(CompilationException) as err: extract_toplevel_blocks(body) self.assertIn( 'Got an unexpected control flow end tag, got endfor but expected endif next (@ 3:4)', str(err.exception))
def test_deceptive_do_statement(self): body = '{% do thing %}{% myblock foo %}hi{% endmyblock %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'myblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, '{% myblock foo %}hi{% endmyblock %}')
def test_set_block(self): body = '{% set x %}1{% endset %}{% myblock foo %}hi{% endmyblock %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'myblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, '{% myblock foo %}hi{% endmyblock %}')
def test_embedded_self_closing_comment_block(self): body = '{% myblock foo %} {#}{% endmyblock %} {#}{% endmyblock %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'myblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, body) self.assertEqual(blocks[0].contents, ' {#}{% endmyblock %} {#}')
def test_crazy_set_statement(self): body = '{% set x = (thing("{% myblock foo %}")) %}{% otherblock bar %}x{% endotherblock %}{% set y = otherthing("{% myblock foo %}") %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'otherblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, '{% otherblock bar %}x{% endotherblock %}') self.assertEqual(blocks[0].block_type_name, 'otherblock')
def test_materialization_parse(self): body = '{% materialization xxx, default %} ... {% endmaterialization %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'materialization'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'materialization') self.assertEqual(blocks[0].block_name, 'xxx') self.assertEqual(blocks[0].full_block, body) body = '{% materialization xxx, adapter="other" %} ... {% endmaterialization %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'materialization'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'materialization') self.assertEqual(blocks[0].block_name, 'xxx') self.assertEqual(blocks[0].full_block, body)
def test_quoted_endblock_within_block(self): body = '{% myblock something -%} {% set x = ("{% endmyblock %}") %} {% endmyblock %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'myblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'myblock') self.assertEqual(blocks[0].contents, '{% set x = ("{% endmyblock %}") %} ')
def test_docs_block_expr(self): body = '{% docs more_doc %} asdf {{ "{% enddocs %}" ~ "}}" }}{% enddocs %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'docs'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'docs') self.assertEqual(blocks[0].contents, ' asdf {{ "{% enddocs %}" ~ "}}" }}') self.assertEqual(blocks[0].block_name, 'more_doc')
def test_do_block(self): body = '{% do %}thing.update(){% enddo %}{% myblock foo %}hi{% endmyblock %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'do', 'myblock'}, collect_raw_data=False) self.assertEqual(len(blocks), 2) self.assertEqual(blocks[0].contents, 'thing.update()') self.assertEqual(blocks[0].block_type_name, 'do') self.assertEqual(blocks[1].full_block, '{% myblock foo %}hi{% endmyblock %}')
def test_macro_with_crazy_args(self): body = '''{% macro foo(a, b=asdf("cool this is 'embedded'" * 3) + external_var, c)%}cool{# block comment with {% endmacro %} in it #} stuff here {% endmacro %}''' blocks = extract_toplevel_blocks(body, allowed_blocks={'macro'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'macro') self.assertEqual(blocks[0].block_name, 'foo') self.assertEqual( blocks[0].contents, 'cool{# block comment with {% endmacro %} in it #} stuff here ')
def test_unclosed_model_quotes(self): # test case for https://github.com/fishtown-analytics/dbt/issues/1533 body = '{% model my_model -%} select * from "something"."something_else{% endmodel %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'model'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'model') self.assertEqual(blocks[0].contents, 'select * from "something"."something_else') self.assertEqual(blocks[0].block_name, 'my_model')
def test_basic(self): body = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n' block_data = ' \n\r\t{%- mytype foo %}' + body + '{%endmytype -%}' blocks = extract_toplevel_blocks(block_data, allowed_blocks={'mytype'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'mytype') self.assertEqual(blocks[0].block_name, 'foo') self.assertEqual(blocks[0].contents, body) self.assertEqual(blocks[0].full_block, block_data)
def _extract_request_data(self, data): data = self.decode_sql(data) macro_blocks = [] data_chunks = [] for block in extract_toplevel_blocks(data): if block.block_type_name == 'macro': macro_blocks.append(block.full_block) else: data_chunks.append(block.full_block) macros = '\n'.join(macro_blocks) sql = ''.join(data_chunks) return sql, macros
def test_evil_comments(self): body = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n' comment = '{# external comment {% othertype bar %} select * from thing.other_thing{% endothertype %} #}' block_data = ' \n\r\t{%- mytype foo %}' + body + '{%endmytype -%}' blocks = extract_toplevel_blocks(comment + block_data, allowed_blocks={'mytype'}, collect_raw_data=False) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].block_type_name, 'mytype') self.assertEqual(blocks[0].block_name, 'foo') self.assertEqual(blocks[0].contents, body) self.assertEqual(blocks[0].full_block, block_data)
def test_docs_block(self): body = '{% docs __my_doc__ %} asdf {# nope {% enddocs %}} #} {% enddocs %} {% docs __my_other_doc__ %} asdf "{% enddocs %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'docs'}, collect_raw_data=False) self.assertEqual(len(blocks), 2) self.assertEqual(blocks[0].block_type_name, 'docs') self.assertEqual(blocks[0].contents, ' asdf {# nope {% enddocs %}} #} ') self.assertEqual(blocks[0].block_name, '__my_doc__') self.assertEqual(blocks[1].block_type_name, 'docs') self.assertEqual(blocks[1].contents, ' asdf "') self.assertEqual(blocks[1].block_name, '__my_other_doc__')
def test_multiple(self): body_one = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n' body_two = ('{{ config(bar=1)}}\r\nselect * from {% if foo %} thing ' '{% else %} other_thing {% endif %}') block_data = (' {% mytype foo %}' + body_one + '{% endmytype %}' + '\r\n{% othertype bar %}' + body_two + '{% endothertype %}') blocks = extract_toplevel_blocks( block_data, allowed_blocks={'mytype', 'othertype'}, collect_raw_data=False) self.assertEqual(len(blocks), 2)
def test_peaceful_macro_coexistence(self): body = '{# my macro #} {% macro foo(a, b) %} do a thing {%- endmacro %} {# my model #} {% a b %} test {% enda %}' blocks = extract_toplevel_blocks(body, allowed_blocks={'macro', 'a'}, collect_raw_data=True) self.assertEqual(len(blocks), 4) self.assertEqual(blocks[0].full_block, '{# my macro #} ') self.assertEqual(blocks[1].block_type_name, 'macro') self.assertEqual(blocks[1].block_name, 'foo') self.assertEqual(blocks[1].contents, ' do a thing') self.assertEqual(blocks[2].full_block, ' {# my model #} ') self.assertEqual(blocks[3].block_type_name, 'a') self.assertEqual(blocks[3].block_name, 'b') self.assertEqual(blocks[3].contents, ' test ')
def test_macro_with_trailing_data(self): body = '{# my macro #} {% macro foo(a, b) %} do a thing {%- endmacro %} {# my model #} {% a b %} test {% enda %} raw data so cool' blocks = extract_toplevel_blocks(body, allowed_blocks={'macro', 'a'}, collect_raw_data=True) self.assertEqual(len(blocks), 5) self.assertEqual(blocks[0].full_block, '{# my macro #} ') self.assertEqual(blocks[1].block_type_name, 'macro') self.assertEqual(blocks[1].block_name, 'foo') self.assertEqual(blocks[1].contents, ' do a thing') self.assertEqual(blocks[2].full_block, ' {# my model #} ') self.assertEqual(blocks[3].block_type_name, 'a') self.assertEqual(blocks[3].block_name, 'b') self.assertEqual(blocks[3].contents, ' test ') self.assertEqual(blocks[4].full_block, ' raw data so cool')
def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]: try: blocks = extract_toplevel_blocks( source_file.contents, allowed_blocks=self.allowed_blocks, collect_raw_data=False) # this makes mypy happy, and this is an invariant we really need for block in blocks: assert isinstance(block, BlockTag) yield block except CompilationException as exc: if exc.node is None: exc.add_node(source_file) raise
def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]: try: blocks = extract_toplevel_blocks( source_file.contents, allowed_blocks=self.allowed_blocks, collect_raw_data=False) # this makes mypy happy, and this is an invariant we really need for block in blocks: assert isinstance(block, BlockTag) yield block except CompilationException as exc: if exc.node is None: # TODO(jeb): attach info about resource type/file path here exc.node = NotImplemented raise
def test_awful_jinja(self): blocks = extract_toplevel_blocks( if_you_do_this_you_are_awful, allowed_blocks={'snapshot', 'materialization'}, collect_raw_data=False) self.assertEqual(len(blocks), 2) self.assertEqual( len([b for b in blocks if b.block_type_name == '__dbt__data']), 0) self.assertEqual(blocks[0].block_type_name, 'snapshot') self.assertEqual( blocks[0].contents, '\n '.join([ '''{% set x = ("{% endsnapshot %}" + (40 * '%})')) %}''', '{# {% endsnapshot %} #}', '{% embedded %}', ' some block data right here', '{% endembedded %}' ])) self.assertEqual(blocks[1].block_type_name, 'materialization') self.assertEqual(blocks[1].contents, '\nhi\n')
def parse(self, docfile): try: blocks = extract_toplevel_blocks(docfile.file_contents, allowed_blocks={'docs'}, collect_raw_data=False) except dbt.exceptions.CompilationException as exc: if exc.node is None: exc.node = docfile raise for block in blocks: try: template = get_template(block.full_block, {}) except dbt.exceptions.CompilationException as e: e.node = docfile raise # in python 3.x this can just be "yield from" isntead of a loop for d in self._parse_template_docs(template, docfile): yield d
def test_complex_file(self): blocks = extract_toplevel_blocks( complex_snapshot_file, allowed_blocks={'mytype', 'myothertype'}, collect_raw_data=False) self.assertEqual(len(blocks), 3) self.assertEqual(blocks[0].block_type_name, 'mytype') self.assertEqual(blocks[0].block_name, 'foo') self.assertEqual(blocks[0].full_block, '{% mytype foo %} some stuff {% endmytype %}') self.assertEqual(blocks[0].contents, ' some stuff ') self.assertEqual(blocks[1].block_type_name, 'mytype') self.assertEqual(blocks[1].block_name, 'bar') self.assertEqual(blocks[1].full_block, bar_block) self.assertEqual(blocks[1].contents, bar_block[16:-15].rstrip()) self.assertEqual(blocks[2].block_type_name, 'myothertype') self.assertEqual(blocks[2].block_name, 'x') self.assertEqual(blocks[2].full_block, x_block.strip()) self.assertEqual( blocks[2].contents, x_block[len('\n{% myothertype x %}' ):-len('{% endmyothertype %}\n')])
def parse_unparsed_macros( self, base_node: UnparsedMacro) -> Iterable[ParsedMacro]: try: blocks: List[jinja.BlockTag] = [ t for t in jinja.extract_toplevel_blocks( base_node.raw_sql, allowed_blocks={'macro', 'materialization'}, collect_raw_data=False, ) if isinstance(t, jinja.BlockTag) ] except CompilationException as exc: exc.add_node(base_node) raise for block in blocks: try: ast = jinja.parse(block.full_block) except CompilationException as e: e.add_node(base_node) raise macro_nodes = list(ast.find_all(jinja2.nodes.Macro)) if len(macro_nodes) != 1: # things have gone disastrously wrong, we thought we only # parsed one block! raise CompilationException( f'Found multiple macros in {block.full_block}, expected 1', node=base_node) macro_name = macro_nodes[0].name if not macro_name.startswith(MACRO_PREFIX): continue name: str = macro_name.replace(MACRO_PREFIX, '') node = self.parse_macro(block, base_node, name) yield node
def test_for_innocuous(self): # no for-loops over macros. body = '{% for x in range(10) %}{% something my_something %} adsf {% endsomething %}{% endfor %}' blocks = extract_toplevel_blocks(body) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, body)
def test_for(self): # no for-loops over macros. body = '{% for x in range(10) %}{% macro my_macro() %} adsf {% endmacro %}{% endfor %}' with self.assertRaises(CompilationException): extract_toplevel_blocks(body)
def test_if_innocuous(self): body = '{% if true %}{% something %}asdfasd{% endsomething %}{% endif %}' blocks = extract_toplevel_blocks(body) self.assertEqual(len(blocks), 1) self.assertEqual(blocks[0].full_block, body)
def test_if(self): # if you conditionally define your macros/models, don't body = '{% if true %}{% macro my_macro() %} adsf {% endmacro %}{% endif %}' with self.assertRaises(CompilationException): extract_toplevel_blocks(body)
def test_comment_block_self_closing(self): # test the case where a comment start looks a lot like it closes itself # (but it doesn't in jinja!) body = '{#} {% myblock foo %} {#}' blocks = extract_toplevel_blocks(body, collect_raw_data=False) self.assertEqual(len(blocks), 0)