def test_endif(self):
body = '{% snapshot foo %}select * from thing{% endsnapshot%}{% endif %}'
with self.assertRaises(CompilationException) as err:
extract_toplevel_blocks(body)
self.assertIn(
'Got an unexpected control flow end tag, got endif but never saw a preceeding if (@ 1:53)',
str(err.exception))
def test_if_endfor_newlines(self):
body = '{% if x %}\n ...\n {% endfor %}\n{% endif %}'
with self.assertRaises(CompilationException) as err:
extract_toplevel_blocks(body)
self.assertIn(
'Got an unexpected control flow end tag, got endfor but expected endif next (@ 3:4)',
str(err.exception))
def test_deceptive_do_statement(self):
body = '{% do thing %}{% myblock foo %}hi{% endmyblock %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'myblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block,
'{% myblock foo %}hi{% endmyblock %}')
def test_set_block(self):
body = '{% set x %}1{% endset %}{% myblock foo %}hi{% endmyblock %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'myblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block,
'{% myblock foo %}hi{% endmyblock %}')
def test_embedded_self_closing_comment_block(self):
body = '{% myblock foo %} {#}{% endmyblock %} {#}{% endmyblock %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'myblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block, body)
self.assertEqual(blocks[0].contents, ' {#}{% endmyblock %} {#}')
def test_crazy_set_statement(self):
body = '{% set x = (thing("{% myblock foo %}")) %}{% otherblock bar %}x{% endotherblock %}{% set y = otherthing("{% myblock foo %}") %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'otherblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block,
'{% otherblock bar %}x{% endotherblock %}')
self.assertEqual(blocks[0].block_type_name, 'otherblock')
def test_materialization_parse(self):
body = '{% materialization xxx, default %} ... {% endmaterialization %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'materialization'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'materialization')
self.assertEqual(blocks[0].block_name, 'xxx')
self.assertEqual(blocks[0].full_block, body)
body = '{% materialization xxx, adapter="other" %} ... {% endmaterialization %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'materialization'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'materialization')
self.assertEqual(blocks[0].block_name, 'xxx')
self.assertEqual(blocks[0].full_block, body)
def test_quoted_endblock_within_block(self):
body = '{% myblock something -%} {% set x = ("{% endmyblock %}") %} {% endmyblock %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'myblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'myblock')
self.assertEqual(blocks[0].contents,
'{% set x = ("{% endmyblock %}") %} ')
def test_docs_block_expr(self):
body = '{% docs more_doc %} asdf {{ "{% enddocs %}" ~ "}}" }}{% enddocs %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'docs'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'docs')
self.assertEqual(blocks[0].contents,
' asdf {{ "{% enddocs %}" ~ "}}" }}')
self.assertEqual(blocks[0].block_name, 'more_doc')
def test_do_block(self):
body = '{% do %}thing.update(){% enddo %}{% myblock foo %}hi{% endmyblock %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'do', 'myblock'},
collect_raw_data=False)
self.assertEqual(len(blocks), 2)
self.assertEqual(blocks[0].contents, 'thing.update()')
self.assertEqual(blocks[0].block_type_name, 'do')
self.assertEqual(blocks[1].full_block,
'{% myblock foo %}hi{% endmyblock %}')
def test_macro_with_crazy_args(self):
body = '''{% macro foo(a, b=asdf("cool this is 'embedded'" * 3) + external_var, c)%}cool{# block comment with {% endmacro %} in it #} stuff here {% endmacro %}'''
blocks = extract_toplevel_blocks(body,
allowed_blocks={'macro'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'macro')
self.assertEqual(blocks[0].block_name, 'foo')
self.assertEqual(
blocks[0].contents,
'cool{# block comment with {% endmacro %} in it #} stuff here ')
def test_unclosed_model_quotes(self):
# test case for https://github.com/fishtown-analytics/dbt/issues/1533
body = '{% model my_model -%} select * from "something"."something_else{% endmodel %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'model'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'model')
self.assertEqual(blocks[0].contents,
'select * from "something"."something_else')
self.assertEqual(blocks[0].block_name, 'my_model')
def test_basic(self):
body = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n'
block_data = ' \n\r\t{%- mytype foo %}' + body + '{%endmytype -%}'
blocks = extract_toplevel_blocks(block_data,
allowed_blocks={'mytype'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'mytype')
self.assertEqual(blocks[0].block_name, 'foo')
self.assertEqual(blocks[0].contents, body)
self.assertEqual(blocks[0].full_block, block_data)
def _extract_request_data(self, data):
data = self.decode_sql(data)
macro_blocks = []
data_chunks = []
for block in extract_toplevel_blocks(data):
if block.block_type_name == 'macro':
macro_blocks.append(block.full_block)
else:
data_chunks.append(block.full_block)
macros = '\n'.join(macro_blocks)
sql = ''.join(data_chunks)
return sql, macros
def test_evil_comments(self):
body = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n'
comment = '{# external comment {% othertype bar %} select * from thing.other_thing{% endothertype %} #}'
block_data = ' \n\r\t{%- mytype foo %}' + body + '{%endmytype -%}'
blocks = extract_toplevel_blocks(comment + block_data,
allowed_blocks={'mytype'},
collect_raw_data=False)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].block_type_name, 'mytype')
self.assertEqual(blocks[0].block_name, 'foo')
self.assertEqual(blocks[0].contents, body)
self.assertEqual(blocks[0].full_block, block_data)
def test_docs_block(self):
body = '{% docs __my_doc__ %} asdf {# nope {% enddocs %}} #} {% enddocs %} {% docs __my_other_doc__ %} asdf "{% enddocs %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'docs'},
collect_raw_data=False)
self.assertEqual(len(blocks), 2)
self.assertEqual(blocks[0].block_type_name, 'docs')
self.assertEqual(blocks[0].contents,
' asdf {# nope {% enddocs %}} #} ')
self.assertEqual(blocks[0].block_name, '__my_doc__')
self.assertEqual(blocks[1].block_type_name, 'docs')
self.assertEqual(blocks[1].contents, ' asdf "')
self.assertEqual(blocks[1].block_name, '__my_other_doc__')
def test_multiple(self):
body_one = '{{ config(foo="bar") }}\r\nselect * from this.that\r\n'
body_two = ('{{ config(bar=1)}}\r\nselect * from {% if foo %} thing '
'{% else %} other_thing {% endif %}')
block_data = (' {% mytype foo %}' + body_one + '{% endmytype %}' +
'\r\n{% othertype bar %}' + body_two +
'{% endothertype %}')
blocks = extract_toplevel_blocks(
block_data,
allowed_blocks={'mytype', 'othertype'},
collect_raw_data=False)
self.assertEqual(len(blocks), 2)
def test_peaceful_macro_coexistence(self):
body = '{# my macro #} {% macro foo(a, b) %} do a thing {%- endmacro %} {# my model #} {% a b %} test {% enda %}'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'macro', 'a'},
collect_raw_data=True)
self.assertEqual(len(blocks), 4)
self.assertEqual(blocks[0].full_block, '{# my macro #} ')
self.assertEqual(blocks[1].block_type_name, 'macro')
self.assertEqual(blocks[1].block_name, 'foo')
self.assertEqual(blocks[1].contents, ' do a thing')
self.assertEqual(blocks[2].full_block, ' {# my model #} ')
self.assertEqual(blocks[3].block_type_name, 'a')
self.assertEqual(blocks[3].block_name, 'b')
self.assertEqual(blocks[3].contents, ' test ')
def test_macro_with_trailing_data(self):
body = '{# my macro #} {% macro foo(a, b) %} do a thing {%- endmacro %} {# my model #} {% a b %} test {% enda %} raw data so cool'
blocks = extract_toplevel_blocks(body,
allowed_blocks={'macro', 'a'},
collect_raw_data=True)
self.assertEqual(len(blocks), 5)
self.assertEqual(blocks[0].full_block, '{# my macro #} ')
self.assertEqual(blocks[1].block_type_name, 'macro')
self.assertEqual(blocks[1].block_name, 'foo')
self.assertEqual(blocks[1].contents, ' do a thing')
self.assertEqual(blocks[2].full_block, ' {# my model #} ')
self.assertEqual(blocks[3].block_type_name, 'a')
self.assertEqual(blocks[3].block_name, 'b')
self.assertEqual(blocks[3].contents, ' test ')
self.assertEqual(blocks[4].full_block, ' raw data so cool')
def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]:
try:
blocks = extract_toplevel_blocks(
source_file.contents,
allowed_blocks=self.allowed_blocks,
collect_raw_data=False)
# this makes mypy happy, and this is an invariant we really need
for block in blocks:
assert isinstance(block, BlockTag)
yield block
except CompilationException as exc:
if exc.node is None:
exc.add_node(source_file)
raise
def extract_blocks(self, source_file: FileBlock) -> Iterable[BlockTag]:
try:
blocks = extract_toplevel_blocks(
source_file.contents,
allowed_blocks=self.allowed_blocks,
collect_raw_data=False)
# this makes mypy happy, and this is an invariant we really need
for block in blocks:
assert isinstance(block, BlockTag)
yield block
except CompilationException as exc:
if exc.node is None:
# TODO(jeb): attach info about resource type/file path here
exc.node = NotImplemented
raise
def test_awful_jinja(self):
blocks = extract_toplevel_blocks(
if_you_do_this_you_are_awful,
allowed_blocks={'snapshot', 'materialization'},
collect_raw_data=False)
self.assertEqual(len(blocks), 2)
self.assertEqual(
len([b for b in blocks if b.block_type_name == '__dbt__data']), 0)
self.assertEqual(blocks[0].block_type_name, 'snapshot')
self.assertEqual(
blocks[0].contents, '\n '.join([
'''{% set x = ("{% endsnapshot %}" + (40 * '%})')) %}''',
'{# {% endsnapshot %} #}', '{% embedded %}',
' some block data right here', '{% endembedded %}'
]))
self.assertEqual(blocks[1].block_type_name, 'materialization')
self.assertEqual(blocks[1].contents, '\nhi\n')
def parse(self, docfile):
try:
blocks = extract_toplevel_blocks(docfile.file_contents,
allowed_blocks={'docs'},
collect_raw_data=False)
except dbt.exceptions.CompilationException as exc:
if exc.node is None:
exc.node = docfile
raise
for block in blocks:
try:
template = get_template(block.full_block, {})
except dbt.exceptions.CompilationException as e:
e.node = docfile
raise
# in python 3.x this can just be "yield from" isntead of a loop
for d in self._parse_template_docs(template, docfile):
yield d
def test_complex_file(self):
blocks = extract_toplevel_blocks(
complex_snapshot_file,
allowed_blocks={'mytype', 'myothertype'},
collect_raw_data=False)
self.assertEqual(len(blocks), 3)
self.assertEqual(blocks[0].block_type_name, 'mytype')
self.assertEqual(blocks[0].block_name, 'foo')
self.assertEqual(blocks[0].full_block,
'{% mytype foo %} some stuff {% endmytype %}')
self.assertEqual(blocks[0].contents, ' some stuff ')
self.assertEqual(blocks[1].block_type_name, 'mytype')
self.assertEqual(blocks[1].block_name, 'bar')
self.assertEqual(blocks[1].full_block, bar_block)
self.assertEqual(blocks[1].contents, bar_block[16:-15].rstrip())
self.assertEqual(blocks[2].block_type_name, 'myothertype')
self.assertEqual(blocks[2].block_name, 'x')
self.assertEqual(blocks[2].full_block, x_block.strip())
self.assertEqual(
blocks[2].contents, x_block[len('\n{% myothertype x %}'
):-len('{% endmyothertype %}\n')])
def parse_unparsed_macros(
self, base_node: UnparsedMacro) -> Iterable[ParsedMacro]:
try:
blocks: List[jinja.BlockTag] = [
t for t in jinja.extract_toplevel_blocks(
base_node.raw_sql,
allowed_blocks={'macro', 'materialization'},
collect_raw_data=False,
) if isinstance(t, jinja.BlockTag)
]
except CompilationException as exc:
exc.add_node(base_node)
raise
for block in blocks:
try:
ast = jinja.parse(block.full_block)
except CompilationException as e:
e.add_node(base_node)
raise
macro_nodes = list(ast.find_all(jinja2.nodes.Macro))
if len(macro_nodes) != 1:
# things have gone disastrously wrong, we thought we only
# parsed one block!
raise CompilationException(
f'Found multiple macros in {block.full_block}, expected 1',
node=base_node)
macro_name = macro_nodes[0].name
if not macro_name.startswith(MACRO_PREFIX):
continue
name: str = macro_name.replace(MACRO_PREFIX, '')
node = self.parse_macro(block, base_node, name)
yield node
def test_for_innocuous(self):
# no for-loops over macros.
body = '{% for x in range(10) %}{% something my_something %} adsf {% endsomething %}{% endfor %}'
blocks = extract_toplevel_blocks(body)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block, body)
def test_for(self):
# no for-loops over macros.
body = '{% for x in range(10) %}{% macro my_macro() %} adsf {% endmacro %}{% endfor %}'
with self.assertRaises(CompilationException):
extract_toplevel_blocks(body)
def test_if_innocuous(self):
body = '{% if true %}{% something %}asdfasd{% endsomething %}{% endif %}'
blocks = extract_toplevel_blocks(body)
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].full_block, body)
def test_if(self):
# if you conditionally define your macros/models, don't
body = '{% if true %}{% macro my_macro() %} adsf {% endmacro %}{% endif %}'
with self.assertRaises(CompilationException):
extract_toplevel_blocks(body)
def test_comment_block_self_closing(self):
# test the case where a comment start looks a lot like it closes itself
# (but it doesn't in jinja!)
body = '{#} {% myblock foo %} {#}'
blocks = extract_toplevel_blocks(body, collect_raw_data=False)
self.assertEqual(len(blocks), 0)
