def assign_after_filter_fails(self):
self.ast_description = """
file: TestTemplate
#def test_function
#set $foo = 'foo'
$foo
#set $foo = 'bar'
$foo
#end def
"""
ast_root, function_node = self._build_function_template()
first_assign = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode('foo'))
function_node.append(first_assign)
first_use = ast.FilterNode(ast.IdentifierNode('foo'))
function_node.append(first_use)
second_assign = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode('bar'))
function_node.append(second_assign)
second_use = ast.FilterNode(ast.IdentifierNode('foo'))
function_node.append(second_use)
optimization_analyzer = optimizer.OptimizationAnalyzer(
ast_root, self.compiler.analyzer_options, self.compiler)
optimization_analyzer.visit_ast = test_util.RecordedFunction(
optimization_analyzer.visit_ast)
self.assertRaises(compiler.Warning, optimization_analyzer.visit_ast,
ast_root)
def test_index_scope_ok(self):
self.ast_description = """
file: TestTemplate
#def test_function
#set $foo = {}
#if True
#set $foo[1] = 1
#end if
#end def
"""
ast_root, function_node = self._build_function_template()
assign_node1 = ast.AssignNode(
ast.IdentifierNode('foo'), ast.DictLiteralNode())
function_node.append(assign_node1)
if_node = ast.IfNode(ast.LiteralNode(True))
assign_node2 = ast.AssignNode(
ast.SliceNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)),
ast.LiteralNode(1))
if_node.append(assign_node2)
function_node.append(if_node)
optimization_analyzer = self._get_analyzer(ast_root)
try:
optimization_analyzer.visit_ast(ast_root)
except analyzer.SemanticAnalyzerError:
self.fail('visit_ast raised SemanticAnalyzerError unexpectedly.')
def build_conditional_body(node):
node.append(ast.AssignNode(
ast.IdentifierNode('_rph_foo'), ast.PlaceholderNode('foo')))
node.append(ast.AssignNode(
ast.IdentifierNode('_fph123'), ast.FilterNode(
ast.IdentifierNode('_rph_foo'))))
node.append(ast.BufferWrite(ast.IdentifierNode('_fph123')))
def test_partial_nested_else_if(self):
self.ast_description = """
file: TestTemplate
#def test_function
#if True
#set $foo = 1
#else
#if True
#set $foo = 2
#end if
#end if
$foo
#end def
"""
ast_root, function_node, if_node = self._build_if_template()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
if_node_2 = ast.IfNode(ast.LiteralNode(True))
if_node_2.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(2)))
if_node.else_.append(if_node_2)
function_node.append(ast.PlaceholderNode('foo'))
optimization_analyzer = self._get_analyzer(ast_root)
self.assertRaises(analyzer.SemanticAnalyzerError,
optimization_analyzer.visit_ast, ast_root)
def test_nested_else(self):
self.ast_description = """
file: TestTemplate
#def test_function
#if True
#set $foo = 1
#else
#if
#set $foo = 2
#else
#set $foo = 3
#end if
#end if
$foo
#end def
"""
ast_root, function_node, if_node = self._build_if_template()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
if_node_2 = ast.IfNode(ast.LiteralNode(True))
if_node_2.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(2)))
if_node_2.else_.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(3)))
if_node.else_.append(if_node_2)
function_node.append(ast.PlaceholderNode('foo'))
optimization_analyzer = self._get_analyzer(ast_root)
try:
optimization_analyzer.visit_ast(ast_root)
except analyzer.SemanticAnalyzerError:
self.fail('visit_ast raised SemanticAnalyzerError unexpectedly.')
def double_assign_ok(self):
self.ast_description = """
file: TestTemplate
#def test_function
#set $foo = 'foo'
#set $foo = 'bar'
$foo
#end def
"""
ast_root, function_node = self._build_function_template()
first_assign = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode('foo'))
function_node.append(first_assign)
second_assign = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode('bar'))
function_node.append(second_assign)
first_use = ast.FilterNode(ast.IdentifierNode('foo'))
function_node.append(first_use)
optimization_analyzer = optimizer.OptimizationAnalyzer(
ast_root, self.compiler.analyzer_options, self.compiler)
optimization_analyzer.visit_ast = test_util.RecordedFunction(
optimization_analyzer.visit_ast)
try:
optimization_analyzer.visit_ast(ast_root)
except compiler.Warning:
self.fail('visit_ast raised WarningError unexpectedly.')
def scope_setter(scope):
scope.local_identifiers.add(ast.IdentifierNode('_rph_foo'))
scope.aliased_expression_map[ast.PlaceholderNode('foo')] = (
ast.IdentifierNode('_rph_foo'))
scope.aliased_expression_map[ast.FilterNode(ast.IdentifierNode(
'_rph_foo'))] = (ast.IdentifierNode('_fph123'))
scope.alias_name_set.add('_fph123')
scope.alias_name_set.add('_rph_foo')
def analyzeCallFunctionNode(self, pnode):
fn = pnode
fname = fn.expression.name
if self.compiler.registry_contains(fname):
# If this is a placeholder that is in the function registry, mark it
# as used so that in the optimizer stage, we can avoid importing
# unused registry values.
self.template.used_function_registry_identifiers.add(fname)
# The fully qualified library function name iff we figure out
# that this is calling into a library.
library_function = None
skip_filter = self.compiler.get_registry_value(fname, 'skip_filter')
if isinstance(fn.expression, ast.PlaceholderNode):
macro_handler_name = 'macro_function_%s' % fn.expression.name
macro_data = self.compiler.macro_registry.get(macro_handler_name)
if macro_data:
macro_function, macro_parse_rule = macro_data
return self.handleMacro(fn, macro_function, macro_parse_rule)
elif fn.expression.name in self.template.template_methods:
fn.sanitization_state = ast.SanitizedState.SANITIZED_STRING
if self.template.library:
# Calling another library function from this library
# function.
library_function = fn.expression.name
elif skip_filter:
# If the function is marked as skip_filter in the registry, we
# know it is sanitized.
fn.sanitization_state = ast.SanitizedState.SANITIZED
elif skip_filter is False:
# If the function is marked as skip_filter=False in the
# registry, we know it is not sanitized.
fn.sanitization_state = ast.SanitizedState.UNSANITIZED
elif isinstance(fn.expression, ast.GetUDNNode):
identifier = [node.name for node in fn.expression.getChildNodes()]
identifier = '.'.join(identifier)
if identifier in self.template.library_identifiers:
# Calling library functions from other templates.
library_function = '%s.%s' % (identifier, fn.expression.name)
if library_function:
# Replace the placeholder node or UDN resolution with a direct
# reference to the library function, either in another imported
# module or here.
fn.expression = ast.IdentifierNode(library_function, pos=pnode.pos)
# Pass the current template instance into the library function.
fn.arg_list.child_nodes.insert(0, ast.IdentifierNode('self'))
# Library functions are spitfire functions so their output is
# sanitized.
fn.sanitization_state = ast.SanitizedState.SANITIZED_STRING
fn.library_function = True
fn.expression = self.build_ast(fn.expression)[0]
fn.arg_list = self.build_ast(fn.arg_list)[0]
return [fn]
def test_duplicate_node_collect(self):
self.ast_description = """
file: TestTemplate
#def test_function
foo
bar
#if True
#set $foo = 1
#end if
baz
boo
#if True
#set $foo = 1
#end if
#end def
NOTE: This test will break if collect_writes is written
using ast.ASTNode.insert_before.
"""
ast_root, function_node = self._build_function_template()
function_node.append(ast.BufferWrite(ast.LiteralNode('foo')))
function_node.append(ast.BufferWrite(ast.LiteralNode('bar')))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
function_node.append(if_node)
function_node.append(ast.BufferWrite(ast.LiteralNode('baz')))
function_node.append(ast.BufferWrite(ast.LiteralNode('boo')))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
optimization_analyzer = self._get_analyzer(ast_root)
ast_root, function_node = self._build_function_template()
tuple_node = ast.TupleLiteralNode()
tuple_node.append(ast.LiteralNode('foo'))
tuple_node.append(ast.LiteralNode('bar'))
function_node.append(ast.BufferExtend(tuple_node))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
function_node.append(if_node)
tuple_node = ast.TupleLiteralNode()
tuple_node.append(ast.LiteralNode('baz'))
tuple_node.append(ast.LiteralNode('boo'))
function_node.append(ast.BufferExtend(tuple_node))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
expected_hash = hash(ast_root)
got_hash = hash(optimization_analyzer.optimize_ast())
self.assertEqual(expected_hash, got_hash)
def handle_repeat(self, dom_node, attr_name):
debug("handle_repeat", dom_node)
expr_pieces = dom_node.getAttribute(attr_name).split()
dom_node.removeAttribute(attr_name)
target = expr_pieces[0]
expr_ast = util.parse(' '.join(expr_pieces[1:]), 'rhs_expression')
node_list = []
# hack - assumes python syntax
fn = ast.ForNode(
ast.TargetListNode([
ast.IdentifierNode("self.repeat['%s']" % target),
ast.IdentifierNode(target)
]),
ast.ExpressionListNode([
ast.CallFunctionNode(ast.IdentifierNode('enumerate'),
ast.ArgListNode([expr_ast]))
]))
if self.has_child_stuff(dom_node):
debug("has_child_stuff:", dom_node)
fn.extend(self.build_ast(dom_node))
#fn.append(self.make_tag_node(dom_node))
#for n in dom_node.childNodes:
# fn.extend(self.build_ast(n))
else:
# print "no children"
fn.extend(self.build_ast(dom_node))
if (dom_node.previousSibling and dom_node.previousSibling.nodeType
== xml.dom.minidom.Node.TEXT_NODE
and not dom_node.previousSibling.nodeValue.strip()):
# inject the previous whitespace sibling to keep the output looking
# ok fixme: a conditional is probably required here - you only want
# to execute this if it's not the last execution of the loop
fn.prepend(self.build_ast(dom_node.previousSibling))
# now remove the previous sibling
#print "node", dom_node
#print "parent", dom_node.parentNode
#print ' '.join("previous", dom_node.previousSibling,
# id(dom_node.previousSibling))
#print "next", dom_node.nextSibling, id(dom_node.nextSibling)
#dom_node.parentNode.removeChild(dom_node.previousSibling)
node_list.append(ast.EatPrevious())
node_list.append(fn)
#fn.extend(self.make_tag_node(dom_node, close=True))
return node_list
def test_collect_writes_join_if(self):
self.ast_description = """
file: TestTemplate
#def test_function
foo
bar
#if True
#set $foo = 1
#end if
baz
boo
#end def
"""
ast_root, function_node = self._build_function_template()
function_node.append(ast.BufferWrite(ast.LiteralNode('foo')))
function_node.append(ast.BufferWrite(ast.LiteralNode('bar')))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
function_node.append(if_node)
function_node.append(ast.BufferWrite(ast.LiteralNode('baz')))
function_node.append(ast.BufferWrite(ast.LiteralNode('boo')))
optimization_analyzer = self._get_analyzer(ast_root)
ast_root, function_node = self._build_function_template()
tuple_node = ast.TupleLiteralNode()
tuple_node.append(ast.LiteralNode('foo'))
tuple_node.append(ast.LiteralNode('bar'))
function_node.append(ast.BufferExtend(tuple_node))
if_node = ast.IfNode()
if_node.append(ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)))
function_node.append(if_node)
tuple_node = ast.TupleLiteralNode()
tuple_node.append(ast.LiteralNode('baz'))
tuple_node.append(ast.LiteralNode('boo'))
function_node.append(ast.BufferExtend(tuple_node))
expected_hash = hash(ast_root)
got_hash = hash(optimization_analyzer.optimize_ast())
self.assertEqual(expected_hash, got_hash)
def analyzeGetUDNNode(self, pnode):
children = pnode.getChildNodes()
if isinstance(children[0], ast.PlaceholderNode):
identifier = '.'.join([node.name for node in children])
# Some modules are trusted not to need UDN resolution.
if self._identifier_can_skip_UDN_resolution(identifier):
expr = '%s.%s' % (identifier, pnode.name)
return [ast.IdentifierNode(expr, pos=pnode.pos)]
expression = self.build_ast(pnode.expression)[0]
return [ast.GetUDNNode(expression, pnode.name, pos=pnode.pos)]
def test_hoist_both(self):
self.ast_description = """
file: TestTemplate
#global $foo
#def test_function
#if True
$foo
#else
$foo
#end if
#end def
"""
def scope_setter(scope):
scope.local_identifiers.add(ast.IdentifierNode('_rph_foo'))
scope.aliased_expression_map[ast.PlaceholderNode('foo')] = (
ast.IdentifierNode('_rph_foo'))
scope.aliased_expression_map[ast.FilterNode(ast.IdentifierNode(
'_rph_foo'))] = (ast.IdentifierNode('_fph123'))
scope.alias_name_set.add('_fph123')
scope.alias_name_set.add('_rph_foo')
def build_conditional_body(node):
node.append(ast.AssignNode(
ast.IdentifierNode('_rph_foo'), ast.PlaceholderNode('foo')))
node.append(ast.AssignNode(
ast.IdentifierNode('_fph123'), ast.FilterNode(
ast.IdentifierNode('_rph_foo'))))
node.append(ast.BufferWrite(ast.IdentifierNode('_fph123')))
ast_root, function_node, if_node = self._build_if_template()
ast_root.global_placeholders.add('foo')
scope_setter(function_node.scope)
function_node.scope.local_identifiers.add(ast.IdentifierNode('self'))
scope_setter(if_node.scope)
scope_setter(if_node.else_.scope)
build_conditional_body(if_node)
build_conditional_body(if_node.else_)
final_pass_analyzer = optimizer.FinalPassAnalyzer(
ast_root, self.compiler.analyzer_options, self.compiler)
final_pass_analyzer.hoist = test_util.RecordedFunction(
final_pass_analyzer.hoist)
final_pass_analyzer.visit_ast(ast_root)
# The 4 calls are hoisting the rph alias and the fph alias out of
# both the if and else clauses.
self.assertEqual(len(final_pass_analyzer.hoist.GetCalls()), 4)
def test_set_error(self):
self.ast_description = """
file: TestTemplate
#implements library
#set $foo = True
#def test_function
#end def
"""
ast_root, def_node = self._build_function_template_library()
assign_node = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(True))
ast_root.insert_before(def_node, assign_node)
semantic_analyzer = self._get_analyzer(ast_root)
self.assertRaises(analyzer.SemanticAnalyzerError,
semantic_analyzer.get_ast)
def test_index_before_assign_error(self):
self.ast_description = """
file: TestTemplate
#def test_function
#set $foo[1] = 1
#end def
"""
ast_root, function_node = self._build_function_template()
assign_node = ast.AssignNode(
ast.SliceNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)),
ast.LiteralNode(1))
function_node.append(assign_node)
optimization_analyzer = self._get_analyzer(ast_root)
self.assertRaises(analyzer.SemanticAnalyzerError,
optimization_analyzer.visit_ast, ast_root)
def analyzePlaceholderNode(self, pnode):
if (self.options.fail_library_searchlist_access
and pnode.name not in self.template.global_placeholders):
if (self.options.strict_global_check
and not self.template.allow_undeclared_globals and
(not self.template.has_identifier(pnode.name)
and pnode.name not in self.compiler.function_name_registry)):
# Break compile if no #loose_resolution and variable is not
# available in any reasonable scope.
err_msg = ('identifier %s is unavailable and is not declared '
'as a #global display variable' % pnode.name)
self.compiler.error(SemanticAnalyzerError(err_msg),
pos=pnode.pos)
elif self.template.library:
# Only do placeholder resolutions for placeholders declared with
# #global in library templates.
identifier_node = ast.IdentifierNode(pnode.name, pos=pnode.pos)
return [identifier_node]
return [pnode]
def __init__(self, classname, parse_root, options, compiler):
self.classname = classname
self.parse_root = parse_root
self.options = options
self.compiler = compiler
self.ast_root = None
self.template = None
self.strip_lines = False
self.uses_raw = False
self.base_extends_identifiers = []
if self.compiler.base_extends_package:
# this means that extends are supposed to all happen relative to
# some other package - this is handy for assuring all templates
# reference within a tree, say for localization, where each locale
# might have its own package
packages = self.compiler.base_extends_package.split('.')
self.base_extends_identifiers = [
ast.IdentifierNode(module_name) for module_name in packages
]
def test_empty_if_full_else_fails(self):
self.ast_description = """
file: TestTemplate
#def test_function
#if True
#else
#set $foo = true
#end if
#end def
"""
ast_root, def_node, if_node = self._build_if_template()
assign_node = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(True))
if_node.else_.append(assign_node)
semantic_analyzer = self._get_analyzer(ast_root)
self.assertRaises(analyzer.SemanticAnalyzerError,
semantic_analyzer.get_ast)
def test_slice_identifier_ok(self):
self.ast_description = """
file: TestTemplate
#def test_function
#set $foo[1] = 1
#end def
"""
ast_root, def_node = self._build_function_template()
assign_node = ast.AssignNode(
ast.SliceNode(
ast.IdentifierNode('foo'), ast.LiteralNode(1)),
ast.LiteralNode(1))
def_node.append(assign_node)
semantic_analyzer = self._get_analyzer(ast_root)
try:
semantic_analyzer.get_ast()
except analyzer.SemanticAnalyzerError:
self.fail('get_ast raised SemanticAnalyzerError unexpectedly.')
def test_hoists_both_from_plus(self):
self.ast_description = """
file: TestTemplate
#global $foo
#def test_function
#set $bar = $foo + $foo
#end def
"""
ast_root, function_node = self._build_function_template()
ast_root.global_placeholders.add('foo')
function_node.append(ast.AssignNode(
ast.IdentifierNode('bar'), ast.BinOpNode('+', ast.PlaceholderNode(
'foo'), ast.PlaceholderNode('foo'))))
optimization_analyzer = self._get_analyzer_and_visit(ast_root)
self.assertEqual(
optimization_analyzer._placeholdernode_replacement.GetResults(),
[True, True])
def test_non_empty_for_ok(self):
self.ast_description = """
file: TestTemplate
#def test_function
#for $i in []
#set $foo = True
#end for
#end def
"""
ast_root, def_node, for_node = self._build_for_template()
assign_node = ast.AssignNode(
ast.IdentifierNode('foo'), ast.LiteralNode(True))
for_node.append(assign_node)
semantic_analyzer = self._get_analyzer(ast_root)
try:
semantic_analyzer.get_ast()
except analyzer.SemanticAnalyzerError:
self.fail('get_ast raised SemanticAnalyzerError unexpectedly.')
def analyzePlaceholderSubstitutionNode(self, pnode):
# print ' '.join(analyzePlaceholderSubstitutionNode', pnode,
# pnode.parameter_list.get_arg_map())
node_list = []
ph_expression = self.build_ast(pnode.expression)[0]
# If the expression contained a macro that was parsed as a
# fragment, the expression is now a statement and can be moved
# outside of the ast.PlaceholderSubstitutionNode.
#
# This is a hack to get around design decisions that were made
# early on. It is up to the macro authors to correctly decide how
# the macro should be parsed and the compiler should throw errors
# if there is an odd state where nodes are somewhere unexpected.
if isinstance(ph_expression, ast.statement_nodes):
return [ph_expression]
arg_map = pnode.parameter_list.get_arg_map()
default_format_string = '%s'
format_string = arg_map.get('format_string', default_format_string)
skip_filter = False
cache_forever = False
registered_function = False
function_has_only_literal_args = False
never_cache = False
if isinstance(ph_expression, ast.CallFunctionNode):
fname = ph_expression.expression.name
if self.compiler.registry_contains(fname):
function_has_only_literal_args = (
ph_expression.arg_list and not [
_arg for _arg in ph_expression.arg_list
if not isinstance(_arg, ast.LiteralNode)
])
skip_filter = self.compiler.get_registry_value(
fname, 'skip_filter')
skip_unless_baked = self.compiler.get_registry_value(
fname, 'skip_filter_unless_baked')
skip_filter = skip_filter or (not self.template.baked
and skip_unless_baked)
cache_forever = self.compiler.get_registry_value(
fname, 'cache_forever')
never_cache = self.compiler.get_registry_value(
fname, 'never_cache')
elif ph_expression.library_function:
# Don't escape function calls into library templates.
skip_filter = True
if (self.compiler.enable_filters
and format_string == default_format_string
and not isinstance(ph_expression, ast.LiteralNode)):
arg_node_map = pnode.parameter_list.get_arg_node_map()
if 'raw' in arg_map:
# If this is a |raw usage and the template does not allow raw,
# raise an error.
self.uses_raw = True
if (self.options.no_raw
and not self.template.explicitly_allow_raw):
err_msg = ('|raw is not allowed in templates compiled '
'with the --no-raw flag.')
self.compiler.error(SemanticAnalyzerError(err_msg),
pos=pnode.pos)
else:
# if we need to filter, wrap up the node and wait for further
# analysis later on
if skip_filter:
# explicitly set the filter to none here - this means we
# will cache expensive pseudo-filtered nodes
ph_expression = ast.FilterNode(ph_expression,
None,
pos=pnode.pos)
else:
ph_expression = ast.FilterNode(
ph_expression,
arg_node_map.get('filter', ast.DefaultFilterFunction),
pos=pnode.pos)
# if this is a literal node, we still might want to filter it
# but the output should always be the same - so do it once and
# cache FIXME: could fold this and apply the function at
# compile-time
if (not never_cache and
(registered_function and function_has_only_literal_args)
or cache_forever or 'cache' in arg_map):
cache_expression = ast.CacheNode(ph_expression,
pos=pnode.pos)
self.template.cached_identifiers.add(cache_expression)
node_list.append(cache_expression)
ph_expression = ast.IdentifierNode(cache_expression.name,
pos=pnode.pos)
if isinstance(ph_expression, ast.LiteralNode):
buffer_write = ast.BufferWrite(ph_expression, pos=pnode.pos)
node_list.append(buffer_write)
elif (self.compiler.enable_filters
and format_string == default_format_string):
# we are already filtering, don't bother creating a new string
buffer_write = ast.BufferWrite(ph_expression, pos=pnode.pos)
node_list.append(buffer_write)
else:
buffer_write = ast.BufferWrite(ast.BinOpNode(
'%', ast.LiteralNode(format_string, pos=pnode.pos),
ph_expression),
pos=pnode.pos)
node_list.append(buffer_write)
return node_list
