def assign_after_filter_fails(self): self.ast_description = """ file: TestTemplate #def test_function #set $foo = 'foo' $foo #set $foo = 'bar' $foo #end def """ ast_root, function_node = self._build_function_template() first_assign = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode('foo')) function_node.append(first_assign) first_use = ast.FilterNode(ast.IdentifierNode('foo')) function_node.append(first_use) second_assign = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode('bar')) function_node.append(second_assign) second_use = ast.FilterNode(ast.IdentifierNode('foo')) function_node.append(second_use) optimization_analyzer = optimizer.OptimizationAnalyzer( ast_root, self.compiler.analyzer_options, self.compiler) optimization_analyzer.visit_ast = test_util.RecordedFunction( optimization_analyzer.visit_ast) self.assertRaises(compiler.Warning, optimization_analyzer.visit_ast, ast_root)
def test_index_scope_ok(self): self.ast_description = """ file: TestTemplate #def test_function #set $foo = {} #if True #set $foo[1] = 1 #end if #end def """ ast_root, function_node = self._build_function_template() assign_node1 = ast.AssignNode( ast.IdentifierNode('foo'), ast.DictLiteralNode()) function_node.append(assign_node1) if_node = ast.IfNode(ast.LiteralNode(True)) assign_node2 = ast.AssignNode( ast.SliceNode( ast.IdentifierNode('foo'), ast.LiteralNode(1)), ast.LiteralNode(1)) if_node.append(assign_node2) function_node.append(if_node) optimization_analyzer = self._get_analyzer(ast_root) try: optimization_analyzer.visit_ast(ast_root) except analyzer.SemanticAnalyzerError: self.fail('visit_ast raised SemanticAnalyzerError unexpectedly.')
def build_conditional_body(node): node.append(ast.AssignNode( ast.IdentifierNode('_rph_foo'), ast.PlaceholderNode('foo'))) node.append(ast.AssignNode( ast.IdentifierNode('_fph123'), ast.FilterNode( ast.IdentifierNode('_rph_foo')))) node.append(ast.BufferWrite(ast.IdentifierNode('_fph123')))
def test_partial_nested_else_if(self): self.ast_description = """ file: TestTemplate #def test_function #if True #set $foo = 1 #else #if True #set $foo = 2 #end if #end if $foo #end def """ ast_root, function_node, if_node = self._build_if_template() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) if_node_2 = ast.IfNode(ast.LiteralNode(True)) if_node_2.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(2))) if_node.else_.append(if_node_2) function_node.append(ast.PlaceholderNode('foo')) optimization_analyzer = self._get_analyzer(ast_root) self.assertRaises(analyzer.SemanticAnalyzerError, optimization_analyzer.visit_ast, ast_root)
def test_nested_else(self): self.ast_description = """ file: TestTemplate #def test_function #if True #set $foo = 1 #else #if #set $foo = 2 #else #set $foo = 3 #end if #end if $foo #end def """ ast_root, function_node, if_node = self._build_if_template() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) if_node_2 = ast.IfNode(ast.LiteralNode(True)) if_node_2.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(2))) if_node_2.else_.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(3))) if_node.else_.append(if_node_2) function_node.append(ast.PlaceholderNode('foo')) optimization_analyzer = self._get_analyzer(ast_root) try: optimization_analyzer.visit_ast(ast_root) except analyzer.SemanticAnalyzerError: self.fail('visit_ast raised SemanticAnalyzerError unexpectedly.')
def double_assign_ok(self): self.ast_description = """ file: TestTemplate #def test_function #set $foo = 'foo' #set $foo = 'bar' $foo #end def """ ast_root, function_node = self._build_function_template() first_assign = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode('foo')) function_node.append(first_assign) second_assign = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode('bar')) function_node.append(second_assign) first_use = ast.FilterNode(ast.IdentifierNode('foo')) function_node.append(first_use) optimization_analyzer = optimizer.OptimizationAnalyzer( ast_root, self.compiler.analyzer_options, self.compiler) optimization_analyzer.visit_ast = test_util.RecordedFunction( optimization_analyzer.visit_ast) try: optimization_analyzer.visit_ast(ast_root) except compiler.Warning: self.fail('visit_ast raised WarningError unexpectedly.')
def scope_setter(scope): scope.local_identifiers.add(ast.IdentifierNode('_rph_foo')) scope.aliased_expression_map[ast.PlaceholderNode('foo')] = ( ast.IdentifierNode('_rph_foo')) scope.aliased_expression_map[ast.FilterNode(ast.IdentifierNode( '_rph_foo'))] = (ast.IdentifierNode('_fph123')) scope.alias_name_set.add('_fph123') scope.alias_name_set.add('_rph_foo')
def analyzeCallFunctionNode(self, pnode): fn = pnode fname = fn.expression.name if self.compiler.registry_contains(fname): # If this is a placeholder that is in the function registry, mark it # as used so that in the optimizer stage, we can avoid importing # unused registry values. self.template.used_function_registry_identifiers.add(fname) # The fully qualified library function name iff we figure out # that this is calling into a library. library_function = None skip_filter = self.compiler.get_registry_value(fname, 'skip_filter') if isinstance(fn.expression, ast.PlaceholderNode): macro_handler_name = 'macro_function_%s' % fn.expression.name macro_data = self.compiler.macro_registry.get(macro_handler_name) if macro_data: macro_function, macro_parse_rule = macro_data return self.handleMacro(fn, macro_function, macro_parse_rule) elif fn.expression.name in self.template.template_methods: fn.sanitization_state = ast.SanitizedState.SANITIZED_STRING if self.template.library: # Calling another library function from this library # function. library_function = fn.expression.name elif skip_filter: # If the function is marked as skip_filter in the registry, we # know it is sanitized. fn.sanitization_state = ast.SanitizedState.SANITIZED elif skip_filter is False: # If the function is marked as skip_filter=False in the # registry, we know it is not sanitized. fn.sanitization_state = ast.SanitizedState.UNSANITIZED elif isinstance(fn.expression, ast.GetUDNNode): identifier = [node.name for node in fn.expression.getChildNodes()] identifier = '.'.join(identifier) if identifier in self.template.library_identifiers: # Calling library functions from other templates. library_function = '%s.%s' % (identifier, fn.expression.name) if library_function: # Replace the placeholder node or UDN resolution with a direct # reference to the library function, either in another imported # module or here. fn.expression = ast.IdentifierNode(library_function, pos=pnode.pos) # Pass the current template instance into the library function. fn.arg_list.child_nodes.insert(0, ast.IdentifierNode('self')) # Library functions are spitfire functions so their output is # sanitized. fn.sanitization_state = ast.SanitizedState.SANITIZED_STRING fn.library_function = True fn.expression = self.build_ast(fn.expression)[0] fn.arg_list = self.build_ast(fn.arg_list)[0] return [fn]
def test_duplicate_node_collect(self): self.ast_description = """ file: TestTemplate #def test_function foo bar #if True #set $foo = 1 #end if baz boo #if True #set $foo = 1 #end if #end def NOTE: This test will break if collect_writes is written using ast.ASTNode.insert_before. """ ast_root, function_node = self._build_function_template() function_node.append(ast.BufferWrite(ast.LiteralNode('foo'))) function_node.append(ast.BufferWrite(ast.LiteralNode('bar'))) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) function_node.append(if_node) function_node.append(ast.BufferWrite(ast.LiteralNode('baz'))) function_node.append(ast.BufferWrite(ast.LiteralNode('boo'))) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) optimization_analyzer = self._get_analyzer(ast_root) ast_root, function_node = self._build_function_template() tuple_node = ast.TupleLiteralNode() tuple_node.append(ast.LiteralNode('foo')) tuple_node.append(ast.LiteralNode('bar')) function_node.append(ast.BufferExtend(tuple_node)) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) function_node.append(if_node) tuple_node = ast.TupleLiteralNode() tuple_node.append(ast.LiteralNode('baz')) tuple_node.append(ast.LiteralNode('boo')) function_node.append(ast.BufferExtend(tuple_node)) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) expected_hash = hash(ast_root) got_hash = hash(optimization_analyzer.optimize_ast()) self.assertEqual(expected_hash, got_hash)
def handle_repeat(self, dom_node, attr_name): debug("handle_repeat", dom_node) expr_pieces = dom_node.getAttribute(attr_name).split() dom_node.removeAttribute(attr_name) target = expr_pieces[0] expr_ast = util.parse(' '.join(expr_pieces[1:]), 'rhs_expression') node_list = [] # hack - assumes python syntax fn = ast.ForNode( ast.TargetListNode([ ast.IdentifierNode("self.repeat['%s']" % target), ast.IdentifierNode(target) ]), ast.ExpressionListNode([ ast.CallFunctionNode(ast.IdentifierNode('enumerate'), ast.ArgListNode([expr_ast])) ])) if self.has_child_stuff(dom_node): debug("has_child_stuff:", dom_node) fn.extend(self.build_ast(dom_node)) #fn.append(self.make_tag_node(dom_node)) #for n in dom_node.childNodes: # fn.extend(self.build_ast(n)) else: # print "no children" fn.extend(self.build_ast(dom_node)) if (dom_node.previousSibling and dom_node.previousSibling.nodeType == xml.dom.minidom.Node.TEXT_NODE and not dom_node.previousSibling.nodeValue.strip()): # inject the previous whitespace sibling to keep the output looking # ok fixme: a conditional is probably required here - you only want # to execute this if it's not the last execution of the loop fn.prepend(self.build_ast(dom_node.previousSibling)) # now remove the previous sibling #print "node", dom_node #print "parent", dom_node.parentNode #print ' '.join("previous", dom_node.previousSibling, # id(dom_node.previousSibling)) #print "next", dom_node.nextSibling, id(dom_node.nextSibling) #dom_node.parentNode.removeChild(dom_node.previousSibling) node_list.append(ast.EatPrevious()) node_list.append(fn) #fn.extend(self.make_tag_node(dom_node, close=True)) return node_list
def test_collect_writes_join_if(self): self.ast_description = """ file: TestTemplate #def test_function foo bar #if True #set $foo = 1 #end if baz boo #end def """ ast_root, function_node = self._build_function_template() function_node.append(ast.BufferWrite(ast.LiteralNode('foo'))) function_node.append(ast.BufferWrite(ast.LiteralNode('bar'))) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) function_node.append(if_node) function_node.append(ast.BufferWrite(ast.LiteralNode('baz'))) function_node.append(ast.BufferWrite(ast.LiteralNode('boo'))) optimization_analyzer = self._get_analyzer(ast_root) ast_root, function_node = self._build_function_template() tuple_node = ast.TupleLiteralNode() tuple_node.append(ast.LiteralNode('foo')) tuple_node.append(ast.LiteralNode('bar')) function_node.append(ast.BufferExtend(tuple_node)) if_node = ast.IfNode() if_node.append(ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(1))) function_node.append(if_node) tuple_node = ast.TupleLiteralNode() tuple_node.append(ast.LiteralNode('baz')) tuple_node.append(ast.LiteralNode('boo')) function_node.append(ast.BufferExtend(tuple_node)) expected_hash = hash(ast_root) got_hash = hash(optimization_analyzer.optimize_ast()) self.assertEqual(expected_hash, got_hash)
def analyzeGetUDNNode(self, pnode): children = pnode.getChildNodes() if isinstance(children[0], ast.PlaceholderNode): identifier = '.'.join([node.name for node in children]) # Some modules are trusted not to need UDN resolution. if self._identifier_can_skip_UDN_resolution(identifier): expr = '%s.%s' % (identifier, pnode.name) return [ast.IdentifierNode(expr, pos=pnode.pos)] expression = self.build_ast(pnode.expression)[0] return [ast.GetUDNNode(expression, pnode.name, pos=pnode.pos)]
def test_hoist_both(self): self.ast_description = """ file: TestTemplate #global $foo #def test_function #if True $foo #else $foo #end if #end def """ def scope_setter(scope): scope.local_identifiers.add(ast.IdentifierNode('_rph_foo')) scope.aliased_expression_map[ast.PlaceholderNode('foo')] = ( ast.IdentifierNode('_rph_foo')) scope.aliased_expression_map[ast.FilterNode(ast.IdentifierNode( '_rph_foo'))] = (ast.IdentifierNode('_fph123')) scope.alias_name_set.add('_fph123') scope.alias_name_set.add('_rph_foo') def build_conditional_body(node): node.append(ast.AssignNode( ast.IdentifierNode('_rph_foo'), ast.PlaceholderNode('foo'))) node.append(ast.AssignNode( ast.IdentifierNode('_fph123'), ast.FilterNode( ast.IdentifierNode('_rph_foo')))) node.append(ast.BufferWrite(ast.IdentifierNode('_fph123'))) ast_root, function_node, if_node = self._build_if_template() ast_root.global_placeholders.add('foo') scope_setter(function_node.scope) function_node.scope.local_identifiers.add(ast.IdentifierNode('self')) scope_setter(if_node.scope) scope_setter(if_node.else_.scope) build_conditional_body(if_node) build_conditional_body(if_node.else_) final_pass_analyzer = optimizer.FinalPassAnalyzer( ast_root, self.compiler.analyzer_options, self.compiler) final_pass_analyzer.hoist = test_util.RecordedFunction( final_pass_analyzer.hoist) final_pass_analyzer.visit_ast(ast_root) # The 4 calls are hoisting the rph alias and the fph alias out of # both the if and else clauses. self.assertEqual(len(final_pass_analyzer.hoist.GetCalls()), 4)
def test_set_error(self): self.ast_description = """ file: TestTemplate #implements library #set $foo = True #def test_function #end def """ ast_root, def_node = self._build_function_template_library() assign_node = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(True)) ast_root.insert_before(def_node, assign_node) semantic_analyzer = self._get_analyzer(ast_root) self.assertRaises(analyzer.SemanticAnalyzerError, semantic_analyzer.get_ast)
def test_index_before_assign_error(self): self.ast_description = """ file: TestTemplate #def test_function #set $foo[1] = 1 #end def """ ast_root, function_node = self._build_function_template() assign_node = ast.AssignNode( ast.SliceNode( ast.IdentifierNode('foo'), ast.LiteralNode(1)), ast.LiteralNode(1)) function_node.append(assign_node) optimization_analyzer = self._get_analyzer(ast_root) self.assertRaises(analyzer.SemanticAnalyzerError, optimization_analyzer.visit_ast, ast_root)
def analyzePlaceholderNode(self, pnode): if (self.options.fail_library_searchlist_access and pnode.name not in self.template.global_placeholders): if (self.options.strict_global_check and not self.template.allow_undeclared_globals and (not self.template.has_identifier(pnode.name) and pnode.name not in self.compiler.function_name_registry)): # Break compile if no #loose_resolution and variable is not # available in any reasonable scope. err_msg = ('identifier %s is unavailable and is not declared ' 'as a #global display variable' % pnode.name) self.compiler.error(SemanticAnalyzerError(err_msg), pos=pnode.pos) elif self.template.library: # Only do placeholder resolutions for placeholders declared with # #global in library templates. identifier_node = ast.IdentifierNode(pnode.name, pos=pnode.pos) return [identifier_node] return [pnode]
def __init__(self, classname, parse_root, options, compiler): self.classname = classname self.parse_root = parse_root self.options = options self.compiler = compiler self.ast_root = None self.template = None self.strip_lines = False self.uses_raw = False self.base_extends_identifiers = [] if self.compiler.base_extends_package: # this means that extends are supposed to all happen relative to # some other package - this is handy for assuring all templates # reference within a tree, say for localization, where each locale # might have its own package packages = self.compiler.base_extends_package.split('.') self.base_extends_identifiers = [ ast.IdentifierNode(module_name) for module_name in packages ]
def test_empty_if_full_else_fails(self): self.ast_description = """ file: TestTemplate #def test_function #if True #else #set $foo = true #end if #end def """ ast_root, def_node, if_node = self._build_if_template() assign_node = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(True)) if_node.else_.append(assign_node) semantic_analyzer = self._get_analyzer(ast_root) self.assertRaises(analyzer.SemanticAnalyzerError, semantic_analyzer.get_ast)
def test_slice_identifier_ok(self): self.ast_description = """ file: TestTemplate #def test_function #set $foo[1] = 1 #end def """ ast_root, def_node = self._build_function_template() assign_node = ast.AssignNode( ast.SliceNode( ast.IdentifierNode('foo'), ast.LiteralNode(1)), ast.LiteralNode(1)) def_node.append(assign_node) semantic_analyzer = self._get_analyzer(ast_root) try: semantic_analyzer.get_ast() except analyzer.SemanticAnalyzerError: self.fail('get_ast raised SemanticAnalyzerError unexpectedly.')
def test_hoists_both_from_plus(self): self.ast_description = """ file: TestTemplate #global $foo #def test_function #set $bar = $foo + $foo #end def """ ast_root, function_node = self._build_function_template() ast_root.global_placeholders.add('foo') function_node.append(ast.AssignNode( ast.IdentifierNode('bar'), ast.BinOpNode('+', ast.PlaceholderNode( 'foo'), ast.PlaceholderNode('foo')))) optimization_analyzer = self._get_analyzer_and_visit(ast_root) self.assertEqual( optimization_analyzer._placeholdernode_replacement.GetResults(), [True, True])
def test_non_empty_for_ok(self): self.ast_description = """ file: TestTemplate #def test_function #for $i in [] #set $foo = True #end for #end def """ ast_root, def_node, for_node = self._build_for_template() assign_node = ast.AssignNode( ast.IdentifierNode('foo'), ast.LiteralNode(True)) for_node.append(assign_node) semantic_analyzer = self._get_analyzer(ast_root) try: semantic_analyzer.get_ast() except analyzer.SemanticAnalyzerError: self.fail('get_ast raised SemanticAnalyzerError unexpectedly.')
def analyzePlaceholderSubstitutionNode(self, pnode): # print ' '.join(analyzePlaceholderSubstitutionNode', pnode, # pnode.parameter_list.get_arg_map()) node_list = [] ph_expression = self.build_ast(pnode.expression)[0] # If the expression contained a macro that was parsed as a # fragment, the expression is now a statement and can be moved # outside of the ast.PlaceholderSubstitutionNode. # # This is a hack to get around design decisions that were made # early on. It is up to the macro authors to correctly decide how # the macro should be parsed and the compiler should throw errors # if there is an odd state where nodes are somewhere unexpected. if isinstance(ph_expression, ast.statement_nodes): return [ph_expression] arg_map = pnode.parameter_list.get_arg_map() default_format_string = '%s' format_string = arg_map.get('format_string', default_format_string) skip_filter = False cache_forever = False registered_function = False function_has_only_literal_args = False never_cache = False if isinstance(ph_expression, ast.CallFunctionNode): fname = ph_expression.expression.name if self.compiler.registry_contains(fname): function_has_only_literal_args = ( ph_expression.arg_list and not [ _arg for _arg in ph_expression.arg_list if not isinstance(_arg, ast.LiteralNode) ]) skip_filter = self.compiler.get_registry_value( fname, 'skip_filter') skip_unless_baked = self.compiler.get_registry_value( fname, 'skip_filter_unless_baked') skip_filter = skip_filter or (not self.template.baked and skip_unless_baked) cache_forever = self.compiler.get_registry_value( fname, 'cache_forever') never_cache = self.compiler.get_registry_value( fname, 'never_cache') elif ph_expression.library_function: # Don't escape function calls into library templates. skip_filter = True if (self.compiler.enable_filters and format_string == default_format_string and not isinstance(ph_expression, ast.LiteralNode)): arg_node_map = pnode.parameter_list.get_arg_node_map() if 'raw' in arg_map: # If this is a |raw usage and the template does not allow raw, # raise an error. self.uses_raw = True if (self.options.no_raw and not self.template.explicitly_allow_raw): err_msg = ('|raw is not allowed in templates compiled ' 'with the --no-raw flag.') self.compiler.error(SemanticAnalyzerError(err_msg), pos=pnode.pos) else: # if we need to filter, wrap up the node and wait for further # analysis later on if skip_filter: # explicitly set the filter to none here - this means we # will cache expensive pseudo-filtered nodes ph_expression = ast.FilterNode(ph_expression, None, pos=pnode.pos) else: ph_expression = ast.FilterNode( ph_expression, arg_node_map.get('filter', ast.DefaultFilterFunction), pos=pnode.pos) # if this is a literal node, we still might want to filter it # but the output should always be the same - so do it once and # cache FIXME: could fold this and apply the function at # compile-time if (not never_cache and (registered_function and function_has_only_literal_args) or cache_forever or 'cache' in arg_map): cache_expression = ast.CacheNode(ph_expression, pos=pnode.pos) self.template.cached_identifiers.add(cache_expression) node_list.append(cache_expression) ph_expression = ast.IdentifierNode(cache_expression.name, pos=pnode.pos) if isinstance(ph_expression, ast.LiteralNode): buffer_write = ast.BufferWrite(ph_expression, pos=pnode.pos) node_list.append(buffer_write) elif (self.compiler.enable_filters and format_string == default_format_string): # we are already filtering, don't bother creating a new string buffer_write = ast.BufferWrite(ph_expression, pos=pnode.pos) node_list.append(buffer_write) else: buffer_write = ast.BufferWrite(ast.BinOpNode( '%', ast.LiteralNode(format_string, pos=pnode.pos), ph_expression), pos=pnode.pos) node_list.append(buffer_write) return node_list