def __init__(self, code, **exception_kwargs): self.codeargs = [] self.args = [] self.declared_identifiers = util.Set() self.undeclared_identifiers = util.Set() class FindTuple(object): def visitTuple(s, node, *args): for n in node.nodes: p = PythonCode(n, **exception_kwargs) self.codeargs.append(p) self.args.append(ExpressionGenerator(n).value()) self.declared_identifiers = self.declared_identifiers.union( p.declared_identifiers) self.undeclared_identifiers = self.undeclared_identifiers.union( p.undeclared_identifiers) if isinstance(code, basestring): if re.match(r"\S", code) and not re.match(r",\s*$", code): # if theres text and no trailing comma, insure its parsed # as a tuple by adding a trailing comma code += "," expr = parse(code, "exec", **exception_kwargs) else: expr = code f = FindTuple() visitor.walk(expr, f)
def is_ternary(self, keyword): """return true if the given keyword is a ternary keyword for this ControlLine""" return keyword in { 'if': util.Set(['else', 'elif']), 'try': util.Set(['except', 'finally']), 'for': util.Set(['else']) }.get(self.keyword, [])
def write_variable_declares(self, identifiers, toplevel=False, limit=None): """write variable declarations at the top of a function. the variable declarations are in the form of callable definitions for defs and/or name lookup within the function's context argument. the names declared are based on the names that are referenced in the function body, which don't otherwise have any explicit assignment operation. names that are assigned within the body are assumed to be locally-scoped variables and are not separately declared. for def callable definitions, if the def is a top-level callable then a 'stub' callable is generated which wraps the current Context into a closure. if the def is not top-level, it is fully rendered as a local closure.""" # collection of all defs available to us in this scope comp_idents = dict([(c.name, c) for c in identifiers.defs]) to_write = util.Set() # write "context.get()" for all variables we are going to need that arent in the namespace yet to_write = to_write.union(identifiers.undeclared) # write closure functions for closures that we define right here to_write = to_write.union(util.Set([c.name for c in identifiers.closuredefs.values()])) # remove identifiers that are declared in the argument signature of the callable to_write = to_write.difference(identifiers.argument_declared) # remove identifiers that we are going to assign to. in this way we mimic Python's behavior, # i.e. assignment to a variable within a block means that variable is now a "locally declared" var, # which cannot be referenced beforehand. to_write = to_write.difference(identifiers.locally_declared) # if a limiting set was sent, constraint to those items in that list # (this is used for the caching decorator) if limit is not None: to_write = to_write.intersection(limit) if toplevel and getattr(self.compiler, 'has_ns_imports', False): self.printer.writeline("_import_ns = {}") self.compiler.has_imports = True for ident, ns in self.compiler.namespaces.iteritems(): if ns.attributes.has_key('import'): self.printer.writeline("_mako_get_namespace(context, %s)._populate(_import_ns, %s)" % (repr(ident), repr(re.split(r'\s*,\s*', ns.attributes['import'])))) for ident in to_write: if ident in comp_idents: comp = comp_idents[ident] if comp.is_root(): self.write_def_decl(comp, identifiers) else: self.write_inline_def(comp, identifiers, nested=True) elif ident in self.compiler.namespaces: self.printer.writeline("%s = _mako_get_namespace(context, %s)" % (ident, repr(ident))) else: if getattr(self.compiler, 'has_ns_imports', False): self.printer.writeline("%s = _import_ns.get(%s, context.get(%s, UNDEFINED))" % (ident, repr(ident), repr(ident))) else: self.printer.writeline("%s = context.get(%s, UNDEFINED)" % (ident, repr(ident))) self.printer.writeline("__M_writer = context.writer()")
def _parse_attributes(self, expressions, nonexpressions): undeclared_identifiers = util.Set() self.parsed_attributes = {} for key in self.attributes: if key in expressions: expr = [] for x in re.split(r'(\${.+?})', self.attributes[key]): m = re.match(r'^\${(.+?)}$', x) if m: code = ast.PythonCode(m.group(1), **self.exception_kwargs) undeclared_identifiers = undeclared_identifiers.union( code.undeclared_identifiers) expr.append(m.group(1)) else: if x: expr.append(repr(x)) self.parsed_attributes[key] = " + ".join(expr) elif key in nonexpressions: if re.search(r'${.+?}', self.attributes[key]): raise exceptions.CompileException( "Attibute '%s' in tag '%s' does not allow embedded expressions" % (key, self.keyword), **self.exception_kwargs) self.parsed_attributes[key] = repr(self.attributes[key]) else: raise exceptions.CompileException( "Invalid attribute for tag '%s': '%s'" % (self.keyword, key), **self.exception_kwargs) self.expression_undeclared_identifiers = undeclared_identifiers
def __init__(self, code, **exception_kwargs): self.codeargs = [] self.args = [] self.declared_identifiers = util.Set() self.undeclared_identifiers = util.Set() if isinstance(code, basestring): if re.match(r"\S", code) and not re.match(r",\s*$", code): # if theres text and no trailing comma, insure its parsed # as a tuple by adding a trailing comma code += "," expr = pyparser.parse(code, "exec", **exception_kwargs) else: expr = code f = pyparser.FindTuple(self, PythonCode, **exception_kwargs) f.visit(expr)
def undeclared_identifiers(self): # TODO: make the "filter" shortcut list configurable at parse/gen time return self.code.undeclared_identifiers.union( self.escapes_code.undeclared_identifiers.difference( util.Set(filters.DEFAULT_ESCAPES.keys()) ) )
def undeclared_identifiers(self): res = [] for c in self.function_decl.defaults: res += list( ast.PythonCode(c, **self.exception_kwargs).undeclared_identifiers) return res + list( self.filter_args.undeclared_identifiers.difference( util.Set(filters.DEFAULT_ESCAPES.keys())))
def __init__(self, code, **exception_kwargs): self.code = code # represents all identifiers which are assigned to at some point in the code self.declared_identifiers = util.Set() # represents all identifiers which are referenced before their assignment, if any self.undeclared_identifiers = util.Set() # note that an identifier can be in both the undeclared and declared lists. # using AST to parse instead of using code.co_varnames, code.co_names has several advantages: # - we can locate an identifier as "undeclared" even if its declared later in the same block of code # - AST is less likely to break with version changes (for example, the behavior of co_names changed a little bit # in python version 2.5) if isinstance(code, basestring): expr = pyparser.parse(code.lstrip(), "exec", **exception_kwargs) else: expr = code f = pyparser.FindIdentifiers(self, **exception_kwargs) f.visit(expr)
def __init__(self, node=None, parent=None, nested=False): if parent is not None: # things that have already been declared in an enclosing namespace (i.e. names we can just use) self.declared = util.Set(parent.declared).union([ c.name for c in parent.closuredefs.values() ]).union(parent.locally_declared).union(parent.argument_declared) # if these identifiers correspond to a "nested" scope, it means whatever the # parent identifiers had as undeclared will have been declared by that parent, # and therefore we have them in our scope. if nested: self.declared = self.declared.union(parent.undeclared) # top level defs that are available self.topleveldefs = util.SetLikeDict(**parent.topleveldefs) else: self.declared = util.Set() self.topleveldefs = util.SetLikeDict() # things within this level that are referenced before they are declared (e.g. assigned to) self.undeclared = util.Set() # things that are declared locally. some of these things could be in the "undeclared" # list as well if they are referenced before declared self.locally_declared = util.Set() # assignments made in explicit python blocks. these will be propigated to # the context of local def calls. self.locally_assigned = util.Set() # things that are declared in the argument signature of the def callable self.argument_declared = util.Set() # closure defs that are defined in this level self.closuredefs = util.SetLikeDict() self.node = node if node is not None: node.accept_visitor(self)
# ast.py # Copyright (C) Mako developers # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """Handles parsing of Python code. Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler module is used. """ from StringIO import StringIO from mako import exceptions, util # words that cannot be assigned to (notably smaller than the total keys in __builtins__) reserved = util.Set(['True', 'False', 'None']) try: import _ast util.restore__ast(_ast) import _ast_util except ImportError: _ast = None from compiler import parse as compiler_parse from compiler import visitor def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" try: if _ast:
def undeclared_identifiers(self): identifiers = self.page_args.undeclared_identifiers.difference( util.Set(["__DUMMY"])) return identifiers.union( super(IncludeTag, self).undeclared_identifiers())
def __init__(self, code, **exception_kwargs): self.code = code # represents all identifiers which are assigned to at some point in the code self.declared_identifiers = util.Set() # represents all identifiers which are referenced before their assignment, if any self.undeclared_identifiers = util.Set() # note that an identifier can be in both the undeclared and declared lists. # using AST to parse instead of using code.co_varnames, code.co_names has several advantages: # - we can locate an identifier as "undeclared" even if its declared later in the same block of code # - AST is less likely to break with version changes (for example, the behavior of co_names changed a little bit # in python version 2.5) if isinstance(code, basestring): expr = parse(code.lstrip(), "exec", **exception_kwargs) else: expr = code class FindIdentifiers(object): def __init__(self): self.in_function = False self.local_ident_stack = {} def _add_declared(s, name): if not s.in_function: self.declared_identifiers.add(name) def visitClass(self, node, *args): self._add_declared(node.name) def visitAssName(self, node, *args): self._add_declared(node.name) def visitAssign(self, node, *args): # flip around the visiting of Assign so the expression gets evaluated first, # in the case of a clause like "x=x+5" (x is undeclared) self.visit(node.expr, *args) for n in node.nodes: self.visit(n, *args) def visitFunction(self, node, *args): self._add_declared(node.name) # push function state onto stack. dont log any # more identifiers as "declared" until outside of the function, # but keep logging identifiers as "undeclared". # track argument names in each function header so they arent counted as "undeclared" saved = {} inf = self.in_function self.in_function = True for arg in node.argnames: if arg in self.local_ident_stack: saved[arg] = True else: self.local_ident_stack[arg] = True for n in node.getChildNodes(): self.visit(n, *args) self.in_function = inf for arg in node.argnames: if arg not in saved: del self.local_ident_stack[arg] def visitFor(self, node, *args): # flip around visit self.visit(node.list, *args) self.visit(node.assign, *args) self.visit(node.body, *args) def visitName(s, node, *args): if node.name not in __builtins__ and node.name not in self.declared_identifiers and node.name not in s.local_ident_stack: self.undeclared_identifiers.add(node.name) def visitImport(self, node, *args): for (mod, alias) in node.names: if alias is not None: self._add_declared(alias) else: self._add_declared(mod.split('.')[0]) def visitFrom(self, node, *args): for (mod, alias) in node.names: if alias is not None: self._add_declared(alias) else: if mod == '*': raise exceptions.CompileException( "'import *' is not supported, since all identifier names must be explicitly declared. Please use the form 'from <modulename> import <name1>, <name2>, ...' instead.", **exception_kwargs) self._add_declared(mod) f = FindIdentifiers() visitor.walk(expr, f) #, walker=walker())
def write_toplevel(self): """traverse a template structure for module-level directives and generate the start of module-level code.""" inherit = [] namespaces = {} module_code = [] encoding =[None] self.compiler.pagetag = None class FindTopLevel(object): def visitInheritTag(s, node): inherit.append(node) def visitNamespaceTag(s, node): namespaces[node.name] = node def visitPageTag(s, node): self.compiler.pagetag = node def visitCode(s, node): if node.ismodule: module_code.append(node) f = FindTopLevel() for n in self.node.nodes: n.accept_visitor(f) self.compiler.namespaces = namespaces module_ident = util.Set() for n in module_code: module_ident = module_ident.union(n.declared_identifiers()) module_identifiers = _Identifiers() module_identifiers.declared = module_ident # module-level names, python code if not self.compiler.generate_unicode and self.compiler.source_encoding: self.printer.writeline("# -*- encoding:%s -*-" % self.compiler.source_encoding) self.printer.writeline("from mako import runtime, filters, cache") self.printer.writeline("UNDEFINED = runtime.UNDEFINED") self.printer.writeline("__M_dict_builtin = dict") self.printer.writeline("__M_locals_builtin = locals") self.printer.writeline("_magic_number = %s" % repr(MAGIC_NUMBER)) self.printer.writeline("_modified_time = %s" % repr(time.time())) self.printer.writeline("_template_filename=%s" % repr(self.compiler.filename)) self.printer.writeline("_template_uri=%s" % repr(self.compiler.uri)) self.printer.writeline("_template_cache=cache.Cache(__name__, _modified_time)") self.printer.writeline("_source_encoding=%s" % repr(self.compiler.source_encoding)) if self.compiler.imports: buf = '' for imp in self.compiler.imports: buf += imp + "\n" self.printer.writeline(imp) impcode = ast.PythonCode(buf, source='', lineno=0, pos=0, filename='template defined imports') else: impcode = None main_identifiers = module_identifiers.branch(self.node) module_identifiers.topleveldefs = module_identifiers.topleveldefs.union(main_identifiers.topleveldefs) [module_identifiers.declared.add(x) for x in ["UNDEFINED"]] if impcode: [module_identifiers.declared.add(x) for x in impcode.declared_identifiers] self.compiler.identifiers = module_identifiers self.printer.writeline("_exports = %s" % repr([n.name for n in main_identifiers.topleveldefs.values()])) self.printer.write("\n\n") if len(module_code): self.write_module_code(module_code) if len(inherit): self.write_namespaces(namespaces) self.write_inherit(inherit[-1]) elif len(namespaces): self.write_namespaces(namespaces) return main_identifiers.topleveldefs.values()
class _Identifiers(object): """tracks the status of identifier names as template code is rendered.""" def __init__(self, node=None, parent=None, nested=False): if parent is not None: # things that have already been declared in an enclosing namespace (i.e. names we can just use) self.declared = util.Set(parent.declared).union([c.name for c in parent.closuredefs.values()]).union(parent.locally_declared).union(parent.argument_declared) # if these identifiers correspond to a "nested" scope, it means whatever the # parent identifiers had as undeclared will have been declared by that parent, # and therefore we have them in our scope. if nested: self.declared = self.declared.union(parent.undeclared) # top level defs that are available self.topleveldefs = util.SetLikeDict(**parent.topleveldefs) else: self.declared = util.Set() self.topleveldefs = util.SetLikeDict() # things within this level that are referenced before they are declared (e.g. assigned to) self.undeclared = util.Set() # things that are declared locally. some of these things could be in the "undeclared" # list as well if they are referenced before declared self.locally_declared = util.Set() # assignments made in explicit python blocks. these will be propigated to # the context of local def calls. self.locally_assigned = util.Set() # things that are declared in the argument signature of the def callable self.argument_declared = util.Set() # closure defs that are defined in this level self.closuredefs = util.SetLikeDict() self.node = node if node is not None: node.accept_visitor(self) def branch(self, node, **kwargs): """create a new Identifiers for a new Node, with this Identifiers as the parent.""" return _Identifiers(node, self, **kwargs) defs = property(lambda self:util.Set(self.topleveldefs.union(self.closuredefs).values())) def __repr__(self): return "Identifiers(declared=%s, locally_declared=%s, undeclared=%s, topleveldefs=%s, closuredefs=%s, argumenetdeclared=%s)" % (repr(list(self.declared)), repr(list(self.locally_declared)), repr(list(self.undeclared)), repr([c.name for c in self.topleveldefs.values()]), repr([c.name for c in self.closuredefs.values()]), repr(self.argument_declared)) def check_declared(self, node): """update the state of this Identifiers with the undeclared and declared identifiers of the given node.""" for ident in node.undeclared_identifiers(): if ident != 'context' and ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.locally_declared.add(ident) def add_declared(self, ident): self.declared.add(ident) if ident in self.undeclared: self.undeclared.remove(ident) def visitExpression(self, node): self.check_declared(node) def visitControlLine(self, node): self.check_declared(node) def visitCode(self, node): if not node.ismodule: self.check_declared(node) self.locally_assigned = self.locally_assigned.union(node.declared_identifiers()) def visitDefTag(self, node): if node.is_root(): self.topleveldefs[node.name] = node elif node is not self.node: self.closuredefs[node.name] = node for ident in node.undeclared_identifiers(): if ident != 'context' and ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) # visit defs only one level deep if node is self.node: for ident in node.declared_identifiers(): self.argument_declared.add(ident) for n in node.nodes: n.accept_visitor(self) def visitIncludeTag(self, node): self.check_declared(node) def visitPageTag(self, node): for ident in node.declared_identifiers(): self.argument_declared.add(ident) self.check_declared(node) def visitCallNamespaceTag(self, node): self.visitCallTag(node) def visitCallTag(self, node): if node is self.node: for ident in node.undeclared_identifiers(): if ident != 'context' and ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.argument_declared.add(ident) for n in node.nodes: n.accept_visitor(self) else: for ident in node.undeclared_identifiers(): if ident != 'context' and ident not in self.declared.union(self.locally_declared): self.undeclared.add(ident)