def parse_subscript_expression(self, node):
"""
Parse a subscript statement. Gets attributes and items from an
object.
"""
lineno = self.stream.lineno
if self.stream.current.type == 'dot':
self.stream.next()
token = self.stream.current
if token.type in ('name', 'integer'):
arg = nodes.ConstantExpression(token.value, token.lineno,
self.filename)
else:
raise TemplateSyntaxError('expected name or number',
token.lineno, self.filename)
self.stream.next()
elif self.stream.current.type == 'lbracket':
self.stream.next()
args = []
while self.stream.current.type != 'rbracket':
if args:
self.stream.expect('comma')
args.append(self.parse_subscribed_expression())
self.stream.expect('rbracket')
if len(args) == 1:
arg = args[0]
else:
arg = nodes.TupleExpression(args, lineno, self.filename)
else:
raise TemplateSyntaxError('expected subscript expression',
self.lineno, self.filename)
return nodes.SubscriptExpression(node, arg, lineno, self.filename)
def process_variable(lineno, token, name):
if token != 'name':
raise TemplateSyntaxError('can only use variable not '
'constants or expressions in '
'translation variable blocks.',
lineno, self.filename)
# plural name without trailing "_"? that's a keyword
if not name.endswith('_'):
raise TemplateSyntaxError('illegal use of keyword \'%s\' as '
'identifier in translatable block.'
% name, lineno, self.filename)
name = name[:-1]
if name not in replacements:
raise TemplateSyntaxError('unregistered translation variable'
" '%s'." % name, lineno,
self.filename)
# check that we don't have an expression here, thus the
# next token *must* be a variable_end token (or a
# block_end token when in no_variable_block mode)
next_token = self.tokenstream.next()[1]
if next_token != 'variable_end' and not \
(self.no_variable_block and next_token == 'block_end'):
raise TemplateSyntaxError('you cannot use variable '
'expressions inside translatable '
'tags. apply filters in the '
'trans header.', lineno,
self.filename)
buf.append('%%(%s)s' % name)
def filter(self, s, filter_nodes):
"""
Apply a filter on an object that already is a python expression.
Used to avoid redundant code in bitor and the filter directive.
"""
filters = []
for n in filter_nodes:
if n.__class__ is ast.CallFunc:
if n.node.__class__ is not ast.Name:
raise TemplateSyntaxError(
'invalid filter. filter must '
'be a hardcoded function name '
'from the filter namespace', n.lineno, n.filename)
args = []
for arg in n.args:
if arg.__class__ is ast.Keyword:
raise TemplateSyntaxError(
'keyword arguments for '
'filters are not supported.', n.lineno, n.filename)
args.append(self.handle_node(arg))
if n.star_args is not None or n.dstar_args is not None:
raise TemplateSyntaxError(
'*args / **kwargs is not supported '
'for filters', n.lineno, n.filename)
filters.append('(%r, %s)' % (n.node.name, self.to_tuple(args)))
elif n.__class__ is ast.Name:
filters.append('(%r, ())' % n.name)
else:
raise TemplateSyntaxError(
'invalid filter. filter must be a '
'hardcoded function name from the '
'filter namespace', n.lineno, n.filename)
self.used_shortcuts.add('apply_filters')
return 'apply_filters(%s, context, %s)' % (s, self.to_tuple(filters))
def handle_block_directive(self, lineno, gen):
"""
Handle block directives used for inheritance.
"""
tokens = list(gen)
if not tokens:
raise TemplateSyntaxError('block requires a name', lineno,
self.filename)
block_name = tokens.pop(0)
if block_name[1] != 'name':
raise TemplateSyntaxError('expected \'name\', got %r' %
block_name[1], lineno, self.filename)
# disallow keywords
if not block_name[2].endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as block name.' % block_name[2],
lineno, self.filename)
name = block_name[2][:-1]
# check if this block does not exist by now.
if name in self.blocks:
raise TemplateSyntaxError('block %r defined twice' %
name, lineno, self.filename)
self.blocks.add(name)
if tokens:
body = nodes.NodeList(lineno, [nodes.Print(lineno,
self.parse_python(lineno, tokens, '(%s)').expr)])
else:
# otherwise parse the body and attach it to the block
body = self.subparse(end_of_block_tag, True)
self.close_remaining_block()
return nodes.Block(lineno, name, body)
def walk(nodes_, stack):
for node in nodes_:
# all names excluding keywords have an trailing underline.
# if we find a name without trailing underline that's a
# keyword and this code raises an error. else strip the
# underline again
if node.__class__ in (ast.AssName, ast.Name, ast.Keyword):
if not node.name.endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as identifier.' %
node.name, node.lineno,
self.filename)
node.name = node.name[:-1]
# same for attributes
elif node.__class__ is ast.Getattr:
if not node.attrname.endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as attribute name.' %
node.name, node.lineno,
self.filename)
node.attrname = node.attrname[:-1]
# if we have a ForLoop we ensure that nobody patches existing
# object using "for foo.bar in seq"
elif node.__class__ is nodes.ForLoop:
def check(node):
if node.__class__ not in (ast.AssName, ast.AssTuple):
raise TemplateSyntaxError('can\'t assign to '
'expression.',
node.lineno,
self.filename)
for n in node.getChildNodes():
check(n)
check(node.item)
# ensure that in child templates block are either top level
# or located inside of another block tag.
elif self.extends is not None and \
node.__class__ is nodes.Block:
if stack[-1] is not body:
for n in stack:
if n.__class__ is nodes.Block:
break
else:
raise TemplateSyntaxError('misplaced block %r, '
'blocks in child '
'templates must be '
'either top level or '
'located in a block '
'tag.' % node.name,
node.lineno,
self.filename)
# now set the filename and continue working on the childnodes
node.filename = self.filename
stack.append(node)
walk(node.getChildNodes(), stack)
stack.pop()
def handle_extends_directive(self, lineno, gen):
"""
Handle the extends directive used for inheritance.
"""
tokens = list(gen)
if len(tokens) != 1 or tokens[0][1] != 'string':
raise TemplateSyntaxError('extends requires a string', lineno,
self.filename)
if self.extends is not None:
raise TemplateSyntaxError('extends called twice', lineno,
self.filename)
self.extends = nodes.Extends(lineno, tokens[0][2][1:-1])
def parse_raw_directive(self):
"""
Handle fake raw directive. (real raw directives are handled by
the lexer. But if there are arguments to raw or the end tag
is missing the parser tries to resolve this directive. In that
case present the user a useful error message.
"""
if self.stream:
raise TemplateSyntaxError('raw directive does not support '
'any arguments.', self.stream.lineno,
self.filename)
raise TemplateSyntaxError('missing end tag for raw directive.',
self.stream.lineno, self.filename)
def handle_macro_directive(self, lineno, gen):
"""
Handle {% macro foo bar, baz %} as well as
{% macro foo(bar, baz) %}.
"""
try:
macro_name = gen.next()
except StopIteration:
raise TemplateSyntaxError('macro requires a name', lineno,
self.filename)
if macro_name[1] != 'name':
raise TemplateSyntaxError('expected \'name\', got %r' %
macro_name[1], lineno,
self.filename)
# disallow keywords as identifiers
elif not macro_name[2].endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as macro name.' % macro_name[2],
lineno, self.filename)
# make the punctuation around arguments optional
arg_list = list(gen)
if arg_list and arg_list[0][1:] == ('operator', '(') and \
arg_list[-1][1:] == ('operator', ')'):
arg_list = arg_list[1:-1]
ast = self.parse_python(lineno, arg_list, 'def %s(%%s):pass' %
str(macro_name[2][:-1]))
body = self.subparse(end_of_macro, True)
self.close_remaining_block()
if ast.varargs or ast.kwargs:
raise TemplateSyntaxError('variable length macro signature '
'not allowed.', lineno,
self.filename)
if ast.argnames:
defaults = [None] * (len(ast.argnames) - len(ast.defaults)) + \
ast.defaults
args = []
for idx, argname in enumerate(ast.argnames):
# disallow keywords as argument names
if not argname.endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as macro argument.' % argname,
lineno, self.filename)
args.append((argname[:-1], defaults[idx]))
else:
args = None
return nodes.Macro(lineno, ast.name, args, body)
def expect(self, token_type, token_value=_missing):
"""Expect a given token type and return it"""
if self.current.type != token_type:
raise TemplateSyntaxError(
"expected token %r, got %r" % (token_type, self.current.type),
self.current.lineno, self.filename)
elif token_value is not _missing and \
self.current.value != token_value:
raise TemplateSyntaxError(
"expected %r, got %r" % (token_value, self.current.value),
self.current.lineno, self.filename)
try:
return self.current
finally:
self.next()
def parse_tuple_expression(self, enforce=False, simplified=False):
"""
Parse multiple expressions into a tuple. This can also return
just one expression which is not a tuple. If you want to enforce
a tuple, pass it enforce=True.
"""
lineno = self.stream.lineno
if simplified:
parse = self.parse_primary_expression
else:
parse = self.parse_expression
args = []
is_tuple = False
while True:
if args:
self.stream.expect('comma')
if self.stream.current.type in tuple_edge_tokens:
break
args.append(parse())
if self.stream.current.type == 'comma':
is_tuple = True
else:
break
if not is_tuple and args:
if enforce:
raise TemplateSyntaxError('tuple expected', lineno,
self.filename)
return args[0]
return nodes.TupleExpression(args, lineno, self.filename)
def process_variable():
var_name = self.stream.expect('name')
if var_name.value not in replacements:
raise TemplateSyntaxError('unregistered translation variable'
" '%s'." % var_name.value,
var_name.lineno, self.filename)
buf.append('%%(%s)s' % var_name.value)
def parse_block_directive(self):
"""
Handle block directives used for inheritance.
"""
token = self.stream.expect('block')
name = self.stream.expect('name').value
# check if this block does not exist by now.
if name in self.blocks:
raise TemplateSyntaxError('block %r defined twice' %
name, token.lineno,
self.filename)
self.blocks.add(name)
if self.stream.current.type != 'block_end':
lineno = self.stream.lineno
expr = self.parse_tuple_expression()
node = nodes.Print(expr, lineno, self.filename)
body = nodes.NodeList([node], lineno, self.filename)
self.stream.expect('block_end')
else:
# otherwise parse the body and attach it to the block
self.stream.expect('block_end')
body = self.subparse(end_of_block_tag, True)
self.stream.expect('block_end')
return nodes.Block(name, body, token.lineno, self.filename)
def parse_extends_directive(self):
"""
Handle the extends directive used for inheritance.
"""
raise TemplateSyntaxError('mispositioned extends tag. extends must '
'be the first tag of a template.',
self.stream.lineno, self.filename)
def test_name(self, name):
"""
Test if a name is not a special constant
"""
if name in ('true', 'false', 'none', 'undefined', '_'):
raise TemplateSyntaxError('expected name not special constant',
self.stream.lineno, self.filename)
def parse_python(self, lineno, gen, template):
"""
Convert the passed generator into a flat string representing
python sourcecode and return an ast node or raise a
TemplateSyntaxError.
"""
tokens = []
for t_lineno, t_token, t_data in gen:
if t_token == 'string':
# because some libraries have problems with unicode
# objects we do some lazy unicode processing here.
# if a string is ASCII only we yield it as string
# in other cases as unicode. This works around
# problems with datetimeobj.strftime()
# also escape newlines in strings
t_data = t_data.replace('\n', '\\n')
try:
str(t_data)
except UnicodeError:
tokens.append('u' + t_data)
continue
tokens.append(t_data)
source = '\xef\xbb\xbf' + (template % (u' '.join(tokens)).
encode('utf-8'))
try:
ast = parse(source, 'exec')
except SyntaxError, e:
raise TemplateSyntaxError('invalid syntax in expression',
lineno + (e.lineno or 0),
self.filename)
def parse_for_loop(self):
"""
Handle a for directive and return a ForLoop node
"""
token = self.stream.expect('for')
item = self.parse_tuple_expression(simplified=True)
if not item.allows_assignments():
raise TemplateSyntaxError('cannot assign to expression',
token.lineno, self.filename)
self.stream.expect('in')
seq = self.parse_tuple_expression()
if self.stream.current.type == 'recursive':
self.stream.next()
recursive = True
else:
recursive = False
self.stream.expect('block_end')
body = self.subparse(switch_for)
# do we have an else section?
if self.stream.current.type == 'else':
self.stream.next()
self.stream.expect('block_end')
else_ = self.subparse(end_of_for, True)
else:
self.stream.next()
else_ = None
self.stream.expect('block_end')
return nodes.ForLoop(item, seq, body, else_, recursive,
token.lineno, self.filename)
def check(node):
if node.__class__ not in (ast.AssName, ast.AssTuple):
raise TemplateSyntaxError('can\'t assign to '
'expression.',
node.lineno,
self.filename)
for n in node.getChildNodes():
check(n)
def close_remaining_block(self):
"""
If we opened a block tag because one of our tags requires an end
tag we can use this method to drop the rest of the block from
the stream. If the next token isn't the block end we throw an
error.
"""
lineno, _, tagname = self.tokenstream.last
try:
lineno, token, data = self.tokenstream.next()
except StopIteration:
raise TemplateSyntaxError('missing closing tag', lineno,
self.filename)
if token != 'block_end':
print token, data, list(self.tokenstream)
raise TemplateSyntaxError('expected empty %s-directive but '
'found additional arguments.' %
tagname, lineno, self.filename)
def handle_set_directive(self, lineno, gen):
"""
Handle {% set foo = 'value of foo' %}.
"""
try:
name = gen.next()
if name[1] != 'name' or gen.next()[1:] != ('operator', '='):
raise ValueError()
except (StopIteration, ValueError):
raise TemplateSyntaxError('invalid syntax for set', lineno,
self.filename)
ast = self.parse_python(lineno, gen, '(%s)')
# disallow keywords
if not name[2].endswith('_'):
raise TemplateSyntaxError('illegal use of keyword %r '
'as identifier in set statement.' %
name[2], lineno, self.filename)
return nodes.Set(lineno, str(name[2][:-1]), ast.expr)
def handle_assign_name(self, node):
"""
Handle name assignments.
"""
if node.name == '_' or \
node.name in self.constants:
raise TemplateSyntaxError('cannot override %r' % node.name,
node.lineno, node.filename)
return 'context[%r]' % node.name
def parse_number_expression(self):
"""
Parse a number literal.
"""
token = self.stream.current
if token.type not in ('integer', 'float'):
raise TemplateSyntaxError('integer or float literal expected',
token.lineno, self.filename)
self.stream.next()
return nodes.ConstantExpression(token.value, token.lineno, self.filename)
def handle_regex(self, node):
"""
Regular expression literals.
"""
if self.environment.disable_regexps:
raise TemplateSyntaxError('regular expressions disabled.')
if node.value in self.compiled_regular_expressions:
return self.compiled_regular_expressions[node.value]
name = 'regex_%d' % len(self.compiled_regular_expressions)
self.compiled_regular_expressions[node.value] = name
return name
def handle_include_directive(self, lineno, gen):
"""
Handle the include directive used for template inclusion.
"""
tokens = list(gen)
# hardcoded include (faster because copied into the bytecode)
if len(tokens) == 1 and tokens[0][1] == 'string':
return nodes.Include(lineno, str(tokens[0][2][1:-1]))
raise TemplateSyntaxError('invalid syntax for include '
'directive. Requires a hardcoded '
'string', lineno,
self.filename)
def handle_set(self, node):
"""
Handle variable assignments.
"""
if node.name == '_' or node.name in self.constants:
raise TemplateSyntaxError('cannot override %r' % node.name,
node.lineno, node.filename)
return self.indent(self.nodeinfo(node)) + '\n' + \
self.indent('context[%r] = %s' % (
node.name,
self.handle_node(node.expr)
))
def handle_call_directive(self, lineno, gen):
"""
Handle {% call foo() %}...{% endcall %}
"""
expr = self.parse_python(lineno, gen, '(%s)').expr
if expr.__class__ is not ast.CallFunc:
raise TemplateSyntaxError('call requires a function or macro '
'call as only argument.', lineno,
self.filename)
body = self.subparse(end_of_call, True)
self.close_remaining_block()
return nodes.Call(lineno, expr, body)
def parse_bool_expression(self):
"""
Parse a boolean literal.
"""
token = self.stream.expect('name')
if token.value == 'true':
value = True
elif token.value == 'false':
value = False
else:
raise TemplateSyntaxError("expected boolean literal",
token.lineno, self.filename)
return nodes.ConstantExpression(value, token.lineno, self.filename)
def filter_stream(self, stream):
paren_stack = 0
for token in stream:
if token.type != "data":
yield token
continue
pos = 0
lineno = token.lineno
while 1:
if not paren_stack:
match = _outside_re.search(token.value, pos)
else:
match = _inside_re.search(token.value, pos)
if match is None:
break
new_pos = match.start()
if new_pos > pos:
preval = token.value[pos:new_pos]
yield Token(lineno, "data", preval)
lineno += count_newlines(preval)
gtok = match.group()
if gtok[0] == "\\":
yield Token(lineno, "data", gtok[1:])
elif not paren_stack:
yield Token(lineno, "block_begin", None)
yield Token(lineno, "name", "trans")
yield Token(lineno, "block_end", None)
paren_stack = 1
else:
if gtok == "(" or paren_stack > 1:
yield Token(lineno, "data", gtok)
paren_stack += gtok == ")" and -1 or 1
if not paren_stack:
yield Token(lineno, "block_begin", None)
yield Token(lineno, "name", "endtrans")
yield Token(lineno, "block_end", None)
pos = match.end()
if pos < len(token.value):
yield Token(lineno, "data", token.value[pos:])
if paren_stack:
raise TemplateSyntaxError(
"unclosed gettext expression",
token.lineno,
stream.name,
stream.filename,
)
def handle_node(self, node):
"""
Handle one node. Resolves the correct callback functions defined
in the callback mapping. This also raises the `TemplateSyntaxError`
for unsupported syntax elements such as list comprehensions.
"""
if node.__class__ in self.handlers:
out = self.handlers[node.__class__](node)
elif node.__class__ in self.unsupported:
raise TemplateSyntaxError(
'unsupported syntax element %r found.' %
self.unsupported[node.__class__], node.lineno, node.filename)
else:
raise AssertionError('unhandled node %r' % node.__class__)
return out
def handle_subscript(self, node):
"""
Handle variable based attribute access foo['bar'].
"""
if len(node.subs) != 1:
raise TemplateSyntaxError(
'attribute access requires one '
'argument', node.lineno, node.filename)
assert node.flags != 'OP_DELETE', 'wtf? do we support that?'
if node.subs[0].__class__ is ast.Sliceobj:
return '%s%s' % (self.handle_node(
node.expr), self.handle_node(node.subs[0]))
self.used_shortcuts.add('get_attribute')
return 'get_attribute(%s, %s)' % (self.handle_node(
node.expr), self.handle_node(node.subs[0]))
def handle_print_directive(self, lineno, gen):
"""
Handle {{ foo }} and {% print foo %}.
"""
ast = self.parse_python(lineno, gen, 'print_(%s)')
# ast is something like Discard(CallFunc(Name('print_'), ...))
# so just use the args
arguments = ast.expr.args
# we only accept one argument
if len(arguments) != 1:
raise TemplateSyntaxError('invalid argument count for print; '
'print requires exactly one argument, '
'got %d.' % len(arguments), lineno,
self.filename)
return nodes.Print(lineno, arguments[0])
