def __next__(self): token = self.stream.current if token.type is TOKEN_EOF: self.stream.close() raise StopIteration() next(self.stream) return token
def parse_statements(self, end_tokens, drop_needle=False): """Parse multiple statements into a list until one of the end tokens is reached. This is used to parse the body of statements as it also parses template data if appropriate. The parser checks first if the current token is a colon and skips it if there is one. Then it checks for the block end and parses until if one of the `end_tokens` is reached. Per default the active token in the stream at the end of the call is the matched end token. If this is not wanted `drop_needle` can be set to `True` and the end token is removed. """ # the first token may be a colon for python compatibility self.stream.skip_if('colon') # in the future it would be possible to add whole code sections # by adding some sort of end of statement token and parsing those here. self.stream.expect('block_end') result = self.subparse(end_tokens) # we reached the end of the template too early, the subparser # does not check for this, so we do that now if self.stream.current.type == 'eof': self.fail_eof(end_tokens) if drop_needle: next(self.stream) return result
def parse_primary(self): token = self.stream.current if token.type == 'name': if token.value in ('true', 'false', 'True', 'False'): node = nodes.Const(token.value in ('true', 'True'), lineno=token.lineno) elif token.value in ('none', 'None'): node = nodes.Const(None, lineno=token.lineno) else: node = nodes.Name(token.value, 'load', lineno=token.lineno) next(self.stream) elif token.type == 'string': next(self.stream) buf = [token.value] lineno = token.lineno while self.stream.current.type == 'string': buf.append(self.stream.current.value) next(self.stream) node = nodes.Const(''.join(buf), lineno=lineno) elif token.type in ('integer', 'float'): next(self.stream) node = nodes.Const(token.value, lineno=token.lineno) elif token.type == 'lparen': next(self.stream) node = self.parse_tuple(explicit_parentheses=True) self.stream.expect('rparen') elif token.type == 'lbracket': node = self.parse_list() elif token.type == 'lbrace': node = self.parse_dict() else: self.fail("unexpected '{0!s}'".format(describe_token(token)), token.lineno) return node
def parse_test(self, node): token = next(self.stream) if self.stream.current.test('name:not'): next(self.stream) negated = True else: negated = False name = self.stream.expect('name').value while self.stream.current.type == 'dot': next(self.stream) name += '.' + self.stream.expect('name').value dyn_args = dyn_kwargs = None kwargs = [] if self.stream.current.type == 'lparen': args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) elif self.stream.current.type in ('name', 'string', 'integer', 'float', 'lparen', 'lbracket', 'lbrace') and not \ self.stream.current.test_any('name:else', 'name:or', 'name:and'): if self.stream.current.test('name:is'): self.fail('You cannot chain multiple tests with is') args = [self.parse_expression()] else: args = [] node = nodes.Test(node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) if negated: node = nodes.Not(node, lineno=token.lineno) return node
def parse_subscript(self, node): token = next(self.stream) if token.type == 'dot': attr_token = self.stream.current next(self.stream) if attr_token.type == 'name': return nodes.Getattr(node, attr_token.value, 'load', lineno=token.lineno) elif attr_token.type != 'integer': self.fail('expected name or number', attr_token.lineno) arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) return nodes.Getitem(node, arg, 'load', lineno=token.lineno) if token.type == 'lbracket': args = [] while self.stream.current.type != 'rbracket': if args: self.stream.expect('comma') args.append(self.parse_subscribed()) self.stream.expect('rbracket') if len(args) == 1: arg = args[0] else: arg = nodes.Tuple(args, 'load', lineno=token.lineno) return nodes.Getitem(node, arg, 'load', lineno=token.lineno) self.fail('expected subscript expression', self.lineno)
def test_buffered_streaming(self): tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index " "}} - {{ item }}</li>{%- endfor %}</ul>") stream = tmpl.stream(seq=list(range(4))) stream.enable_buffering(size=3) self.assert_equal(next(stream), u'<ul><li>1 - 0</li><li>2 - 1</li>') self.assert_equal(next(stream), u'<li>3 - 2</li><li>4 - 3</li></ul>')
def __init__(self, generator, name, filename): self._iter = iter(generator) self._pushed = deque() self.name = name self.filename = filename self.closed = False self.current = Token(1, TOKEN_INITIAL, '') next(self)
def test_operators(self): from jinja2.lexer import operators for test, expect in iteritems(operators): if test in '([{}])': continue stream = env.lexer.tokenize('{{{{ {0!s} }}}}'.format(test)) next(stream) assert stream.current.type == expect
def parse_add(self): lineno = self.stream.current.lineno left = self.parse_sub() while self.stream.current.type == 'add': next(self.stream) right = self.parse_sub() left = nodes.Add(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def parse_sub(self): lineno = self.stream.current.lineno left = self.parse_concat() while self.stream.current.type == 'sub': next(self.stream) right = self.parse_concat() left = nodes.Sub(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def parse_concat(self): lineno = self.stream.current.lineno args = [self.parse_mul()] while self.stream.current.type == 'tilde': next(self.stream) args.append(self.parse_mul()) if len(args) == 1: return args[0] return nodes.Concat(args, lineno=lineno)
def parse_mul(self): lineno = self.stream.current.lineno left = self.parse_div() while self.stream.current.type == 'mul': next(self.stream) right = self.parse_div() left = nodes.Mul(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def parse_floordiv(self): lineno = self.stream.current.lineno left = self.parse_mod() while self.stream.current.type == 'floordiv': next(self.stream) right = self.parse_mod() left = nodes.FloorDiv(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def parse_mod(self): lineno = self.stream.current.lineno left = self.parse_pow() while self.stream.current.type == 'mod': next(self.stream) right = self.parse_pow() left = nodes.Mod(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def parse_pow(self): lineno = self.stream.current.lineno left = self.parse_unary() while self.stream.current.type == 'pow': next(self.stream) right = self.parse_unary() left = nodes.Pow(left, right, lineno=lineno) lineno = self.stream.current.lineno return left
def test_cycler(self): items = 1, 2, 3 c = Cycler(*items) for item in items + items: assert c.current == item assert next(c) == item next(c) assert c.current == 2 c.reset() assert c.current == 1
def test_find_refererenced_templates(self): ast = env.parse('{% extends "layout.html" %}{% include helper %}') i = meta.find_referenced_templates(ast) assert next(i) == 'layout.html' assert next(i) is None assert list(i) == [] ast = env.parse('{% extends "layout.html" %}' '{% from "test.html" import a, b as c %}' '{% import "meh.html" as meh %}' '{% include "muh.html" %}') i = meta.find_referenced_templates(ast) assert list(i) == ['layout.html', 'test.html', 'meh.html', 'muh.html']
def test_simple(self): ts = TokenStream(self.test_tokens, "foo", "bar") assert ts.current.type is TOKEN_BLOCK_BEGIN assert bool(ts) assert not bool(ts.eos) next(ts) assert ts.current.type is TOKEN_BLOCK_END assert bool(ts) assert not bool(ts.eos) next(ts) assert ts.current.type is TOKEN_EOF assert not bool(ts) assert bool(ts.eos)
def parse_unary(self, with_filter=True): token_type = self.stream.current.type lineno = self.stream.current.lineno if token_type == 'sub': next(self.stream) node = nodes.Neg(self.parse_unary(False), lineno=lineno) elif token_type == 'add': next(self.stream) node = nodes.Pos(self.parse_unary(False), lineno=lineno) else: node = self.parse_primary() node = self.parse_postfix(node) if with_filter: node = self.parse_filter_expr(node) return node
def parse_set(self): """Parse an assign statement.""" lineno = next(self.stream).lineno target = self.parse_assign_target() self.stream.expect('assign') expr = self.parse_tuple() return nodes.Assign(target, expr, lineno=lineno)
def parse_context(): if self.stream.current.value in ('with', 'without') and \ self.stream.look().test('name:context'): node.with_context = next(self.stream).value == 'with' self.stream.skip() return True return False
def parse_macro(self): node = nodes.Macro(lineno=next(self.stream).lineno) node.name = self.parse_assign_target(name_only=True).name self.parse_signature(node) node.body = self.parse_statements(('name:endmacro',), drop_needle=True) return node
def look(self): """Look at the next token.""" old_token = next(self) result = self.current self.push(result) self.current = old_token return result
def parse(self, parser): return nodes.Output([self.call_method('_dump', [ nodes.EnvironmentAttribute('sandboxed'), self.attr('ext_attr'), nodes.ImportedName(__name__ + '.importable_object'), nodes.ContextReference() ])]).set_lineno(next(parser.stream).lineno)
def parse(self, parser): node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno) node.options = [ nodes.Keyword('autoescape', parser.parse_expression()) ] node.body = parser.parse_statements(('name:endautoescape',), drop_needle=True) return nodes.Scope([node])
def parse_call(self, node): token = self.stream.expect('lparen') args = [] kwargs = [] dyn_args = dyn_kwargs = None require_comma = False def ensure(expr): if not expr: self.fail('invalid syntax for function call expression', token.lineno) while self.stream.current.type != 'rparen': if require_comma: self.stream.expect('comma') # support for trailing comma if self.stream.current.type == 'rparen': break if self.stream.current.type == 'mul': ensure(dyn_args is None and dyn_kwargs is None) next(self.stream) dyn_args = self.parse_expression() elif self.stream.current.type == 'pow': ensure(dyn_kwargs is None) next(self.stream) dyn_kwargs = self.parse_expression() else: ensure(dyn_args is None and dyn_kwargs is None) if self.stream.current.type == 'name' and \ self.stream.look().type == 'assign': key = self.stream.current.value self.stream.skip(2) value = self.parse_expression() kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) else: ensure(not kwargs) args.append(self.parse_expression()) require_comma = True self.stream.expect('rparen') if node is None: return args, kwargs, dyn_args, dyn_kwargs return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
def parse_import_context(self, node, default): if self.stream.current.test_any('name:with', 'name:without') and \ self.stream.look().test('name:context'): node.with_context = next(self.stream).value == 'with' self.stream.skip() else: node.with_context = default return node
def parse_print(self): node = nodes.Output(lineno=next(self.stream).lineno) node.nodes = [] while self.stream.current.type != 'block_end': if node.nodes: self.stream.expect('comma') node.nodes.append(self.parse_expression()) return node
def expect(self, expr): """Expect a given token type and return it. This accepts the same argument as :meth:`jinja2.lexer.Token.test`. """ if not self.current.test(expr): expr = describe_token_expr(expr) if self.current.type is TOKEN_EOF: raise TemplateSyntaxError('unexpected end of template, ' 'expected %r.' % expr, self.current.lineno, self.name, self.filename) raise TemplateSyntaxError("expected token {0!r}, got {1!r}".format(expr, describe_token(self.current)), self.current.lineno, self.name, self.filename) try: return self.current finally: next(self)
def parse_filter(self, node, start_inline=False): while self.stream.current.type == 'pipe' or start_inline: if not start_inline: next(self.stream) token = self.stream.expect('name') name = token.value while self.stream.current.type == 'dot': next(self.stream) name += '.' + self.stream.expect('name').value if self.stream.current.type == 'lparen': args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) else: args = [] kwargs = [] dyn_args = dyn_kwargs = None node = nodes.Filter(node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) start_inline = False return node
def parse_from(self): node = nodes.FromImport(lineno=next(self.stream).lineno) node.template = self.parse_expression() self.stream.expect('name:import') node.names = [] def parse_context(): if self.stream.current.value in ('with', 'without') and \ self.stream.look().test('name:context'): node.with_context = next(self.stream).value == 'with' self.stream.skip() return True return False while 1: if node.names: self.stream.expect('comma') if self.stream.current.type == 'name': if parse_context(): break target = self.parse_assign_target(name_only=True) if target.name.startswith('_'): self.fail( 'names starting with an underline can not ' 'be imported', target.lineno, exc=TemplateAssertionError) if self.stream.skip_if('name:as'): alias = self.parse_assign_target(name_only=True) node.names.append((target.name, alias.name)) else: node.names.append(target.name) if parse_context() or self.stream.current.type != 'comma': break else: break if not hasattr(node, 'with_context'): node.with_context = False self.stream.skip_if('comma') return node
def __call__(self, *args, **kwargs): # try to consume the positional arguments arguments = list(args[:self._argument_count]) off = len(arguments) # if the number of arguments consumed is not the number of # arguments expected we start filling in keyword arguments # and defaults. if off != self._argument_count: for idx, name in enumerate(self.arguments[len(arguments):]): try: value = kwargs.pop(name) except KeyError: try: value = self.defaults[idx - self._argument_count + off] except IndexError: value = self._environment.undefined( 'parameter %r was not provided' % name, name=name) arguments.append(value) # it's important that the order of these arguments does not change # if not also changed in the compiler's `function_scoping` method. # the order is caller, keyword arguments, positional arguments! if self.caller: caller = kwargs.pop('caller', None) if caller is None: caller = self._environment.undefined('No caller defined', name='caller') arguments.append(caller) if self.catch_kwargs: arguments.append(kwargs) elif kwargs: raise TypeError('macro %r takes no keyword argument %r' % (self.name, next(iter(kwargs)))) if self.catch_varargs: arguments.append(args[self._argument_count:]) elif len(args) > self._argument_count: raise TypeError('macro %r takes not more than %d argument(s)' % (self.name, len(self.arguments))) return self._func(*arguments)
def subparse(self, end_tokens=None): body = [] data_buffer = [] add_data = data_buffer.append if end_tokens is not None: self._end_token_stack.append(end_tokens) def flush_data(): if data_buffer: lineno = data_buffer[0].lineno body.append(nodes.Output(data_buffer[:], lineno=lineno)) del data_buffer[:] try: while self.stream: token = self.stream.current if token.type == 'data': if token.value: add_data(nodes.TemplateData(token.value, lineno=token.lineno)) next(self.stream) elif token.type == 'variable_begin': next(self.stream) add_data(self.parse_tuple(with_condexpr=True)) self.stream.expect('variable_end') elif token.type == 'block_begin': flush_data() next(self.stream) if end_tokens is not None and \ self.stream.current.test_any(*end_tokens): return body rv = self.parse_statement() if isinstance(rv, list): body.extend(rv) else: body.append(rv) self.stream.expect('block_end') else: raise AssertionError('internal parsing error') flush_data() finally: if end_tokens is not None: self._end_token_stack.pop() return body
def parse_for(self): """Parse a for loop.""" lineno = self.stream.expect('name:for').lineno target = self.parse_assign_target(extra_end_rules=('name:in', )) self.stream.expect('name:in') iter = self.parse_tuple(with_condexpr=False, extra_end_rules=('name:recursive', )) test = None if self.stream.skip_if('name:if'): test = self.parse_expression() recursive = self.stream.skip_if('name:recursive') body = self.parse_statements(('name:endfor', 'name:else')) if next(self.stream).value == 'endfor': else_ = [] else: else_ = self.parse_statements(('name:endfor', ), drop_needle=True) return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
def parse_not(self): if self.stream.current.test('name:not'): lineno = next(self.stream).lineno return nodes.Not(self.parse_not(), lineno=lineno) return self.parse_compare()
def parse(self, parser): node = nodes.ExprStmt(lineno=next(parser.stream).lineno) node.node = parser.parse_tuple() return node
def parse(self, parser): token = next(parser.stream) if token.value == 'break': return nodes.Break(lineno=token.lineno) return nodes.Continue(lineno=token.lineno)
def parse_filter_block(self): node = nodes.FilterBlock(lineno=next(self.stream).lineno) node.filter = self.parse_filter(None, start_inline=True) node.body = self.parse_statements(('name:endfilter',), drop_needle=True) return node
def skip(self, n=1): """Got n tokens ahead.""" for x in range(n): next(self)
def next_if(self, expr): """Perform the token test and return the token if it matched. Otherwise the return value is `None`. """ if self.current.test(expr): return next(self)
def do_last(environment, seq): """Return the last item of a sequence.""" try: return next(iter(reversed(seq))) except StopIteration: return environment.undefined('No last item, sequence was empty.')
def parse_extends(self): node = nodes.Extends(lineno=next(self.stream).lineno) node.template = self.parse_expression() return node
def parse(self, parser): """Parse a translatable tag.""" lineno = next(parser.stream).lineno num_called_num = False # find all the variables referenced. Additionally a variable can be # defined in the body of the trans block too, but this is checked at # a later state. plural_expr = None plural_expr_assignment = None variables = {} while parser.stream.current.type != 'block_end': if variables: parser.stream.expect('comma') # skip colon for python compatibility if parser.stream.skip_if('colon'): break name = parser.stream.expect('name') if name.value in variables: parser.fail('translatable variable %r defined twice.' % name.value, name.lineno, exc=TemplateAssertionError) # expressions if parser.stream.current.type == 'assign': next(parser.stream) variables[name.value] = var = parser.parse_expression() else: variables[name.value] = var = nodes.Name(name.value, 'load') if plural_expr is None: if isinstance(var, nodes.Call): plural_expr = nodes.Name('_trans', 'load') variables[name.value] = plural_expr plural_expr_assignment = nodes.Assign( nodes.Name('_trans', 'store'), var) else: plural_expr = var num_called_num = name.value == 'num' parser.stream.expect('block_end') plural = plural_names = None have_plural = False referenced = set() # now parse until endtrans or pluralize singular_names, singular = self._parse_block(parser, True) if singular_names: referenced.update(singular_names) if plural_expr is None: plural_expr = nodes.Name(singular_names[0], 'load') num_called_num = singular_names[0] == 'num' # if we have a pluralize block, we parse that too if parser.stream.current.test('name:pluralize'): have_plural = True next(parser.stream) if parser.stream.current.type != 'block_end': name = parser.stream.expect('name') if name.value not in variables: parser.fail('unknown variable %r for pluralization' % name.value, name.lineno, exc=TemplateAssertionError) plural_expr = variables[name.value] num_called_num = name.value == 'num' parser.stream.expect('block_end') plural_names, plural = self._parse_block(parser, False) next(parser.stream) referenced.update(plural_names) else: next(parser.stream) # register free names as simple name expressions for var in referenced: if var not in variables: variables[var] = nodes.Name(var, 'load') if not have_plural: plural_expr = None elif plural_expr is None: parser.fail('pluralize without variables', lineno) node = self._make_node(singular, plural, variables, plural_expr, bool(referenced), num_called_num and have_plural) node.set_lineno(lineno) if plural_expr_assignment is not None: return [plural_expr_assignment, node] else: return node
def _safe_next(self): try: return next(self._iterator) except StopIteration: return _last_iteration
def parse_import(self): node = nodes.Import(lineno=next(self.stream).lineno) node.template = self.parse_expression() self.stream.expect('name:as') node.target = self.parse_assign_target(name_only=True).name return self.parse_import_context(node, False)