def reset(self):
"""
Reset translation variables such as indention or cycle id
"""
#: current level of indention
self.indention = 0
#: each {% cycle %} tag has a unique ID which increments
#: automatically for each tag.
self.last_cycle_id = 0
#: set of used shortcuts jinja has to make local automatically
self.used_shortcuts = set(['undefined_singleton'])
#: set of used datastructures jinja has to import
self.used_data_structures = set()
#: set of used utils jinja has to import
self.used_utils = set()
#: flags for runtime error
self.require_runtime_error = False
def __init__(self, environment, source, filename=None):
#XXX: with Jinja 1.3 call becomes a keyword. Add it also
# to the lexer.py file.
self.environment = environment
if isinstance(source, str):
source = source.decode(environment.template_charset, 'ignore')
if isinstance(filename, unicode):
filename = filename.encode('utf-8')
self.source = source
self.filename = filename
#: if this template has a parent template it's stored here
#: after parsing
self.extends = None
#: set for blocks in order to keep them unique
self.blocks = set()
#: mapping of directives that require special treatment
self.directives = {
'raw': self.handle_raw_directive,
'for': self.handle_for_directive,
'if': self.handle_if_directive,
'cycle': self.handle_cycle_directive,
'set': self.handle_set_directive,
'filter': self.handle_filter_directive,
'print': self.handle_print_directive,
'macro': self.handle_macro_directive,
'call_': self.handle_call_directive,
'block': self.handle_block_directive,
'extends': self.handle_extends_directive,
'include': self.handle_include_directive,
'trans': self.handle_trans_directive
}
#: set of directives that are only available in a certain
#: context.
self.context_directives = set([
'elif', 'else', 'endblock', 'endfilter', 'endfor', 'endif',
'endmacro', 'endraw', 'endtrans', 'pluralize'
])
#: get the `no_variable_block` flag
self.no_variable_block = self.environment.lexer.no_variable_block
self.tokenstream = environment.lexer.tokenize(source, filename)
def __init__(self, environment, source, filename=None):
self.environment = environment
if isinstance(source, str):
source = source.decode(environment.template_charset, 'ignore')
if isinstance(filename, unicode):
filename = filename.encode('utf-8')
self.source = source
self.filename = filename
self.closed = False
#: set for blocks in order to keep them unique
self.blocks = set()
#: mapping of directives that require special treatment
self.directives = {
# "fake" directives that just trigger errors
'raw': self.parse_raw_directive,
'extends': self.parse_extends_directive,
# real directives
'for': self.parse_for_loop,
'if': self.parse_if_condition,
'cycle': self.parse_cycle_directive,
'call': self.parse_call_directive,
'set': self.parse_set_directive,
'filter': self.parse_filter_directive,
'print': self.parse_print_directive,
'macro': self.parse_macro_directive,
'block': self.parse_block_directive,
'include': self.parse_include_directive,
'trans': self.parse_trans_directive
}
#: set of directives that are only available in a certain
#: context.
self.context_directives = set([
'elif', 'else', 'endblock', 'endfilter', 'endfor', 'endif',
'endmacro', 'endraw', 'endtrans', 'pluralize'
])
#: get the `no_variable_block` flag
self.no_variable_block = self.environment.lexer.no_variable_block
self.stream = environment.lexer.tokenize(source, filename)
def parse_compare_expression(self):
"""
Parse something like {{ foo == bar }}.
"""
known_operators = set(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq', 'in'])
lineno = self.stream.lineno
expr = self.parse_add_expression()
ops = []
while True:
if self.stream.current.type in known_operators:
op = self.stream.current.type
self.stream.next()
ops.append([op, self.parse_add_expression()])
elif self.stream.current.type == 'not' and \
self.stream.look().type == 'in':
self.stream.skip(2)
ops.append(['not in', self.parse_add_expression()])
else:
break
if not ops:
return expr
return nodes.CompareExpression(expr, ops, lineno, self.filename)
# static regular expressions
whitespace_re = re.compile(r'\s+(?um)')
name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)')
integer_re = re.compile(r'\d+')
float_re = re.compile(r'\d+\.\d+')
regex_re = re.compile(r'\@/([^/\\]*(?:\\.[^/\\]*)*)*/[a-z]*(?ms)')
# set of used keywords
keywords = set(['and', 'block', 'cycle', 'elif', 'else', 'endblock',
'endfilter', 'endfor', 'endif', 'endmacro', 'endraw',
'endtrans', 'extends', 'filter', 'for', 'if', 'in',
'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw',
'recursive', 'set', 'trans', 'print', 'call', 'endcall'])
# bind operators to token types
operators = {
'+': 'add',
'-': 'sub',
'/': 'div',
'//': 'floordiv',
'*': 'mul',
'%': 'mod',
'**': 'pow',
'~': 'tilde',
'!': 'bang',
'@': 'at',
end_of_comment = StateTest.expect_token('comment_end',
msg='expected end of comment')
# internal tag callbacks
switch_for = StateTest.expect_token('else', 'endfor')
end_of_for = StateTest.expect_token('endfor')
switch_if = StateTest.expect_token('else', 'elif', 'endif')
end_of_if = StateTest.expect_token('endif')
end_of_filter = StateTest.expect_token('endfilter')
end_of_macro = StateTest.expect_token('endmacro')
end_of_call = StateTest.expect_token('endcall')
end_of_block_tag = StateTest.expect_token('endblock')
end_of_trans = StateTest.expect_token('endtrans')
# this ends a tuple
tuple_edge_tokens = set(['rparen', 'block_end', 'variable_end', 'in',
'recursive'])
class Parser(object):
"""
The template parser class.
Transforms sourcecode into an abstract syntax tree.
"""
def __init__(self, environment, source, filename=None):
self.environment = environment
if isinstance(source, str):
source = source.decode(environment.template_charset, 'ignore')
if isinstance(filename, unicode):
filename = filename.encode('utf-8')
def handle_template(self, node):
"""
Handle the overall template node. This node is the first node and
ensures that we get the bootstrapping code. It also knows about
inheritance information. It only occours as outer node, never in
the tree itself.
"""
self.indention = 1
# if there is a parent template we parse the parent template and
# update the blocks there. Once this is done we drop the current
# template in favor of the new one. Do that until we found the
# root template.
parent = None
overwrites = {}
blocks = {}
requirements = []
outer_filename = node.filename or '<template>'
# this set is required in order to not add blocks to the block
# dict a second time if they were not overridden in one template
# in the template chain.
already_registered_block = set()
while node.extends is not None:
# the direct child nodes in a template that are not blocks
# are processed as template globals, thus executed *before*
# the master layout template is loaded. This can be used
# for further processing. The output of those nodes does
# not appear in the final template.
requirements += [child for child in node.body.get_child_nodes()
if child.__class__ not in (nodes.Text,
nodes.Block)]
# load the template we inherit from and add not known blocks.
# this also marks the templates on the controlled loader but
# are never removed. that's no problem because we don't allow
# parents we extend from as includes and the controlled loader
# is only used for this templated
parent = self.loader.parse(node.extends,
node.filename)
# look up all block nodes in the current template and
# add them to the override dict.
for n in get_nodes(nodes.Block, node):
overwrites[n.name] = n
# handle direct overrides
for n in get_nodes(nodes.Block, parent):
# an overwritten block for the parent template. handle that
# override in the template and register it in the deferred
# block dict.
if n.name in overwrites and n not in already_registered_block:
blocks.setdefault(n.name, []).append(n.clone())
n.replace(overwrites[n.name])
already_registered_block.add(n)
# make the parent node the new node
node = parent
# handle requirements code
if requirements:
requirement_lines = ['def bootstrap(context):']
for n in requirements:
requirement_lines.append(self.handle_node(n))
requirement_lines.append(' if 0: yield None\n')
# handle body in order to get the used shortcuts
body_code = self.handle_node(node.body)
# same for blocks in callables
block_lines = []
block_items = blocks.items()
block_items.sort()
dict_lines = []
for name, items in block_items:
tmp = []
for idx, item in enumerate(items):
# ensure that the indention is correct
self.indention = 1
func_name = 'block_%s_%s' % (name, idx)
data = self.handle_block(item, idx + 1)
# blocks with data
if data:
block_lines.extend([
'def %s(context):' % func_name,
self.indent(self.nodeinfo(item, True)),
data,
' if 0: yield None\n'
])
tmp.append('buffereater(%s)' % func_name)
self.used_utils.add('buffereater')
# blocks without data, can default to something
# from utils
else:
tmp.append('(lambda x: "")')
dict_lines.append(' %r: %s' % (
str(name),
self.to_tuple(tmp)
))
# bootstrapping code
lines = ['# Essential imports', 'from __future__ import division']
if self.used_utils:
lines.append('from jinja.utils import %s' % \
', '.join(tuple(self.used_utils)))
if self.require_runtime_error:
lines.append('from jinja.exceptions import TemplateRuntimeError')
if self.used_data_structures:
lines.append('from jinja.datastructure import %s' % ', '.
join(self.used_data_structures))
if self.need_set_import:
lines.append('from jinja.utils import set')
# compile regular expressions
if self.compiled_regular_expressions:
lines.append('import re')
lines.append('\n# Compile used regular expressions')
for regex, name in self.compiled_regular_expressions.iteritems():
lines.append('%s = re.compile(%r)' % (name, regex))
lines.append(
'\n# Aliases for some speedup\n'
'%s\n\n'
'# Marker for Jinja templates\n'
'__jinja_template__ = True\n\n'
'# Name for disabled debugging\n'
'__name__ = %r\n\n'
'def generate(context):\n'
' assert environment is context.environment' % (
'\n'.join([
'%s = environment.%s' % (item, item) for item in
self.used_shortcuts
]),
outer_filename
)
)
# the template body
if requirements:
lines.append(' for item in bootstrap(context): pass')
lines.append(body_code)
lines.append(' if 0: yield None\n')
# now write the bootstrapping (requirements) core if there is one
if requirements:
lines.append('# Bootstrapping code')
lines.extend(requirement_lines)
# blocks must always be defined. even if it's empty. some
# features depend on it
if block_lines:
lines.append('# Superable blocks')
lines.extend(block_lines)
lines.append('# Block mapping')
if dict_lines:
lines.append('blocks = {\n%s\n}\n' % ',\n'.join(dict_lines))
else:
lines.append('blocks = {}\n')
# now get the real source lines and map the debugging symbols
debug_mapping = []
file_mapping = {}
last = None
offset = -1
sourcelines = ('\n'.join(lines)).splitlines()
result = []
for idx, line in enumerate(sourcelines):
m = _debug_re.search(line)
if m is not None:
d = m.groupdict()
filename = d['filename'] or None
if isinstance(filename, unicode):
filename = filename.encode('utf-8')
if filename in file_mapping:
file_id = file_mapping[filename]
else:
file_id = file_mapping[filename] = 'F%d' % \
len(file_mapping)
this = (file_id, int(d['lineno']))
# if it's the same as the line before we ignore it
if this != last:
debug_mapping.append('(%r, %s, %r)' % ((idx - offset,) + this))
last = this
# for each debug symbol the line number and so the offset
# changes by one.
offset += 1
else:
result.append(line)
# now print file mapping and debug info
# the debug info:
# debug_info binds template line numbers to generated
# source lines. this information is always
# present and part of the bytecode.
# template_source only available if loaded from string to
# get debug source code. Because this is
# dumped too it's a bad idea to dump templates
# loaded from a string.
result.append('\n# Debug Information')
file_mapping = file_mapping.items()
file_mapping.sort(lambda a, b: cmp(a[1], b[1]))
for filename, file_id in file_mapping:
result.append('%s = %r' % (file_id, filename))
result.append('debug_info = %s' % self.to_tuple(debug_mapping))
result.append('template_source = %r' % self.source)
return '\n'.join(result)
def __init__(self, environment, node, source):
self.environment = environment
self.loader = environment.loader.get_controlled_loader()
self.node = node
self.source = source
self.closed = False
#: current level of indention
self.indention = 0
#: each {% cycle %} tag has a unique ID which increments
#: automatically for each tag.
self.last_cycle_id = 0
#: set of used shortcuts jinja has to make local automatically
self.used_shortcuts = set(['undefined_singleton'])
#: set of used datastructures jinja has to import
self.used_data_structures = set()
#: set of used utils jinja has to import
self.used_utils = set()
#: flags for runtime error
self.require_runtime_error = False
#: do wee need a "set" object?
self.need_set_import = False
#: flag for regular expressions
self.compiled_regular_expressions = {}
#: bind the nodes to the callback functions. There are
#: some missing! A few are specified in the `unhandled`
#: mapping in order to disallow their usage, some of them
#: will not appear in the jinja parser output because
#: they are filtered out.
self.handlers = {
# block nodes
nodes.Template: self.handle_template,
nodes.Text: self.handle_template_text,
nodes.NodeList: self.handle_node_list,
nodes.ForLoop: self.handle_for_loop,
nodes.IfCondition: self.handle_if_condition,
nodes.Cycle: self.handle_cycle,
nodes.Print: self.handle_print,
nodes.Macro: self.handle_macro,
nodes.Call: self.handle_call,
nodes.Set: self.handle_set,
nodes.Filter: self.handle_filter,
nodes.Block: self.handle_block,
nodes.Include: self.handle_include,
nodes.Trans: self.handle_trans,
# expression nodes
nodes.NameExpression: self.handle_name,
nodes.CompareExpression: self.handle_compare,
nodes.TestExpression: self.handle_test,
nodes.ConstantExpression: self.handle_const,
nodes.RegexExpression: self.handle_regex,
nodes.SubscriptExpression: self.handle_subscript,
nodes.FilterExpression: self.handle_filter_expr,
nodes.CallExpression: self.handle_call_expr,
nodes.AddExpression: BinaryOperator('+', self),
nodes.SubExpression: BinaryOperator('-', self),
nodes.ConcatExpression: self.handle_concat,
nodes.DivExpression: BinaryOperator('/', self),
nodes.FloorDivExpression: BinaryOperator('//', self),
nodes.MulExpression: BinaryOperator('*', self),
nodes.ModExpression: BinaryOperator('%', self),
nodes.PosExpression: UnaryOperator('+', self),
nodes.NegExpression: UnaryOperator('-', self),
nodes.PowExpression: BinaryOperator('**', self),
nodes.DictExpression: self.handle_dict,
nodes.SetExpression: self.handle_set_expr,
nodes.ListExpression: self.handle_list,
nodes.TupleExpression: self.handle_tuple,
nodes.UndefinedExpression: self.handle_undefined,
nodes.AndExpression: BinaryOperator(' and ', self),
nodes.OrExpression: BinaryOperator(' or ', self),
nodes.NotExpression: UnaryOperator(' not ', self),
nodes.SliceExpression: self.handle_slice,
nodes.ConditionalExpression: self.handle_conditional_expr
}
# environments with the same lexer
_lexer_cache = WeakValueDictionary()
# static regular expressions
whitespace_re = re.compile(r'\s+(?um)')
name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
r'|"([^"\\]*(?:\\.[^"\\]*)*)")(?ms)')
integer_re = re.compile(r'\d+')
float_re = re.compile(r'\d+\.\d+')
regex_re = re.compile(r'@/([^/\\]*(?:\\.[^/\\]*)*)*/[a-z]*(?ms)')
# set of used keywords
keywords = set([
'and', 'block', 'cycle', 'elif', 'else', 'endblock', 'endfilter', 'endfor',
'endif', 'endmacro', 'endraw', 'endtrans', 'extends', 'filter', 'for',
'if', 'in', 'include', 'is', 'macro', 'not', 'or', 'pluralize', 'raw',
'recursive', 'set', 'trans', 'print', 'call', 'endcall'
])
# bind operators to token types
operators = {
'+': 'add',
'-': 'sub',
'/': 'div',
'//': 'floordiv',
'*': 'mul',
'%': 'mod',
'**': 'pow',
'~': 'tilde',
'!': 'bang',
'@': 'at',
def handle_template(self, node):
"""
Handle the overall template node. This node is the first node and
ensures that we get the bootstrapping code. It also knows about
inheritance information. It only occours as outer node, never in
the tree itself.
"""
self.indention = 1
# if there is a parent template we parse the parent template and
# update the blocks there. Once this is done we drop the current
# template in favor of the new one. Do that until we found the
# root template.
parent = None
overwrites = {}
blocks = {}
requirements = []
outer_filename = node.filename or '<template>'
# this set is required in order to not add blocks to the block
# dict a second time if they were not overridden in one template
# in the template chain.
already_registered_block = set()
while node.extends is not None:
# the direct child nodes in a template that are not blocks
# are processed as template globals, thus executed *before*
# the master layout template is loaded. This can be used
# for further processing. The output of those nodes does
# not appear in the final template.
requirements += [
child for child in node.getChildNodes()
if child.__class__ not in (nodes.Text, nodes.Block,
nodes.Extends)
]
# load the template we inherit from and add not known blocks
parent = self.environment.loader.parse(node.extends.template,
node.filename)
# look up all block nodes in the current template and
# add them to the override dict.
for n in get_nodes(nodes.Block, node):
overwrites[n.name] = n
# handle direct overrides
for n in get_nodes(nodes.Block, parent):
# an overwritten block for the parent template. handle that
# override in the template and register it in the deferred
# block dict.
if n.name in overwrites and not n in already_registered_block:
blocks.setdefault(n.name, []).append(n.clone())
n.replace(overwrites[n.name])
already_registered_block.add(n)
# make the parent node the new node
node = parent
# handle requirements code
if requirements:
requirement_lines = ['def bootstrap(context):']
for n in requirements:
requirement_lines.append(self.handle_node(n))
requirement_lines.append(' if 0: yield None\n')
# handle body in order to get the used shortcuts
body_lines = [self.handle_node(n) for n in node]
# same for blocks in callables
block_lines = []
block_items = blocks.items()
block_items.sort()
dict_lines = []
for name, items in block_items:
tmp = []
for idx, item in enumerate(items):
# ensure that the indention is correct
self.indention = 1
func_name = 'block_%s_%s' % (name, idx)
data = self.handle_block(item, idx + 1)
# blocks with data
if data:
block_lines.extend([
'def %s(context):' % func_name,
self.indent(self.nodeinfo(item, True)), data,
' if 0: yield None\n'
])
tmp.append('buffereater(%s)' % func_name)
self.used_utils.add('buffereater')
# blocks without data, can default to something
# from utils
else:
tmp.append('empty_block')
self.used_utils.add('empty_block')
dict_lines.append(' %r: %s' % (str(name), self.to_tuple(tmp)))
# bootstrapping code
lines = ['# Essential imports', 'from __future__ import division']
if self.used_utils:
lines.append('from jinja.utils import %s' %
', '.join(self.used_utils))
if self.require_runtime_error:
lines.append('from jinja.exceptions import TemplateRuntimeError')
if self.used_data_structures:
lines.append('from jinja.datastructure import %s' %
', '.join(self.used_data_structures))
lines.append(
'\n# Aliases for some speedup\n'
'%s\n\n'
'# Name for disabled debugging\n'
'__name__ = %r\n\n'
'def generate(context):\n'
' assert environment is context.environment' % ('\n'.join([
'%s = environment.%s' % (item, item) for item in [
'get_attribute', 'perform_test', 'apply_filters',
'call_function', 'call_function_simple', 'finish_var',
'undefined_singleton'
] if item in self.used_shortcuts
]), outer_filename))
# the template body
if requirements:
lines.append(' for item in bootstrap(context): pass')
lines.extend(body_lines)
lines.append(' if 0: yield None\n')
# now write the bootstrapping (requirements) core if there is one
if requirements:
lines.append('# Bootstrapping code')
lines.extend(requirement_lines)
# blocks must always be defined. even if it's empty. some
# features depend on it
if block_lines:
lines.append('# Superable blocks')
lines.extend(block_lines)
lines.append('# Block mapping')
if dict_lines:
lines.append('blocks = {\n%s\n}\n' % ',\n'.join(dict_lines))
else:
lines.append('blocks = {}\n')
# now get the real source lines and map the debugging symbols
debug_mapping = []
file_mapping = {}
last = None
offset = -1
sourcelines = ('\n'.join(lines)).splitlines()
result = []
for idx, line in enumerate(sourcelines):
m = _debug_re.search(line)
if m is not None:
d = m.groupdict()
filename = d['filename'] or None
if isinstance(filename, unicode):
filename = filename.encode('utf-8')
if filename in file_mapping:
file_id = file_mapping[filename]
else:
file_id = file_mapping[filename] = 'F%d' % \
len(file_mapping)
this = (file_id, int(d['lineno']))
# if it's the same as the line before we ignore it
if this != last:
debug_mapping.append('(%r, %s, %r)' %
((idx - offset, ) + this))
last = this
# for each debug symbol the line number and so the offset
# changes by one.
offset += 1
else:
result.append(line)
# now print file mapping and debug info
result.append('\n# Debug Information')
file_mapping = file_mapping.items()
file_mapping.sort(lambda a, b: cmp(a[1], b[1]))
for filename, file_id in file_mapping:
result.append('%s = %r' % (file_id, filename))
result.append('debug_info = %s' % self.to_tuple(debug_mapping))
return '\n'.join(result)
def __init__(self, environment, node, source):
self.environment = environment
self.loader = environment.loader.get_controlled_loader()
self.node = node
self.source = source
self.closed = False
#: current level of indention
self.indention = 0
#: each {% cycle %} tag has a unique ID which increments
#: automatically for each tag.
self.last_cycle_id = 0
#: set of used shortcuts jinja has to make local automatically
self.used_shortcuts = set(['undefined_singleton'])
#: set of used datastructures jinja has to import
self.used_data_structures = set()
#: set of used utils jinja has to import
self.used_utils = set()
#: flags for runtime error
self.require_runtime_error = False
#: do wee need a "set" object?
self.need_set_import = False
#: flag for regular expressions
self.compiled_regular_expressions = {}
#: bind the nodes to the callback functions. There are
#: some missing! A few are specified in the `unhandled`
#: mapping in order to disallow their usage, some of them
#: will not appear in the jinja parser output because
#: they are filtered out.
self.handlers = {
# block nodes
nodes.Template: self.handle_template,
nodes.Text: self.handle_template_text,
nodes.NodeList: self.handle_node_list,
nodes.ForLoop: self.handle_for_loop,
nodes.IfCondition: self.handle_if_condition,
nodes.Cycle: self.handle_cycle,
nodes.Print: self.handle_print,
nodes.Macro: self.handle_macro,
nodes.Call: self.handle_call,
nodes.Set: self.handle_set,
nodes.Filter: self.handle_filter,
nodes.Block: self.handle_block,
nodes.Include: self.handle_include,
nodes.Trans: self.handle_trans,
# expression nodes
nodes.NameExpression: self.handle_name,
nodes.CompareExpression: self.handle_compare,
nodes.TestExpression: self.handle_test,
nodes.ConstantExpression: self.handle_const,
nodes.RegexExpression: self.handle_regex,
nodes.SubscriptExpression: self.handle_subscript,
nodes.FilterExpression: self.handle_filter_expr,
nodes.CallExpression: self.handle_call_expr,
nodes.AddExpression: self.handle_add,
nodes.SubExpression: self.handle_sub,
nodes.ConcatExpression: self.handle_concat,
nodes.DivExpression: self.handle_div,
nodes.FloorDivExpression: self.handle_floor_div,
nodes.MulExpression: self.handle_mul,
nodes.ModExpression: self.handle_mod,
nodes.PosExpression: self.handle_pos,
nodes.NegExpression: self.handle_neg,
nodes.PowExpression: self.handle_pow,
nodes.DictExpression: self.handle_dict,
nodes.SetExpression: self.handle_set_expr,
nodes.ListExpression: self.handle_list,
nodes.TupleExpression: self.handle_tuple,
nodes.UndefinedExpression: self.handle_undefined,
nodes.AndExpression: self.handle_and,
nodes.OrExpression: self.handle_or,
nodes.NotExpression: self.handle_not,
nodes.SliceExpression: self.handle_slice,
nodes.ConditionalExpression: self.handle_conditional_expr
}