def filter_stream(self, stream):
"""
We convert
{{ some.variable | filter1 | filter 2}}
to
{{ some.variable | filter1 | filter 2 | bind}}
... for all variable declarations in the template
This function is called by jinja2 immediately
after the lexing stage, but before the parser is called.
"""
while not stream.eos:
token = next(stream)
if token.test("variable_begin"):
var_expr = []
while not token.test("variable_end"):
var_expr.append(token)
token = next(stream)
variable_end = token
last_token = var_expr[-1]
if (not last_token.test("name")
or not last_token.value in ('bind', 'inclause', 'sqlsafe')):
# don't bind twice
var_expr.append(Token(10, 'pipe', u'|'))
var_expr.append(Token(10, 'name', u'bind'))
var_expr.append(variable_end)
for token in var_expr:
yield token
else:
yield token
def filter_stream(self, stream):
"""
We convert
{{ some.variable | filter1 | filter 2}}
to
{{ some.variable | filter1 | filter 2 | yaml}}
... for all variable declarations in the template
This function is called by jinja2 immediately
after the lexing stage, but before the parser is called.
"""
while not stream.eos:
token = next(stream)
if token.test('variable_begin'):
var_expr = []
while not token.test('variable_end'):
var_expr.append(token)
token = next(stream)
variable_end = token
last_token = var_expr[-1]
if last_token.test('name') and last_token.value == 'yaml':
# don't yaml twice
continue
var_expr.append(Token(10, 'pipe', u'|'))
var_expr.append(Token(10, 'name', u'yaml'))
var_expr.append(variable_end)
for token in var_expr:
yield token
else:
yield token
def filter_stream(self, stream: "TokenStream") -> Iterable[Token]:
"""Modify the colon to lparen and rparen tokens"""
# expect a colon
# 0: don't expect to change any {{a | filter: arg}}
# to {{a | filter(arg)}}
# 1: expect a filter
# 2: expect the colon
# 3: expect rparen
flag = 0
for token in stream:
# print(token.value, token.type)
if flag == 0 and token.type is TOKEN_PIPE:
flag = 1
yield token
elif token.type is TOKEN_NAME and flag == 1:
flag = 2
yield token
elif token.type is TOKEN_COLON and flag == 2:
flag = 3
yield Token(token.lineno, TOKEN_LPAREN, None)
elif token.type is TOKEN_COLON and flag == 3:
# {{ a | filter: 1, x: 2}} => {{ a | filter: 1, x=2}}
yield Token(token.lineno, TOKEN_ASSIGN, None)
elif (
token.type in (TOKEN_VARIABLE_END, TOKEN_BLOCK_END, TOKEN_PIPE)
and flag == 3
):
flag = 1 if token.type is TOKEN_PIPE else 0
yield Token(token.lineno, TOKEN_RPAREN, None)
yield token
else:
yield token
def filter_stream(self, stream: TokenStream):
for token in stream:
if token.type != "data":
yield token
continue
pos = 0
lineno = token.lineno
while match := REGEX_DUNDER.search(token.value, pos):
new_pos = match.start()
if new_pos > pos:
preval = token.value[pos:new_pos]
yield Token(lineno, "data", preval)
lineno += count_newlines(preval)
yield Token(lineno, "variable_begin", "")
yield Token(lineno, "name", match[1])
yield Token(lineno, "pipe", "|")
yield Token(lineno, "name", "default")
yield Token(lineno, "lparen", "(")
yield Token(lineno, "string", match[0])
yield Token(lineno, "rparen", ")")
yield Token(lineno, "variable_end", "")
pos = match.end()
if pos < len(token.value):
yield Token(lineno, "data", token.value[pos:])
def filter_stream(self, stream):
in_trans = False
in_variable = False
for token in stream:
# Check if we are inside a trans block - we cannot use filters there!
if token.type == 'block_begin':
block_name = stream.current.value
if block_name == 'trans':
in_trans = True
elif block_name == 'endtrans':
in_trans = False
elif token.type == 'variable_begin':
in_variable = True
if in_variable and not in_trans:
if token.type == 'variable_end' or (token.type == 'name'
and token.value == 'if'):
yield Token(token.lineno, 'pipe', '|')
yield Token(token.lineno, 'name', 'latex')
if token.type == 'variable_end':
in_variable = False
# Original token
yield token
class TokenStreamTestCase(JinjaTestCase):
test_tokens = [
Token(1, TOKEN_BLOCK_BEGIN, ''),
Token(2, TOKEN_BLOCK_END, ''),
]
def test_simple(self):
ts = TokenStream(self.test_tokens, "foo", "bar")
assert ts.current.type is TOKEN_BLOCK_BEGIN
assert bool(ts)
assert not bool(ts.eos)
next(ts)
assert ts.current.type is TOKEN_BLOCK_END
assert bool(ts)
assert not bool(ts.eos)
next(ts)
assert ts.current.type is TOKEN_EOF
assert not bool(ts)
assert bool(ts.eos)
def test_iter(self):
token_types = [
t.type for t in TokenStream(self.test_tokens, "foo", "bar")
]
assert token_types == [
'block_begin',
'block_end',
]
class TestTokenStream:
test_tokens = [
Token(1, TOKEN_BLOCK_BEGIN, ""),
Token(2, TOKEN_BLOCK_END, ""),
]
def test_simple(self, env):
ts = TokenStream(self.test_tokens, "foo", "bar")
assert ts.current.type is TOKEN_BLOCK_BEGIN
assert bool(ts)
assert not bool(ts.eos)
next(ts)
assert ts.current.type is TOKEN_BLOCK_END
assert bool(ts)
assert not bool(ts.eos)
next(ts)
assert ts.current.type is TOKEN_EOF
assert not bool(ts)
assert bool(ts.eos)
def test_iter(self, env):
token_types = [
t.type for t in TokenStream(self.test_tokens, "foo", "bar")
]
assert token_types == [
"block_begin",
"block_end",
]
def _template_call(stream, function_name):
lineno = stream.current.lineno
yield from (
Token(lineno, "variable_begin", "{{"),
Token(lineno, "name", function_name),
Token(lineno, "lparen", "("),
Token(lineno, "rparen", ")"),
Token(lineno, "variable_end", "}}"),
)
def filter_stream(self, stream):
for token in stream:
if token.type == 'variable_end':
yield Token(token.lineno, 'rparen', ')')
yield Token(token.lineno, 'pipe', '|')
yield Token(token.lineno, 'name', 'assert_safe')
yield token
if token.type == 'variable_begin':
yield Token(token.lineno, 'lparen', '(')
def filter_stream(self, stream):
# The token stream looks like this:
# ------------------------
# variable_begin {{
# name event
# dot .
# name getTitle
# lparen (
# rparen )
# pipe |
# name safe
# variable_end }}
# ------------------------
# Intercepting the end of the actual variable is hard but it's rather easy to get the end of
# the variable tag or the start of the first filter. As filters are optional we need to check
# both cases. If we inject the code before the first filter we *probably* don't need to run
# it again later assuming our filters are nice and only return unicode. If that's not the
# case we can simply remove the `variable_done` checks.
# Due to the way Jinja works it is pretty much impossible to apply the filter to arguments
# passed inside a {% trans foo=..., bar=... %} argument list - we have nothing to detect the
# end of an argument as the 'comma' token might be inside a function call. So in that case#
# people simply need to unicodify the strings manually. :(
variable_done = False
in_trans = False
in_variable = False
for token in stream:
# Check if we are inside a trans block - we cannot use filters there!
if token.type == 'block_begin':
block_name = stream.current.value
if block_name == 'trans':
in_trans = True
elif block_name == 'endtrans':
in_trans = False
elif token.type == 'variable_begin':
in_variable = True
if not in_trans and in_variable:
if token.type == 'pipe':
# Inject our filter call before the first filter
yield Token(token.lineno, 'pipe', '|')
yield Token(token.lineno, 'name', 'ensure_unicode')
variable_done = True
elif token.type == 'variable_end' or (token.type == 'name'
and token.value == 'if'):
if not variable_done:
# Inject our filter call if we haven't injected it right after the variable
yield Token(token.lineno, 'pipe', '|')
yield Token(token.lineno, 'name', 'ensure_unicode')
variable_done = False
if token.type == 'variable_end':
in_variable = False
# Original token
yield token
def filter_njk_stream(self, stream):
for token in stream:
# patch strict equality operator `===`
if token.test("eq:==") and stream.current.test("assign:="):
yield Token(token.lineno, "name", "is")
yield Token(token.lineno, "name", "sameas")
stream.skip(1)
else:
yield token
def filter_stream(self, stream):
djedi_init = [
Token(0, 'block_begin', '{%'),
Token(0, 'name', DJEDI_INIT_TAG),
Token(0, 'block_end', '%}'),
Token(0, 'data', '\n')
]
for token in djedi_init:
yield token
for token in stream:
yield token
def filter_stream(self, stream):
paren_stack = 0
for token in stream:
if token.type != "data":
yield token
continue
pos = 0
lineno = token.lineno
while 1:
if not paren_stack:
match = _outside_re.search(token.value, pos)
else:
match = _inside_re.search(token.value, pos)
if match is None:
break
new_pos = match.start()
if new_pos > pos:
preval = token.value[pos:new_pos]
yield Token(lineno, "data", preval)
lineno += count_newlines(preval)
gtok = match.group()
if gtok[0] == "\\":
yield Token(lineno, "data", gtok[1:])
elif not paren_stack:
yield Token(lineno, "block_begin", None)
yield Token(lineno, "name", "trans")
yield Token(lineno, "block_end", None)
paren_stack = 1
else:
if gtok == "(" or paren_stack > 1:
yield Token(lineno, "data", gtok)
paren_stack += gtok == ")" and -1 or 1
if not paren_stack:
yield Token(lineno, "block_begin", None)
yield Token(lineno, "name", "endtrans")
yield Token(lineno, "block_end", None)
pos = match.end()
if pos < len(token.value):
yield Token(lineno, "data", token.value[pos:])
if paren_stack:
raise TemplateSyntaxError(
"unclosed gettext expression",
token.lineno,
stream.name,
stream.filename,
)
def filter_stream(self, stream):
if not self.custom_autoescape_enabled:
if hasattr(self.environment, 'custom_autoescape_filter_name'):
if hasattr(self.environment, 'custom_select_autoescape'):
self.custom_autoescape_enabled = True
autoescape = self.custom_autoescape_enabled and self.environment.custom_select_autoescape(
stream.name)
for token in stream:
if token.type == 'variable_end' and autoescape:
yield Token(token.lineno, 'pipe', '|')
yield Token(token.lineno, 'name',
self.environment.custom_autoescape_filter_name)
yield token
def filter_stream(self, stream):
paren_stack = 0
for token in stream:
if token.type is not 'data':
yield token
continue
pos = 0
lineno = token.lineno
while 1:
if not paren_stack:
match = _outside_re.search(token.value, pos)
else:
match = _inside_re.search(token.value, pos)
if match is None:
break
new_pos = match.start()
if new_pos > pos:
preval = token.value[pos:new_pos]
yield Token(lineno, 'data', preval)
lineno += count_newlines(preval)
gtok = match.group()
if gtok[0] == '\\':
yield Token(lineno, 'data', gtok[1:])
elif not paren_stack:
yield Token(lineno, 'block_begin', None)
yield Token(lineno, 'name', 'trans')
yield Token(lineno, 'block_end', None)
paren_stack = 1
else:
if gtok == '(' or paren_stack > 1:
yield Token(lineno, 'data', gtok)
paren_stack += gtok == ')' and -1 or 1
if not paren_stack:
yield Token(lineno, 'block_begin', None)
yield Token(lineno, 'name', 'endtrans')
yield Token(lineno, 'block_end', None)
pos = match.end()
if pos < len(token.value):
yield Token(lineno, 'data', token.value[pos:])
if paren_stack:
raise TemplateSyntaxError('unclosed gettext expression',
token.lineno, stream.name,
stream.filename)
def filter_stream(self, stream):
ctx = StreamProcessContext(stream)
strip_depth = 0
while True:
if stream.current.type == 'block_begin':
if stream.look().test('name:strip') or \
stream.look().test('name:endstrip'):
stream.skip()
if stream.current.value == 'strip':
strip_depth += 1
else:
strip_depth -= 1
if strip_depth < 0:
ctx.fail('Unexpected tag endstrip')
stream.skip()
if stream.current.type != 'block_end':
ctx.fail('expected end of block, got %s' %
describe_token(stream.current))
stream.skip()
if strip_depth > 0 and stream.current.type == 'data':
ctx.token = stream.current
value = self.normalize(ctx)
yield Token(stream.current.lineno, 'data', value)
else:
yield stream.current
next(stream)
def filter_stream(self, stream):
compressor = Compressor()
strip_depth = 0
while 1:
if stream.current.type == 'block_begin':
if stream.look().test('name:strip') or \
stream.look().test('name:endstrip'):
stream.skip()
if stream.current.value == 'strip':
strip_depth += 1
else:
strip_depth -= 1
if strip_depth < 0:
self.fail('Unexpected tag endstrip', stream, token)
stream.skip()
if stream.current.type != 'block_end':
self.fail('expected end of block, got %s' %
describe_token(stream.current), stream, token)
stream.skip()
if strip_depth > 0 and stream.current.type == 'data':
token = stream.current
try:
value = compressor.compress(token.value)
except CompressError as e:
self.fail(e.message, stream, token)
else:
yield Token(stream.current.lineno, 'data', value)
else:
yield stream.current
next(stream)
def filter_stream(self, stream):
ctx = StreamProcessContext(stream)
strip_depth = 0
while True:
if stream.current.type == "block_begin":
if stream.look().test("name:strip") or stream.look().test(
"name:endstrip"):
stream.skip()
if stream.current.value == "strip":
strip_depth += 1
else:
strip_depth -= 1
if strip_depth < 0:
ctx.fail("Unexpected tag endstrip")
stream.skip()
if stream.current.type != "block_end":
ctx.fail("expected end of block, got %s" %
describe_token(stream.current))
stream.skip()
if strip_depth > 0 and stream.current.type == "data":
ctx.token = stream.current
value = self.normalize(ctx)
yield Token(stream.current.lineno, "data", value)
else:
yield stream.current
next(stream)
def filter_stream(self, stream):
self.environment.epfl_compoext_data[stream.name] = {
"component_exports": [], # these values are accessed from
"component_parents": {}, # epfl-page
"component_children": {}
}
lineno = 0
for token in stream:
yield token
lineno = token.lineno + 1
yield Token(lineno, 'block_begin', None)
yield Token(lineno, 'name', 'component_exports')
yield Token(lineno, 'block_end', None)
def _visit_data(stream, state):
if not state["processing"]:
yield from SqlvmExtension._pass_along_visitor(stream, state)
return
lines = stream.current.value.split("\n")
for line in lines[:-1]:
if line and not line.isspace():
yield Token(stream.current.lineno, "data", line)
state["statement_started"] = True
if state["statement_started"]:
yield from SqlvmExtension._template_call(stream, "statement")
state["statement_started"] = False
yield Token(stream.current.lineno, "data", lines[-1])
next(stream)
def next_if_name(self, stream):
name = ''
while stream.current.test(TOKEN_SUB) or stream.current.test(
TOKEN_NAME) or stream.current.test(
TOKEN_COLON) or stream.current.test(
TOKEN_DOT) or stream.current.test(TOKEN_DOT):
name += next(stream).value
return Token(stream.current.lineno, TOKEN_NAME, name) if name else None
def filter_stream(self, stream):
ctx = StreamProcessContext(stream)
for token in stream:
if token.type != 'data':
yield token
continue
ctx.token = token
value = self.normalize(ctx)
yield Token(token.lineno, 'data', value)
def filter_stream(self, stream: Any):
lineno = 0
for token in stream:
if token.type == "data":
prefix, newline, value = token.value.partition("\n")
if token.lineno > lineno:
prefix = dedent(prefix)
token = Token(token.lineno, "data",
prefix + newline + dedent(value))
yield token
lineno = token.lineno
def filter_stream(self, stream):
compressor = Compressor()
for token in stream:
if token.type != 'data':
yield token
continue
try:
value = compressor.compress(token.value)
except CompressError as e:
self.fail(e.message, stream, token)
else:
yield Token(token.lineno, 'data', value)
def arg_tokenizer(self, token:Token):
for mo in DEF_ARG_RE.finditer(token.value):
print(mo)
print(mo['name'], mo['double'])
yield token
return
if not mo:
yield token
return
while mo:
if mo.lastgroup == 'double':
yield Token(token.lineno, TOKEN_DATA, DEF_PREFIX)
else:
yield from (
Token(token.lineno, TOKEN_VARIABLE_BEGIN, "{{"),
Token(token.lineno, TOKEN_NAME, mo['name']),
Token(token.lineno, TOKEN_VARIABLE_END, "}}"),
)
mo = DEF_ARG_RE.search(token.value, mo.endpos)
def filter_stream(self, stream):
"""
We convert
{{ some.variable | filter1 | filter 2}}
to
{{ some.variable | filter1 | filter 2 | yaml}}
... for all variable declarations in the template
This function is called by jinja2 immediately
after the lexing stage, but before the parser is called.
"""
while not stream.eos:
token = next(stream)
if token.test("variable_begin"):
var_expr = []
while not token.test("variable_end"):
var_expr.append(token)
token = next(stream)
variable_end = token
last_token = var_expr[-1]
if last_token.test("name") and last_token.value == "yaml":
# don't yaml twice
continue
# Wrap the whole expression between the `variable_begin`
# and `variable_end` marks in parens:
var_expr.insert(1, Token(var_expr[0].lineno, "lparen", None))
var_expr.append(Token(var_expr[-1].lineno, "rparen", None))
var_expr.append(Token(token.lineno, "pipe", "|"))
var_expr.append(Token(token.lineno, "name", "yaml"))
var_expr.append(variable_end)
for token in var_expr:
yield token
else:
yield token
def filter_stream(self, stream):
if not self.page:
self.page = str(self.filename)
self.name = stream.name
self.filename = stream.filename
for token in stream:
if token.type != 'data':
yield token
continue
self.lineno = token.lineno
self.normalize(token.value)
yield Token(token.lineno, 'data', ''.join(self.buf))
def filter_stream(self, stream):
super_stream = super().filter_stream(stream)
for token in super_stream:
if token.type != 'data':
yield token
continue
value = re.sub(r'\n', '', token.value)
value = re.sub(r'(>)(\s+)(<)', r'\1\3', value)
value = re.sub(r'\s+', r' ', value)
value = re.sub(r'(")(\s+)(/>)', r'\1\3', value)
yield Token(token.lineno, token.type, value)
def interpolate(self, token):
pos = 0
end = len(token.value)
lineno = token.lineno
while 1:
match = _gettext_re.search(token.value, pos)
if match is None:
break
value = token.value[pos:match.start()]
if value:
yield Token(lineno, "data", value)
lineno += count_newlines(token.value)
yield Token(lineno, "variable_begin", None)
yield Token(lineno, "name", "gettext")
yield Token(lineno, "lparen", None)
yield Token(lineno, "string", match.group(1))
yield Token(lineno, "rparen", None)
yield Token(lineno, "variable_end", None)
pos = match.end()
if pos < end:
yield Token(lineno, "data", token.value[pos:])
def interpolate(self, token):
pos = 0
end = len(token.value)
lineno = token.lineno
while 1:
match = _gettext_re.search(token.value, pos)
if match is None:
break
value = token.value[pos:match.start()]
if value:
yield Token(lineno, 'data', value)
lineno += count_newlines(token.value)
yield Token(lineno, 'variable_begin', None)
yield Token(lineno, 'name', 'gettext')
yield Token(lineno, 'lparen', None)
yield Token(lineno, 'string', match.group(1))
yield Token(lineno, 'rparen', None)
yield Token(lineno, 'variable_end', None)
pos = match.end()
if pos < end:
yield Token(lineno, 'data', token.value[pos:])