def _get_parser():
import django
parser = Parser([])
if django.VERSION >= (1, 9):
import django.template.defaultfilters
parser.add_library(django.template.defaultfilters.register)
return parser
def compile_nodelist(self):
"""
Pass template name to parser instance
"""
if self.engine.debug:
lexer = DebugLexer19(self.source)
else:
lexer = Lexer(self.source)
tokens = lexer.tokenize()
parser = Parser(
tokens,
self.engine.template_libraries,
self.engine.template_builtins,
)
parser.template_name = self.origin.template_name
try:
return parser.parse()
except Exception as e:
if self.engine.debug:
e.template_debug = self.get_exception_info(e, e.token)
raise
def compile_nodelist(self):
"""
Pass template name to parser instance
"""
if self.engine.debug:
lexer = DebugLexer19(self.source)
else:
lexer = Lexer(self.source)
tokens = lexer.tokenize()
parser = Parser(
tokens,
self.engine.template_libraries,
self.engine.template_builtins,
)
parser.template_name = self.origin.template_name
try:
return parser.parse()
except Exception as e:
if self.engine.debug:
e.template_debug = self.get_exception_info(e, e.token)
raise
def linebreakless(parser: Parser, token: Token):
library = template.Library()
library.tag(LinebreaklessNode.KEEP_LINEBREAK_TAG,
LinebreaklessNode.keeplinebreak)
parser.add_library(library)
nodelist = parser.parse(("endlinebreakless", ))
parser.delete_first_token()
return LinebreaklessNode(token, nodelist)
def build_migration(self, src, filename=None, charset='utf-8'):
"""Create a migration for a Django template file to Transifex Native syntax.
The returned object contains every change separately, so that
it can be reviewed string by string.
:param unicode src: the whole Django template
:param str filename: the filename of the original template
:param str charset: the character set to use
:return: a FileMigration instance
:rtype: FileMigration
"""
self._reset()
src = force_text(src, charset)
# Using the DebugLexer because we need the positional information
# of each token (start/end pos). It is slower than Lexer, but Lexer
# doesn't provide that information
tokens = DebugLexer(src).tokenize()
parser = Parser(tokens, libraries={}, builtins=[], origin=filename)
# Since no template libraries are loaded when this code is running,
# we need to override the find function in order to use the
# functionality of the Parser class. The overridden function returns
# the object as given.
# Without the override, a KeyError would be raised inside the parser.
parser.find_filter = find_filter_identity
# Create a migration object for this template; we'll add stuff to it
# as we go
migration = FileMigration(filename, src)
while parser.tokens:
token = parser.next_token()
start, end = token.position
# Parse the current token. This may or may not return a migration.
# Also it may return a list of tokens that were consumed,
# additionally to the current token. If this happens,
# `_parse_token()` will have made sure that `parser` has moved
# forward, consuming those tokens, so that they don't appear again
# in the while loop.
string_migration, extra_consumed_tokens = self._parse_token(
token, parser, original_string=src[start:end])
if not string_migration:
continue
# If additional tokens were consumed, we need to add
# them in the migration, so that the StringMigration object
# includes the information of what part of the original template
# was migrated to the new syntax, for this particular translatable
# string
if extra_consumed_tokens:
for extra_token in extra_consumed_tokens:
start, end = extra_token.position
string_migration.update(src[start:end], '')
migration.add_string(string_migration)
return migration
def test_can_load_from_missing_taglib(self):
template = ('{% load friendly_loader %}'
'{% friendly_load error from error_tags %}')
lexer = Lexer(template)
parser = Parser(lexer.tokenize(), engine.template_libraries,
engine.template_builtins)
parser.parse()
self.assertTrue(isinstance(Template(template), Template),
'Expected template to initialize')
def test_can_load_taglib_using_friendly_load(self):
template = '{% load friendly_loader %}{% friendly_load flatpages %}'
lexer = Lexer(template)
parser = Parser(lexer.tokenize(), engine.template_libraries,
engine.template_builtins)
parser.parse()
self.assertTrue(
'get_flatpages' in parser.tags,
'Expected flatpages taglib to load and provide the get_flatpages tag'
)
def __str__(self):
my_lexer = get_lexer(self.title)
my_tokens = my_lexer.tokenize()
# Deliberately strip off template tokens that are not text or variable.
for my_token in my_tokens:
if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR):
my_tokens.remove(my_token)
my_parser = Parser(my_tokens)
return my_parser.parse().render(get_sitetree().current_page_context)
def __str__(self):
my_lexer = get_lexer(self.title)
my_tokens = my_lexer.tokenize()
# Deliberately strip off template tokens that are not text or variable.
for my_token in my_tokens:
if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR):
my_tokens.remove(my_token)
my_parser = Parser(my_tokens)
return my_parser.parse().render(get_sitetree().current_page_context)
def __str__(self):
my_lexer = Lexer(self.title, UNKNOWN_SOURCE)
my_tokens = my_lexer.tokenize()
# Deliberately strip off template tokens that are not text or variable.
for my_token in my_tokens:
if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR):
my_tokens.remove(my_token)
my_parser = Parser(my_tokens)
return my_parser.parse().render(SiteTree.get_global_context())
def __str__(self):
my_lexer = Lexer(self.title, UNKNOWN_SOURCE)
my_tokens = my_lexer.tokenize()
# Deliberately strip off template tokens that are not text or variable.
for my_token in my_tokens:
if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR):
my_tokens.remove(my_token)
my_parser = Parser(my_tokens)
return my_parser.parse().render(SiteTree.get_global_context())
def test_raises_template_syntaxerror(self):
self.assertRaises(TemplateSyntaxError, self.render, "{% host_url %}")
self.assertRaises(TemplateSyntaxError, self.render,
"{% host_url 'simple-direct' host %}")
self.assertRaises(TemplateSyntaxError, self.render,
"{% host_url 'simple-direct' as %}")
self.assertRaises(TemplateSyntaxError, self.render,
"{% host_url simple-direct %}")
self.assertRaises(TemplateSyntaxError, parse_params, 'host_url',
Parser(['']), "username=='johndoe'")
self.assertRaises(TemplateSyntaxError, parse_params, 'host_url',
Parser(['']), "\n='johndoe'")
def capture(parser: Parser, token: Token):
bits = token.contents.split()
if len(bits) > 2:
raise TemplateSyntaxError(
f"'capture' tag got unexpected arguments {bits[2:]!r}")
name: str = bits[1]
if "'" in name or '"' in name:
raise TypeError(
f"'capture' tag got a quoted name {name}. use a name without quotes"
)
nodelist = parser.parse(parse_until=['endcapture'])
parser.delete_first_token()
return CaptureNode(name=name, nodelist=nodelist)
def parse(self, parser, stream, container, nextargs=None):
stream_copy = copy(stream)
if hasattr(parser, 'tokens'):
parser_copy = DjangoParser(list(parser.tokens))
parser_copy.tags = copy(parser.tags)
parser_copy.filters = copy(parser.filters)
else:
parser_copy = FakeParser(parser)
try:
self._do_parse(parser_copy, stream_copy, Container(), nextargs)
except BaseError, e:
pass
def second_pass_render(request, content):
"""
Split on the secret delimiter and generate the token list by passing
through text outside of phased blocks as single text tokens and tokenizing
text inside the phased blocks. This ensures that nothing outside of the
phased blocks is tokenized, thus eliminating the possibility of a template
code injection vulnerability.
"""
result = tokens = []
for index, bit in enumerate(content.split(
settings.PHASED_SECRET_DELIMITER)):
if index % 2:
tokens = Lexer(bit, None).tokenize()
else:
tokens.append(Token(TOKEN_TEXT, bit))
context = RequestContext(
request, restore_csrf_token(request, unpickle_context(bit)))
rendered = Parser(tokens).parse().render(context)
if settings.PHASED_SECRET_DELIMITER in rendered:
rendered = second_pass_render(request, rendered)
result.append(rendered)
return "".join(result)
def test_filter_parsing(self):
c = {"article": {"section": "News"}}
p = Parser("", builtins=[filter_library])
def fe_test(s, val):
self.assertEqual(FilterExpression(s, p).resolve(c), val)
fe_test("article.section", "News")
fe_test("article.section|upper", "NEWS")
fe_test('"News"', "News")
fe_test("'News'", "News")
fe_test(r'"Some \"Good\" News"', 'Some "Good" News')
fe_test(r'"Some \"Good\" News"', 'Some "Good" News')
fe_test(r"'Some \'Bad\' News'", "Some 'Bad' News")
fe = FilterExpression(r'"Some \"Good\" News"', p)
self.assertEqual(fe.filters, [])
self.assertEqual(fe.var, 'Some "Good" News')
# Filtered variables should reject access of attributes beginning with
# underscores.
msg = (
"Variables and attributes may not begin with underscores: 'article._hidden'"
)
with self.assertRaisesMessage(TemplateSyntaxError, msg):
FilterExpression("article._hidden|upper", p)
def test_unicorn_args_and_kwargs():
# args after the component name get ignored
token = Token(TokenType.TEXT,
"unicorn 'todo' '1' 2 hello='world' test=3 '4'")
unicorn_node = unicorn(Parser([]), token)
assert unicorn_node.kwargs == {"hello": "world", "test": 3}
def next_token():
token = Parser.next_token(parser)
if token.token_type == TOKEN_VAR:
token.token_type = TOKEN_TEXT
token.contents = (appsettings.ANGULAR_TOKENS['VARIABLE_OPEN'] +
token.contents +
appsettings.ANGULAR_TOKENS['VARIABLE_CLOSE'])
return token
def try_func(parser: Parser, token: Token):
nodelist_try: NodeList
nodelist_except: Optional[NodeList] = None
catch_exceptions: Optional[List] = None
# Try
nodelist_try: Token = parser.parse(("except", "endtry"))
next_token = parser.next_token()
next_token_name = next_token.contents.lstrip().split(" ")[0].lower()
# Catch
if next_token_name == "except":
# Catch
contents = next_token.contents.split(" ")[1:]
if len(contents) > 0:
catch_exceptions = contents
else:
catch_exceptions = []
nodelist_except = parser.parse(("endtry", ))
else:
parser.parse(("endtry",))
parser.next_token()
return TryNode(
nodelist_try=nodelist_try,
nodelist_except=nodelist_except,
catch_exceptions=catch_exceptions,
)
def test_token_kwargs(self):
parser = Parser('')
self.assertEqual(token_kwargs([], parser), {})
bits = ['a="A"']
token_kwargs(bits, parser)
self.assertEqual(bits, [])
bits = ['a="A"', 'invalid']
token_kwargs(bits, parser)
self.assertEqual(bits, ['invalid'])
def clean(self):
template = DjangoTemplate('')
template.source = self.changed_content
try:
engine = Engine.get_default()
lexer = DebugLexer(self.changed_content)
tokens = lexer.tokenize()
parser = Parser(tokens, engine.template_libraries,
engine.template_builtins, Origin(UNKNOWN_SOURCE))
parser.parse()
except TemplateSyntaxError as e:
exception_info = template.get_exception_info(e, e.token)
raise ValidationError({
'changed_content':
mark_safe(
'{message}<br/>Line {line}:\n<pre>{before}<b>{during}</b>{after}</pre>'
.format(**exception_info))
}) from e
def url(self, sitetree_item, context=None):
"""Resolves item's URL.
:param TreeItemBase sitetree_item: TreeItemBase heir object, 'url' property of which
is processed as URL pattern or simple URL.
:param Context context:
:rtype: str|unicode
"""
context = context or self.current_page_context
resolve_var = self.resolve_var
if not isinstance(sitetree_item, MODEL_TREE_ITEM_CLASS):
sitetree_item = resolve_var(sitetree_item, context)
resolved_url = self._items_urls.get(sitetree_item)
if resolved_url is not None:
return resolved_url
# Resolve only if item's URL is marked as pattern.
if sitetree_item.urlaspattern:
url = sitetree_item.url
view_path = url
all_arguments = []
if ' ' in url:
view_path = url.split(' ')
# We should try to resolve URL parameters from site tree item.
for view_argument in view_path[1:]:
resolved = resolve_var(view_argument)
# We enclose arg in double quotes as already resolved.
all_arguments.append('"%s"' % resolved)
view_path = view_path[0].strip('"\' ')
url_pattern = "'%s' %s" % (view_path, ' '.join(all_arguments))
else:
url_pattern = '%s' % sitetree_item.url
if sitetree_item.urlaspattern:
# Form token to pass to Django 'url' tag.
url_token = 'url %s as item.url_resolved' % url_pattern
url_tag(Parser(None),
Token(token_type=TOKEN_BLOCK,
contents=url_token)).render(context)
resolved_url = context[
'item.url_resolved'] or UNRESOLVED_ITEM_MARKER
else:
resolved_url = url_pattern
self._items_urls[sitetree_item] = resolved_url
return resolved_url
def parse(self, parser, stream, container, nextargs=None):
stream_copy = copy(stream)
if hasattr(parser, 'tokens'):
parser_copy = DjangoParser(list(parser.tokens))
parser_copy.tags = copy(parser.tags)
parser_copy.filters = copy(parser.filters)
else:
parser_copy = FakeParser(parser)
reps = ListValue()
add_reps = True
while (add_reps):
try:
self._do_parse(parser_copy, stream_copy, Container(), nextargs)
except BaseError, e:
add_reps = False
except Exception, e:
raise
def test_token_kwargs(self):
parser = Parser("")
self.assertEqual(token_kwargs([], parser), {})
bits = ['a="A"']
token_kwargs(bits, parser)
self.assertEqual(bits, [])
bits = ['a="A"', "invalid"]
token_kwargs(bits, parser)
self.assertEqual(bits, ["invalid"])
def test_filter_args_count(self):
p = Parser("")
l = Library()
@l.filter
def no_arguments(value):
pass
@l.filter
def one_argument(value, arg):
pass
@l.filter
def one_opt_argument(value, arg=False):
pass
@l.filter
def two_arguments(value, arg, arg2):
pass
@l.filter
def two_one_opt_arg(value, arg, arg2=False):
pass
p.add_library(l)
for expr in (
'1|no_arguments:"1"',
'1|two_arguments',
'1|two_arguments:"1"',
'1|two_one_opt_arg',
):
with self.assertRaises(TemplateSyntaxError):
FilterExpression(expr, p)
for expr in (
# Correct number of arguments
'1|no_arguments',
'1|one_argument:"1"',
# One optional
'1|one_opt_argument',
'1|one_opt_argument:"1"',
# Not supplying all
'1|two_one_opt_arg:"1"',
):
FilterExpression(expr, p)
def url(self, sitetree_item: Union['TreeItemBase', FilterExpression], context: Context = None) -> str:
"""Resolves item's URL.
:param sitetree_item: TreeItemBase heir object, 'url' property of which
is processed as URL pattern or simple URL.
:param context:
"""
context = context or self.current_page_context
resolve_var = self.resolve_var
if not isinstance(sitetree_item, MODEL_TREE_ITEM_CLASS):
sitetree_item = resolve_var(sitetree_item, context)
resolved_url = self._items_urls.get(sitetree_item)
if resolved_url is not None:
return resolved_url
# Resolve only if item's URL is marked as pattern.
if sitetree_item.urlaspattern:
url = sitetree_item.url
view_path = url
all_arguments = []
if ' ' in url:
view_path = url.split(' ')
# We should try to resolve URL parameters from site tree item.
for view_argument in view_path[1:]:
resolved = resolve_var(view_argument)
# We enclose arg in double quotes as already resolved.
all_arguments.append(f'"{resolved}"')
view_path = view_path[0].strip('"\' ')
url_pattern = f"'{view_path}' {' '.join(all_arguments)}"
else:
url_pattern = f'{sitetree_item.url}'
if sitetree_item.urlaspattern:
# Form token to pass to Django 'url' tag.
url_token = f'url {url_pattern} as item.url_resolved'
url_tag(
Parser([]),
Token(token_type=TOKEN_BLOCK, contents=url_token)
).render(context)
resolved_url = context['item.url_resolved'] or UNRESOLVED_ITEM_MARKER
else:
resolved_url = url_pattern
self._items_urls[sitetree_item] = resolved_url
return resolved_url
def loremi(count, method, random=False):
from django.template.base import Context, Token, Parser, TOKEN_TEXT
from django.template.defaulttags import lorem
c = Context()
lorem_str = "lorem %s %s" % (count, method)
if random:
lorem_str += " random"
t = Token(TOKEN_TEXT, lorem_str)
p = Parser(t)
return lorem(p, t).render(c)
def test_unicorn_render_with_invalid_html():
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargsWithHtmlEntity' test_kwarg=test_var.nested",
)
unicorn_node = unicorn(Parser([]), token)
context = {"test_var": {"nested": "variable!"}}
actual = unicorn_node.render(Context(context))
assert "->variable!<-" in actual
def test_filter_args_count(self):
parser = Parser("")
register = Library()
@register.filter
def no_arguments(value):
pass
@register.filter
def one_argument(value, arg):
pass
@register.filter
def one_opt_argument(value, arg=False):
pass
@register.filter
def two_arguments(value, arg, arg2):
pass
@register.filter
def two_one_opt_arg(value, arg, arg2=False):
pass
parser.add_library(register)
for expr in (
'1|no_arguments:"1"',
'1|two_arguments',
'1|two_arguments:"1"',
'1|two_one_opt_arg',
):
with self.assertRaises(TemplateSyntaxError):
FilterExpression(expr, parser)
for expr in (
# Correct number of arguments
'1|no_arguments',
'1|one_argument:"1"',
# One optional
'1|one_opt_argument',
'1|one_opt_argument:"1"',
# Not supplying all
'1|two_one_opt_arg:"1"',
):
FilterExpression(expr, parser)
def test_unicorn_render_kwarg():
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' test_kwarg='tested!'",
)
unicorn_node = unicorn(Parser([]), token)
context = {}
actual = unicorn_node.render(Context(context))
assert "<b>tested!</b>" in actual
def test_unicorn_render_id_use_pk():
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentModel' model_id=model.id",
)
unicorn_node = unicorn(Parser([]), token)
context = {"model": {"pk": 123}}
actual = unicorn_node.render(Context(context))
assert "==123==" in actual
def test_unicorn_render_context_variable():
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' test_kwarg=test_var.nested",
)
unicorn_node = unicorn(Parser([]), token)
context = {"test_var": {"nested": "variable!"}}
actual = unicorn_node.render(Context(context))
assert "<b>variable!</b>" in actual
def test_unicorn_render_child_component_no_script_tag(settings):
settings.DEBUG = True
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' parent=view",
)
unicorn_node = unicorn(Parser([]), token)
view = FakeComponentParent(component_name="test", component_id="asdf")
context = {"view": view}
html = unicorn_node.render(Context(context))
assert "<script" not in html
def test_unicorn_render_parent_component_one_script_tag(settings):
settings.DEBUG = True
token = Token(
TokenType.TEXT,
"unicorn 'tests.templatetags.test_unicorn_render.FakeComponentParent'",
)
unicorn_node = unicorn(Parser([]), token)
context = {}
html = unicorn_node.render(Context(context))
assert '<script type="module"' in html
assert len(re.findall('<script type="module"', html)) == 1
def extract_transifex_template_strings(
src, origin=None, charset='utf-8', fkeygen=generate_key):
"""Parse the given template and extract translatable content
based on the syntax supported by Transifex Native.
Supports the {% t %} and {% ut %} template tags.
:param unicode src: the whole Django template
:param str origin: an optional context for the filename of the source,
e.g. the file name
:param str charset: the character set to use
:param func fkeygen: key generator function
:return: a list of SourceString objects
:rtype: list
"""
src = force_text(src, charset)
tokens = Lexer(src).tokenize()
parser = Parser(tokens, {}, [], origin)
# Since no template libraries are loaded when this code is running,
# we need to override the find function in order to use the functionality
# of the Parser class. The overridden function returns the object as given.
# Without the override, a KeyError is raised inside the parser.
parser.find_filter = find_filter_identity
strings = []
while parser.tokens:
token = parser.next_token()
if (token.token_type == TOKEN_BLOCK and
token.split_contents()[0] in ('t', 'ut')):
tnode = do_t(parser, token)
source_string = tnode_to_source_string(tnode, fkeygen)
if source_string is None:
continue
if token.lineno and origin:
source_string.occurrences = [
"{}:{}".format(origin, token.lineno)]
strings.append(source_string)
return strings
{% else %}
<span class="mini quiet">{% trans 'Unknown content' %}</span>
{% endif %}
</li>
{% endfor %}
</ul>
{% endif %}
</div>
</div>
{% endblock %}
"""
settings.configure()
apps.populate((
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
))
elapsed = 0
lexer = Lexer(template_source, None)
tokens = lexer.tokenize()
for i in xrange(500):
parser = Parser(list(tokens))
parser.parse()
