def parse_source(self):
source_lines = set()
lexer = Lexer(self.text, "<string>")
tokens = lexer.tokenize()
comment = False
for token in tokens:
assert isinstance(token, Token)
if token.token_type == TOKEN_BLOCK:
if token.contents == 'comment':
comment = True
continue
elif token.contents == 'endcomment':
comment = False
continue
if comment:
continue
if token.token_type == TOKEN_BLOCK or token.token_type == TOKEN_VAR:
if token.token_type == TOKEN_BLOCK and token.contents.startswith(
'end'):
continue
source_lines.add(token.lineno)
return tuple(sorted(source_lines)), ()
def parse_source(self):
source_lines = set()
lexer = Lexer(self.text, "<string>")
tokens = lexer.tokenize()
comment = False
for token in tokens:
assert isinstance(token, Token)
if token.token_type == TOKEN_BLOCK:
if token.contents == 'comment':
comment = True
continue
elif token.contents == 'endcomment':
comment = False
continue
if comment:
continue
if token.token_type == TOKEN_BLOCK or token.token_type == TOKEN_VAR:
if token.token_type == TOKEN_BLOCK and token.contents.startswith('end'):
continue
source_lines.add(token.lineno)
return tuple(sorted(source_lines)), ()
def validate_template(self, template_string):
# We want to tokenize like normal, then use a custom parser.
lexer = Lexer(template_string, None)
tokens = lexer.tokenize()
parser = TemplateValidationParser(tokens, self.allow, self.disallow, self.secure)
for node in parser.parse():
template = getattr(node, LOADED_TEMPLATE_ATTR, None)
def validate_template(self, template_string):
# We want to tokenize like normal, then use a custom parser.
lexer = Lexer(template_string, None)
tokens = lexer.tokenize()
parser = TemplateValidationParser(tokens, self.allow, self.disallow,
self.secure)
for node in parser.parse():
template = getattr(node, LOADED_TEMPLATE_ATTR, None)
def tokenize():
"""
Returns a stream of Django Token() entities
"""
for template in get_templates():
with open(template) as fp:
template_content = fp.read()
lexer = Lexer(template_content, None)
for token in lexer.tokenize():
yield token
def tokenize():
"""
Returns a stream of Django Token() entities
"""
for template in get_templates():
with open(template) as fp:
template_content = fp.read()
lexer = Lexer(template_content, None)
for token in lexer.tokenize():
yield token
def load_blocks(self):
"""Loads the asset blocks defined in the template
handles:
* extends - to track template hierachy
* css,javascript - start of asset
* endcss, endjavascript - end of asset
* {{ .. }} - expansion of variables to settings variables according to VAR_EXPANSIONS
"""
try:
template_string, _filepath = filesystem.load_template_source(self.templatepath)
except TemplateDoesNotExist:
template_string, _filepath = app_directories.load_template_source(self.templatepath)
self.content_hash = hash(template_string)
try:
result = TemplateAssetBucket()
l = Lexer(template_string, self.templatepath)
within = None
texts = []
for m in l.tokenize():
if m.token_type == TOKEN_BLOCK:
split = m.split_contents()
typ = split[0]
if typ == "extends":
if split[1].endswith('"') or split[1].endswith("'"):
self.extends = split[1].strip('"').strip("'")
else:
pass #TODO figure out support for variable expansion
elif typ in TemplateAssetBlock.BLOCKTAGS:
within = typ
prop = _parse_asset_parameters(m.split_contents())
elif typ.startswith('end'):
if typ[3:] == within:
within = None
result.append(TemplateAssetBlock(''.join(texts), template=self, **prop))
elif typ[3:] in TemplateAssetBlock.BLOCKTAGS:
assert false, "encountered dangling %s tag in '%s'" % (typ,self.templatepath)
elif within:
if m.token_type == TOKEN_TEXT:
texts.append(m.contents)
elif m.token_type == TOKEN_VAR:
v = VAR_EXPANSIONS.get(m.contents,'')
if v:
texts.append(v)
#? else:
#assert False, "Variable replacement in client side magic not yet supported"
return result
except UnicodeDecodeError:
return "/* could not load %s as a template */\n" % templatepath
def _render_html(self, template_string, context={}):
# :(
if DJANGO_VERSION > (1,2):
from django.template import import_library
tag_lib = import_library('beproud.django.commons.tests.test_tags')
else:
from django.template import get_library
tag_lib = get_library('beproud.django.commons.tests.test_tags')
lexer = Lexer(template_string, self._make_origin())
parser = Parser(lexer.tokenize())
parser.add_library(tag_lib)
nodelist = parser.parse()
return nodelist.render(Context(context))
def _load_all_templates(directory):
"""
Loads all templates in a directory (recursively) and yields tuples of
template tokens and template paths.
"""
if os.path.exists(directory):
for name in os.listdir(directory):
path = os.path.join(directory, name)
if os.path.isdir(path):
for template in _load_all_templates(path):
yield template
elif path.endswith('.html'):
with open(path, 'rb') as fobj:
source = fobj.read().decode(settings.FILE_CHARSET)
lexer = Lexer(source, path)
yield lexer.tokenize(), path
def render_custom_content(body, context_data={}):
"""Renders custom content for the payload using Django templating.
This will take the custom payload content template provided by
the user and render it using a stripped down version of Django's
templating system.
In order to keep the payload safe, we use a limited Context along with a
custom Parser that blocks certain template tags. This gives us
tags like {% for %} and {% if %}, but blacklists tags like {% load %}
and {% include %}.
"""
lexer = Lexer(body, origin=None)
parser = CustomPayloadParser(lexer.tokenize())
nodes = parser.parse()
return nodes.render(Context(context_data))
def render_custom_content(body, context_data={}):
"""Renders custom content for the payload using Django templating.
This will take the custom payload content template provided by
the user and render it using a stripped down version of Django's
templating system.
In order to keep the payload safe, we use a limited Context along with a
custom Parser that blocks certain template tags. This gives us
tags like {% for %} and {% if %}, but blacklists tags like {% load %}
and {% include %}.
"""
lexer = Lexer(body, origin=None)
parser = CustomPayloadParser(lexer.tokenize())
nodes = parser.parse()
return nodes.render(Context(context_data))
def _render_html(self, template_string, context={}):
# :(
if DJANGO_VERSION > (1, 9):
from django.template.library import import_library
tag_lib = import_library('testapp.tags')
else: # DJANGO_VERSION > (1,7):
from django.template.base import import_library
tag_lib = import_library('testapp.tags')
if DJANGO_VERSION > (1, 9):
lexer = Lexer(template_string)
else:
lexer = Lexer(template_string, self._make_origin())
parser = Parser(lexer.tokenize())
parser.add_library(tag_lib)
nodelist = parser.parse()
return nodelist.render(Context(context))
def _fix_html_type(request, html, filetype):
for group, files in requested_assets[request][filetype].items():
# parse the content for the individual file tokens
indices = []
def sub_func(matchobj):
indices.append(int(matchobj.group(2)))
return ""
regex = token_regexes[filetype][group]
html = regex.sub(sub_func, html)
# replace the 'replace me' tag with actual list of
# 'tags' (ie <link href="foo.css">)
file_html = u""
uncompressible_html = u""
for index in indices:
fileObj = files[index]
if fileObj.isCompressible():
file_html += fileObj.render()
else:
uncompressible_html += fileObj.render()
# try to use the provided 'compress' app to compress the output
if hasattr(settings, 'COMPRESS') and settings.COMPRESS:
# Currently this only supports the django-css app we use
from django.template import Lexer,Parser,Token,TOKEN_TEXT
file_html += "{% endcompress %}"
lexer = Lexer(file_html, None)
from compressor.templatetags.compress import compress
file_html = compress(
Parser(lexer.tokenize()),
Token(TOKEN_TEXT, "compress " + filetype)
).render({})
file_html = uncompressible_html + file_html
tag = ASSET_DEFS[filetype]['destination_tag'].get(group, None)
if tag:
html = smart_unicode(html)
html = html.replace(tag, file_html + tag)
return html
def compile_string(template_string):
lexer = Lexer(template_string, None)
parser = ProcessingParser(lexer.tokenize())
return parser.parse()
def extract_django(fileobj, keywords, comment_tags, options):
"""Extract messages from Django template files.
:param fileobj: the file-like object the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
intrans = False
inplural = False
message_context = None
singular = []
plural = []
lineno = 1
encoding = options.get('encoding', 'utf8')
text = fileobj.read().decode(encoding)
try:
text_lexer = Lexer(text)
except TypeError:
# Django 1.9 changed the way we invoke Lexer; older versions
# require two parameters.
text_lexer = Lexer(text, None)
for t in text_lexer.tokenize():
lineno += t.contents.count('\n')
if intrans:
if t.token_type == TOKEN_BLOCK:
endbmatch = endblock_re.match(t.contents)
pluralmatch = plural_re.match(t.contents)
if endbmatch:
if inplural:
if message_context:
yield (
lineno,
'npgettext',
[smart_text(message_context),
smart_text(u''.join(singular)),
smart_text(u''.join(plural))],
[],
)
else:
yield (
lineno,
'ngettext',
(smart_text(u''.join(singular)),
smart_text(u''.join(plural))),
[])
else:
if message_context:
yield (
lineno,
'pgettext',
[smart_text(message_context),
smart_text(u''.join(singular))],
[],
)
else:
yield (
lineno,
None,
smart_text(u''.join(singular)),
[])
intrans = False
inplural = False
message_context = None
singular = []
plural = []
elif pluralmatch:
inplural = True
else:
raise SyntaxError('Translation blocks must not include '
'other block tags: %s' % t.contents)
elif t.token_type == TOKEN_VAR:
if inplural:
plural.append('%%(%s)s' % t.contents)
else:
singular.append('%%(%s)s' % t.contents)
elif t.token_type == TOKEN_TEXT:
if inplural:
plural.append(t.contents)
else:
singular.append(t.contents)
else:
if t.token_type == TOKEN_BLOCK:
imatch = inline_re.match(t.contents)
bmatch = block_re.match(t.contents)
cmatches = constant_re.findall(t.contents)
if imatch:
g = imatch.group(1)
if g[0] == '"':
g = g.strip('"')
elif g[0] == "'":
g = g.strip("'")
message_context = imatch.group(3)
if message_context:
# strip quotes
message_context = message_context[1:-1]
yield (
lineno,
'pgettext',
[smart_text(message_context), smart_text(g)],
[],
)
message_context = None
else:
yield lineno, None, smart_text(g), []
elif bmatch:
if bmatch.group(2):
message_context = bmatch.group(2)[1:-1]
for fmatch in constant_re.findall(t.contents):
yield lineno, None, smart_text(fmatch), []
intrans = True
inplural = False
singular = []
plural = []
elif cmatches:
for cmatch in cmatches:
yield lineno, None, smart_text(cmatch), []
elif t.token_type == TOKEN_VAR:
parts = t.contents.split('|')
cmatch = constant_re.match(parts[0])
if cmatch:
yield lineno, None, smart_text(cmatch.group(1)), []
for p in parts[1:]:
if p.find(':_(') >= 0:
p1 = p.split(':', 1)[1]
if p1[0] == '_':
p1 = p1[1:]
if p1[0] == '(':
p1 = p1.strip('()')
if p1[0] == "'":
p1 = p1.strip("'")
elif p1[0] == '"':
p1 = p1.strip('"')
yield lineno, None, smart_text(p1), []
def extract_django(fileobj, keywords, comment_tags, options):
"""Extract messages from Django template files.
:param fileobj: the file-like object the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should
be recognized as translation functions
:param comment_tags: a list of translator tags to search for and
include in the results
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)``
tuples
:rtype: ``iterator``
"""
intrans = False
inplural = False
trimmed = False
message_context = None
singular = []
plural = []
lineno = 1
encoding = options.get('encoding', 'utf8')
text = fileobj.read().decode(encoding)
try:
text_lexer = Lexer(text)
except TypeError:
# Django 1.9 changed the way we invoke Lexer; older versions
# require two parameters.
text_lexer = Lexer(text, None)
for t in text_lexer.tokenize():
lineno += t.contents.count('\n')
if intrans:
if t.token_type == TOKEN_BLOCK:
endbmatch = endblock_re.match(t.contents)
pluralmatch = plural_re.match(t.contents)
if endbmatch:
if inplural:
if message_context:
yield (
lineno,
'npgettext',
[
smart_text(message_context),
smart_text(join_tokens(singular, trimmed)),
smart_text(join_tokens(plural, trimmed))
],
[],
)
else:
yield (lineno, 'ngettext',
(smart_text(join_tokens(singular, trimmed)),
smart_text(join_tokens(plural,
trimmed))), [])
else:
if message_context:
yield (
lineno,
'pgettext',
[
smart_text(message_context),
smart_text(join_tokens(singular, trimmed))
],
[],
)
else:
yield (lineno, None,
smart_text(join_tokens(singular,
trimmed)), [])
intrans = False
inplural = False
message_context = None
singular = []
plural = []
elif pluralmatch:
inplural = True
else:
raise SyntaxError('Translation blocks must not include '
'other block tags: %s' % t.contents)
elif t.token_type == TOKEN_VAR:
if inplural:
plural.append('%%(%s)s' % t.contents)
else:
singular.append('%%(%s)s' % t.contents)
elif t.token_type == TOKEN_TEXT:
if inplural:
plural.append(t.contents)
else:
singular.append(t.contents)
else:
if t.token_type == TOKEN_BLOCK:
imatch = inline_re.match(t.contents)
bmatch = block_re.match(t.contents)
cmatches = constant_re.findall(t.contents)
if imatch:
g = imatch.group(1)
g = strip_quotes(g)
message_context = imatch.group(3)
if message_context:
# strip quotes
message_context = message_context[1:-1]
yield (
lineno,
'pgettext',
[smart_text(message_context),
smart_text(g)],
[],
)
message_context = None
else:
yield lineno, None, smart_text(g), []
elif bmatch:
if bmatch.group(2):
message_context = bmatch.group(2)[1:-1]
for fmatch in constant_re.findall(t.contents):
stripped_fmatch = strip_quotes(fmatch)
yield lineno, None, smart_text(stripped_fmatch), []
intrans = True
inplural = False
trimmed = 'trimmed' in t.split_contents()
singular = []
plural = []
elif cmatches:
for cmatch in cmatches:
stripped_cmatch = strip_quotes(cmatch)
yield lineno, None, smart_text(stripped_cmatch), []
elif t.token_type == TOKEN_VAR:
parts = t.contents.split('|')
cmatch = constant_re.match(parts[0])
if cmatch:
stripped_cmatch = strip_quotes(cmatch.group(1))
yield lineno, None, smart_text(stripped_cmatch), []
for p in parts[1:]:
if p.find(':_(') >= 0:
p1 = p.split(':', 1)[1]
if p1[0] == '_':
p1 = p1[1:]
if p1[0] == '(':
p1 = p1.strip('()')
p1 = strip_quotes(p1)
yield lineno, None, smart_text(p1), []
