def compile_nodelist(self): """ Pass template name to parser instance """ if self.engine.debug: lexer = DebugLexer19(self.source) else: lexer = Lexer(self.source) tokens = lexer.tokenize() parser = Parser( tokens, self.engine.template_libraries, self.engine.template_builtins, ) parser.template_name = self.origin.template_name try: return parser.parse() except Exception as e: if self.engine.debug: e.template_debug = self.get_exception_info(e, e.token) raise
def get(key): cache_key = CACHE_PREFIX + 'image' + get_language() + key content = cache.get(cache_key) if content is None: obj, created = Chunk.objects.get_or_create( key=key, defaults={'content': key}) #a content can to contain other chunks as vars lexer = Lexer(obj.content,0) content = ''.join(map(Chunk.process_token,lexer.tokenize())) cache.set(cache_key, content) return content
def __str__(self): my_lexer = Lexer(self.title, UNKNOWN_SOURCE) my_tokens = my_lexer.tokenize() # Deliberately strip off template tokens that are not text or variable. for my_token in my_tokens: if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR): my_tokens.remove(my_token) my_parser = Parser(my_tokens) return my_parser.parse().render(SiteTree.get_global_context())
def _load_all_templates(directory): """ Loads all templates in a directory (recursively) and yields tuples of template tokens and template paths. """ if os.path.exists(directory): for name in os.listdir(directory): path = os.path.join(directory, name) if os.path.isdir(path): for template in _load_all_templates(path): yield template elif path.endswith('.html'): with open(path, 'rb') as fobj: source = fobj.read().decode(settings.FILE_CHARSET) lexer = Lexer(source, path) yield lexer.tokenize(), path
def render_custom_content(body, context_data={}): """Renders custom content for the payload using Django templating. This will take the custom payload content template provided by the user and render it using a stripped down version of Django's templating system. In order to keep the payload safe, we use a limited Context along with a custom Parser that blocks certain template tags. This gives us tags like {% for %} and {% if %}, but blacklists tags like {% load %} and {% include %}. """ lexer = Lexer(body, origin=None) parser = CustomPayloadParser(lexer.tokenize()) template = Template('') template.nodelist = parser.parse() return template.render(Context(context_data))
def _load_all_templates(directory): """ Loads all templates in a directory (recursively) and yields tuples of template tokens and template paths. """ if os.path.exists(directory): for name in os.listdir(directory): path = os.path.join(directory, name) if os.path.isdir(path): for template in _load_all_templates(path): yield template elif path.endswith('.html'): with open(path, 'rb') as fobj: source = fobj.read().decode(settings.FILE_CHARSET) if DJANGO_1_8: lexer = Lexer(source, path) else: lexer = Lexer(source) yield lexer.tokenize(), path
def render_custom_content(body, context_data={}): """Render custom content for the payload using Django templating. This will take the custom payload content template provided by the user and render it using a stripped down version of Django's templating system. In order to keep the payload safe, we use a limited Context along with a custom Parser that blocks certain template tags. This gives us tags like ``{% for %}`` and ``{% if %}``, but blacklists tags like ``{% load %}`` and ``{% include %}``. Args: body (unicode): The template content to render. context_data (dict, optional): Context data for the template. Returns: unicode: The rendered template. Raises: django.template.TemplateSyntaxError: There was a syntax error in the template. """ template = Template('') if django.VERSION >= (1, 9): lexer = Lexer(body) parser_args = (template.engine.template_libraries, template.engine.template_builtins, template.origin) else: lexer = Lexer(body, origin=None) parser_args = () parser = CustomPayloadParser(lexer.tokenize(), *parser_args) template.nodelist = parser.parse() return template.render(Context(context_data))
def extract_django(fileobj, keywords, comment_tags, options): """Extract messages from Django template files. :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ intrans = False inplural = False trimmed = False message_context = None singular = [] plural = [] lineno = 1 encoding = options.get('encoding', 'utf8') text = fileobj.read().decode(encoding) try: text_lexer = Lexer(text) except TypeError: # Django 1.9 changed the way we invoke Lexer; older versions # require two parameters. text_lexer = Lexer(text, None) for t in text_lexer.tokenize(): lineno += t.contents.count('\n') if intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: if message_context: yield ( lineno, 'npgettext', [ smart_text(message_context), smart_text(join_tokens(singular, trimmed)), smart_text(join_tokens(plural, trimmed)) ], [], ) else: yield (lineno, 'ngettext', (smart_text(join_tokens(singular, trimmed)), smart_text(join_tokens(plural, trimmed))), []) else: if message_context: yield ( lineno, 'pgettext', [ smart_text(message_context), smart_text(join_tokens(singular, trimmed)) ], [], ) else: yield (lineno, None, smart_text(join_tokens(singular, trimmed)), []) intrans = False inplural = False message_context = None singular = [] plural = [] elif pluralmatch: inplural = True else: raise SyntaxError('Translation blocks must not include ' 'other block tags: %s' % t.contents) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: if inplural: plural.append(t.contents) else: singular.append(t.contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) g = strip_quotes(g) message_context = imatch.group(3) if message_context: # strip quotes message_context = message_context[1:-1] yield ( lineno, 'pgettext', [smart_text(message_context), smart_text(g)], [], ) message_context = None else: yield lineno, None, smart_text(g), [] elif bmatch: if bmatch.group(2): message_context = bmatch.group(2)[1:-1] for fmatch in constant_re.findall(t.contents): stripped_fmatch = strip_quotes(fmatch) yield lineno, None, smart_text(stripped_fmatch), [] intrans = True inplural = False trimmed = 'trimmed' in t.split_contents() singular = [] plural = [] elif cmatches: for cmatch in cmatches: stripped_cmatch = strip_quotes(cmatch) yield lineno, None, smart_text(stripped_cmatch), [] elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: stripped_cmatch = strip_quotes(cmatch.group(1)) yield lineno, None, smart_text(stripped_cmatch), [] for p in parts[1:]: if p.find(':_(') >= 0: p1 = p.split(':', 1)[1] if p1[0] == '_': p1 = p1[1:] if p1[0] == '(': p1 = p1.strip('()') p1 = strip_quotes(p1) yield lineno, None, smart_text(p1), []
def extract_django(fileobj, keywords, comment_tags, options): """Extract messages from Django template files. Adapted from https://github.com/python-babel/django-babel/blob/master/django_babel/extract.py :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ intrans = False inplural = False trimmed = False message_context = None singular = [] plural = [] lineno = 1 encoding = options.get("encoding", "utf8") text = fileobj.read().decode(encoding) try: text_lexer = Lexer(text) except TypeError: # Django 1.9 changed the way we invoke Lexer; older versions # require two parameters. text_lexer = Lexer(text, None) # raise SystemError([t.contents for t in text_lexer.tokenize()]) for t in text_lexer.tokenize(): lineno += t.contents.count("\n") if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) if imatch: g = imatch.group(1) g = strip_quotes(g) default_message = imatch.group(3) if default_message: comments = [ COMMENT_TAG_FOR_DEFAULT_MESSAGE + ": " + strip_quotes(default_message) ] else: comments = [] comment = imatch.group(7) if comment: comments.append(strip_quotes(comment)) message_context = imatch.group(5) if message_context: # strip quotes message_context = message_context[1:-1] yield ( lineno, "pgettext", [smart_text(message_context), smart_text(g)], comments, ) message_context = None else: yield lineno, None, smart_text(g), comments
<p>{% trans 'None available' %}</p> {% else %} <ul class="actionlist"> {% for entry in admin_log %} <li class="{% if entry.is_addition %}addlink{% endif %}{% if entry.is_change %}changelink{% endif %}{% if entry.is_deletion %}deletelink{% endif %}"> {% if entry.is_deletion or not entry.get_admin_url %} {{ entry.object_repr }} {% else %} <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a> {% endif %} <br/> {% if entry.content_type %} <span class="mini quiet">{% filter capfirst %}{% trans entry.content_type.name %}{% endfilter %}</span> {% else %} <span class="mini quiet">{% trans 'Unknown content' %}</span> {% endif %} </li> {% endfor %} </ul> {% endif %} </div> </div> {% endblock %} """ elapsed = 0 for i in xrange(5000): # print i lexer = Lexer(template_source, None) lexer.tokenize()
<p>{% trans 'None available' %}</p> {% else %} <ul class="actionlist"> {% for entry in admin_log %} <li class="{% if entry.is_addition %}addlink{% endif %}{% if entry.is_change %}changelink{% endif %}{% if entry.is_deletion %}deletelink{% endif %}"> {% if entry.is_deletion or not entry.get_admin_url %} {{ entry.object_repr }} {% else %} <a href="{{ entry.get_admin_url }}">{{ entry.object_repr }}</a> {% endif %} <br/> {% if entry.content_type %} <span class="mini quiet">{% filter capfirst %}{% trans entry.content_type.name %}{% endfilter %}</span> {% else %} <span class="mini quiet">{% trans 'Unknown content' %}</span> {% endif %} </li> {% endfor %} </ul> {% endif %} </div> </div> {% endblock %} """ elapsed = 0 for i in xrange(10000): # print i lexer = Lexer(template_source, None) lexer.tokenize()
def lines(self): source_lines = set() if SHOW_PARSING: print("-------------- {}".format(self.filename)) if django.VERSION >= (1, 9): lexer = Lexer(self.source()) else: lexer = Lexer(self.source(), self.filename) tokens = lexer.tokenize() # Are we inside a comment? comment = False # Is this a template that extends another template? extends = False # Are we inside a block? inblock = False for token in tokens: if SHOW_PARSING: print( "%10s %2d: %r" % ( TOKEN_MAPPING[token.token_type], token.lineno, token.contents, ) ) if token.token_type == TOKEN_BLOCK: if token.contents == "endcomment": comment = False continue if comment: continue if token.token_type == TOKEN_BLOCK: if token.contents.startswith("endblock"): inblock = False elif token.contents.startswith("block"): inblock = True if extends: continue if extends and not inblock: # In an inheriting tempalte, ignore all tags outside of # blocks. continue if token.contents == "comment": comment = True if token.contents.startswith("end"): continue elif token.contents in ("else", "empty"): continue elif token.contents.startswith("elif"): # NOTE: I don't like this, I want to be able to trace elif # nodes, but the Django template engine doesn't track them # in a way that we can get useful information from them. continue elif token.contents.startswith("extends"): extends = True source_lines.add(token.lineno) elif token.token_type == TOKEN_VAR: source_lines.add(token.lineno) elif token.token_type == TOKEN_TEXT: if extends and not inblock: continue # Text nodes often start with newlines, but we don't want to # consider that first line to be part of the text. lineno = token.lineno lines = token.contents.splitlines(True) num_lines = len(lines) if lines[0].isspace(): lineno += 1 num_lines -= 1 source_lines.update(range(lineno, lineno+num_lines)) if SHOW_PARSING: print("\t\t\tNow source_lines is: {!r}".format(source_lines)) return source_lines
def extract_django(fileobj, keywords, comment_tags, options): """Extract messages from Django template files. :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ intrans = False inplural = False trimmed = False message_context = None singular = [] plural = [] lineno = 1 encoding = options.get('encoding', 'utf8') text = fileobj.read().decode(encoding) try: text_lexer = Lexer(text) except TypeError: # Django 1.9 changed the way we invoke Lexer; older versions # require two parameters. text_lexer = Lexer(text, None) for t in text_lexer.tokenize(): lineno += t.contents.count('\n') if intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: if message_context: yield ( lineno, 'npgettext', [smart_text(message_context), smart_text(join_tokens(singular, trimmed)), smart_text(join_tokens(plural, trimmed))], [], ) else: yield ( lineno, 'ngettext', (smart_text(join_tokens(singular, trimmed)), smart_text(join_tokens(plural, trimmed))), []) else: if message_context: yield ( lineno, 'pgettext', [smart_text(message_context), smart_text(join_tokens(singular, trimmed))], [], ) else: yield ( lineno, None, smart_text(join_tokens(singular, trimmed)), []) intrans = False inplural = False message_context = None singular = [] plural = [] elif pluralmatch: inplural = True else: raise SyntaxError('Translation blocks must not include ' 'other block tags: %s' % t.contents) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: if inplural: plural.append(t.contents) else: singular.append(t.contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) g = strip_quotes(g) message_context = imatch.group(3) if message_context: # strip quotes message_context = message_context[1:-1] yield ( lineno, 'pgettext', [smart_text(message_context), smart_text(g)], [], ) message_context = None else: yield lineno, None, smart_text(g), [] elif bmatch: if bmatch.group(2): message_context = bmatch.group(2)[1:-1] for fmatch in constant_re.findall(t.contents): stripped_fmatch = strip_quotes(fmatch) yield lineno, None, smart_text(stripped_fmatch), [] intrans = True inplural = False trimmed = 'trimmed' in t.split_contents() singular = [] plural = [] elif cmatches: for cmatch in cmatches: stripped_cmatch = strip_quotes(cmatch) yield lineno, None, smart_text(stripped_cmatch), [] elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: stripped_cmatch = strip_quotes(cmatch.group(1)) yield lineno, None, smart_text(stripped_cmatch), [] for p in parts[1:]: if p.find(':_(') >= 0: p1 = p.split(':', 1)[1] if p1[0] == '_': p1 = p1[1:] if p1[0] == '(': p1 = p1.strip('()') p1 = strip_quotes(p1) yield lineno, None, smart_text(p1), []
{% else %} <span class="mini quiet">{% trans 'Unknown content' %}</span> {% endif %} </li> {% endfor %} </ul> {% endif %} </div> </div> {% endblock %} """ settings.configure() apps.populate(( 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', )) elapsed = 0 lexer = Lexer(template_source, None) tokens = lexer.tokenize() for i in xrange(500): parser = Parser(list(tokens)) parser.parse()
def lines(self): source_lines = set() if SHOW_PARSING: print("-------------- {}".format(self.filename)) if django.VERSION >= (1, 9): lexer = Lexer(self.source()) else: lexer = Lexer(self.source(), self.filename) tokens = lexer.tokenize() # Are we inside a comment? comment = False # Is this a template that extends another template? extends = False # Are we inside a block? inblock = False for token in tokens: if SHOW_PARSING: print("%10s %2d: %r" % ( _token_name(token.token_type), token.lineno, token.contents, )) if token.token_type == TokenType.BLOCK: if token.contents == "endcomment": comment = False continue if comment: continue if token.token_type == TokenType.BLOCK: if token.contents.startswith("endblock"): inblock = False elif token.contents.startswith("block"): inblock = True if extends: continue if extends and not inblock: # In an inheriting tempalte, ignore all tags outside of # blocks. continue if token.contents == "comment": comment = True if token.contents.startswith("end"): continue elif token.contents in ("else", "empty"): continue elif token.contents.startswith("elif"): # NOTE: I don't like this, I want to be able to trace elif # nodes, but the Django template engine doesn't track them # in a way that we can get useful information from them. continue elif token.contents.startswith("extends"): extends = True source_lines.add(token.lineno) elif token.token_type == TokenType.VAR: source_lines.add(token.lineno) elif token.token_type == TokenType.TEXT: if extends and not inblock: continue # Text nodes often start with newlines, but we don't want to # consider that first line to be part of the text. lineno = token.lineno lines = token.contents.splitlines(True) num_lines = len(lines) if lines[0].isspace(): lineno += 1 num_lines -= 1 source_lines.update(range(lineno, lineno + num_lines)) if SHOW_PARSING: print("\t\t\tNow source_lines is: {!r}".format(source_lines)) return source_lines