def _get_parser(): import django parser = Parser([]) if django.VERSION >= (1, 9): import django.template.defaultfilters parser.add_library(django.template.defaultfilters.register) return parser
def compile_nodelist(self): """ Pass template name to parser instance """ if self.engine.debug: lexer = DebugLexer19(self.source) else: lexer = Lexer(self.source) tokens = lexer.tokenize() parser = Parser( tokens, self.engine.template_libraries, self.engine.template_builtins, ) parser.template_name = self.origin.template_name try: return parser.parse() except Exception as e: if self.engine.debug: e.template_debug = self.get_exception_info(e, e.token) raise
def linebreakless(parser: Parser, token: Token): library = template.Library() library.tag(LinebreaklessNode.KEEP_LINEBREAK_TAG, LinebreaklessNode.keeplinebreak) parser.add_library(library) nodelist = parser.parse(("endlinebreakless", )) parser.delete_first_token() return LinebreaklessNode(token, nodelist)
def build_migration(self, src, filename=None, charset='utf-8'): """Create a migration for a Django template file to Transifex Native syntax. The returned object contains every change separately, so that it can be reviewed string by string. :param unicode src: the whole Django template :param str filename: the filename of the original template :param str charset: the character set to use :return: a FileMigration instance :rtype: FileMigration """ self._reset() src = force_text(src, charset) # Using the DebugLexer because we need the positional information # of each token (start/end pos). It is slower than Lexer, but Lexer # doesn't provide that information tokens = DebugLexer(src).tokenize() parser = Parser(tokens, libraries={}, builtins=[], origin=filename) # Since no template libraries are loaded when this code is running, # we need to override the find function in order to use the # functionality of the Parser class. The overridden function returns # the object as given. # Without the override, a KeyError would be raised inside the parser. parser.find_filter = find_filter_identity # Create a migration object for this template; we'll add stuff to it # as we go migration = FileMigration(filename, src) while parser.tokens: token = parser.next_token() start, end = token.position # Parse the current token. This may or may not return a migration. # Also it may return a list of tokens that were consumed, # additionally to the current token. If this happens, # `_parse_token()` will have made sure that `parser` has moved # forward, consuming those tokens, so that they don't appear again # in the while loop. string_migration, extra_consumed_tokens = self._parse_token( token, parser, original_string=src[start:end]) if not string_migration: continue # If additional tokens were consumed, we need to add # them in the migration, so that the StringMigration object # includes the information of what part of the original template # was migrated to the new syntax, for this particular translatable # string if extra_consumed_tokens: for extra_token in extra_consumed_tokens: start, end = extra_token.position string_migration.update(src[start:end], '') migration.add_string(string_migration) return migration
def test_can_load_from_missing_taglib(self): template = ('{% load friendly_loader %}' '{% friendly_load error from error_tags %}') lexer = Lexer(template) parser = Parser(lexer.tokenize(), engine.template_libraries, engine.template_builtins) parser.parse() self.assertTrue(isinstance(Template(template), Template), 'Expected template to initialize')
def test_can_load_taglib_using_friendly_load(self): template = '{% load friendly_loader %}{% friendly_load flatpages %}' lexer = Lexer(template) parser = Parser(lexer.tokenize(), engine.template_libraries, engine.template_builtins) parser.parse() self.assertTrue( 'get_flatpages' in parser.tags, 'Expected flatpages taglib to load and provide the get_flatpages tag' )
def __str__(self): my_lexer = get_lexer(self.title) my_tokens = my_lexer.tokenize() # Deliberately strip off template tokens that are not text or variable. for my_token in my_tokens: if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR): my_tokens.remove(my_token) my_parser = Parser(my_tokens) return my_parser.parse().render(get_sitetree().current_page_context)
def __str__(self): my_lexer = Lexer(self.title, UNKNOWN_SOURCE) my_tokens = my_lexer.tokenize() # Deliberately strip off template tokens that are not text or variable. for my_token in my_tokens: if my_token.token_type not in (TOKEN_TEXT, TOKEN_VAR): my_tokens.remove(my_token) my_parser = Parser(my_tokens) return my_parser.parse().render(SiteTree.get_global_context())
def test_raises_template_syntaxerror(self): self.assertRaises(TemplateSyntaxError, self.render, "{% host_url %}") self.assertRaises(TemplateSyntaxError, self.render, "{% host_url 'simple-direct' host %}") self.assertRaises(TemplateSyntaxError, self.render, "{% host_url 'simple-direct' as %}") self.assertRaises(TemplateSyntaxError, self.render, "{% host_url simple-direct %}") self.assertRaises(TemplateSyntaxError, parse_params, 'host_url', Parser(['']), "username=='johndoe'") self.assertRaises(TemplateSyntaxError, parse_params, 'host_url', Parser(['']), "\n='johndoe'")
def capture(parser: Parser, token: Token): bits = token.contents.split() if len(bits) > 2: raise TemplateSyntaxError( f"'capture' tag got unexpected arguments {bits[2:]!r}") name: str = bits[1] if "'" in name or '"' in name: raise TypeError( f"'capture' tag got a quoted name {name}. use a name without quotes" ) nodelist = parser.parse(parse_until=['endcapture']) parser.delete_first_token() return CaptureNode(name=name, nodelist=nodelist)
def parse(self, parser, stream, container, nextargs=None): stream_copy = copy(stream) if hasattr(parser, 'tokens'): parser_copy = DjangoParser(list(parser.tokens)) parser_copy.tags = copy(parser.tags) parser_copy.filters = copy(parser.filters) else: parser_copy = FakeParser(parser) try: self._do_parse(parser_copy, stream_copy, Container(), nextargs) except BaseError, e: pass
def second_pass_render(request, content): """ Split on the secret delimiter and generate the token list by passing through text outside of phased blocks as single text tokens and tokenizing text inside the phased blocks. This ensures that nothing outside of the phased blocks is tokenized, thus eliminating the possibility of a template code injection vulnerability. """ result = tokens = [] for index, bit in enumerate(content.split( settings.PHASED_SECRET_DELIMITER)): if index % 2: tokens = Lexer(bit, None).tokenize() else: tokens.append(Token(TOKEN_TEXT, bit)) context = RequestContext( request, restore_csrf_token(request, unpickle_context(bit))) rendered = Parser(tokens).parse().render(context) if settings.PHASED_SECRET_DELIMITER in rendered: rendered = second_pass_render(request, rendered) result.append(rendered) return "".join(result)
def test_filter_parsing(self): c = {"article": {"section": "News"}} p = Parser("", builtins=[filter_library]) def fe_test(s, val): self.assertEqual(FilterExpression(s, p).resolve(c), val) fe_test("article.section", "News") fe_test("article.section|upper", "NEWS") fe_test('"News"', "News") fe_test("'News'", "News") fe_test(r'"Some \"Good\" News"', 'Some "Good" News') fe_test(r'"Some \"Good\" News"', 'Some "Good" News') fe_test(r"'Some \'Bad\' News'", "Some 'Bad' News") fe = FilterExpression(r'"Some \"Good\" News"', p) self.assertEqual(fe.filters, []) self.assertEqual(fe.var, 'Some "Good" News') # Filtered variables should reject access of attributes beginning with # underscores. msg = ( "Variables and attributes may not begin with underscores: 'article._hidden'" ) with self.assertRaisesMessage(TemplateSyntaxError, msg): FilterExpression("article._hidden|upper", p)
def test_unicorn_args_and_kwargs(): # args after the component name get ignored token = Token(TokenType.TEXT, "unicorn 'todo' '1' 2 hello='world' test=3 '4'") unicorn_node = unicorn(Parser([]), token) assert unicorn_node.kwargs == {"hello": "world", "test": 3}
def next_token(): token = Parser.next_token(parser) if token.token_type == TOKEN_VAR: token.token_type = TOKEN_TEXT token.contents = (appsettings.ANGULAR_TOKENS['VARIABLE_OPEN'] + token.contents + appsettings.ANGULAR_TOKENS['VARIABLE_CLOSE']) return token
def try_func(parser: Parser, token: Token): nodelist_try: NodeList nodelist_except: Optional[NodeList] = None catch_exceptions: Optional[List] = None # Try nodelist_try: Token = parser.parse(("except", "endtry")) next_token = parser.next_token() next_token_name = next_token.contents.lstrip().split(" ")[0].lower() # Catch if next_token_name == "except": # Catch contents = next_token.contents.split(" ")[1:] if len(contents) > 0: catch_exceptions = contents else: catch_exceptions = [] nodelist_except = parser.parse(("endtry", )) else: parser.parse(("endtry",)) parser.next_token() return TryNode( nodelist_try=nodelist_try, nodelist_except=nodelist_except, catch_exceptions=catch_exceptions, )
def test_token_kwargs(self): parser = Parser('') self.assertEqual(token_kwargs([], parser), {}) bits = ['a="A"'] token_kwargs(bits, parser) self.assertEqual(bits, []) bits = ['a="A"', 'invalid'] token_kwargs(bits, parser) self.assertEqual(bits, ['invalid'])
def clean(self): template = DjangoTemplate('') template.source = self.changed_content try: engine = Engine.get_default() lexer = DebugLexer(self.changed_content) tokens = lexer.tokenize() parser = Parser(tokens, engine.template_libraries, engine.template_builtins, Origin(UNKNOWN_SOURCE)) parser.parse() except TemplateSyntaxError as e: exception_info = template.get_exception_info(e, e.token) raise ValidationError({ 'changed_content': mark_safe( '{message}<br/>Line {line}:\n<pre>{before}<b>{during}</b>{after}</pre>' .format(**exception_info)) }) from e
def url(self, sitetree_item, context=None): """Resolves item's URL. :param TreeItemBase sitetree_item: TreeItemBase heir object, 'url' property of which is processed as URL pattern or simple URL. :param Context context: :rtype: str|unicode """ context = context or self.current_page_context resolve_var = self.resolve_var if not isinstance(sitetree_item, MODEL_TREE_ITEM_CLASS): sitetree_item = resolve_var(sitetree_item, context) resolved_url = self._items_urls.get(sitetree_item) if resolved_url is not None: return resolved_url # Resolve only if item's URL is marked as pattern. if sitetree_item.urlaspattern: url = sitetree_item.url view_path = url all_arguments = [] if ' ' in url: view_path = url.split(' ') # We should try to resolve URL parameters from site tree item. for view_argument in view_path[1:]: resolved = resolve_var(view_argument) # We enclose arg in double quotes as already resolved. all_arguments.append('"%s"' % resolved) view_path = view_path[0].strip('"\' ') url_pattern = "'%s' %s" % (view_path, ' '.join(all_arguments)) else: url_pattern = '%s' % sitetree_item.url if sitetree_item.urlaspattern: # Form token to pass to Django 'url' tag. url_token = 'url %s as item.url_resolved' % url_pattern url_tag(Parser(None), Token(token_type=TOKEN_BLOCK, contents=url_token)).render(context) resolved_url = context[ 'item.url_resolved'] or UNRESOLVED_ITEM_MARKER else: resolved_url = url_pattern self._items_urls[sitetree_item] = resolved_url return resolved_url
def parse(self, parser, stream, container, nextargs=None): stream_copy = copy(stream) if hasattr(parser, 'tokens'): parser_copy = DjangoParser(list(parser.tokens)) parser_copy.tags = copy(parser.tags) parser_copy.filters = copy(parser.filters) else: parser_copy = FakeParser(parser) reps = ListValue() add_reps = True while (add_reps): try: self._do_parse(parser_copy, stream_copy, Container(), nextargs) except BaseError, e: add_reps = False except Exception, e: raise
def test_token_kwargs(self): parser = Parser("") self.assertEqual(token_kwargs([], parser), {}) bits = ['a="A"'] token_kwargs(bits, parser) self.assertEqual(bits, []) bits = ['a="A"', "invalid"] token_kwargs(bits, parser) self.assertEqual(bits, ["invalid"])
def test_filter_args_count(self): p = Parser("") l = Library() @l.filter def no_arguments(value): pass @l.filter def one_argument(value, arg): pass @l.filter def one_opt_argument(value, arg=False): pass @l.filter def two_arguments(value, arg, arg2): pass @l.filter def two_one_opt_arg(value, arg, arg2=False): pass p.add_library(l) for expr in ( '1|no_arguments:"1"', '1|two_arguments', '1|two_arguments:"1"', '1|two_one_opt_arg', ): with self.assertRaises(TemplateSyntaxError): FilterExpression(expr, p) for expr in ( # Correct number of arguments '1|no_arguments', '1|one_argument:"1"', # One optional '1|one_opt_argument', '1|one_opt_argument:"1"', # Not supplying all '1|two_one_opt_arg:"1"', ): FilterExpression(expr, p)
def url(self, sitetree_item: Union['TreeItemBase', FilterExpression], context: Context = None) -> str: """Resolves item's URL. :param sitetree_item: TreeItemBase heir object, 'url' property of which is processed as URL pattern or simple URL. :param context: """ context = context or self.current_page_context resolve_var = self.resolve_var if not isinstance(sitetree_item, MODEL_TREE_ITEM_CLASS): sitetree_item = resolve_var(sitetree_item, context) resolved_url = self._items_urls.get(sitetree_item) if resolved_url is not None: return resolved_url # Resolve only if item's URL is marked as pattern. if sitetree_item.urlaspattern: url = sitetree_item.url view_path = url all_arguments = [] if ' ' in url: view_path = url.split(' ') # We should try to resolve URL parameters from site tree item. for view_argument in view_path[1:]: resolved = resolve_var(view_argument) # We enclose arg in double quotes as already resolved. all_arguments.append(f'"{resolved}"') view_path = view_path[0].strip('"\' ') url_pattern = f"'{view_path}' {' '.join(all_arguments)}" else: url_pattern = f'{sitetree_item.url}' if sitetree_item.urlaspattern: # Form token to pass to Django 'url' tag. url_token = f'url {url_pattern} as item.url_resolved' url_tag( Parser([]), Token(token_type=TOKEN_BLOCK, contents=url_token) ).render(context) resolved_url = context['item.url_resolved'] or UNRESOLVED_ITEM_MARKER else: resolved_url = url_pattern self._items_urls[sitetree_item] = resolved_url return resolved_url
def loremi(count, method, random=False): from django.template.base import Context, Token, Parser, TOKEN_TEXT from django.template.defaulttags import lorem c = Context() lorem_str = "lorem %s %s" % (count, method) if random: lorem_str += " random" t = Token(TOKEN_TEXT, lorem_str) p = Parser(t) return lorem(p, t).render(c)
def test_unicorn_render_with_invalid_html(): token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargsWithHtmlEntity' test_kwarg=test_var.nested", ) unicorn_node = unicorn(Parser([]), token) context = {"test_var": {"nested": "variable!"}} actual = unicorn_node.render(Context(context)) assert "->variable!<-" in actual
def test_filter_args_count(self): parser = Parser("") register = Library() @register.filter def no_arguments(value): pass @register.filter def one_argument(value, arg): pass @register.filter def one_opt_argument(value, arg=False): pass @register.filter def two_arguments(value, arg, arg2): pass @register.filter def two_one_opt_arg(value, arg, arg2=False): pass parser.add_library(register) for expr in ( '1|no_arguments:"1"', '1|two_arguments', '1|two_arguments:"1"', '1|two_one_opt_arg', ): with self.assertRaises(TemplateSyntaxError): FilterExpression(expr, parser) for expr in ( # Correct number of arguments '1|no_arguments', '1|one_argument:"1"', # One optional '1|one_opt_argument', '1|one_opt_argument:"1"', # Not supplying all '1|two_one_opt_arg:"1"', ): FilterExpression(expr, parser)
def test_unicorn_render_kwarg(): token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' test_kwarg='tested!'", ) unicorn_node = unicorn(Parser([]), token) context = {} actual = unicorn_node.render(Context(context)) assert "<b>tested!</b>" in actual
def test_unicorn_render_id_use_pk(): token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentModel' model_id=model.id", ) unicorn_node = unicorn(Parser([]), token) context = {"model": {"pk": 123}} actual = unicorn_node.render(Context(context)) assert "==123==" in actual
def test_unicorn_render_context_variable(): token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' test_kwarg=test_var.nested", ) unicorn_node = unicorn(Parser([]), token) context = {"test_var": {"nested": "variable!"}} actual = unicorn_node.render(Context(context)) assert "<b>variable!</b>" in actual
def test_unicorn_render_child_component_no_script_tag(settings): settings.DEBUG = True token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentKwargs' parent=view", ) unicorn_node = unicorn(Parser([]), token) view = FakeComponentParent(component_name="test", component_id="asdf") context = {"view": view} html = unicorn_node.render(Context(context)) assert "<script" not in html
def test_unicorn_render_parent_component_one_script_tag(settings): settings.DEBUG = True token = Token( TokenType.TEXT, "unicorn 'tests.templatetags.test_unicorn_render.FakeComponentParent'", ) unicorn_node = unicorn(Parser([]), token) context = {} html = unicorn_node.render(Context(context)) assert '<script type="module"' in html assert len(re.findall('<script type="module"', html)) == 1
def extract_transifex_template_strings( src, origin=None, charset='utf-8', fkeygen=generate_key): """Parse the given template and extract translatable content based on the syntax supported by Transifex Native. Supports the {% t %} and {% ut %} template tags. :param unicode src: the whole Django template :param str origin: an optional context for the filename of the source, e.g. the file name :param str charset: the character set to use :param func fkeygen: key generator function :return: a list of SourceString objects :rtype: list """ src = force_text(src, charset) tokens = Lexer(src).tokenize() parser = Parser(tokens, {}, [], origin) # Since no template libraries are loaded when this code is running, # we need to override the find function in order to use the functionality # of the Parser class. The overridden function returns the object as given. # Without the override, a KeyError is raised inside the parser. parser.find_filter = find_filter_identity strings = [] while parser.tokens: token = parser.next_token() if (token.token_type == TOKEN_BLOCK and token.split_contents()[0] in ('t', 'ut')): tnode = do_t(parser, token) source_string = tnode_to_source_string(tnode, fkeygen) if source_string is None: continue if token.lineno and origin: source_string.occurrences = [ "{}:{}".format(origin, token.lineno)] strings.append(source_string) return strings
{% else %} <span class="mini quiet">{% trans 'Unknown content' %}</span> {% endif %} </li> {% endfor %} </ul> {% endif %} </div> </div> {% endblock %} """ settings.configure() apps.populate(( 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', )) elapsed = 0 lexer = Lexer(template_source, None) tokens = lexer.tokenize() for i in xrange(500): parser = Parser(list(tokens)) parser.parse()