def second_pass_render(request, content): """ Split on the secret delimiter and generate the token list by passing through text outside of phased blocks as single text tokens and tokenizing text inside the phased blocks. This ensures that nothing outside of the phased blocks is tokenized, thus eliminating the possibility of a template code injection vulnerability. """ result = tokens = [] for index, bit in enumerate(content.split( settings.PHASED_SECRET_DELIMITER)): if index % 2: tokens = Lexer(bit, None).tokenize() context = RequestContext( request, restore_csrf_token(request, unpickle_context(bit))) else: tokens = [Token(TOKEN_TEXT, bit)] context = None rendered = Parser(tokens).parse().render(context) if settings.PHASED_SECRET_DELIMITER in rendered: rendered = second_pass_render(request, rendered) result.append(rendered) return "".join(result)
def parse_source(self): source_lines = set() lexer = Lexer(self.text, "<string>") tokens = lexer.tokenize() comment = False for token in tokens: assert isinstance(token, Token) if token.token_type == TOKEN_BLOCK: if token.contents == 'comment': comment = True continue elif token.contents == 'endcomment': comment = False continue if comment: continue if token.token_type == TOKEN_BLOCK or token.token_type == TOKEN_VAR: if token.token_type == TOKEN_BLOCK and token.contents.startswith( 'end'): continue source_lines.add(token.lineno) return tuple(sorted(source_lines)), ()
def _render_html(self, template_string, context={}): # :( if DJANGO_VERSION > (1, 9): from django.template.library import import_library tag_lib = import_library('testapp.tags') else: # DJANGO_VERSION > (1,7): from django.template.base import import_library tag_lib = import_library('testapp.tags') if DJANGO_VERSION > (1, 9): lexer = Lexer(template_string) else: lexer = Lexer(template_string, self._make_origin()) parser = Parser(lexer.tokenize()) parser.add_library(tag_lib) nodelist = parser.parse() return nodelist.render(Context(context))
def validate_template(self, template_string): # We want to tokenize like normal, then use a custom parser. lexer = Lexer(template_string, None) tokens = lexer.tokenize() parser = TemplateValidationParser(tokens, self.allow, self.disallow, self.secure) for node in parser.parse(): template = getattr(node, LOADED_TEMPLATE_ATTR, None)
def tokenize(): """ Returns a stream of Django Token() entities """ for template in get_templates(): with open(template) as fp: template_content = fp.read() lexer = Lexer(template_content, None) for token in lexer.tokenize(): yield token
def _get_completion_ppp(self, text): """ Return tuple containing the prefix, pivot, and partial of the current line of input. >>> completer._get_completion_ppp('{{') ('{', '{', '') >>> completer._get_completion_ppp('{{ var }}{% get_') ('{{ var }}{', '%', ' get_') How it works: 1. Tokenize text, add first n-1 tokens to "prefix". 2. Split on final "|%{:". Call it "pivot". 3. Any text after pivot is called the "partial". 4. Text prior to the pivot but after the first n-1 tokens is appended to the prefix. """ if len(text) == 0: return ('', '', '') prefix = '' partial = '' pivot = '' tokens = Lexer(text, None).tokenize() if tokens[-1].token_type != TOKEN_TEXT: return (text, '', '') prefix_tokens = tokens[:-1] working_area = tokens[-1].contents prefix = text[:-len(working_area)] # Iterate backwards through string, finding the first # occurrence of any of the chars "|%{:". Call it the pivot. for index, char in list(enumerate(working_area))[::-1]: if char == ' ': if ' ' in working_area[:index]: pivot = char break if char in '|%{:': pivot = char break # No pivot was found if len(pivot) == 0: return (text, '', '') pieces = working_area.split(pivot) prefix += pivot.join(pieces[:-1]) partial = pieces[-1] return (prefix, pivot, partial)
def _render_html(self, template_string, context={}): # :( if DJANGO_VERSION > (1,2): from django.template import import_library tag_lib = import_library('beproud.django.commons.tests.test_tags') else: from django.template import get_library tag_lib = get_library('beproud.django.commons.tests.test_tags') lexer = Lexer(template_string, self._make_origin()) parser = Parser(lexer.tokenize()) parser.add_library(tag_lib) nodelist = parser.parse() return nodelist.render(Context(context))
def _load_all_templates(directory): """ Loads all templates in a directory (recursively) and yields tuples of template tokens and template paths. """ if os.path.exists(directory): for name in os.listdir(directory): path = os.path.join(directory, name) if os.path.isdir(path): for template in _load_all_templates(path): yield template elif path.endswith('.html'): with open(path, 'rb') as fobj: source = fobj.read().decode(settings.FILE_CHARSET) lexer = Lexer(source, path) yield lexer.tokenize(), path
def render_custom_content(body, context_data={}): """Renders custom content for the payload using Django templating. This will take the custom payload content template provided by the user and render it using a stripped down version of Django's templating system. In order to keep the payload safe, we use a limited Context along with a custom Parser that blocks certain template tags. This gives us tags like {% for %} and {% if %}, but blacklists tags like {% load %} and {% include %}. """ lexer = Lexer(body, origin=None) parser = CustomPayloadParser(lexer.tokenize()) nodes = parser.parse() return nodes.render(Context(context_data))
def runsource(self, source, filename="<input>", symbol="single"): """ readline calls this method with the current source buffer. This method can return True to instruct readline to capture another line of input using the "..." prompt or return False to tell readline to clear the source buffer and capture a new phrase. How it works: 1. Tokenize input. 2. Load parser with tokens. 3. Attempt to parse, loading a list with nodes. 4. If unclosed tag exception is raised, get more user input. 5. If everything went smoothly, print output, otherwise print exception. """ if source == 'exit': raise ExitREPL() if not source: return False tokens = Lexer(source, None).tokenize() self.parser.tokens = tokens nodes = [] try: try: for node in self.parser.parse(): nodes.append(node) except TemplateSyntaxError as e: if e.args[0].startswith('Unclosed tags'): # inside block, so ask for more input return True else: raise for node in nodes: self.output.write('%s' % (node.render(self.context), )) self.output.write('\n') return False except: self.showtraceback() return False
def templatize(src, origin=None): """ Turns a Django template into something that is understood by xgettext. It does so by translating the Django translation tags into standard gettext function invocations. """ from django.conf import settings from django.template import (Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK) src = force_text(src, settings.FILE_CHARSET) out = StringIO() message_context = None intrans = False inplural = False singular = [] plural = [] incomment = False comment = [] lineno_comment_map = {} comment_lineno_cache = None for t in Lexer(src, origin).tokenize(): if incomment: if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment': content = ''.join(comment) translators_comment_start = None for lineno, line in enumerate(content.splitlines(True)): if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK): translators_comment_start = lineno for lineno, line in enumerate(content.splitlines(True)): if translators_comment_start is not None and lineno >= translators_comment_start: out.write(' # %s' % line) else: out.write(' #\n') incomment = False comment = [] else: comment.append(t.contents) elif intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: if message_context: out.write(' npgettext(%r, %r, %r,count) ' % (message_context, ''.join(singular), ''.join(plural))) else: out.write(' ngettext(%r, %r, count) ' % (''.join(singular), ''.join(plural))) for part in singular: out.write(blankout(part, 'S')) for part in plural: out.write(blankout(part, 'P')) else: if message_context: out.write(' pgettext(%r, %r) ' % (message_context, ''.join(singular))) else: out.write(' gettext(%r) ' % ''.join(singular)) for part in singular: out.write(blankout(part, 'S')) message_context = None intrans = False inplural = False singular = [] plural = [] elif pluralmatch: inplural = True else: filemsg = '' if origin: filemsg = 'file %s, ' % origin raise SyntaxError( "Translation blocks must not include other block tags: %s (%sline %d)" % (t.contents, filemsg, t.lineno)) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: contents = one_percent_re.sub('%%', t.contents) if inplural: plural.append(contents) else: singular.append(contents) else: # Handle comment tokens (`{# ... #}`) plus other constructs on # the same line: if comment_lineno_cache is not None: cur_lineno = t.lineno + t.contents.count('\n') if comment_lineno_cache == cur_lineno: if t.token_type != TOKEN_COMMENT: for c in lineno_comment_map[comment_lineno_cache]: filemsg = '' if origin: filemsg = 'file %s, ' % origin warn_msg = ( "The translator-targeted comment '%s' " "(%sline %d) was ignored, because it wasn't the last item " "on the line.") % (c, filemsg, comment_lineno_cache) warnings.warn(warn_msg, TranslatorCommentWarning) lineno_comment_map[comment_lineno_cache] = [] else: out.write( '# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache])) comment_lineno_cache = None if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) if g[0] == '"': g = g.strip('"') elif g[0] == "'": g = g.strip("'") g = one_percent_re.sub('%%', g) if imatch.group(2): # A context is provided context_match = context_re.match(imatch.group(2)) message_context = context_match.group(1) if message_context[0] == '"': message_context = message_context.strip('"') elif message_context[0] == "'": message_context = message_context.strip("'") out.write(' pgettext(%r, %r) ' % (message_context, g)) message_context = None else: out.write(' gettext(%r) ' % g) elif bmatch: for fmatch in constant_re.findall(t.contents): out.write(' _(%s) ' % fmatch) if bmatch.group(1): # A context is provided context_match = context_re.match(bmatch.group(1)) message_context = context_match.group(1) if message_context[0] == '"': message_context = message_context.strip('"') elif message_context[0] == "'": message_context = message_context.strip("'") intrans = True inplural = False singular = [] plural = [] elif cmatches: for cmatch in cmatches: out.write(' _(%s) ' % cmatch) elif t.contents == 'comment': incomment = True else: out.write(blankout(t.contents, 'B')) elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: out.write(' _(%s) ' % cmatch.group(1)) for p in parts[1:]: if p.find(':_(') >= 0: out.write(' %s ' % p.split(':', 1)[1]) else: out.write(blankout(p, 'F')) elif t.token_type == TOKEN_COMMENT: if t.contents.lstrip().startswith(TRANSLATOR_COMMENT_MARK): lineno_comment_map.setdefault(t.lineno, []).append(t.contents) comment_lineno_cache = t.lineno else: out.write(blankout(t.contents, 'X')) return force_str(out.getvalue())
def extract_django(fileobj, keywords, comment_tags, options): """Extract messages from Django template files. :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ intrans = False inplural = False message_context = None singular = [] plural = [] lineno = 1 encoding = options.get('encoding', 'utf8') text = fileobj.read().decode(encoding) for t in Lexer(text, None).tokenize(): lineno += t.contents.count('\n') if intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: if message_context: yield ( lineno, 'npgettext', [ smart_text(message_context), smart_text(u''.join(singular)), smart_text(u''.join(plural)) ], [], ) else: yield (lineno, 'ngettext', (smart_text(u''.join(singular)), smart_text(u''.join(plural))), []) else: if message_context: yield ( lineno, 'pgettext', [ smart_text(message_context), smart_text(u''.join(singular)) ], [], ) else: yield (lineno, None, smart_text(u''.join(singular)), []) intrans = False inplural = False message_context = None singular = [] plural = [] elif pluralmatch: inplural = True else: raise SyntaxError('Translation blocks must not include ' 'other block tags: %s' % t.contents) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: if inplural: plural.append(t.contents) else: singular.append(t.contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) if g[0] == '"': g = g.strip('"') elif g[0] == "'": g = g.strip("'") message_context = imatch.group(3) if message_context: # strip quotes message_context = message_context[1:-1] yield ( lineno, 'pgettext', [smart_text(message_context), smart_text(g)], [], ) message_context = None else: yield lineno, None, smart_text(g), [] elif bmatch: if bmatch.group(2): message_context = bmatch.group(2)[1:-1] for fmatch in constant_re.findall(t.contents): yield lineno, None, smart_text(fmatch), [] intrans = True inplural = False singular = [] plural = [] elif cmatches: for cmatch in cmatches: yield lineno, None, smart_text(cmatch), [] elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: yield lineno, None, smart_text(cmatch.group(1)), [] for p in parts[1:]: if p.find(':_(') >= 0: p1 = p.split(':', 1)[1] if p1[0] == '_': p1 = p1[1:] if p1[0] == '(': p1 = p1.strip('()') if p1[0] == "'": p1 = p1.strip("'") elif p1[0] == '"': p1 = p1.strip('"') yield lineno, None, smart_text(p1), []
def templatize(src, origin=None): """ Turns a Django template into something that is understood by xgettext. It does so by translating the Django translation tags into standard gettext function invocations. """ from django.template import (Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK) out = StringIO() message_context = None intrans = False inplural = False singular = [] plural = [] incomment = False comment = [] for t in Lexer(src, origin).tokenize(): if incomment: if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment': content = ''.join(comment) translators_comment_start = None for lineno, line in enumerate(content.splitlines(True)): if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK): translators_comment_start = lineno for lineno, line in enumerate(content.splitlines(True)): if translators_comment_start is not None and lineno >= translators_comment_start: out.write(' # %s' % line) else: out.write(' #\n') incomment = False comment = [] else: comment.append(t.contents) elif intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: if message_context: out.write(' npgettext(%r, %r, %r,count) ' % (message_context, ''.join(singular), ''.join(plural))) else: out.write(' ngettext(%r, %r, count) ' % (''.join(singular), ''.join(plural))) for part in singular: out.write(blankout(part, 'S')) for part in plural: out.write(blankout(part, 'P')) else: if message_context: out.write(' pgettext(%r, %r) ' % (message_context, ''.join(singular))) else: out.write(' gettext(%r) ' % ''.join(singular)) for part in singular: out.write(blankout(part, 'S')) message_context = None intrans = False inplural = False singular = [] plural = [] elif pluralmatch: inplural = True else: filemsg = '' if origin: filemsg = 'file %s, ' % origin raise SyntaxError( "Translation blocks must not include other block tags: %s (%sline %d)" % (t.contents, filemsg, t.lineno)) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: contents = one_percent_re.sub('%%', t.contents) if inplural: plural.append(contents) else: singular.append(contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) if g[0] == '"': g = g.strip('"') elif g[0] == "'": g = g.strip("'") g = one_percent_re.sub('%%', g) if imatch.group(2): # A context is provided context_match = context_re.match(imatch.group(2)) message_context = context_match.group(1) if message_context[0] == '"': message_context = message_context.strip('"') elif message_context[0] == "'": message_context = message_context.strip("'") out.write(' pgettext(%r, %r) ' % (message_context, g)) message_context = None else: out.write(' gettext(%r) ' % g) elif bmatch: for fmatch in constant_re.findall(t.contents): out.write(' _(%s) ' % fmatch) if bmatch.group(1): # A context is provided context_match = context_re.match(bmatch.group(1)) message_context = context_match.group(1) if message_context[0] == '"': message_context = message_context.strip('"') elif message_context[0] == "'": message_context = message_context.strip("'") intrans = True inplural = False singular = [] plural = [] elif cmatches: for cmatch in cmatches: out.write(' _(%s) ' % cmatch) elif t.contents == 'comment': incomment = True else: out.write(blankout(t.contents, 'B')) elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: out.write(' _(%s) ' % cmatch.group(1)) for p in parts[1:]: if p.find(':_(') >= 0: out.write(' %s ' % p.split(':', 1)[1]) else: out.write(blankout(p, 'F')) elif t.token_type == TOKEN_COMMENT: out.write(' # %s' % t.contents) else: out.write(blankout(t.contents, 'X')) return out.getvalue()
def templatize(src): """ Turns a Django template into something that is understood by xgettext. It does so by translating the Django translation tags into standard gettext function invocations. """ from django.template import Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK out = StringIO() intrans = False inplural = False singular = [] plural = [] for t in Lexer(src, None).tokenize(): if intrans: if t.token_type == TOKEN_BLOCK: endbmatch = endblock_re.match(t.contents) pluralmatch = plural_re.match(t.contents) if endbmatch: if inplural: out.write(' ngettext(%r,%r,count) ' % (''.join(singular), ''.join(plural))) for part in singular: out.write(blankout(part, 'S')) for part in plural: out.write(blankout(part, 'P')) else: out.write(' gettext(%r) ' % ''.join(singular)) for part in singular: out.write(blankout(part, 'S')) intrans = False inplural = False singular = [] plural = [] elif pluralmatch: inplural = True else: raise SyntaxError("Translation blocks must not include other block tags: %s" % t.contents) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: if inplural: plural.append(t.contents) else: singular.append(t.contents) else: if t.token_type == TOKEN_BLOCK: imatch = inline_re.match(t.contents) bmatch = block_re.match(t.contents) cmatches = constant_re.findall(t.contents) if imatch: g = imatch.group(1) if g[0] == '"': g = g.strip('"') elif g[0] == "'": g = g.strip("'") out.write(' gettext(%r) ' % g) elif bmatch: for fmatch in constant_re.findall(t.contents): out.write(' _(%s) ' % fmatch) intrans = True inplural = False singular = [] plural = [] elif cmatches: for cmatch in cmatches: out.write(' _(%s) ' % cmatch) else: out.write(blankout(t.contents, 'B')) elif t.token_type == TOKEN_VAR: parts = t.contents.split('|') cmatch = constant_re.match(parts[0]) if cmatch: out.write(' _(%s) ' % cmatch.group(1)) for p in parts[1:]: if p.find(':_(') >= 0: out.write(' %s ' % p.split(':',1)[1]) else: out.write(blankout(p, 'F')) else: out.write(blankout(t.contents, 'X')) return out.getvalue()
def my_templatize(src, origin=None): # Jinja2 spaceless src = strip_whitespaces(src) """ Turns a Django template into something that is understood by xgettext. It does so by translating the Django translation tags into standard gettext function invocations. """ from django.template import (Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK, TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK) out = StringIO() intrans = False inplural = False singular = [] plural = [] incomment = False comment = [] for t in Lexer(src, origin).tokenize(): if incomment: if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment': content = ''.join(comment) translators_comment_start = None for lineno, line in enumerate( content.splitlines(True)): if line.lstrip().startswith( TRANSLATOR_COMMENT_MARK): translators_comment_start = lineno for lineno, line in enumerate( content.splitlines(True)): if translators_comment_start is not None and lineno >= translators_comment_start: out.write(' # %s' % line) else: out.write(' #\n') incomment = False comment = [] else: comment.append(t.contents) elif intrans: if t.token_type == TOKEN_BLOCK: endbmatch = trans_real.endblock_re.match(t.contents) pluralmatch = trans_real.plural_re.match(t.contents) if endbmatch: if inplural: out.write(' ngettext(%r,%r,count) ' % (''.join(singular), ''.join(plural))) for part in singular: out.write(trans_real.blankout(part, 'S')) for part in plural: out.write(trans_real.blankout(part, 'P')) else: out.write(' gettext(%r) ' % ''.join(singular)) for part in singular: out.write(trans_real.blankout(part, 'S')) intrans = False inplural = False singular = [] plural = [] elif pluralmatch: inplural = True else: filemsg = '' if origin: filemsg = 'file %s, ' % origin raise SyntaxError( "Translation blocks must not include other block tags: %s (%sline %d)" % (t.contents, filemsg, t.lineno)) elif t.token_type == TOKEN_VAR: if inplural: plural.append('%%(%s)s' % t.contents) else: singular.append('%%(%s)s' % t.contents) elif t.token_type == TOKEN_TEXT: contents = t.contents.replace('%', '%%') if inplural: plural.append(contents) else: singular.append(contents) else: if t.token_type == TOKEN_BLOCK: imatch = trans_real.inline_re.match(t.contents) bmatch = trans_real.block_re.match(t.contents) cmatches = trans_real.constant_re.findall(t.contents) if imatch: g = imatch.group(1) if g[0] == '"': g = g.strip('"') elif g[0] == "'": g = g.strip("'") out.write(' gettext(%r) ' % g) elif bmatch: for fmatch in trans_real.constant_re.findall( t.contents): out.write(' _(%s) ' % fmatch) intrans = True inplural = False singular = [] plural = [] elif cmatches: for cmatch in cmatches: out.write(' _(%s) ' % cmatch) elif t.contents == 'comment': incomment = True else: out.write(trans_real.blankout(t.contents, 'B')) elif t.token_type == TOKEN_VAR: cmatches = trans_real.constant_re.findall(t.contents) if cmatches: for cmatch in cmatches: out.write(' _(%s) ' % cmatch) # findall is necessary for macros having translation constants as parameters # original django code: # # parts = t.contents.split('|') # cmatch = constant_re.match(parts[0]) # if cmatch: # out.write(' _(%s) ' % cmatch.group(1)) # for p in parts[1:]: # if p.find(':_(') >= 0: # out.write(' %s ' % p.split(':',1)[1]) # else: # out.write(trans_real.blankout(p, 'F')) elif t.token_type == TOKEN_COMMENT: out.write(' # %s' % t.contents) else: out.write(trans_real.blankout(t.contents, 'X')) return out.getvalue()
def extract(fileobj, keywords, comment_tags, options): """Extracts translation messages from underscore template files. This method does also extract django templates. If a template does not contain any django translation tags we always fallback to underscore extraction. This is a plugin to Babel, written according to http://babel.pocoo.org/docs/messages/#writing-extraction-methods :param fileobj: the file-like object the messages should be extracted from :param keywords: a list of keywords (i.e. function names) that should be recognized as translation functions :param comment_tags: a list of translator tags to search for and include in the results :param options: a dictionary of additional options (optional) :return: an iterator over ``(lineno, funcname, message, comments)`` tuples :rtype: ``iterator`` """ encoding = options.get('encoding', 'utf-8') original_position = fileobj.tell() text = fileobj.read().decode(encoding) # TODO: There must be another way. Find a way to fix the ordering # in babel directly! vars = [ token.token_type != TOKEN_TEXT for token in Lexer(text, None).tokenize() ] could_be_django = any(list(vars)) if could_be_django: fileobj.seek(original_position) iterator = extract_django(fileobj, keywords, comment_tags, options) for lineno, funcname, message, comments in iterator: yield lineno, funcname, message, comments else: # Underscore template extraction comments = [] fileobj.seek(original_position) for lineno, line in enumerate(fileobj, 1): funcname = None stream = TokenStream.from_tuple_iter( tokenize(line, underscore.rules)) while not stream.eof: if stream.current.type == 'gettext_begin': stream.expect('gettext_begin') funcname = stream.expect('func_name').value args, kwargs = parse_arguments(stream, 'gettext_end') strings = [] for arg in args: try: arg = int(arg) except ValueError: pass if isinstance(arg, six.string_types): strings.append(force_text(arg)) else: strings.append(None) for arg in kwargs: strings.append(None) if len(strings) == 1: strings = strings[0] else: strings = tuple(strings) yield lineno, funcname, strings, [] stream.next()