コード例 #1
0
ファイル: dt_report.py プロジェクト: manabu/dtcov
    def parse_source(self):
        source_lines = set()

        lexer = Lexer(self.text, "<string>")

        tokens = lexer.tokenize()

        comment = False
        for token in tokens:
            assert isinstance(token, Token)
            if token.token_type == TOKEN_BLOCK:
                if token.contents == 'comment':
                    comment = True
                    continue
                elif token.contents == 'endcomment':
                    comment = False
                    continue

            if comment:
                continue

            if token.token_type == TOKEN_BLOCK or token.token_type == TOKEN_VAR:
                if token.token_type == TOKEN_BLOCK and token.contents.startswith(
                        'end'):
                    continue

                source_lines.add(token.lineno)

        return tuple(sorted(source_lines)), ()
コード例 #2
0
ファイル: dt_report.py プロジェクト: manabu/dtcov
    def parse_source(self):
        source_lines = set()

        lexer = Lexer(self.text, "<string>")

        tokens = lexer.tokenize()

        comment = False
        for token in tokens:
            assert isinstance(token, Token)
            if token.token_type == TOKEN_BLOCK:
                if token.contents == 'comment':
                    comment = True
                    continue
                elif token.contents == 'endcomment':
                    comment = False
                    continue

            if comment:
                continue

            if token.token_type == TOKEN_BLOCK or token.token_type == TOKEN_VAR:
                if token.token_type == TOKEN_BLOCK and token.contents.startswith('end'):
                    continue

                source_lines.add(token.lineno)

        return tuple(sorted(source_lines)), ()
コード例 #3
0
    def validate_template(self, template_string):
        # We want to tokenize like normal, then use a custom parser.
        lexer = Lexer(template_string, None)
        tokens = lexer.tokenize()
        parser = TemplateValidationParser(tokens, self.allow, self.disallow, self.secure)

        for node in parser.parse():
            template = getattr(node, LOADED_TEMPLATE_ATTR, None)
コード例 #4
0
ファイル: validators.py プロジェクト: somair/philo
    def validate_template(self, template_string):
        # We want to tokenize like normal, then use a custom parser.
        lexer = Lexer(template_string, None)
        tokens = lexer.tokenize()
        parser = TemplateValidationParser(tokens, self.allow, self.disallow,
                                          self.secure)

        for node in parser.parse():
            template = getattr(node, LOADED_TEMPLATE_ATTR, None)
コード例 #5
0
 def tokenize():
     """
     Returns a stream of Django Token() entities
     """
     for template in get_templates():
         with open(template) as fp:
             template_content = fp.read()
         lexer = Lexer(template_content, None)
         for token in lexer.tokenize():
             yield token
コード例 #6
0
 def tokenize():
     """
     Returns a stream of Django Token() entities
     """
     for template in get_templates():
         with open(template) as fp:
             template_content = fp.read()
         lexer = Lexer(template_content, None)
         for token in lexer.tokenize():
             yield token
コード例 #7
0
ファイル: templateassets.py プロジェクト: thepian/themaestro
    def load_blocks(self):
        """Loads the asset blocks defined in the template
        handles:
         * extends - to track template hierachy
         * css,javascript - start of asset
         * endcss, endjavascript - end of asset
         * {{ .. }} - expansion of variables to settings variables according to VAR_EXPANSIONS
         """
        try:
            template_string, _filepath = filesystem.load_template_source(self.templatepath)
        except TemplateDoesNotExist:
            template_string, _filepath = app_directories.load_template_source(self.templatepath)

        self.content_hash = hash(template_string)

        try:
            result = TemplateAssetBucket()

            l = Lexer(template_string, self.templatepath)
            within = None
            texts = []
            for m in l.tokenize():
                if m.token_type == TOKEN_BLOCK:
                    split = m.split_contents()
                    typ = split[0]

                    if typ == "extends":
                        if split[1].endswith('"') or split[1].endswith("'"):
                            self.extends = split[1].strip('"').strip("'")
                        else:
                            pass #TODO figure out support for variable expansion
                    elif typ in TemplateAssetBlock.BLOCKTAGS:
                        within = typ
                        prop = _parse_asset_parameters(m.split_contents())
                    elif typ.startswith('end'):
                        if typ[3:] == within:
                            within = None
                            result.append(TemplateAssetBlock(''.join(texts), template=self, **prop))
                        elif typ[3:] in TemplateAssetBlock.BLOCKTAGS:
                            assert false, "encountered dangling %s tag in '%s'" % (typ,self.templatepath)
                elif within:
                    if m.token_type == TOKEN_TEXT:
                        texts.append(m.contents)
                    elif m.token_type == TOKEN_VAR:
                        v = VAR_EXPANSIONS.get(m.contents,'')
                        if v:
                            texts.append(v)
                        #? else:
                        #assert False, "Variable replacement in client side magic not yet supported"

            return result
        except UnicodeDecodeError:
            return "/* could not load %s as a template */\n" % templatepath
コード例 #8
0
ファイル: test_template.py プロジェクト: aodag/bpcommons
    def _render_html(self, template_string, context={}):
        # :(
        if DJANGO_VERSION > (1,2):
            from django.template import import_library
            tag_lib = import_library('beproud.django.commons.tests.test_tags')
        else:
            from django.template import get_library
            tag_lib = get_library('beproud.django.commons.tests.test_tags')

        lexer = Lexer(template_string, self._make_origin())
        parser = Parser(lexer.tokenize())
        parser.add_library(tag_lib)
        nodelist = parser.parse()

        return nodelist.render(Context(context))
コード例 #9
0
ファイル: check.py プロジェクト: sundayliu/django-cms-1
def _load_all_templates(directory):
    """
    Loads all templates in a directory (recursively) and yields tuples of
    template tokens and template paths.
    """
    if os.path.exists(directory):
        for name in os.listdir(directory):
            path = os.path.join(directory, name)
            if os.path.isdir(path):
                for template in _load_all_templates(path):
                    yield template
            elif path.endswith('.html'):
                with open(path, 'rb') as fobj:
                    source = fobj.read().decode(settings.FILE_CHARSET)
                    lexer = Lexer(source, path)
                    yield lexer.tokenize(), path
コード例 #10
0
def render_custom_content(body, context_data={}):
    """Renders custom content for the payload using Django templating.

    This will take the custom payload content template provided by
    the user and render it using a stripped down version of Django's
    templating system.

    In order to keep the payload safe, we use a limited Context along with a
    custom Parser that blocks certain template tags. This gives us
    tags like {% for %} and {% if %}, but blacklists tags like {% load %}
    and {% include %}.
    """
    lexer = Lexer(body, origin=None)
    parser = CustomPayloadParser(lexer.tokenize())
    nodes = parser.parse()

    return nodes.render(Context(context_data))
コード例 #11
0
ファイル: webhooks.py プロジェクト: teotikalki/reviewboard
def render_custom_content(body, context_data={}):
    """Renders custom content for the payload using Django templating.

    This will take the custom payload content template provided by
    the user and render it using a stripped down version of Django's
    templating system.

    In order to keep the payload safe, we use a limited Context along with a
    custom Parser that blocks certain template tags. This gives us
    tags like {% for %} and {% if %}, but blacklists tags like {% load %}
    and {% include %}.
    """
    lexer = Lexer(body, origin=None)
    parser = CustomPayloadParser(lexer.tokenize())
    nodes = parser.parse()

    return nodes.render(Context(context_data))
コード例 #12
0
    def _render_html(self, template_string, context={}):
        # :(
        if DJANGO_VERSION > (1, 9):
            from django.template.library import import_library
            tag_lib = import_library('testapp.tags')
        else:  # DJANGO_VERSION > (1,7):
            from django.template.base import import_library
            tag_lib = import_library('testapp.tags')

        if DJANGO_VERSION > (1, 9):
            lexer = Lexer(template_string)
        else:
            lexer = Lexer(template_string, self._make_origin())
        parser = Parser(lexer.tokenize())
        parser.add_library(tag_lib)
        nodelist = parser.parse()

        return nodelist.render(Context(context))
コード例 #13
0
def _fix_html_type(request, html, filetype):
    for group, files in requested_assets[request][filetype].items():

        # parse the content for the individual file tokens
        indices = []
        def sub_func(matchobj):
            indices.append(int(matchobj.group(2)))
            return ""

        regex = token_regexes[filetype][group]
        html = regex.sub(sub_func, html)

        # replace the 'replace me' tag with actual list of
        # 'tags' (ie <link href="foo.css">)
        file_html = u""
        uncompressible_html = u""
        for index in indices:
            fileObj = files[index]
            if fileObj.isCompressible():
                file_html += fileObj.render()
            else:
                uncompressible_html += fileObj.render()

        # try to use the provided 'compress' app to compress the output
        if hasattr(settings, 'COMPRESS') and settings.COMPRESS:
            # Currently this only supports the django-css app we use

            from django.template import Lexer,Parser,Token,TOKEN_TEXT
            file_html += "{% endcompress %}"
            lexer = Lexer(file_html, None)
            from compressor.templatetags.compress import compress
            file_html = compress(
                Parser(lexer.tokenize()),
                Token(TOKEN_TEXT, "compress " + filetype)
            ).render({})

        file_html = uncompressible_html + file_html
        tag = ASSET_DEFS[filetype]['destination_tag'].get(group, None)
        if tag:
            html = smart_unicode(html)
            html = html.replace(tag, file_html + tag)

    return html
コード例 #14
0
ファイル: processing.py プロジェクト: pombredanne/turbion
def compile_string(template_string):
    lexer = Lexer(template_string, None)
    parser = ProcessingParser(lexer.tokenize())
    return parser.parse()
コード例 #15
0
ファイル: extract.py プロジェクト: robhudson/django-babel
def extract_django(fileobj, keywords, comment_tags, options):
    """Extract messages from Django template files.

    :param fileobj: the file-like object the messages should be extracted from
    :param keywords: a list of keywords (i.e. function names) that should
                     be recognized as translation functions
    :param comment_tags: a list of translator tags to search for and
                         include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
             tuples
    :rtype: ``iterator``
    """
    intrans = False
    inplural = False
    message_context = None
    singular = []
    plural = []
    lineno = 1

    encoding = options.get('encoding', 'utf8')
    text = fileobj.read().decode(encoding)

    try:
        text_lexer = Lexer(text)
    except TypeError:
        # Django 1.9 changed the way we invoke Lexer; older versions
        # require two parameters.
        text_lexer = Lexer(text, None)

    for t in text_lexer.tokenize():
        lineno += t.contents.count('\n')
        if intrans:
            if t.token_type == TOKEN_BLOCK:
                endbmatch = endblock_re.match(t.contents)
                pluralmatch = plural_re.match(t.contents)
                if endbmatch:
                    if inplural:
                        if message_context:
                            yield (
                                lineno,
                                'npgettext',
                                [smart_text(message_context),
                                 smart_text(u''.join(singular)),
                                 smart_text(u''.join(plural))],
                                [],
                            )
                        else:
                            yield (
                                lineno,
                                'ngettext',
                                (smart_text(u''.join(singular)),
                                 smart_text(u''.join(plural))),
                                [])
                    else:
                        if message_context:
                            yield (
                                lineno,
                                'pgettext',
                                [smart_text(message_context),
                                 smart_text(u''.join(singular))],
                                [],
                            )
                        else:
                            yield (
                                lineno,
                                None,
                                smart_text(u''.join(singular)),
                                [])

                    intrans = False
                    inplural = False
                    message_context = None
                    singular = []
                    plural = []
                elif pluralmatch:
                    inplural = True
                else:
                    raise SyntaxError('Translation blocks must not include '
                                      'other block tags: %s' % t.contents)
            elif t.token_type == TOKEN_VAR:
                if inplural:
                    plural.append('%%(%s)s' % t.contents)
                else:
                    singular.append('%%(%s)s' % t.contents)
            elif t.token_type == TOKEN_TEXT:
                if inplural:
                    plural.append(t.contents)
                else:
                    singular.append(t.contents)
        else:
            if t.token_type == TOKEN_BLOCK:
                imatch = inline_re.match(t.contents)
                bmatch = block_re.match(t.contents)
                cmatches = constant_re.findall(t.contents)
                if imatch:
                    g = imatch.group(1)
                    if g[0] == '"':
                        g = g.strip('"')
                    elif g[0] == "'":
                        g = g.strip("'")
                    message_context = imatch.group(3)
                    if message_context:
                        # strip quotes
                        message_context = message_context[1:-1]
                        yield (
                            lineno,
                            'pgettext',
                            [smart_text(message_context), smart_text(g)],
                            [],
                        )
                        message_context = None
                    else:
                        yield lineno, None, smart_text(g), []
                elif bmatch:
                    if bmatch.group(2):
                        message_context = bmatch.group(2)[1:-1]
                    for fmatch in constant_re.findall(t.contents):
                        yield lineno, None, smart_text(fmatch), []
                    intrans = True
                    inplural = False
                    singular = []
                    plural = []
                elif cmatches:
                    for cmatch in cmatches:
                        yield lineno, None, smart_text(cmatch), []
            elif t.token_type == TOKEN_VAR:
                parts = t.contents.split('|')
                cmatch = constant_re.match(parts[0])
                if cmatch:
                    yield lineno, None, smart_text(cmatch.group(1)), []
                for p in parts[1:]:
                    if p.find(':_(') >= 0:
                        p1 = p.split(':', 1)[1]
                        if p1[0] == '_':
                            p1 = p1[1:]
                        if p1[0] == '(':
                            p1 = p1.strip('()')
                        if p1[0] == "'":
                            p1 = p1.strip("'")
                        elif p1[0] == '"':
                            p1 = p1.strip('"')
                        yield lineno, None, smart_text(p1), []
コード例 #16
0
def extract_django(fileobj, keywords, comment_tags, options):
    """Extract messages from Django template files.

    :param fileobj: the file-like object the messages should be extracted from
    :param keywords: a list of keywords (i.e. function names) that should
                     be recognized as translation functions
    :param comment_tags: a list of translator tags to search for and
                         include in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)``
             tuples
    :rtype: ``iterator``
    """
    intrans = False
    inplural = False
    trimmed = False
    message_context = None
    singular = []
    plural = []
    lineno = 1

    encoding = options.get('encoding', 'utf8')
    text = fileobj.read().decode(encoding)

    try:
        text_lexer = Lexer(text)
    except TypeError:
        # Django 1.9 changed the way we invoke Lexer; older versions
        # require two parameters.
        text_lexer = Lexer(text, None)

    for t in text_lexer.tokenize():
        lineno += t.contents.count('\n')
        if intrans:
            if t.token_type == TOKEN_BLOCK:
                endbmatch = endblock_re.match(t.contents)
                pluralmatch = plural_re.match(t.contents)
                if endbmatch:
                    if inplural:
                        if message_context:
                            yield (
                                lineno,
                                'npgettext',
                                [
                                    smart_text(message_context),
                                    smart_text(join_tokens(singular, trimmed)),
                                    smart_text(join_tokens(plural, trimmed))
                                ],
                                [],
                            )
                        else:
                            yield (lineno, 'ngettext',
                                   (smart_text(join_tokens(singular, trimmed)),
                                    smart_text(join_tokens(plural,
                                                           trimmed))), [])
                    else:
                        if message_context:
                            yield (
                                lineno,
                                'pgettext',
                                [
                                    smart_text(message_context),
                                    smart_text(join_tokens(singular, trimmed))
                                ],
                                [],
                            )
                        else:
                            yield (lineno, None,
                                   smart_text(join_tokens(singular,
                                                          trimmed)), [])

                    intrans = False
                    inplural = False
                    message_context = None
                    singular = []
                    plural = []
                elif pluralmatch:
                    inplural = True
                else:
                    raise SyntaxError('Translation blocks must not include '
                                      'other block tags: %s' % t.contents)
            elif t.token_type == TOKEN_VAR:
                if inplural:
                    plural.append('%%(%s)s' % t.contents)
                else:
                    singular.append('%%(%s)s' % t.contents)
            elif t.token_type == TOKEN_TEXT:
                if inplural:
                    plural.append(t.contents)
                else:
                    singular.append(t.contents)
        else:
            if t.token_type == TOKEN_BLOCK:
                imatch = inline_re.match(t.contents)
                bmatch = block_re.match(t.contents)
                cmatches = constant_re.findall(t.contents)
                if imatch:
                    g = imatch.group(1)
                    g = strip_quotes(g)
                    message_context = imatch.group(3)
                    if message_context:
                        # strip quotes
                        message_context = message_context[1:-1]
                        yield (
                            lineno,
                            'pgettext',
                            [smart_text(message_context),
                             smart_text(g)],
                            [],
                        )
                        message_context = None
                    else:
                        yield lineno, None, smart_text(g), []
                elif bmatch:
                    if bmatch.group(2):
                        message_context = bmatch.group(2)[1:-1]
                    for fmatch in constant_re.findall(t.contents):
                        stripped_fmatch = strip_quotes(fmatch)
                        yield lineno, None, smart_text(stripped_fmatch), []
                    intrans = True
                    inplural = False
                    trimmed = 'trimmed' in t.split_contents()
                    singular = []
                    plural = []
                elif cmatches:
                    for cmatch in cmatches:
                        stripped_cmatch = strip_quotes(cmatch)
                        yield lineno, None, smart_text(stripped_cmatch), []
            elif t.token_type == TOKEN_VAR:
                parts = t.contents.split('|')
                cmatch = constant_re.match(parts[0])
                if cmatch:
                    stripped_cmatch = strip_quotes(cmatch.group(1))
                    yield lineno, None, smart_text(stripped_cmatch), []
                for p in parts[1:]:
                    if p.find(':_(') >= 0:
                        p1 = p.split(':', 1)[1]
                        if p1[0] == '_':
                            p1 = p1[1:]
                        if p1[0] == '(':
                            p1 = p1.strip('()')
                        p1 = strip_quotes(p1)
                        yield lineno, None, smart_text(p1), []