コード例 #1
0
    def extract_nodes(self, nodes):
        translator_comments = []
        in_translator_comments = False
        input_encoding = self.config["encoding"] or "ascii"
        comment_tags = list(
            filter(None, re.split(r"\s+", self.config["comment-tags"])))

        for node in nodes:
            child_nodes = None
            if (in_translator_comments and isinstance(node, parsetree.Text)
                    and not node.content.strip()):
                # Ignore whitespace within translator comments
                continue

            if isinstance(node, parsetree.Comment):
                value = node.text.strip()
                if in_translator_comments:
                    translator_comments.extend(
                        self._split_comment(node.lineno, value))
                    continue
                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        in_translator_comments = True
                        translator_comments.extend(
                            self._split_comment(node.lineno, value))
                continue

            if isinstance(node, parsetree.DefTag):
                code = node.function_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.BlockTag):
                code = node.body_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.CallTag):
                code = node.code.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.PageTag):
                code = node.body_decl.code
            elif isinstance(node, parsetree.CallNamespaceTag):
                code = node.expression
                child_nodes = node.nodes
            elif isinstance(node, parsetree.ControlLine):
                if node.isend:
                    in_translator_comments = False
                    continue
                code = node.text
            elif isinstance(node, parsetree.Code):
                in_translator_comments = False
                code = node.code.code
            elif isinstance(node, parsetree.Expression):
                code = node.code.code
            else:
                continue

            # Comments don't apply unless they immediately precede the message
            if (translator_comments
                    and translator_comments[-1][0] < node.lineno - 1):
                translator_comments = []

            translator_strings = [
                comment[1] for comment in translator_comments
            ]

            if isinstance(code, compat.text_type):
                code = code.encode(input_encoding, "backslashreplace")

            used_translator_comments = False
            # We add extra newline to work around a pybabel bug
            # (see python-babel/babel#274, parse_encoding dies if the first
            # input string of the input is non-ascii)
            # Also, because we added it, we have to subtract one from
            # node.lineno
            code = compat.byte_buffer(compat.b("\n") + code)

            for message in self.process_python(code, node.lineno - 1,
                                               translator_strings):
                yield message
                used_translator_comments = True

            if used_translator_comments:
                translator_comments = []
            in_translator_comments = False

            if child_nodes:
                for extracted in self.extract_nodes(child_nodes):
                    yield extracted
コード例 #2
0
ファイル: extract.py プロジェクト: Allifreyr/plexpy
    def extract_nodes(self, nodes):
        translator_comments = []
        in_translator_comments = False
        comment_tags = list(
            filter(None, re.split(r'\s+', self.config['comment-tags'])))

        for node in nodes:
            child_nodes = None
            if in_translator_comments and \
                    isinstance(node, parsetree.Text) and \
                    not node.content.strip():
                # Ignore whitespace within translator comments
                continue

            if isinstance(node, parsetree.Comment):
                value = node.text.strip()
                if in_translator_comments:
                    translator_comments.extend(
                        self._split_comment(node.lineno, value))
                    continue
                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        in_translator_comments = True
                        translator_comments.extend(
                            self._split_comment(node.lineno, value))
                continue

            if isinstance(node, parsetree.DefTag):
                code = node.function_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.BlockTag):
                code = node.body_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.CallTag):
                code = node.code.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.PageTag):
                code = node.body_decl.code
            elif isinstance(node, parsetree.CallNamespaceTag):
                code = node.expression
                child_nodes = node.nodes
            elif isinstance(node, parsetree.ControlLine):
                if node.isend:
                    in_translator_comments = False
                    continue
                code = node.text
            elif isinstance(node, parsetree.Code):
                in_translator_comments = False
                code = node.code.code
            elif isinstance(node, parsetree.Expression):
                code = node.code.code
            else:
                continue

            # Comments don't apply unless they immediately preceed the message
            if translator_comments and \
                    translator_comments[-1][0] < node.lineno - 1:
                translator_comments = []

            translator_strings = [
                comment[1] for comment in translator_comments]

            if isinstance(code, compat.text_type):
                code = code.encode('ascii', 'backslashreplace')

            used_translator_comments = False
            code = compat.byte_buffer(code)

            for message in self.process_python(
                    code, node.lineno, translator_strings):
                yield message
                used_translator_comments = True

            if used_translator_comments:
                translator_comments = []
            in_translator_comments = False

            if child_nodes:
                for extracted in self.extract_nodes(child_nodes):
                    yield extracted
コード例 #3
0
ファイル: babelplugin.py プロジェクト: 1025ryu/sns
def extract_nodes(nodes, keywords, comment_tags, options):
    """Extract messages from Mako's lexer node objects

    :param nodes: an iterable of Mako parsetree.Node objects to extract from
    :param keywords: a list of keywords (i.e. function names) that should be
                     recognized as translation functions
    :param comment_tags: a list of translator tags to search for and include
                         in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples
    :rtype: ``iterator``
    """
    translator_comments = []
    in_translator_comments = False

    for node in nodes:
        child_nodes = None
        if in_translator_comments and isinstance(node, parsetree.Text) and \
                not node.content.strip():
            # Ignore whitespace within translator comments
            continue

        if isinstance(node, parsetree.Comment):
            value = node.text.strip()
            if in_translator_comments:
                translator_comments.extend(_split_comment(node.lineno, value))
                continue
            for comment_tag in comment_tags:
                if value.startswith(comment_tag):
                    in_translator_comments = True
                    translator_comments.extend(_split_comment(node.lineno,
                                                              value))
            continue

        if isinstance(node, parsetree.DefTag):
            code = node.function_decl.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.BlockTag):
            code = node.body_decl.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.CallTag):
            code = node.code.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.PageTag):
            code = node.body_decl.code
        elif isinstance(node, parsetree.CallNamespaceTag):
            code = node.expression
            child_nodes = node.nodes
        elif isinstance(node, parsetree.ControlLine):
            if node.isend:
                in_translator_comments = False
                continue
            code = node.text
        elif isinstance(node, parsetree.Code):
            in_translator_comments = False
            code = node.code.code
        elif isinstance(node, parsetree.Expression):
            code = node.code.code
        else:
            continue

        # Comments don't apply unless they immediately preceed the message
        if translator_comments and \
                translator_comments[-1][0] < node.lineno - 1:
            translator_comments = []

        translator_strings = [comment[1] for comment in translator_comments]

        if isinstance(code, compat.text_type):
            code = code.encode('ascii', 'backslashreplace')

        used_translator_comments = False
        code = compat.byte_buffer(code)
        for lineno, funcname, messages, python_translator_comments \
                in extract_python(code, keywords, comment_tags, options):
            yield (node.lineno + (lineno - 1), funcname, messages,
                   translator_strings + python_translator_comments)
            used_translator_comments = True

        if used_translator_comments:
            translator_comments = []
        in_translator_comments = False

        if child_nodes:
            for extracted in extract_nodes(child_nodes, keywords, comment_tags,
                                           options):
                yield extracted
コード例 #4
0
ファイル: babelplugin.py プロジェクト: fractal520/dbops
def extract_nodes(nodes, keywords, comment_tags, options):
    """Extract messages from Mako's lexer node objects

    :param nodes: an iterable of Mako parsetree.Node objects to extract from
    :param keywords: a list of keywords (i.e. function names) that should be
                     recognized as translation functions
    :param comment_tags: a list of translator tags to search for and include
                         in the results
    :param options: a dictionary of additional options (optional)
    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples
    :rtype: ``iterator``
    """
    translator_comments = []
    in_translator_comments = False

    for node in nodes:
        child_nodes = None
        if in_translator_comments and isinstance(node, parsetree.Text) and \
                not node.content.strip():
            # Ignore whitespace within translator comments
            continue

        if isinstance(node, parsetree.Comment):
            value = node.text.strip()
            if in_translator_comments:
                translator_comments.extend(_split_comment(node.lineno, value))
                continue
            for comment_tag in comment_tags:
                if value.startswith(comment_tag):
                    in_translator_comments = True
                    translator_comments.extend(
                        _split_comment(node.lineno, value))
            continue

        if isinstance(node, parsetree.DefTag):
            code = node.function_decl.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.BlockTag):
            code = node.body_decl.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.CallTag):
            code = node.code.code
            child_nodes = node.nodes
        elif isinstance(node, parsetree.PageTag):
            code = node.body_decl.code
        elif isinstance(node, parsetree.CallNamespaceTag):
            code = node.expression
            child_nodes = node.nodes
        elif isinstance(node, parsetree.ControlLine):
            if node.isend:
                in_translator_comments = False
                continue
            code = node.text
        elif isinstance(node, parsetree.Code):
            in_translator_comments = False
            code = node.code.code
        elif isinstance(node, parsetree.Expression):
            code = node.code.code
        else:
            continue

        # Comments don't apply unless they immediately preceed the message
        if translator_comments and \
                translator_comments[-1][0] < node.lineno - 1:
            translator_comments = []

        translator_strings = [comment[1] for comment in translator_comments]

        if isinstance(code, compat.text_type):
            code = code.encode('ascii', 'backslashreplace')

        used_translator_comments = False
        code = compat.byte_buffer(code)
        for lineno, funcname, messages, python_translator_comments \
                in extract_python(code, keywords, comment_tags, options):
            yield (node.lineno + (lineno - 1), funcname, messages,
                   translator_strings + python_translator_comments)
            used_translator_comments = True

        if used_translator_comments:
            translator_comments = []
        in_translator_comments = False

        if child_nodes:
            for extracted in extract_nodes(child_nodes, keywords, comment_tags,
                                           options):
                yield extracted
コード例 #5
0
ファイル: extract.py プロジェクト: marcosptf/fedora
    def extract_nodes(self, nodes):
        translator_comments = []
        in_translator_comments = False
        input_encoding = self.config['encoding'] or 'ascii'
        comment_tags = list(
            filter(None, re.split(r'\s+', self.config['comment-tags'])))

        for node in nodes:
            child_nodes = None
            if in_translator_comments and \
                    isinstance(node, parsetree.Text) and \
                    not node.content.strip():
                # Ignore whitespace within translator comments
                continue

            if isinstance(node, parsetree.Comment):
                value = node.text.strip()
                if in_translator_comments:
                    translator_comments.extend(
                        self._split_comment(node.lineno, value))
                    continue
                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        in_translator_comments = True
                        translator_comments.extend(
                            self._split_comment(node.lineno, value))
                continue

            if isinstance(node, parsetree.DefTag):
                code = node.function_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.BlockTag):
                code = node.body_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.CallTag):
                code = node.code.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.PageTag):
                code = node.body_decl.code
            elif isinstance(node, parsetree.CallNamespaceTag):
                code = node.expression
                child_nodes = node.nodes
            elif isinstance(node, parsetree.ControlLine):
                if node.isend:
                    in_translator_comments = False
                    continue
                code = node.text
            elif isinstance(node, parsetree.Code):
                in_translator_comments = False
                code = node.code.code
            elif isinstance(node, parsetree.Expression):
                code = node.code.code
            else:
                continue

            # Comments don't apply unless they immediately preceed the message
            if translator_comments and \
                    translator_comments[-1][0] < node.lineno - 1:
                translator_comments = []

            translator_strings = [
                comment[1] for comment in translator_comments]

            if isinstance(code, compat.text_type):
                code = code.encode(input_encoding, 'backslashreplace')

            used_translator_comments = False
            # We add extra newline to work around a pybabel bug
            # (see python-babel/babel#274, parse_encoding dies if the first
            # input string of the input is non-ascii)
            # Also, because we added it, we have to subtract one from
            # node.lineno
            code = compat.byte_buffer(compat.b('\n') + code)

            for message in self.process_python(
                    code, node.lineno - 1, translator_strings):
                yield message
                used_translator_comments = True

            if used_translator_comments:
                translator_comments = []
            in_translator_comments = False

            if child_nodes:
                for extracted in self.extract_nodes(child_nodes):
                    yield extracted
コード例 #6
0
    def extract_nodes(self, nodes):
        translator_comments = []
        in_translator_comments = False
        comment_tags = list(
            filter(None, re.split(r'\s+', self.config['comment-tags'])))

        for node in nodes:
            child_nodes = None
            if in_translator_comments and \
                    isinstance(node, parsetree.Text) and \
                    not node.content.strip():
                # Ignore whitespace within translator comments
                continue

            if isinstance(node, parsetree.Comment):
                value = node.text.strip()
                if in_translator_comments:
                    translator_comments.extend(
                        self._split_comment(node.lineno, value))
                    continue
                for comment_tag in comment_tags:
                    if value.startswith(comment_tag):
                        in_translator_comments = True
                        translator_comments.extend(
                            self._split_comment(node.lineno, value))
                continue

            if isinstance(node, parsetree.DefTag):
                code = node.function_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.BlockTag):
                code = node.body_decl.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.CallTag):
                code = node.code.code
                child_nodes = node.nodes
            elif isinstance(node, parsetree.PageTag):
                code = node.body_decl.code
            elif isinstance(node, parsetree.CallNamespaceTag):
                code = node.expression
                child_nodes = node.nodes
            elif isinstance(node, parsetree.ControlLine):
                if node.isend:
                    in_translator_comments = False
                    continue
                code = node.text
            elif isinstance(node, parsetree.Code):
                in_translator_comments = False
                code = node.code.code
            elif isinstance(node, parsetree.Expression):
                code = node.code.code
            else:
                continue

            # Comments don't apply unless they immediately preceed the message
            if translator_comments and \
                    translator_comments[-1][0] < node.lineno - 1:
                translator_comments = []

            translator_strings = [
                comment[1] for comment in translator_comments
            ]

            if isinstance(code, compat.text_type):
                code = code.encode('ascii', 'backslashreplace')

            used_translator_comments = False
            code = compat.byte_buffer(code)

            for message in self.process_python(code, node.lineno,
                                               translator_strings):
                yield message
                used_translator_comments = True

            if used_translator_comments:
                translator_comments = []
            in_translator_comments = False

            if child_nodes:
                for extracted in self.extract_nodes(child_nodes):
                    yield extracted