Пример #1
0
    def n_mkfunc(self, node):
        """If the function has a docstring (this is found in the code
        constants), pull that out and make it part of the syntax
        tree. When generating the source string that AST node rather
        than the code field is seen and used.
        """

        code = find_code_node(node, -3).attr

        if (
            node[-1].pattr != "closure"
            and len(code.co_consts) > 0
            and code.co_consts[0] is not None
        ):
            docstring_node = SyntaxTree(
                "docstring",
                [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
                transformed_by="n_mkfunc",
            )
            node = SyntaxTree(
                "mkfunc",
                node[:-1] + [docstring_node, node[-1]],
                transformed_by="n_mkfunc",
            )

        return node
Пример #2
0
    def n_mkfunc(self, node):
        """If the function has a docstring (this is found in the code
        constants), pull that out and make it part of the syntax
        tree. When generating the source string that AST node rather
        than the code field is seen and used.
        """

        code = find_code_node(node, -3).attr

        mkfunc_pattr = node[-1].pattr
        if isinstance(mkfunc_pattr, tuple):
            assert isinstance(mkfunc_pattr, tuple)
            assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int)
            is_closure = node[-1].pattr[3] != 0
        else:
            # FIXME: This is what we had before. It is hoaky and probably wrong.
            is_closure = mkfunc_pattr == "closure"

        if ((not is_closure) and len(code.co_consts) > 0
                and isinstance(code.co_consts[0], str)):
            docstring_node = SyntaxTree(
                "docstring",
                [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])],
                transformed_by="n_mkfunc",
            )
            node = SyntaxTree(
                "mkfunc",
                node[:-1] + [docstring_node, node[-1]],
                transformed_by="n_mkfunc",
            )

        return node
Пример #3
0
    def transform(self, ast):
        self.maybe_show_tree(ast)
        self.ast = copy(ast)
        self.ast = self.traverse(self.ast, is_lambda=False)

        try:
            for i in range(len(self.ast)):
                if is_docstring(self.ast[i]):
                    docstring_ast = SyntaxTree(
                        "docstring",
                        [
                            Token(
                                "LOAD_STR",
                                has_arg=True,
                                offset=0,
                                pattr=self.ast[i][0][0][0][0].attr,
                            )
                        ],
                        transformed_by="transform",
                    )
                    del self.ast[i]
                    self.ast.insert(0, docstring_ast)
                    break

            if self.ast[-1] == RETURN_NONE:
                self.ast.pop()  # remove last node
                # todo: if empty, add 'pass'
        except:
            pass

        return self.ast
Пример #4
0
def test_token():
    # Test token formatting of: LOAD_CONST None
    t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
    expect = "                0  LOAD_CONST               None"
    # print(t.format())
    assert t
    assert t.format() == expect

    # Make sure equality testing of tokens ignores offset
    t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
    assert t2 == t

    # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
    # of co_consts.
    t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
    expect = "                1  LOAD_CONST               False"
    assert t.format() == expect, t.format()

    # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
    expect = "(005)                1  LOAD_CONST               False"
    assert t.format(token_num=5) == expect, t.format(token_num=5)
Пример #5
0
    def transform(self, ast, code):
        self.maybe_show_tree(ast)
        self.ast = copy(ast)
        self.ast = self.traverse(self.ast, is_lambda=False)
        n = len(self.ast)

        try:
            # Disambiguate a string (expression) which appears as a "call_stmt" at
            # the beginning of a function versus a docstring. Seems pretty academic,
            # but this is Python.
            call_stmt = ast[0][0]
            if is_not_docstring(call_stmt):
                call_stmt.kind = "string_at_beginning"
                call_stmt.transformed_by = "transform"
                pass
        except:
            pass
        try:

            for i in range(n):
                if is_docstring(self.ast[i], self.version, code.co_consts):
                    load_const = self.ast[i].first_child()
                    docstring_ast = SyntaxTree(
                        "docstring",
                        [
                            Token(
                                "LOAD_STR",
                                has_arg=True,
                                offset=0,
                                attr=load_const.attr,
                                pattr=load_const.pattr,
                            )
                        ],
                        transformed_by="transform",
                    )
                    del self.ast[i]
                    self.ast.insert(0, docstring_ast)
                    break

            if self.ast[-1] == RETURN_NONE:
                self.ast.pop()  # remove last node
                # todo: if empty, add 'pass'
        except:
            pass

        return self.ast
Пример #6
0
def test_token():
    # Test token formatting of: LOAD_CONST None
    t = Token('LOAD_CONST', offset=0, attr=None, pattr=None, has_arg=True)
    expect = '           0  LOAD_CONST               None'
    # print(t.format())
    assert t
    assert t.format() == expect

    # Make sure equality testing of tokens ignores offset
    t2 = Token('LOAD_CONST', offset=2, attr=None, pattr=None, has_arg=True)
    assert t2 == t

    # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
    # of co_consts.
    t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
    expect = '           1  LOAD_CONST            0  False'
    assert t.format() == expect
Пример #7
0
    "list": 0,  # [expressions...]
    "list_comp": 0,
    "set_comp": 0,
    "set_comp_expr": 0,
    "unary_convert": 0,
}

LINE_LENGTH = 80

# Some parse trees created below are used for comparing code
# fragments (like "return None" at the end of functions).

NONE = SyntaxTree("expr", [NoneToken])

RETURN_NONE = SyntaxTree(
    "stmt", [SyntaxTree("return", [NONE, Token("RETURN_VALUE")])])

PASS = SyntaxTree(
    "stmts",
    [SyntaxTree("sstmt", [SyntaxTree("stmt", [SyntaxTree("pass", [])])])])

ASSIGN_DOC_STRING = lambda doc_string: \
  SyntaxTree("stmt",
      [ SyntaxTree("assign",
            [ SyntaxTree("expr", [ Token("LOAD_STR", pattr=doc_string) ]),
              SyntaxTree("store", [ Token("STORE_NAME", pattr="__doc__")])
            ])])

NAME_MODULE = SyntaxTree("assign", [
    SyntaxTree("expr",
               [Token("LOAD_NAME", pattr="__name__", offset=0, has_arg=True)]),
Пример #8
0
    'dict_comp': 0,
    'generator_exp': 0,  # (expressions...)
    'list': 0,  # [expressions...]
    'list_comp': 0,
    'set_comp': 0,
    'set_comp_expr': 0,
    'unary_convert': 0,
}

LINE_LENGTH = 80

# Some parse trees created below are used for comparing code
# fragments (like 'return None' at the end of functions).

RETURN_LOCALS = SyntaxTree('return', [
    SyntaxTree('ret_expr', [SyntaxTree('expr', [Token('LOAD_LOCALS')])]),
    Token('RETURN_VALUE')
])

NONE = SyntaxTree('expr', [NoneToken])

RETURN_NONE = SyntaxTree(
    'stmt', [SyntaxTree('return', [NONE, Token('RETURN_VALUE')])])

PASS = SyntaxTree(
    'stmts',
    [SyntaxTree('sstmt', [SyntaxTree('stmt', [SyntaxTree('pass', [])])])])

ASSIGN_DOC_STRING = lambda doc_string: \
  SyntaxTree('stmt',
      [ SyntaxTree('assign',