def n_mkfunc(self, node): """If the function has a docstring (this is found in the code constants), pull that out and make it part of the syntax tree. When generating the source string that AST node rather than the code field is seen and used. """ if self.version >= (3, 7): code_index = -3 else: code_index = -2 code = find_code_node(node, code_index).attr mkfunc_pattr = node[-1].pattr if isinstance(mkfunc_pattr, tuple): assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int) is_closure = node[-1].pattr[3] != 0 else: # FIXME: This is what we had before. It is hoaky and probably wrong. is_closure = mkfunc_pattr == "closure" if ((not is_closure) and len(code.co_consts) > 0 and isinstance(code.co_consts[0], str)): docstring_node = SyntaxTree( "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])]) docstring_node.transformed_by = "n_mkfunc" node = SyntaxTree("mkfunc", node[:-1] + [docstring_node, node[-1]]) node.transformed_by = "n_mkfunc" return node
def n_mkfunc(self, node): """If the function has a docstring (this is found in the code constants), pull that out and make it part of the syntax tree. When generating the source string that AST node rather than the code field is seen and used. """ if self.version >= 3.7: code_index = -3 else: code_index = -2 code = find_code_node(node, code_index).attr if ( node[-1].pattr != "closure" and len(code.co_consts) > 0 and code.co_consts[0] is not None ): docstring_node = SyntaxTree( "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])] ) docstring_node.transformed_by = "n_mkfunc" node = SyntaxTree("mkfunc", node[:-1] + [docstring_node, node[-1]]) node.transformed_by = "n_mkfunc" return node
def transform(self, ast, code): self.maybe_show_tree(ast) self.ast = copy(ast) self.ast = self.traverse(self.ast, is_lambda=False) try: # Disambiguate a string (expression) which appears as a "call_stmt" at # the beginning of a function versus a docstring. Seems pretty academic, # but this is Python. call_stmt = ast[0][0] if is_not_docstring(call_stmt): call_stmt.kind = "string_at_beginning" call_stmt.transformed_by = "transform" pass except: pass try: for i in range(len(self.ast)): sstmt = ast[i] if len(sstmt) == 1 and sstmt == "sstmt": self.ast[i] = self.ast[i][0] if is_docstring(self.ast[i], self.version, code.co_consts): load_const = self.ast[i].first_child() docstring_ast = SyntaxTree( "docstring", [ Token( "LOAD_STR", has_arg=True, offset=0, attr=load_const.attr, pattr=load_const.pattr, ) ], ) docstring_ast.transformed_by = "transform" del self.ast[i] self.ast.insert(0, docstring_ast) break if self.ast[-1] == RETURN_NONE: self.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass return self.ast
def test_token(): # Test token formatting of: LOAD_CONST None t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True) expect = " 0 LOAD_CONST None" # print(t.format()) assert t assert t.format() == expect # Make sure equality testing of tokens ignores offset t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True) assert t2 == t # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index # of co_consts. t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True) expect = " 1 LOAD_CONST False" assert t.format() == expect
"list": 0, # [expressions...] "list_comp": 0, "set_comp": 0, "set_comp_expr": 0, "unary_convert": 0, } LINE_LENGTH = 80 # Some parse trees created below are used for comparing code # fragments (like "return None" at the end of functions). RETURN_LOCALS = SyntaxTree( "return", [ SyntaxTree("ret_expr", [SyntaxTree("expr", [Token("LOAD_LOCALS")])]), Token("RETURN_VALUE"), ], ) NONE = SyntaxTree("expr", [NoneToken]) RETURN_NONE = SyntaxTree( "stmt", [SyntaxTree("return", [NONE, Token("RETURN_VALUE")])]) PASS = SyntaxTree( "stmts", [SyntaxTree("sstmt", [SyntaxTree("stmt", [SyntaxTree("pass", [])])])]) ASSIGN_DOC_STRING = lambda doc_string, doc_load: SyntaxTree( "stmt",
def customize_for_version(self, is_pypy, version): if is_pypy: ######################## # PyPy changes ####################### TABLE_DIRECT.update({ 'assert_pypy': ('%|assert %c\n', (1, 'assert_expr')), # This is as a result of an if transoration 'assert0_pypy': ('%|assert %c\n', (0, 'assert_expr')), 'assert_not_pypy': ('%|assert not %c\n', (1, 'assert_exp')), 'assert2_not_pypy': ('%|assert not %c, %c\n', (1, 'assert_exp'), (4, 'expr')), 'assert2_pypy': ('%|assert %c, %c\n', (1, 'assert_expr'), (4, 'expr')), 'try_except_pypy': ('%|try:\n%+%c%-%c\n\n', 1, 2), 'tryfinallystmt_pypy': ('%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3), 'assign3_pypy': ('%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2), 'assign2_pypy': ('%|%c, %c = %c, %c\n', 3, 2, 0, 1), }) else: ######################## # Without PyPy ####################### TABLE_DIRECT.update({ # "assert" and "assert_expr" are added via transform rules. "assert": ("%|assert %c\n", 0), "assert2": ("%|assert %c, %c\n", 0, 3), # Created only via transformation "assertnot": ("%|assert not %p\n", (0, PRECEDENCE['unary_not'])), "assert2not": ("%|assert not %p, %c\n", (0, PRECEDENCE['unary_not']), 3), "assign2": ("%|%c, %c = %c, %c\n", 3, 4, 0, 1), "assign3": ("%|%c, %c, %c = %c, %c, %c\n", 5, 6, 7, 0, 1, 2), "try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3), }) if version >= 3.0: if version >= 3.2: TABLE_DIRECT.update({ "del_deref_stmt": ("%|del %c\n", 0), "DELETE_DEREF": ("%{pattr}", 0) }) from uncompyle6.semantics.customize3 import customize_for_version3 customize_for_version3(self, version) else: # < 3.0 TABLE_DIRECT.update({ "except_cond3": ("%|except %c, %c:\n", (1, "expr"), (-2, "store")) }) if version <= 2.6: TABLE_DIRECT["testtrue_then"] = TABLE_DIRECT["testtrue"] if 2.4 <= version <= 2.6: TABLE_DIRECT.update({"comp_for": (" for %c in %c", 3, 1)}) else: TABLE_DIRECT.update({"comp_for": (" for %c in %c%c", 2, 0, 3)}) if version >= 2.5: from uncompyle6.semantics.customize25 import customize_for_version25 customize_for_version25(self, version) if version >= 2.6: from uncompyle6.semantics.customize26_27 import ( customize_for_version26_27, ) customize_for_version26_27(self, version) pass else: # < 2.5 global NAME_MODULE NAME_MODULE = SyntaxTree( "stmt", [ SyntaxTree( "assign", [ SyntaxTree( "expr", [ Token( "LOAD_GLOBAL", pattr="__name__", offset=0, has_arg=True, ) ], ), SyntaxTree( "store", [ Token( "STORE_NAME", pattr="__module__", offset=3, has_arg=True, ) ], ), ], ) ], ) TABLE_DIRECT.update({ "importmultiple": ("%|import %c%c\n", 2, 3), "import_cont": (", %c", 2), "tryfinallystmt": ( "%|try:\n%+%c%-%|finally:\n%+%c%-", (1, "suite_stmts_opt"), (5, "suite_stmts_opt"), ), }) if version == 2.4: def n_iftrue_stmt24(node): self.template_engine(("%c", 0), node) self.default(node) self.prune() self.n_iftrue_stmt24 = n_iftrue_stmt24 else: # version <= 2.3: TABLE_DIRECT.update({"if1_stmt": ("%|if 1\n%+%c%-", 5)}) if version <= 2.1: TABLE_DIRECT.update({ "importmultiple": ("%c", 2), # FIXME: not quite right. We have indiividual imports # when there is in fact one: "import a, b, ..." "imports_cont": ("%C%,", (1, 100, "\n")), }) pass pass pass # < 2.5 # < 3.0 continues TABLE_R.update({ "STORE_SLICE+0": ("%c[:]", 0), "STORE_SLICE+1": ("%c[%p:]", 0, (1, -1)), "STORE_SLICE+2": ("%c[:%p]", 0, (1, -1)), "STORE_SLICE+3": ("%c[%p:%p]", 0, (1, -1), (2, -1)), "DELETE_SLICE+0": ("%|del %c[:]\n", 0), "DELETE_SLICE+1": ("%|del %c[%c:]\n", 0, 1), "DELETE_SLICE+2": ("%|del %c[:%c]\n", 0, 1), "DELETE_SLICE+3": ("%|del %c[%c:%c]\n", 0, 1, 2), }) TABLE_DIRECT.update({"raise_stmt2": ("%|raise %c, %c\n", 0, 1)}) # exec as a built-in statement is only in Python 2.x def n_exec_stmt(node): """ exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT """ self.write(self.indent, "exec ") self.preorder(node[0]) if not node[1][0].isNone(): sep = " in " for subnode in node[1]: self.write(sep) sep = ", " self.preorder(subnode) self.println() self.prune() # stop recursing self.n_exec_smt = n_exec_stmt pass # < 3.0 return
def customize_for_version(self, is_pypy, version): if is_pypy: ######################## # PyPy changes ####################### TABLE_DIRECT.update({ 'assert_pypy': ('%|assert %c\n', 1), 'assert2_pypy': ('%|assert %c, %c\n', 1, 4), 'try_except_pypy': ('%|try:\n%+%c%-%c\n\n', 1, 2), 'tryfinallystmt_pypy': ('%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3), 'assign3_pypy': ('%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2), 'assign2_pypy': ('%|%c, %c = %c, %c\n', 3, 2, 0, 1), }) else: ######################## # Without PyPy ####################### TABLE_DIRECT.update({ 'assert': ('%|assert %c\n', 0), 'assert2': ('%|assert %c, %c\n', 0, 3), 'try_except': ('%|try:\n%+%c%-%c\n\n', 1, 3), 'assign2': ('%|%c, %c = %c, %c\n', 3, 4, 0, 1), 'assign3': ('%|%c, %c, %c = %c, %c, %c\n', 5, 6, 7, 0, 1, 2), }) if version < 3.0: TABLE_R.update({ 'STORE_SLICE+0': ('%c[:]', 0), 'STORE_SLICE+1': ('%c[%p:]', 0, (1, -1)), 'STORE_SLICE+2': ('%c[:%p]', 0, (1, -1)), 'STORE_SLICE+3': ('%c[%p:%p]', 0, (1, -1), (2, -1)), 'DELETE_SLICE+0': ('%|del %c[:]\n', 0), 'DELETE_SLICE+1': ('%|del %c[%c:]\n', 0, 1), 'DELETE_SLICE+2': ('%|del %c[:%c]\n', 0, 1), 'DELETE_SLICE+3': ('%|del %c[%c:%c]\n', 0, 1, 2), }) TABLE_DIRECT.update({ 'raise_stmt2': ('%|raise %c, %c\n', 0, 1), }) # exec as a built-in statement is only in Python 2.x def n_exec_stmt(self, node): """ exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT """ self.write(self.indent, 'exec ') self.preorder(node[0]) if not node[1][0].isNone(): sep = ' in ' for subnode in node[1]: self.write(sep) sep = ", " self.preorder(subnode) self.println() self.prune() # stop recursing self.n_exec_smt = n_exec_stmt else: TABLE_DIRECT.update({ # Gotta love Python for its futzing around with syntax like this 'raise_stmt2': ('%|raise %c from %c\n', 0, 1), }) if version >= 3.2: TABLE_DIRECT.update({ 'del_deref_stmt': ('%|del %c\n', 0), 'DELETE_DEREF': ('%{pattr}', 0), }) if version <= 2.4: TABLE_DIRECT.update({ 'importmultiple': ('%|import %c%c\n', 2, 3), 'import_cont': (', %c', 2), 'tryfinallystmt': ('%|try:\n%+%c%-%|finally:\n%+%c%-', (1, 'suite_stmts_opt'), (5, 'suite_stmts_opt')) }) if version == 2.3: TABLE_DIRECT.update({'if1_stmt': ('%|if 1\n%+%c%-', 5)}) global NAME_MODULE NAME_MODULE = AST('stmt', [ AST('assign', [ AST('expr', [ Token('LOAD_GLOBAL', pattr='__name__', offset=0, has_arg=True) ]), AST('store', [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True) ]) ]) ]) pass if version <= 2.3: if version <= 2.1: TABLE_DIRECT.update({ 'importmultiple': ('%c', 2), # FIXME: not quite right. We have indiividual imports # when there is in fact one: "import a, b, ..." 'imports_cont': ('%C%,', (1, 100, '\n')), }) pass pass pass elif version >= 2.5: ######################## # Import style for 2.5+ ######################## TABLE_DIRECT.update({ 'importmultiple': ('%|import %c%c\n', 2, 3), 'import_cont': (', %c', 2), # With/as is allowed as "from future" thing in 2.5 # Note: It is safe to put the variables after "as" in parenthesis, # and sometimes it is needed. 'withstmt': ('%|with %c:\n%+%c%-', 0, 3), 'withasstmt': ('%|with %c as (%c):\n%+%c%-', 0, 2, 3), }) # In 2.5+ "except" handlers and the "finally" can appear in one # "try" statement. So the below has the effect of combining the # "tryfinally" with statement with the "try_except" statement def tryfinallystmt(node): if len(node[1][0]) == 1 and node[1][0][0] == 'stmt': if node[1][0][0][0] == 'try_except': node[1][0][0][0].kind = 'tf_try_except' if node[1][0][0][0] == 'tryelsestmt': node[1][0][0][0].kind = 'tf_tryelsestmt' self.default(node) self.n_tryfinallystmt = tryfinallystmt ######################################## # Python 2.6+ # except <condition> as <var> # vs. older: # except <condition> , <var> # # For 2.6 we use the older syntax which # matches how we parse this in bytecode ######################################## if version > 2.6: TABLE_DIRECT.update({ 'except_cond2': ('%|except %c as %c:\n', 1, 5), }) else: TABLE_DIRECT.update({ 'except_cond3': ('%|except %c, %c:\n', 1, 6), 'testtrue_then': ('not %p', (0, 22)), }) if 2.4 <= version <= 2.6: TABLE_DIRECT.update({ 'comp_for': (' for %c in %c', 3, 1), }) else: TABLE_DIRECT.update({ 'comp_for': (' for %c in %c%c', 2, 0, 3), }) if version >= 3.0: TABLE_DIRECT.update({ 'function_def_annotate': ('\n\n%|def %c%c\n', -1, 0), 'store_locals': ('%|# inspect.currentframe().f_locals = __locals__\n', ), }) def n_mkfunc_annotate(node): if self.version >= 3.3 or node[-2] == 'kwargs': # LOAD_CONST code object .. # LOAD_CONST 'x0' if >= 3.3 # EXTENDED_ARG # MAKE_FUNCTION .. code = node[-4] elif node[-3] == 'expr': code = node[-3][0] else: # LOAD_CONST code object .. # MAKE_FUNCTION .. code = node[-3] self.indent_more() for annotate_last in range(len(node) - 1, -1, -1): if node[annotate_last] == 'annotate_tuple': break # FIXME: the real situation is that when derived from # function_def_annotate we the name has been filled in. # But when derived from funcdefdeco it hasn't Would like a better # way to distinquish. if self.f.getvalue()[-4:] == 'def ': self.write(code.attr.co_name) # FIXME: handle and pass full annotate args make_function3_annotate(self, node, is_lambda=False, codeNode=code, annotate_last=annotate_last) if len(self.param_stack) > 1: self.write('\n\n') else: self.write('\n\n\n') self.indent_less() self.prune() # stop recursing self.n_mkfunc_annotate = n_mkfunc_annotate if version >= 3.4: ######################## # Python 3.4+ Additions ####################### TABLE_DIRECT.update({ 'LOAD_CLASSDEREF': ('%{pattr}', ), }) ######################## # Python 3.5+ Additions ####################### if version >= 3.5: TABLE_DIRECT.update({ 'await_expr': ('await %c', 0), 'await_stmt': ('%|%c\n', 0), 'async_for_stmt': ('%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25), 'async_forelse_stmt': ('%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28), 'async_with_stmt': ('%|async with %c:\n%+%c%-', 0, 7), 'async_with_as_stmt': ('%|async with %c as %c:\n%+%c%-', 0, 6, 7), 'unmap_dict': ('{**%C}', (0, -1, ', **')), # 'unmapexpr': ( '{**%c}', 0), # done by n_unmapexpr }) def async_call(node): self.f.write('async ') node.kind == 'call' p = self.prec self.prec = 80 self.template_engine(('%c(%P)', 0, (1, -4, ', ', 100)), node) self.prec = p node.kind == 'async_call' self.prune() self.n_async_call = async_call self.n_build_list_unpack = self.n_list if version == 3.5: def n_call(node): mapping = self._get_mapping(node) table = mapping[0] key = node for i in mapping[1:]: key = key[i] pass if key.kind.startswith('CALL_FUNCTION_VAR_KW'): # Python 3.5 changes the stack position of *args. kwargs come # after *args whereas in earlier Pythons, *args is at the end # which simplifies things from our perspective. # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX # We will just swap the order to make it look like earlier Python 3. entry = table[key.kind] kwarg_pos = entry[2][1] args_pos = kwarg_pos - 1 # Put last node[args_pos] after subsequent kwargs while node[ kwarg_pos] == 'kwarg' and kwarg_pos < len( node): # swap node[args_pos] with node[kwargs_pos] node[kwarg_pos], node[args_pos] = node[ args_pos], node[kwarg_pos] args_pos = kwarg_pos kwarg_pos += 1 elif key.kind.startswith('CALL_FUNCTION_VAR'): nargs = node[-1].attr & 0xFF if nargs > 0: template = ('%c(%C, ', 0, (1, nargs + 1, ', ')) else: template = ('%c(', 0) self.template_engine(template, node) args_node = node[-2] if args_node == 'pos_arg': args_node = args_node[0] if args_node == 'expr': args_node = args_node[0] if args_node == 'build_list_unpack': template = ('*%P)', (0, len(args_node) - 1, ', *', 100)) self.template_engine(template, args_node) else: template = ('*%c)', -2) self.template_engine(template, node) self.prune() self.default(node) self.n_call = n_call def n_function_def(node): if self.version == 3.6: code_node = node[0][0] else: code_node = node[0][1] is_code = hasattr(code_node, 'attr') and iscode( code_node.attr) if (is_code and (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])): self.template_engine(('\n\n%|async def %c\n', -2), node) else: self.template_engine(('\n\n%|def %c\n', -2), node) self.prune() self.n_function_def = n_function_def def unmapexpr(node): last_n = node[0][-1] for n in node[0]: self.preorder(n) if n != last_n: self.f.write(', **') pass pass self.prune() pass self.n_unmapexpr = unmapexpr if version >= 3.6: ######################## # Python 3.6+ Additions ####################### # Value 100 is important; it is exactly # module/function precidence. PRECEDENCE['call_kw'] = 100 TABLE_DIRECT.update({ 'tryfinally36': ('%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', (1, 'returns'), 3), 'fstring_expr': ("{%c%{conversion}}", 0), # FIXME: the below assumes the format strings # don't have ''' in them. Fix this properly 'fstring_single': ("f'''{%c%{conversion}}'''", 0), 'fstring_multi': ("f'''%c'''", 0), 'func_args36': ("%c(**", 0), 'try_except36': ('%|try:\n%+%c%-%c\n\n', 1, 2), 'unpack_list': ('*%c', (0, 'list')), 'call_ex': ('%c(%c)', (0, 'expr'), 1), 'call_ex_kw': ('%c(%c)', (0, 'expr'), 2), }) TABLE_R.update({ 'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)), # Not quite right 'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')), }) def build_unpack_tuple_with_call(node): if node[0] == 'expr': tup = node[0][0] else: tup = node[0] pass assert tup == 'tuple' self.call36_tuple(tup) buwc = node[-1] assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL') for n in node[1:-1]: self.f.write(', *') self.preorder(n) pass self.prune() return self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call def build_unpack_map_with_call(node): n = node[0] if n == 'expr': n = n[0] if n == 'dict': self.call36_dict(n) first = 1 sep = ', **' else: first = 0 sep = '**' for n in node[first:-1]: self.f.write(sep) self.preorder(n) sep = ', **' pass self.prune() return self.n_build_map_unpack_with_call = build_unpack_map_with_call def call_ex_kw2(node): """Handle CALL_FUNCTION_EX 2 (have KW) but with BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL""" # This is weird shit. Thanks Python! self.preorder(node[0]) self.write('(') assert node[1] == 'build_tuple_unpack_with_call' btuwc = node[1] tup = btuwc[0] if tup == 'expr': tup = tup[0] assert tup == 'tuple' self.call36_tuple(tup) assert node[2] == 'build_map_unpack_with_call' self.write(', ') d = node[2][0] if d == 'expr': d = d[0] assert d == 'dict' self.call36_dict(d) args = btuwc[1] self.write(', *') self.preorder(args) self.write(', **') star_star_args = node[2][1] if star_star_args == 'expr': star_star_args = star_star_args[0] self.preorder(star_star_args) self.write(')') self.prune() self.n_call_ex_kw2 = call_ex_kw2 def call_ex_kw3(node): """Handle CALL_FUNCTION_EX 1 (have KW) but without BUILD_MAP_UNPACK_WITH_CALL""" self.preorder(node[0]) self.write('(') args = node[1][0] if args == 'expr': args = args[0] if args == 'tuple': if self.call36_tuple(args) > 0: self.write(', ') pass pass self.write('*') self.preorder(node[1][1]) self.write(', ') kwargs = node[2] if kwargs == 'expr': kwargs = kwargs[0] self.write('**') self.preorder(kwargs) self.write(')') self.prune() self.n_call_ex_kw3 = call_ex_kw3 def call_ex_kw4(node): """Handle CALL_FUNCTION_EX 2 (have KW) but without BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL""" self.preorder(node[0]) self.write('(') args = node[1][0] if args == 'tuple': if self.call36_tuple(args) > 0: self.write(', ') pass pass else: self.write('*') self.preorder(args) self.write(', ') pass kwargs = node[2] if kwargs == 'expr': kwargs = kwargs[0] self.write('**') self.preorder(kwargs) self.write(')') self.prune() self.n_call_ex_kw4 = call_ex_kw4 def call36_tuple(node): """ A tuple used in a call, these are like normal tuples but they don't have the enclosing parenthesis. """ assert node == 'tuple' # Note: don't iterate over last element which is a # BUILD_TUPLE... flat_elems = flatten_list(node[:-1]) self.indent_more(INDENT_PER_LEVEL) sep = '' for elem in flat_elems: if elem in ('ROT_THREE', 'EXTENDED_ARG'): continue assert elem == 'expr' line_number = self.line_number value = self.traverse(elem) if line_number != self.line_number: sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1] self.write(sep, value) sep = ', ' self.indent_less(INDENT_PER_LEVEL) return len(flat_elems) self.call36_tuple = call36_tuple def call36_dict(node): """ A dict used in a call_ex_kw2, which are a dictionary items expressed in a call. This should format to: a=1, b=2 In other words, no braces, no quotes around keys and ":" becomes "=". We will source-code use line breaks to guide us when to break. """ p = self.prec self.prec = 100 self.indent_more(INDENT_PER_LEVEL) sep = INDENT_PER_LEVEL[:-1] line_number = self.line_number if node[0].kind.startswith('kvlist'): # Python 3.5+ style key/value list in dict kv_node = node[0] l = list(kv_node) i = 0 length = len(l) # FIXME: Parser-speed improved grammars will have BUILD_MAP # at the end. So in the future when everything is # complete, we can do an "assert" instead of "if". if kv_node[-1].kind.startswith("BUILD_MAP"): length -= 1 # Respect line breaks from source while i < length: self.write(sep) name = self.traverse(l[i], indent='') # Strip off beginning and trailing quotes in name name = name[1:-1] if i > 0: line_number = self.indent_if_source_nl( line_number, self.indent + INDENT_PER_LEVEL[:-1]) line_number = self.line_number self.write(name, '=') value = self.traverse(l[i + 1], indent=self.indent + (len(name) + 2) * ' ') self.write(value) sep = ", " if line_number != self.line_number: sep += "\n" + self.indent + INDENT_PER_LEVEL[: -1] line_number = self.line_number i += 2 pass elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'): keys_node = node[-2] keys = keys_node.attr # from trepan.api import debug; debug() assert keys_node == 'LOAD_CONST' and isinstance( keys, tuple) for i in range(node[-1].attr): self.write(sep) self.write(keys[i], '=') value = self.traverse(node[i], indent='') self.write(value) sep = ", " if line_number != self.line_number: sep += "\n" + self.indent + INDENT_PER_LEVEL[: -1] line_number = self.line_number pass pass else: assert False, "Don't known to to untangle dictionary" self.prec = p self.indent_less(INDENT_PER_LEVEL) return self.call36_dict = call36_dict FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'} def n_except_suite_finalize(node): if node[1] == 'returns' and self.hide_internal: # Process node[1] only. # The code after "returns", e.g. node[3], is dead code. # Adding it is wrong as it dedents and another # exception handler "except_stmt" afterwards. # Note it is also possible that the grammar is wrong here. # and this should not be "except_stmt". self.indent_more() self.preorder(node[1]) self.indent_less() else: self.default(node) self.prune() self.n_except_suite_finalize = n_except_suite_finalize def n_formatted_value(node): if node[0] == 'LOAD_CONST': self.write(node[0].attr) self.prune() else: self.default(node) self.n_formatted_value = n_formatted_value def f_conversion(node): node.conversion = FSTRING_CONVERSION_MAP.get( node.data[1].attr, '') def fstring_expr(node): f_conversion(node) self.default(node) self.n_fstring_expr = fstring_expr def fstring_single(node): f_conversion(node) self.default(node) self.n_fstring_single = fstring_single # def kwargs_only_36(node): # keys = node[-1].attr # num_kwargs = len(keys) # values = node[:num_kwargs] # for i, (key, value) in enumerate(zip(keys, values)): # self.write(key + '=') # self.preorder(value) # if i < num_kwargs: # self.write(',') # self.prune() # return # self.n_kwargs_only_36 = kwargs_only_36 def kwargs_36(node): self.write('(') keys = node[-1].attr num_kwargs = len(keys) num_posargs = len(node) - (num_kwargs + 1) n = len(node) assert n >= len(keys)+1, \ 'not enough parameters keyword-tuple values' # try: # assert n >= len(keys)+1, \ # 'not enough parameters keyword-tuple values' # except: # from trepan.api import debug; debug() sep = '' # FIXME: adjust output for line breaks? for i in range(num_posargs): self.write(sep) self.preorder(node[i]) sep = ', ' i = num_posargs j = 0 # FIXME: adjust output for line breaks? while i < n - 1: self.write(sep) self.write(keys[j] + '=') self.preorder(node[i]) sep = ', ' i += 1 j += 1 self.write(')') self.prune() return self.n_kwargs_36 = kwargs_36 def starred(node): l = len(node) assert l > 0 pos_args = node[0] if pos_args == 'expr': pos_args = pos_args[0] if pos_args == 'tuple': star_start = 1 template = '%C', (0, -1, ', ') self.template_engine(template, pos_args) self.write(', ') else: star_start = 0 if l > 1: template = ('*%C', (star_start, -1, ', *')) else: template = ('*%c', (star_start, 'expr')) self.template_engine(template, node) self.prune() self.n_starred = starred def return_closure(node): # Nothing should be output here self.prune() return self.n_return_closure = return_closure pass # version >= 3.6 pass # version >= 3.4 pass # version >= 3.0 return
from uncompyle6.scanners.tok import Token, NoneToken if PYTHON3: minint = -sys.maxsize - 1 maxint = sys.maxsize else: minint = -sys.maxint - 1 maxint = sys.maxint LINE_LENGTH = 80 # Some parse trees created below are used for comparing code # fragments (like 'return None' at the end of functions). RETURN_LOCALS = AST('return', [ AST('ret_expr', [AST('expr', [Token('LOAD_LOCALS')])]), Token('RETURN_VALUE') ]) NONE = AST('expr', [NoneToken]) RETURN_NONE = AST('stmt', [AST('return', [NONE, Token('RETURN_VALUE')])]) PASS = AST('stmts', [AST('sstmt', [AST('stmt', [AST('pass', [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ AST('stmt', [ AST('assign', [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]), AST('store', [ Token('STORE_NAME', pattr='__doc__')]) ])])
'dict_comp': 0, 'generator_exp': 0, # (expressions...) 'list': 0, # [expressions...] 'list_comp': 0, 'set_comp': 0, 'set_comp_expr': 0, 'unary_convert': 0, } LINE_LENGTH = 80 # Some parse trees created below are used for comparing code # fragments (like 'return None' at the end of functions). RETURN_LOCALS = SyntaxTree('return', [ SyntaxTree('ret_expr', [SyntaxTree('expr', [ Token('LOAD_LOCALS') ])]), Token('RETURN_VALUE')]) NONE = SyntaxTree('expr', [ NoneToken ] ) RETURN_NONE = SyntaxTree('stmt', [ SyntaxTree('return', [ NONE, Token('RETURN_VALUE')]) ]) PASS = SyntaxTree('stmts', [ SyntaxTree('sstmt', [ SyntaxTree('stmt', [ SyntaxTree('pass', [])])])]) ASSIGN_DOC_STRING = lambda doc_string, doc_load: \ SyntaxTree('stmt',
def customize_for_version(self, is_pypy, version): if is_pypy: ######################## # PyPy changes ####################### TABLE_DIRECT.update({ 'assert_pypy': ( '%|assert %c\n' , 1 ), 'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ), 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), 'assign2_pypy': ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1), }) else: ######################## # Without PyPy ####################### TABLE_DIRECT.update({ 'assert': ( '%|assert %c\n' , 0 ), 'assert2': ( '%|assert %c, %c\n' , 0, 3 ), 'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ), 'assign2': ( '%|%c, %c = %c, %c\n', 3, 4, 0, 1 ), 'assign3': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 6, 7, 0, 1, 2 ), }) if version < 3.0: if version == 2.4: def n_iftrue_stmt24(node): self.template_engine(('%|%c', 0), node) self.default(node) self.prune() self.n_iftrue_stmt24 = n_iftrue_stmt24 TABLE_R.update({ 'STORE_SLICE+0': ( '%c[:]', 0 ), 'STORE_SLICE+1': ( '%c[%p:]', 0, (1, -1) ), 'STORE_SLICE+2': ( '%c[:%p]', 0, (1, -1) ), 'STORE_SLICE+3': ( '%c[%p:%p]', 0, (1, -1), (2, -1) ), 'DELETE_SLICE+0': ( '%|del %c[:]\n', 0 ), 'DELETE_SLICE+1': ( '%|del %c[%c:]\n', 0, 1 ), 'DELETE_SLICE+2': ( '%|del %c[:%c]\n', 0, 1 ), 'DELETE_SLICE+3': ( '%|del %c[%c:%c]\n', 0, 1, 2 ), }) TABLE_DIRECT.update({ 'raise_stmt2': ( '%|raise %c, %c\n', 0, 1), }) # exec as a built-in statement is only in Python 2.x def n_exec_stmt(node): """ exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT """ self.write(self.indent, 'exec ') self.preorder(node[0]) if not node[1][0].isNone(): sep = ' in ' for subnode in node[1]: self.write(sep); sep = ", " self.preorder(subnode) self.println() self.prune() # stop recursing self.n_exec_smt = n_exec_stmt else: TABLE_DIRECT.update({ # Gotta love Python for its futzing around with syntax like this 'raise_stmt2': ( '%|raise %c from %c\n', 0, 1), }) if version >= 3.2: TABLE_DIRECT.update({ 'del_deref_stmt': ( '%|del %c\n', 0), 'DELETE_DEREF': ( '%{pattr}', 0 ), }) if version <= 2.4: TABLE_DIRECT.update({ 'importmultiple': ( '%|import %c%c\n', 2, 3), 'import_cont' : ( ', %c', 2), 'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-', (1, 'suite_stmts_opt') , (5, 'suite_stmts_opt') ) }) if version == 2.3: TABLE_DIRECT.update({ 'if1_stmt': ( '%|if 1\n%+%c%-', 5 ) }) global NAME_MODULE NAME_MODULE = SyntaxTree('stmt', [ SyntaxTree('assign', [ SyntaxTree('expr', [Token('LOAD_GLOBAL', pattr='__name__', offset=0, has_arg=True)]), SyntaxTree('store', [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)]) ])]) pass if version <= 2.3: if version <= 2.1: TABLE_DIRECT.update({ 'importmultiple': ( '%c', 2 ), # FIXME: not quite right. We have indiividual imports # when there is in fact one: "import a, b, ..." 'imports_cont': ( '%C%,', (1, 100, '\n') ), }) pass pass pass elif version >= 2.5: ######################## # Import style for 2.5+ ######################## TABLE_DIRECT.update({ 'importmultiple': ( '%|import %c%c\n', 2, 3 ), 'import_cont' : ( ', %c', 2 ), # With/as is allowed as "from future" thing in 2.5 # Note: It is safe to put the variables after "as" in parenthesis, # and sometimes it is needed. 'withstmt': ( '%|with %c:\n%+%c%-', 0, 3), 'withasstmt': ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3), }) # In 2.5+ "except" handlers and the "finally" can appear in one # "try" statement. So the below has the effect of combining the # "tryfinally" with statement with the "try_except" statement def tryfinallystmt(node): if len(node[1][0]) == 1 and node[1][0][0] == 'stmt': if node[1][0][0][0] == 'try_except': node[1][0][0][0].kind = 'tf_try_except' if node[1][0][0][0] == 'tryelsestmt': node[1][0][0][0].kind = 'tf_tryelsestmt' self.default(node) self.n_tryfinallystmt = tryfinallystmt ######################################## # Python 2.6+ # except <condition> as <var> # vs. older: # except <condition> , <var> # # For 2.6 we use the older syntax which # matches how we parse this in bytecode ######################################## if version > 2.6: TABLE_DIRECT.update({ 'except_cond2': ( '%|except %c as %c:\n', 1, 5 ), }) else: TABLE_DIRECT.update({ 'except_cond3': ( '%|except %c, %c:\n', 1, 6 ), 'testtrue_then': ( 'not %p', (0, 22) ), }) if 2.4 <= version <= 2.6: TABLE_DIRECT.update({ 'comp_for': ( ' for %c in %c', 3, 1 ), }) else: TABLE_DIRECT.update({ 'comp_for': ( ' for %c in %c%c', 2, 0, 3 ), }) if version >= 3.0: from uncompyle6.semantics.customize3 import customize_for_version3 customize_for_version3(self, version) return
def customize_for_version(self, is_pypy, version): if is_pypy: ######################## # PyPy changes ####################### TABLE_DIRECT.update({ 'assert_pypy': ( '%|assert %c\n' , 1 ), 'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ), 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), 'assign2_pypy': ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1), }) else: ######################## # Without PyPy ####################### TABLE_DIRECT.update({ 'assert': ( '%|assert %c\n' , 0 ), 'assert2': ( '%|assert %c, %c\n' , 0, 3 ), 'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ), 'assign2': ( '%|%c, %c = %c, %c\n', 3, 4, 0, 1 ), 'assign3': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 6, 7, 0, 1, 2 ), }) if version >= 3.0: TABLE_DIRECT.update({ # Gotta love Python for its futzing around with syntax like this 'raise_stmt2': ( '%|raise %c from %c\n', 0, 1), }) if version >= 3.2: TABLE_DIRECT.update({ 'del_deref_stmt': ( '%|del %c\n', 0), 'DELETE_DEREF': ( '%{pattr}', 0 ), }) from uncompyle6.semantics.customize3 import customize_for_version3 customize_for_version3(self, version) else: # < 3.0 if 2.4 <= version <= 2.6: TABLE_DIRECT.update({ 'comp_for': ( ' for %c in %c', 3, 1 ), }) else: TABLE_DIRECT.update({ 'comp_for': ( ' for %c in %c%c', 2, 0, 3 ), }) if version >= 2.5: from uncompyle6.semantics.customize25 import customize_for_version25 customize_for_version25(self, version) if version >= 2.6: from uncompyle6.semantics.customize26_27 import customize_for_version26_27 customize_for_version26_27(self, version) pass else: # < 2.5 global NAME_MODULE NAME_MODULE = SyntaxTree('stmt', [ SyntaxTree('assign', [ SyntaxTree('expr', [Token('LOAD_GLOBAL', pattr='__name__', offset=0, has_arg=True)]), SyntaxTree('store', [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)]) ])]) TABLE_DIRECT.update({ 'importmultiple': ( '%|import %c%c\n', 2, 3), 'import_cont' : ( ', %c', 2), 'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-', (1, 'suite_stmts_opt') , (5, 'suite_stmts_opt') ) }) if version == 2.4: def n_iftrue_stmt24(node): self.template_engine(('%c', 0), node) self.default(node) self.prune() self.n_iftrue_stmt24 = n_iftrue_stmt24 else: # version <= 2.3: TABLE_DIRECT.update({ 'if1_stmt': ( '%|if 1\n%+%c%-', 5 ) }) if version <= 2.1: TABLE_DIRECT.update({ 'importmultiple': ( '%c', 2 ), # FIXME: not quite right. We have indiividual imports # when there is in fact one: "import a, b, ..." 'imports_cont': ( '%C%,', (1, 100, '\n') ), }) pass pass pass # < 2.5 # < 3.0 continues TABLE_R.update({ 'STORE_SLICE+0': ( '%c[:]', 0 ), 'STORE_SLICE+1': ( '%c[%p:]', 0, (1, -1) ), 'STORE_SLICE+2': ( '%c[:%p]', 0, (1, -1) ), 'STORE_SLICE+3': ( '%c[%p:%p]', 0, (1, -1), (2, -1) ), 'DELETE_SLICE+0': ( '%|del %c[:]\n', 0 ), 'DELETE_SLICE+1': ( '%|del %c[%c:]\n', 0, 1 ), 'DELETE_SLICE+2': ( '%|del %c[:%c]\n', 0, 1 ), 'DELETE_SLICE+3': ( '%|del %c[%c:%c]\n', 0, 1, 2 ), }) TABLE_DIRECT.update({ 'raise_stmt2': ( '%|raise %c, %c\n', 0, 1), }) # exec as a built-in statement is only in Python 2.x def n_exec_stmt(node): """ exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT """ self.write(self.indent, 'exec ') self.preorder(node[0]) if not node[1][0].isNone(): sep = ' in ' for subnode in node[1]: self.write(sep); sep = ", " self.preorder(subnode) self.println() self.prune() # stop recursing self.n_exec_smt = n_exec_stmt pass # < 3.0 return