def align_deparse_code(version, co, out=sys.stderr, showasm=False, showast=False, showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False): """ ingests and deparses a given code block 'co' """ assert iscode(co) # store final output stream for case of error scanner = get_scanner(version, is_pypy=is_pypy) tokens, customize = scanner.ingest(co, code_objects=code_objects) maybe_show_asm(showasm, tokens) debug_parser = dict(PARSER_DEFAULT_DEBUG) if showgrammar: debug_parser['reduce'] = showgrammar debug_parser['errorstack'] = True # Build AST from disassembly. deparsed = AligningWalker(version, scanner, out, showast=showast, debug_parser=debug_parser, compile_mode=compile_mode, is_pypy = is_pypy) isTopLevel = co.co_name == '<module>' deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel) assert deparsed.ast == 'stmts', 'Should have parsed grammar start' del tokens # save memory deparsed.mod_globs = find_globals(deparsed.ast, set()) # convert leading '__doc__ = "..." into doc string try: if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]): deparsed.print_docstring('', co.co_consts[0]) del deparsed.ast[0] if deparsed.ast[-1] == RETURN_NONE: deparsed.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass # What we've been waiting for: Generate source from AST! deparsed.gen_source(deparsed.ast, co.co_name, customize) for g in deparsed.mod_globs: deparsed.write('# global %s ## Warning: Unused global' % g) if deparsed.ERROR: raise SourceWalkerError("Deparsing stopped due to parse error") return deparsed
def deparse_code(version, co, out=StringIO(), showasm=False, showast=False, showgrammar=False): assert iscode(co) # store final output stream for case of error scanner = get_scanner(version) tokens, customize = scanner.disassemble(co) tokens, customize = scanner.disassemble(co) if showasm: for t in tokens: print(t) debug_parser = dict(PARSER_DEFAULT_DEBUG) debug_parser['reduce'] = showgrammar # Build AST from disassembly. # deparsed = pysource.FragmentsWalker(out, scanner, showast=showast) deparsed = FragmentsWalker(version, scanner, showast=showast, debug_parser=debug_parser) deparsed.ast = deparsed.build_ast(tokens, customize) assert deparsed.ast == 'stmts', 'Should have parsed grammar start' del tokens # save memory # convert leading '__doc__ = "..." into doc string assert deparsed.ast == 'stmts' deparsed.mod_globs = pysource.find_globals(deparsed.ast, set()) # Just when you think we've forgotten about what we # were supposed to to: Generate source from AST! deparsed.gen_source(deparsed.ast, co.co_name, customize) deparsed.set_pos_info(deparsed.ast, 0, len(deparsed.text)) deparsed.fixup_parents(deparsed.ast, None) for g in deparsed.mod_globs: deparsed.write('# global %s ## Warning: Unused global' % g) if deparsed.ERROR: raise deparsed.ERROR return deparsed
def deparse_code(version, co, out=StringIO(), showasm=False, showast=False, showgrammar=False): assert iscode(co) # store final output stream for case of error scanner = get_scanner(version) tokens, customize = scanner.disassemble(co) tokens, customize = scanner.disassemble(co) if showasm: for t in tokens: print(t) debug_parser = dict(PARSER_DEFAULT_DEBUG) if showgrammar: debug_parser['reduce'] = showgrammar debug_parser['errorstack'] = True # Build AST from disassembly. # deparsed = pysource.FragmentsWalker(out, scanner, showast=showast) deparsed = FragmentsWalker(version, scanner, showast=showast, debug_parser=debug_parser) deparsed.ast = deparsed.build_ast(tokens, customize) assert deparsed.ast == 'stmts', 'Should have parsed grammar start' del tokens # save memory # convert leading '__doc__ = "..." into doc string assert deparsed.ast == 'stmts' deparsed.mod_globs = pysource.find_globals(deparsed.ast, set()) # Just when you think we've forgotten about what we # were supposed to to: Generate source from AST! deparsed.gen_source(deparsed.ast, co.co_name, customize) deparsed.set_pos_info(deparsed.ast, 0, len(deparsed.text)) deparsed.fixup_parents(deparsed.ast, None) for g in deparsed.mod_globs: deparsed.write('# global %s ## Warning: Unused global' % g) if deparsed.ERROR: raise deparsed.ERROR return deparsed
def make_function(self, node, isLambda, nested=1, code_index=-2): """Dump function defintion, doc string, and function body.""" def build_param(ast, name, default): """build parameters: - handle defaults - handle format tuple parameters """ if self.version < 3.0: # if formal parameter is a tuple, the paramater name # starts with a dot (eg. '.1', '.2') if name.startswith('.'): # replace the name with the tuple-string name = self.get_tuple_parameter(ast, name) pass pass if default: if self.showast: print() print('--', name) print(default) print('--') pass result = '%s=' % name old_last_finish = self.last_finish self.last_finish = len(result) value = self.traverse(default, indent='') self.last_finish = old_last_finish result += value if result[-2:] == '= ': # default was 'LOAD_CONST None' result += 'None' return result else: return name # node[-1] == MAKE_FUNCTION_n args_node = node[-1] if isinstance(args_node.attr, tuple): defparams = node[:args_node.attr[0]] pos_args, kw_args, annotate_args = args_node.attr else: defparams = node[:args_node.attr] kw_args, annotate_args = (0, 0) pos_args = args_node.attr pass code = node[code_index].attr assert iscode(code) code = Code(code, self.scanner, self.currentclass) # add defaults values to parameter names argc = code.co_argcount paramnames = list(code.co_varnames[:argc]) # defaults are for last n parameters, thus reverse paramnames.reverse(); defparams.reverse() try: ast = self.build_ast(code._tokens, code._customize, isLambda = isLambda, noneInNames = ('None' in code.co_names)) except ParserError as p: self.write(str(p)) self.ERROR = p return # build parameters params = [build_param(ast, name, default) for name, default in zip_longest(paramnames, defparams, fillvalue=None)] params.reverse() # back to correct order if 4 & code.co_flags: # flag 2 -> variable number of args params.append('*%s' % code.co_varnames[argc]) argc += 1 if 8 & code.co_flags: # flag 3 -> keyword args params.append('**%s' % code.co_varnames[argc]) argc += 1 # dump parameter list (with default values) indent = self.indent if isLambda: self.write("lambda ", ", ".join(params)) else: self.write("(", ", ".join(params)) # self.println(indent, '#flags:\t', int(code.co_flags)) if kw_args > 0: if argc > 0: self.write(", *, ") else: self.write("*, ") for n in node: if n == 'pos_arg': continue self.preorder(n) break pass if isLambda: self.write(": ") else: self.println("):") if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly # docstring exists, dump it self.print_docstring(indent, code.co_consts[0]) code._tokens = None # save memory assert ast == 'stmts' all_globals = find_all_globals(ast, set()) for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): self.println(self.indent, 'global ', g) self.mod_globs -= all_globals rn = ('None' in code.co_names) and not find_none(ast) self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, returnNone=rn) code._tokens = None; code._customize = None # save memory
def make_function(self, node, isLambda, nested=1, code_index=-2): """Dump function defintion, doc string, and function body.""" def build_param(ast, name, default): """build parameters: - handle defaults - handle format tuple parameters """ # if formal parameter is a tuple, the paramater name # starts with a dot (eg. '.1', '.2') if name.startswith('.'): # replace the name with the tuple-string name = self.get_tuple_parameter(ast, name) if default: if self.showast: print('--', name) print(default) print('--') pass result = '%s = ' % name old_last_finish = self.last_finish self.last_finish = len(result) value = self.traverse(default, indent='') self.last_finish = old_last_finish result += value if result[-2:] == '= ': # default was 'LOAD_CONST None' result += 'None' return result else: return name # node[-1] == MAKE_xxx_n defparams = node[:node[-1].attr] code = node[code_index].attr assert type(code) == CodeType code = Code(code, self.scanner, self.currentclass) # assert isinstance(code, Code) # add defaults values to parameter names argc = code.co_argcount paramnames = list(code.co_varnames[:argc]) # defaults are for last n parameters, thus reverse paramnames.reverse(); defparams.reverse() try: ast = self.build_ast(code._tokens, code._customize, isLambda = isLambda, noneInNames = ('None' in code.co_names)) except ParserError as p: self.write( str(p)) self.ERROR = p return # build parameters params = [build_param(ast, name, default) for name, default in zip_longest(paramnames, defparams, fillvalue=None)] # params = [ build_param(ast, name, default) for # name, default in zip(paramnames, defparams) ] # params = [] # for i, name in enumerate(paramnames): # default = defparams[i] if len(defparams) > i else None # params.append( build_param(ast, name, default) ) params.reverse() # back to correct order if 4 & code.co_flags: # flag 2 -> variable number of args params.append('*%s' % code.co_varnames[argc]) argc += 1 if 8 & code.co_flags: # flag 3 -> keyword args params.append('**%s' % code.co_varnames[argc]) argc += 1 # dump parameter list (with default values) indent = self.indent if isLambda: self.write("lambda ", ", ".join(params), ": ") else: self.print_("(", ", ".join(params), "):") # self.print_(indent, '#flags:\t', int(code.co_flags)) if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly # docstring exists, dump it self.print_docstring(indent, code.co_consts[0]) code._tokens = None # save memory assert ast == 'stmts' all_globals = find_all_globals(ast, set()) for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): self.print_(self.indent, 'global ', g) self.mod_globs -= all_globals rn = ('None' in code.co_names) and not find_none(ast) self.gen_source(ast, code.co_name, code._customize, isLambda=isLambda, returnNone=rn) code._tokens = None; code._customize = None # save memory
def code_deparse_align(co, out=sys.stderr, version=None, is_pypy=None, debug_opts=DEFAULT_DEBUG_OPTS, code_objects={}, compile_mode='exec'): """ ingests and deparses a given code block 'co' """ assert iscode(co) if version is None: version = float(sys.version[0:3]) if is_pypy is None: is_pypy = IS_PYPY # store final output stream for case of error scanner = get_scanner(version, is_pypy=is_pypy) tokens, customize = scanner.ingest(co, code_objects=code_objects) show_asm = debug_opts.get('asm', None) maybe_show_asm(show_asm, tokens) debug_parser = dict(PARSER_DEFAULT_DEBUG) show_grammar = debug_opts.get('grammar', None) show_grammar = debug_opts.get('grammar', None) if show_grammar: debug_parser['reduce'] = show_grammar debug_parser['errorstack'] = True # Build a parse tree from tokenized and massaged disassembly. show_ast = debug_opts.get('ast', None) deparsed = AligningWalker(version, scanner, out, showast=show_ast, debug_parser=debug_parser, compile_mode=compile_mode, is_pypy=is_pypy) isTopLevel = co.co_name == '<module>' deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel) assert deparsed.ast == 'stmts', 'Should have parsed grammar start' del tokens # save memory deparsed.mod_globs = find_globals(deparsed.ast, set()) # convert leading '__doc__ = "..." into doc string try: if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]): deparsed.print_docstring('', co.co_consts[0]) del deparsed.ast[0] if deparsed.ast[-1] == RETURN_NONE: deparsed.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass # What we've been waiting for: Generate Python source from the parse tree! deparsed.gen_source(deparsed.ast, co.co_name, customize) for g in sorted(deparsed.mod_globs): deparsed.write('# global %s ## Warning: Unused global\n' % g) if deparsed.ERROR: raise SourceWalkerError("Deparsing stopped due to parse error") return deparsed