def __init__(self, label=''): self.insts = [] self.inEdges = misc.Set() self.outEdges = misc.Set() self.label = label self.bid = Block._count self.next = [] Block._count = Block._count + 1
def __init__(self, name, filename, args=(), optimized=0, klass=None): self.super_init() self.hasjrel = misc.Set() self.hasjabs = misc.Set() for i in dis.hasjrel: self.hasjrel.add(dis.opname[i]) for i in dis.hasjabs: self.hasjabs.add(dis.opname[i]) self.name = name self.filename = filename self.docstring = None self.args = args # XXX self.argcount = getArgCount(args) self.klass = klass if optimized: self.flags = CO_OPTIMIZED | CO_NEWLOCALS else: self.flags = 0 self.consts = [] self.names = [] # Free variables found by the symbol table scan, including # variables used only in nested scopes, are included here. self.freevars = [] self.cellvars = [] # The closure list is used to track the order of cell # variables and free variables in the resulting code object. # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both # kinds of variables. self.closure = [] self.varnames = list(args) or [] for i in range(len(self.varnames)): var = self.varnames[i] if isinstance(var, TupleArg): self.varnames[i] = var.getName() self.stage = RAW for name, obj in dir(self): if name[:9] == "_convert_": opname = name[9:] self._converters[opname] = obj for num in range(len(dis.opname)): opnum[dis.opname[num]] = num
class PyFlowGraph(FlowGraph): super_init = FlowGraph.__init__ def __init__(self, name, filename, args=(), optimized=0, klass=None): self.super_init() self.name = name self.filename = filename self.docstring = None self.args = args # XXX self.argcount = getArgCount(args) self.klass = klass if optimized: self.flags = CO_OPTIMIZED | CO_NEWLOCALS else: self.flags = 0 self.consts = [] self.names = [] # Free variables found by the symbol table scan, including # variables used only in nested scopes, are included here. self.freevars = [] self.cellvars = [] # The closure list is used to track the order of cell # variables and free variables in the resulting code object. # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both # kinds of variables. self.closure = [] self.varnames = list(args) or [] for i in range(len(self.varnames)): var = self.varnames[i] if isinstance(var, TupleArg): self.varnames[i] = var.getName() self.stage = RAW def setDocstring(self, doc): self.docstring = doc def setFlag(self, flag): self.flags = self.flags | flag if flag == CO_VARARGS: self.argcount = self.argcount - 1 def checkFlag(self, flag): if self.flags & flag: return 1 def setFreeVars(self, names): self.freevars = list(names) def setCellVars(self, names): self.cellvars = names def getCode(self): """Get a Python code object""" assert self.stage == RAW self.computeStackDepth() self.flattenGraph() assert self.stage == FLAT self.convertArgs() assert self.stage == CONV self.makeByteCode() assert self.stage == DONE return self.newCodeObject() def dump(self, io=None): if io: save = sys.stdout sys.stdout = io pc = 0 for t in self.insts: opname = t[0] if opname == "SET_LINENO": print if len(t) == 1: print "\t", "%3d" % pc, opname pc = pc + 1 else: print "\t", "%3d" % pc, opname, t[1] pc = pc + 3 if io: sys.stdout = save def computeStackDepth(self): """Compute the max stack depth. Approach is to compute the stack effect of each basic block. Then find the path through the code with the largest total effect. """ depth = {} exit = None for b in self.getBlocks(): depth[b] = findDepth(b.getInstructions()) seen = {} def max_depth(b, d): if b in seen: return d seen[b] = 1 d = d + depth[b] children = b.get_children() if children: return max([max_depth(c, d) for c in children]) else: if not b.label == "exit": return max_depth(self.exit, d) else: return d self.stacksize = max_depth(self.entry, 0) def flattenGraph(self): """Arrange the blocks in order and resolve jumps""" assert self.stage == RAW self.insts = insts = [] pc = 0 begin = {} end = {} for b in self.getBlocksInOrder(): begin[b] = pc for inst in b.getInstructions(): insts.append(inst) if len(inst) == 1: pc = pc + 1 elif inst[0] != "SET_LINENO": # arg takes 2 bytes pc = pc + 3 end[b] = pc pc = 0 for i in range(len(insts)): inst = insts[i] if len(inst) == 1: pc = pc + 1 elif inst[0] != "SET_LINENO": pc = pc + 3 opname = inst[0] if self.hasjrel.has_elt(opname): oparg = inst[1] offset = begin[oparg] - pc insts[i] = opname, offset elif self.hasjabs.has_elt(opname): insts[i] = opname, begin[inst[1]] self.stage = FLAT hasjrel = misc.Set() for i in dis.hasjrel: hasjrel.add(dis.opname[i]) hasjabs = misc.Set() for i in dis.hasjabs: hasjabs.add(dis.opname[i]) def convertArgs(self): """Convert arguments from symbolic to concrete form""" assert self.stage == FLAT self.consts.insert(0, self.docstring) self.sort_cellvars() for i in range(len(self.insts)): t = self.insts[i] if len(t) == 2: opname, oparg = t conv = self._converters.get(opname, None) if conv: self.insts[i] = opname, conv(self, oparg) self.stage = CONV def sort_cellvars(self): """Sort cellvars in the order of varnames and prune from freevars. """ cells = {} for name in self.cellvars: cells[name] = 1 self.cellvars = [name for name in self.varnames if name in cells] for name in self.cellvars: del cells[name] self.cellvars = self.cellvars + cells.keys() self.closure = self.cellvars + self.freevars def _lookupName(self, name, list): """Return index of name in list, appending if necessary This routine uses a list instead of a dictionary, because a dictionary can't store two different keys if the keys have the same value but different types, e.g. 2 and 2L. The compiler must treat these two separately, so it does an explicit type comparison before comparing the values. """ t = type(name) for i in range(len(list)): if t == type(list[i]) and list[i] == name: return i end = len(list) list.append(name) return end _converters = {} def _convert_LOAD_CONST(self, arg): if hasattr(arg, 'getCode'): arg = arg.getCode() return self._lookupName(arg, self.consts) def _convert_LOAD_FAST(self, arg): self._lookupName(arg, self.names) return self._lookupName(arg, self.varnames) _convert_STORE_FAST = _convert_LOAD_FAST _convert_DELETE_FAST = _convert_LOAD_FAST def _convert_LOAD_NAME(self, arg): if self.klass is None: self._lookupName(arg, self.varnames) return self._lookupName(arg, self.names) def _convert_NAME(self, arg): if self.klass is None: self._lookupName(arg, self.varnames) return self._lookupName(arg, self.names) _convert_STORE_NAME = _convert_NAME _convert_DELETE_NAME = _convert_NAME _convert_IMPORT_NAME = _convert_NAME _convert_IMPORT_FROM = _convert_NAME _convert_STORE_ATTR = _convert_NAME _convert_LOAD_ATTR = _convert_NAME _convert_DELETE_ATTR = _convert_NAME _convert_LOAD_GLOBAL = _convert_NAME _convert_STORE_GLOBAL = _convert_NAME _convert_DELETE_GLOBAL = _convert_NAME def _convert_DEREF(self, arg): self._lookupName(arg, self.names) self._lookupName(arg, self.varnames) return self._lookupName(arg, self.closure) _convert_LOAD_DEREF = _convert_DEREF _convert_STORE_DEREF = _convert_DEREF def _convert_LOAD_CLOSURE(self, arg): self._lookupName(arg, self.varnames) return self._lookupName(arg, self.closure) _cmp = list(dis.cmp_op) def _convert_COMPARE_OP(self, arg): return self._cmp.index(arg) # similarly for other opcodes... for name, obj in locals().items(): if name[:9] == "_convert_": opname = name[9:] _converters[opname] = obj del name, obj, opname def makeByteCode(self): assert self.stage == CONV self.lnotab = lnotab = LineAddrTable() for t in self.insts: opname = t[0] if len(t) == 1: lnotab.addCode(self.opnum[opname]) else: oparg = t[1] if opname == "SET_LINENO": lnotab.nextLine(oparg) continue hi, lo = twobyte(oparg) try: lnotab.addCode(self.opnum[opname], lo, hi) except ValueError: print opname, oparg print self.opnum[opname], lo, hi raise self.stage = DONE opnum = {} for num in range(len(dis.opname)): opnum[dis.opname[num]] = num del num def newCodeObject(self): assert self.stage == DONE if (self.flags & CO_NEWLOCALS) == 0: nlocals = 0 else: nlocals = len(self.varnames) argcount = self.argcount if self.flags & CO_VARKEYWORDS: argcount = argcount - 1 return types.CodeType(argcount, nlocals, self.stacksize, self.flags, self.lnotab.getCode(), self.getConsts(), tuple(self.names), tuple(self.varnames), self.filename, self.name, self.lnotab.firstline, self.lnotab.getTable(), tuple(self.freevars), tuple(self.cellvars)) def getConsts(self): """Return a tuple for the const slot of the code object Must convert references to code (MAKE_FUNCTION) to code objects recursively. """ l = [] for elt in self.consts: if isinstance(elt, PyFlowGraph): elt = elt.getCode() l.append(elt) return tuple(l)
def __init__(self): self.current = self.entry = Block() self.exit = Block("exit") self.blocks = misc.Set() self.blocks.add(self.entry) self.blocks.add(self.exit)
def __init__(self, names=()): self.names = misc.Set() self.globals = misc.Set() for name in names: self.names.add(name)
"""A flow graph representation for Python bytecode"""
class PyFlowGraph(FlowGraph): super_init = FlowGraph.__init__ def __init__(self, name, filename, args=(), optimized=0): self.super_init() self.name = name self.filename = filename self.docstring = None self.args = args # XXX self.argcount = getArgCount(args) if optimized: self.flags = CO_OPTIMIZED | CO_NEWLOCALS else: self.flags = 0 self.consts = [] self.names = [] self.varnames = list(args) or [] for i in range(len(self.varnames)): var = self.varnames[i] if isinstance(var, TupleArg): self.varnames[i] = var.getName() self.stage = RAW def setDocstring(self, doc): self.docstring = doc self.consts.insert(0, doc) def setFlag(self, flag): self.flags = self.flags | flag if flag == CO_VARARGS: self.argcount = self.argcount - 1 def getCode(self): """Get a Python code object""" if self.stage == RAW: self.flattenGraph() if self.stage == FLAT: self.convertArgs() if self.stage == CONV: self.makeByteCode() if self.stage == DONE: return self.newCodeObject() raise RuntimeError, "inconsistent PyFlowGraph state" def dump(self, io=None): if io: save = sys.stdout sys.stdout = io pc = 0 for t in self.insts: opname = t[0] if opname == "SET_LINENO": print if len(t) == 1: print "\t", "%3d" % pc, opname pc = pc + 1 else: print "\t", "%3d" % pc, opname, t[1] pc = pc + 3 if io: sys.stdout = save def flattenGraph(self): """Arrange the blocks in order and resolve jumps""" assert self.stage == RAW self.insts = insts = [] pc = 0 begin = {} end = {} for b in self.getBlocks(): begin[b] = pc for inst in b.getInstructions(): insts.append(inst) if len(inst) == 1: pc = pc + 1 else: # arg takes 2 bytes pc = pc + 3 end[b] = pc pc = 0 for i in range(len(insts)): inst = insts[i] if len(inst) == 1: pc = pc + 1 else: pc = pc + 3 opname = inst[0] if self.hasjrel.has_elt(opname): oparg = inst[1] offset = begin[oparg] - pc insts[i] = opname, offset elif self.hasjabs.has_elt(opname): insts[i] = opname, begin[inst[1]] self.stacksize = findDepth(self.insts) self.stage = FLAT hasjrel = misc.Set() for i in dis.hasjrel: hasjrel.add(dis.opname[i]) hasjabs = misc.Set() for i in dis.hasjabs: hasjabs.add(dis.opname[i]) def convertArgs(self): """Convert arguments from symbolic to concrete form""" assert self.stage == FLAT for i in range(len(self.insts)): t = self.insts[i] if len(t) == 2: opname = t[0] oparg = t[1] conv = self._converters.get(opname, None) if conv: self.insts[i] = opname, conv(self, oparg) self.stage = CONV def _lookupName(self, name, list): """Return index of name in list, appending if necessary""" if name in list: i = list.index(name) # this is cheap, but incorrect in some cases, e.g 2 vs. 2L if type(name) == type(list[i]): return i for i in range(len(list)): elt = list[i] if type(elt) == type(name) and elt == name: return i end = len(list) list.append(name) return end _converters = {} def _convert_LOAD_CONST(self, arg): return self._lookupName(arg, self.consts) def _convert_LOAD_FAST(self, arg): self._lookupName(arg, self.names) return self._lookupName(arg, self.varnames) _convert_STORE_FAST = _convert_LOAD_FAST _convert_DELETE_FAST = _convert_LOAD_FAST def _convert_NAME(self, arg): return self._lookupName(arg, self.names) _convert_LOAD_NAME = _convert_NAME _convert_STORE_NAME = _convert_NAME _convert_DELETE_NAME = _convert_NAME _convert_IMPORT_NAME = _convert_NAME _convert_IMPORT_FROM = _convert_NAME _convert_STORE_ATTR = _convert_NAME _convert_LOAD_ATTR = _convert_NAME _convert_DELETE_ATTR = _convert_NAME _convert_LOAD_GLOBAL = _convert_NAME _convert_STORE_GLOBAL = _convert_NAME _convert_DELETE_GLOBAL = _convert_NAME _cmp = list(dis.cmp_op) def _convert_COMPARE_OP(self, arg): return self._cmp.index(arg) # similarly for other opcodes... for name, obj in locals().items(): if name[:9] == "_convert_": opname = name[9:] _converters[opname] = obj del name, obj, opname def makeByteCode(self): assert self.stage == CONV self.lnotab = lnotab = LineAddrTable() for t in self.insts: opname = t[0] if len(t) == 1: lnotab.addCode(self.opnum[opname]) else: oparg = t[1] if opname == "SET_LINENO": lnotab.nextLine(oparg) hi, lo = twobyte(oparg) try: lnotab.addCode(self.opnum[opname], lo, hi) except ValueError: print opname, oparg print self.opnum[opname], lo, hi raise self.stage = DONE opnum = {} for num in range(len(dis.opname)): opnum[dis.opname[num]] = num del num def newCodeObject(self): assert self.stage == DONE if self.flags == 0: nlocals = 0 else: nlocals = len(self.varnames) argcount = self.argcount if self.flags & CO_VARKEYWORDS: argcount = argcount - 1 return new.code(argcount, nlocals, self.stacksize, self.flags, self.lnotab.getCode(), self.getConsts(), tuple(self.names), tuple(self.varnames), self.filename, self.name, self.lnotab.firstline, self.lnotab.getTable()) def getConsts(self): """Return a tuple for the const slot of the code object Must convert references to code (MAKE_FUNCTION) to code objects recursively. """ l = [] for elt in self.consts: if isinstance(elt, PyFlowGraph): elt = elt.getCode() l.append(elt) return tuple(l)
import imp