def _get_instructions(func, definition_line=None): instrs = [] for ins in get_instructions(func, first_line=None): argval = None if isinstance(ins.argval, str) or isinstance(ins.argval, int): argval = ins.argval fake = FakeBytecodeInstruction(ins.offset, ins.opname, argval, ins.starts_line) instrs.append(fake) return instrs
def print_structured_flow(fn, dom_tree, bb_list): """Print structure skeleton""" print("\n" + ('-' * 40)) bb_num = 0 for bb in bb_list: bb.num = bb_num bb_num += 1 offset2bb_start = {bb.start_offset: bb for bb in bb_list} offset2bb_end = {} for bb in bb_list: if not hasattr(bb, 'reach_offset'): # Dead code continue if bb.reach_offset not in offset2bb_end: offset2bb_end[bb.reach_offset] = [bb] else: offset2bb_end[bb.reach_offset].append(bb) for inst in get_instructions(fn): offset = inst.offset bb_start = offset2bb_start.get(offset, None) if bb_start: for flag in bb_start.flags: if flag == BB_LOOP: print("LOOP") elif flag == BB_FOR: print("FOR") elif flag == BB_FINALLY: print("FINALLY") elif flag == BB_EXCEPT: print("EXCEPT") pass pass pass print(inst.disassemble()) if offset in offset2bb_end: for bb in offset2bb_end[offset]: print("END of block range: BB num: %s" % (bb.num + 1, )) pass
def print_structured_flow(fn, cfg, current, cs_marks): """Print structure skeleton""" print("\n" + ('-' * 40)) for inst in get_instructions(fn): offset = inst.offset remain = [] if offset in cs_marks: for item in cs_marks[offset]: if not item.startswith('end'): print(item.upper()) else: if item == 'end_continue': item = 'CONTINUE' remain.append(item) pass pass pass print(inst.disassemble()) for item in remain: print(item.upper()) pass pass return
def test_get_instructions(): actual = list(dis.get_instructions(TEST_SOURCE_CODE)) actual_len = len(actual) assert actual_len > 0
def basic_blocks(version, is_pypy, fn): """Create a list of basic blocks found in a code object """ BB = BBMgr(version, is_pypy) # Get jump targets jump_targets = set() for inst in get_instructions(fn): op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if op in BB.JUMP_INSTRUCTIONS: if op in BB.JABS_INSTRUCTIONS: jump_offset = inst.arg else: jump_offset = follow_offset + inst.arg jump_targets.add(jump_offset) pass start_offset = 0 end_offset = -1 jump_offsets = set() prev_offset = -1 endloop_offsets = [-1] flags = set([BB_ENTRY]) for inst in get_instructions(fn): prev_offset = end_offset end_offset = inst.offset op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if op == BB.opcode.SETUP_LOOP: jump_offset = follow_offset + inst.arg endloop_offsets.append(jump_offset) elif offset == endloop_offsets[-1]: endloop_offsets.pop() pass if op in BB.LOOP_INSTRUCTIONS: flags.add(BB_LOOP) elif op in BB.BREAK_INSTRUCTIONS: flags.add(BB_BREAK) jump_offsets.add(endloop_offsets[-1]) flags, jump_offsets = BB.add_bb(start_offset, end_offset, follow_offset, flags, jump_offsets) start_offset = follow_offset if offset in jump_targets: # Fallthrough path and jump target path. # This instruction definitely starts a new basic block # Close off any prior basic block if start_offset < end_offset: flags, jump_offsets = BB.add_bb(start_offset, prev_offset, end_offset, flags, jump_offsets) start_offset = end_offset # Add block flags for certain classes of instructions if op in BB.BLOCK_INSTRUCTIONS: flags.add(BB_BLOCK) elif op in BB.EXCEPT_INSTRUCTIONS: flags.add(BB_EXCEPT) elif op in BB.FINALLY_INSTRUCTIONS: flags.add(BB_FINALLY) elif op in BB.FOR_INSTRUCTIONS: flags.add(BB_FOR) elif op in BB.JUMP_INSTRUCTIONS: # Some sort of jump instruction. # While in theory an absolute jump could be part of the # same (extened) basic block, for our purposes we would like to # call them two basic blocks as that probably mirrors # the code more simply. # Figure out where we jump to amd add it to this # basic block's jump offsets. if op in BB.JABS_INSTRUCTIONS: jump_offset = inst.arg else: jump_offset = follow_offset + inst.arg jump_offsets.add(jump_offset) if op in BB.JUMP_UNCONDITONAL: flags.add(BB_JUMP_UNCONDITIONAL) flags, jump_offsets = BB.add_bb(start_offset, end_offset, follow_offset, flags, jump_offsets) start_offset = follow_offset elif op != BB.opcode.SETUP_LOOP: flags, jump_offsets = BB.add_bb(start_offset, end_offset, follow_offset, flags, jump_offsets) start_offset = follow_offset pass elif op in BB.NOFOLLOW_INSTRUCTIONS: flags.add(BB_NOFOLLOW) flags, jump_offsets = BB.add_bb(start_offset, end_offset, follow_offset, flags, jump_offsets) start_offset = follow_offset pass pass if len(BB.bb_list): BB.bb_list[-1].follow_offset = None # Add remaining instructions? if start_offset <= end_offset: BB.bb_list.append( BasicBlock(start_offset, end_offset, None, flags=flags, jump_offsets=jump_offsets)) return BB.bb_list
def test_iteration(self): for obj in [_f, _C(1).__init__, "a=1", _f.__code__]: with self.subTest(obj=obj): via_object = list(dis.Bytecode(obj)) via_generator = list(dis.get_instructions(obj)) self.assertEqual(via_object, via_generator)
def test_doubly_nested(self): with captured_stdout(): inner = outer()() actual = dis.get_instructions(inner, first_line=expected_inner_line) self.assertEqual(list(actual), expected_opinfo_inner)
def test_nested(self): with captured_stdout(): f = outer() actual = dis.get_instructions(f, first_line=expected_f_line) self.assertEqual(list(actual), expected_opinfo_f)
def test_outer(self): actual = dis.get_instructions(outer, first_line=expected_outer_line) self.assertEqual(list(actual), expected_opinfo_outer)
def test_first_line_set_to_None(self): actual = dis.get_instructions(simple, first_line=None) self.assertEqual(list(actual), expected_opinfo_simple)
def test_default_first_line(self): actual = dis.get_instructions(simple) self.assertEqual(list(actual), expected_opinfo_simple)
def basic_blocks(version, is_pypy, fn, first_line=None): """Create a list of basic blocks found in a code object """ BB = BBMgr(version, is_pypy) # Get jump targets jump_targets = set() instructions = list(get_instructions(fn, first_line=first_line)) for inst in instructions: op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if op in BB.JUMP_INSTRUCTIONS: if op in BB.JABS_INSTRUCTIONS: jump_offset = inst.arg else: jump_offset = follow_offset + inst.arg jump_targets.add(jump_offset) pass start_offset = 0 end_offset = -1 jump_offsets = set() prev_offset = -1 endloop_offsets = [-1] flags = set([BB_ENTRY]) end_try_offset_stack = [] try_stack = [] end_try_offset = None loop_offset = None for i, inst in enumerate(instructions): prev_offset = end_offset end_offset = inst.offset op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if offset == end_try_offset: if len(end_try_offset_stack): end_try_offset = end_try_offset_stack[-1] end_try_offset_stack.pop() else: end_try_offset = None if op in BB.LOOP_INSTRUCTIONS: jump_offset = follow_offset + inst.arg endloop_offsets.append(jump_offset) loop_offset = offset elif offset == endloop_offsets[-1]: endloop_offsets.pop() pass if op in BB.LOOP_INSTRUCTIONS: flags.add(BB_LOOP) elif op in BB.BREAK_INSTRUCTIONS: flags.add(BB_BREAK) jump_offsets.add(endloop_offsets[-1]) block, flags, jump_offsets = BB.add_bb(start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) start_offset = follow_offset if offset in jump_targets: # Fallthrough path and jump target path. # This instruction definitely starts a new basic block # Close off any prior basic block if start_offset < end_offset: block, flags, jump_offsets = BB.add_bb(start_offset, prev_offset, loop_offset, end_offset, flags, jump_offsets) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) pass start_offset = end_offset pass # Add block flags for certain classes of instructions if op in BB.JUMP_CONDITONAL: flags.add(BB_JUMP_CONDITIONAL) if op in BB.POP_BLOCK_INSTRUCTIONS: flags.add(BB_POP_BLOCK) if start_offset == offset: flags.add(BB_STARTS_POP_BLOCK) flags.remove(BB_POP_BLOCK) elif op in BB.EXCEPT_INSTRUCTIONS: if (sys.version_info[0:2] <= (2, 7)): # In Python up to 2.7, thre'POP_TOP'S at the beginning of a block # indicate an exception handler. We also check # that we are nested inside a "try". if len(try_stack) == 0 or start_offset != offset: continue pass if (instructions[i+1].opcode != BB.opcode.opmap['POP_TOP'] or instructions[i+2].opcode != BB.opcode.opmap['POP_TOP']): continue flags.add(BB_EXCEPT) try_stack[-1].exception_offsets.add(start_offset) pass elif op in BB.TRY_INSTRUCTIONS: end_try_offset_stack.append(inst.argval) flags.add(BB_TRY) elif op in BB.END_FINALLY_INSTRUCTIONS: flags.add(BB_END_FINALLY) try_stack[-1].exception_offsets.add(start_offset) elif op in BB.FOR_INSTRUCTIONS: flags.add(BB_FOR) jump_offsets.add(inst.argval) block, flags, jump_offsets = BB.add_bb(start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets) loop_offset = None start_offset = follow_offset elif op in BB.JUMP_INSTRUCTIONS: # Some sort of jump instruction. # Figure out where we jump to amd add it to this # basic block's jump offsets. if op in BB.JABS_INSTRUCTIONS: jump_offset = inst.arg else: jump_offset = inst.argval jump_offsets.add(jump_offset) if op in BB.JUMP_UNCONDITONAL: flags.add(BB_JUMP_UNCONDITIONAL) if jump_offset == follow_offset: flags.add(BB_JUMP_TO_FALLTHROUGH) pass block, flags, jump_offsets = BB.add_bb(start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) pass start_offset = follow_offset elif op != BB.opcode.SETUP_LOOP: if op in BB.FINALLY_INSTRUCTIONS: flags.add(BB_FINALLY) block, flags, jump_offsets = BB.add_bb(start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) start_offset = follow_offset pass elif op in BB.NOFOLLOW_INSTRUCTIONS: flags.add(BB_NOFOLLOW) last_block, flags, jump_offsets = BB.add_bb(start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets) loop_offset = None start_offset = follow_offset pass pass if len(BB.bb_list): BB.bb_list[-1].follow_offset = None BB.start_block = BB.bb_list[0] # Add remaining instructions? if start_offset <= end_offset: BB.bb_list.append(BasicBlock(start_offset, end_offset, loop_offset, None, flags=flags, jump_offsets=jump_offsets)) loop_offset = None pass # Add an artificial block where we can link the exits of other blocks # to. This helps in computing reverse dominators. BB.add_bb(end_offset+1, end_offset+1, None, None, set([BB_EXIT]), []) return BB
def augment_instructions(fn, cfg, version_tuple): """Augment instructions in fn with dominator information""" current_block = cfg.entry_node dom_tree = cfg.dom_tree bb2dom_node = {node.bb: node for node in dom_tree.nodes} # block_stack = [current_block] starts = {current_block.start_offset: current_block} dom_reach_ends = {} ends = {current_block.end_offset: current_block} augmented_instrs = [] bb = None dom = None offset = 0 for inst in get_instructions(fn): offset = inst.offset new_bb = starts.get(offset, None) if new_bb: bb = new_bb # FIXME: if a basic block is only its own dominator we don't have that # listed separately new_dom = bb2dom_node.get(bb, dom) if new_dom is not None: dom = new_dom dom_number = dom.bb.number reach_ends = dom_reach_ends.get(dom.reach_offset, []) reach_ends.append(dom) dom_reach_ends[dom.reach_offset] = reach_ends pseudo_inst = ExtendedInstruction( "DOM_START", 1000, "pseudo", 0, dom_number, dom_number, f"Dominator {dom_number}", True, offset, None, False, False, bb, dom, ) augmented_instrs.append(pseudo_inst) pseudo_inst = ExtendedInstruction( "BB_START", 1001, "pseudo", 0, bb.number, bb.number, f"Basic Block {bb.number}", True, offset, None, False, False, bb, dom, ) augmented_instrs.append(pseudo_inst) if bb.follow_offset: follow_bb = cfg.offset2block[bb.follow_offset].bb starts[follow_bb.start_offset] = follow_bb ends[follow_bb.end_offset] = follow_bb for successor_bb in bb.successors: starts[successor_bb.start_offset] = successor_bb ends[successor_bb.end_offset] = successor_bb pass pass elif bb is None: # FIXME: this shouldn't be needed bb = dom.bb extended_inst = ExtendedInstruction( inst.opname, inst.opcode, inst.optype, inst.inst_size, inst.arg, inst.argval, inst.argrepr, inst.has_arg, inst.offset, inst.starts_line, inst.is_jump_target, inst.has_extended_arg, bb, dom, ) augmented_instrs.append(extended_inst) pass bb = ends.get(offset, None) if bb: pseudo_inst = ExtendedInstruction( "BB_END", 1002, "pseudo", 0, bb.number, bb.number, f"Basic Block {bb.number}", True, offset, None, False, False, bb, dom, ) augmented_instrs.append(pseudo_inst) dom_list = dom_reach_ends.get(offset, None) if dom_list is not None: for dom in reversed(dom_list): dom_number = dom.bb.number pseudo_inst = ExtendedInstruction( "DOM_END", 1003, "pseudo", 0, dom_number, dom_number, f"Basic Block {dom_number}", True, offset, None, False, False, dom.bb, dom, ) augmented_instrs.append(pseudo_inst) pass pass # We have a dummy bb at the end+1. # Add the end dominator info for that which should exist if version_tuple >= (3, 6): offset += 2 else: offset += 1 # FIXME: DRY with above dom_list = dom_reach_ends.get(offset, None) if dom_list is not None: for dom in reversed(dom_list): dom_number = dom.bb.number pseudo_inst = ExtendedInstruction( "DOM_END", 1003, "pseudo", 0, dom_number, dom_number, f"Basic Block {dom_number}", True, offset, None, False, False, dom.bb, dom, ) augmented_instrs.append(pseudo_inst) pass # for inst in augmented_instrs: # print(inst) return augmented_instrs
def test_get_instructions(): actual = list(dis.get_instructions(TEST_SOURCE_CODE)) actual_len = len(actual) assert actual_len > 0
def basic_blocks( fn_or_code, version=PYTHON_VERSION_TRIPLE, is_pypy=IS_PYPY, more_precise_returns=False, print_instructions=False, ): """Create a list of basic blocks found in a code object. `more_precise_returns` indicates whether the RETURN_VALUE should modeled as a jump to the end of the enclosing function or not. See comment in code as to why this might be useful. """ BB = BBMgr(version, is_pypy) # Get jump targets jump_targets = set() instructions = list(get_instructions(fn_or_code)) for inst in instructions: op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if op in BB.JUMP_INSTRUCTIONS: jump_value = get_jump_val(inst.arg, version) if op in BB.JABS_INSTRUCTIONS: jump_offset = jump_value else: jump_offset = follow_offset + jump_value jump_targets.add(jump_offset) pass # Add an artificial block where we can link the exits of other blocks # to. This helps when there is a "raise" not in any try block and # in computing reverse dominators. end_offset = instructions[-1].offset if version >= (3, 6): end_bb_offset = end_offset + 2 else: end_bb_offset = end_offset + 1 end_block, _, _ = BB.add_bb( end_bb_offset, end_bb_offset, None, None, set([BB_EXIT]), [] ) start_offset = 0 end_offset = -1 jump_offsets = set() prev_offset = -1 endloop_offsets = [-1] flags = set([BB_ENTRY]) end_try_offset_stack = [] try_stack = [end_block] end_try_offset = None loop_offset = None return_blocks = [] for i, inst in enumerate(instructions): if print_instructions: print(inst) prev_offset = end_offset end_offset = inst.offset op = inst.opcode offset = inst.offset follow_offset = next_offset(op, BB.opcode, offset) if offset == end_try_offset: if len(end_try_offset_stack): end_try_offset = end_try_offset_stack[-1] end_try_offset_stack.pop() else: end_try_offset = None if op in BB.LOOP_INSTRUCTIONS: jump_offset = follow_offset + inst.arg endloop_offsets.append(jump_offset) loop_offset = offset elif offset == endloop_offsets[-1]: endloop_offsets.pop() pass if op in BB.LOOP_INSTRUCTIONS: flags.add(BB_LOOP) elif op in BB.BREAK_INSTRUCTIONS: flags.add(BB_BREAK) jump_offsets.add(endloop_offsets[-1]) block, flags, jump_offsets = BB.add_bb( start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets, ) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) start_offset = follow_offset if offset in jump_targets: # Fallthrough path and jump target path. # This instruction definitely starts a new basic block # Close off any prior basic block if start_offset < end_offset: block, flags, jump_offsets = BB.add_bb( start_offset, prev_offset, loop_offset, end_offset, flags, jump_offsets, ) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) pass start_offset = end_offset pass # Add block flags for certain classes of instructions if op in BB.JUMP_CONDITONAL: flags.add(BB_JUMP_CONDITIONAL) if op in BB.POP_BLOCK_INSTRUCTIONS: flags.add(BB_POP_BLOCK) if start_offset == offset: flags.add(BB_STARTS_POP_BLOCK) flags.remove(BB_POP_BLOCK) elif op in BB.EXCEPT_INSTRUCTIONS: if sys.version_info[0:2] <= (2, 7): # In Python up to 2.7, thre'POP_TOP'S at the beginning of a block # indicate an exception handler. We also check # that we are nested inside a "try". if len(try_stack) == 0 or start_offset != offset: continue pass if ( instructions[i + 1].opcode != BB.opcode.opmap["POP_TOP"] or instructions[i + 2].opcode != BB.opcode.opmap["POP_TOP"] ): continue flags.add(BB_EXCEPT) try_stack[-1].exception_offsets.add(start_offset) pass elif op in BB.TRY_INSTRUCTIONS: end_try_offset_stack.append(inst.argval) flags.add(BB_TRY) elif op in BB.END_FINALLY_INSTRUCTIONS: flags.add(BB_END_FINALLY) try_stack[-1].exception_offsets.add(start_offset) elif op in BB.FOR_INSTRUCTIONS: flags.add(BB_FOR) jump_offsets.add(inst.argval) block, flags, jump_offsets = BB.add_bb( start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets, ) loop_offset = None start_offset = follow_offset elif op in BB.JUMP_INSTRUCTIONS: # Some sort of jump instruction. # Figure out where we jump to amd add it to this # basic block's jump offsets. jump_offset = inst.argval jump_offsets.add(jump_offset) if op in BB.JUMP_UNCONDITONAL: flags.add(BB_JUMP_UNCONDITIONAL) if jump_offset == follow_offset: flags.add(BB_JUMP_TO_FALLTHROUGH) pass block, flags, jump_offsets = BB.add_bb( start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets, ) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) pass start_offset = follow_offset elif version[:2] >= (3, 9) or ( version[:2] < (3, 8) and op != BB.opcode.SETUP_LOOP ): if op in BB.FINALLY_INSTRUCTIONS: flags.add(BB_FINALLY) block, flags, jump_offsets = BB.add_bb( start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets, ) loop_offset = None if BB_TRY in block.flags: try_stack.append(block) start_offset = follow_offset pass elif op in BB.NOFOLLOW_INSTRUCTIONS: flags.add(BB_NOFOLLOW) if op in BB.RETURN_INSTRUCTIONS: flags.add(BB_RETURN) last_block, flags, jump_offsets = BB.add_bb( start_offset, end_offset, loop_offset, follow_offset, flags, jump_offsets, ) loop_offset = None start_offset = follow_offset if op in BB.RETURN_INSTRUCTIONS: return_blocks.append(last_block) pass pass # If the bytecode comes from Python, then there is possibly an # advantage in treating a return in a block as an instruction # which flows to the next instruction, since that will treat # blocks with unreachable instructions the way Python source # does - the code after that exists. # # However if you care about analysis, then # Hook RETURN_VALUE instructions to the exit block offset if more_precise_returns: for block in return_blocks: block.jump_offsets.add(end_bb_offset) block.edge_count += 1 if len(BB.bb_list): BB.bb_list[-1].follow_offset = None BB.start_block = BB.bb_list[0] # Add remaining instructions? if start_offset <= end_offset: BB.bb_list.append( BasicBlock( start_offset, end_offset, loop_offset, None, flags=flags, jump_offsets=jump_offsets, ) ) loop_offset = None pass return BB