def ROT_TWO(self, instr): one = self.ast_stack.pop() two = self.ast_stack.pop() if self.ilst[0].opname == 'STORE_NAME': kw = dict(lineno=instr.lineno, col_offset=0) stores = [] while self.ilst[0].opname == 'STORE_NAME': stores.append(self.ilst.pop(0)) assert len(stores) <= 3, stores elts_load = [one, two] if len(stores) == 3: elts_load.insert(0, self.ast_stack.pop()) tup_load = _ast.Tuple(elts=elts_load[::-1], ctx=_ast.Load(), **kw) elts_store = [ _ast.Name(id=store.arg, ctx=_ast.Store(), **kw) for store in stores ] tup_store = _ast.Tuple(elts=elts_store, ctx=_ast.Store(), **kw) assgn = _ast.Assign(value=tup_load, targets=[tup_store], **kw) self.ast_stack.append(assgn) # self.ast_stack.append(tup_store) else: self.ast_stack.append(one) self.ast_stack.append(two)
def make_assign_unpack(i, bytecode, unpack_num=-1): if unpack_num < 1: logger.error("Could not find the number of unpacked items. ") return i, None store_exprs = [] value_exprs = [] store_state, value_state = True, False while i >= 0: op, arg = bytecode[i][2], bytecode[i][3] if store_state: if op == UNPACK_SEQUENCE: store_state = False prev_op = bytecode[i - 1][2] if i > 0 else -1 if prev_op == BUILD_TUPLE: value_state = True else: i, value_exprs = Statement.make_expr(i - 1, bytecode) break elif op in STORE_OPCODES: i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store()) store_exprs.insert(0, store_stmt) elif value_state: i, value_stmt = Statement.make_expr(i, bytecode) value_exprs.insert(0, value_stmt) i -= 1 store_exprs = _ast.Tuple(store_exprs, _ast.Store()) if not isinstance(value_exprs, _ast.AST): value_exprs = _ast.Tuple(value_exprs, _ast.Load()) return i, _ast.Assign([store_exprs], value_exprs)
def STORE_NAME(self, instr): value = self.ast_stack.pop() value = self.process_ifexpr(value) if isinstance(value, _ast.Import): if value.from_: assert isinstance(self.ast_stack[-1], _ast.ImportFrom) from_ = self.ast_stack.pop() as_name = instr.arg name = from_.names[0].name if as_name != name: from_.names[0].asname = as_name self.ast_stack.append(from_) else: as_name = instr.arg if value.names[0].asname is None: base_name = value.names[0].name.split('.')[0] if base_name != as_name: value.names[0].asname = as_name self.ast_stack.append(value) elif isinstance(value, (_ast.Attribute)) and isinstance( value.value, (_ast.Import)): asname = instr.arg value = value.value value.names[0].asname = asname self.ast_stack.append(value) elif isinstance(value, (_ast.ClassDef, _ast.FunctionDef)): as_name = instr.arg value.name = as_name self.ast_stack.append(value) elif isinstance(value, _ast.AugAssign): self.ast_stack.append(value) elif isinstance(value, _ast.Assign): _ = self.ast_stack.pop() assname = _ast.Name(instr.arg, _ast.Store(), lineno=instr.lineno, col_offset=0) value.targets.append(assname) self.ast_stack.append(value) else: assname = _ast.Name(instr.arg, _ast.Store(), lineno=instr.lineno, col_offset=0) assign = _ast.Assign(targets=[assname], value=value, lineno=instr.lineno, col_offset=0) self.ast_stack.append(assign)
def make_assign_opt_unpack(i, bytecode): store_exprs = [] value_exprs = [] store_state, value_state = True, False while i >= 0: op, arg = bytecode[i][2], bytecode[i][3] if store_state: if op == ROT_TWO: prev_op = bytecode[i - 1][2] if i > 0 else -1 if prev_op == ROT_THREE: i -= 1 value_state = True store_state = False elif op in STORE_OPCODES: i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store()) store_exprs.insert(0, store_stmt) elif value_state: i, value_stmt = Statement.make_expr(i, bytecode) value_exprs.insert(0, value_stmt) i -= 1 store_exprs = _ast.Tuple(store_exprs, _ast.Store()) if not isinstance(value_exprs, _ast.AST): value_exprs = _ast.Tuple(value_exprs, _ast.Load()) return i, _ast.Assign([store_exprs], value_exprs)
def UNPACK_SEQUENCE(self, instr): nargs = instr.oparg nodes = [] ast_tuple = _ast.Tuple(elts=nodes, ctx=_ast.Store(), lineno=instr.lineno, col_offset=0) for i in range(nargs): nex_instr = self.ilst.pop(0) self.ast_stack.append(None) self.visit(nex_instr) node = self.ast_stack.pop() nodes.append(node.targets[0]) expr = self.ast_stack.pop() if isinstance(expr, _ast.Assign): assgn = expr assgn.targets.append(ast_tuple) value_dup = self.ast_stack.pop() assert cmp_ast(assgn.value, value_dup) else: assgn = _ast.Assign(targets=[ast_tuple], value=expr, lineno=instr.lineno, col_offset=0) self.ast_stack.append(assgn)
def test_constructor(self): ast = self.ast body = [] mod = ast.Module(body) assert mod.body is body target = ast.Name("hi", ast.Store()) expr = ast.Name("apples", ast.Load()) otherwise = [] fr = ast.For(target, expr, body, otherwise, lineno=0, col_offset=1) assert fr.target is target assert fr.iter is expr assert fr.orelse is otherwise assert fr.body is body assert fr.lineno == 0 assert fr.col_offset == 1 fr = ast.For(body=body, target=target, iter=expr, col_offset=1, lineno=0, orelse=otherwise) assert fr.target is target assert fr.iter is expr assert fr.orelse is otherwise assert fr.body is body assert fr.lineno == 0 assert fr.col_offset == 1 exc = raises(TypeError, ast.Module, 1, 2).value msg = str(exc) assert msg == "Module constructor takes either 0 or 1 positional argument" ast.Module(nothing=23)
def make_subscript(i, bytecode, context=None): op = bytecode[i][2] if op == STORE_SUBSCR: # TOS1[TOS] = TOS2 i, index_expr = Statement.make_expr(i - 1, bytecode) i, arr_expr = Statement.make_expr(i - 1, bytecode, context=_ast.Store()) i, rhs_expr = Statement.make_expr(i - 1, bytecode) lhs_expr = _ast.Subscript(arr_expr, index_expr, _ast.Store()) return i, _ast.Assign([lhs_expr], rhs_expr) else: if context is None: context = _ast.Load() # BINARY_SUBSCR: TOS1[TOS] and DELETE_SUBSCR TOS1[TOS] i, index_expr = Statement.make_expr(i - 1, bytecode) i, arr_expr = Statement.make_expr(i - 1, bytecode) return i, _ast.Subscript(arr_expr, index_expr, context)
def INPLACE_OP(self, instr): right = self.ast_stack.pop() left = self.ast_stack.pop() left.ctx = _ast.Store() aug_assign = _ast.AugAssign(target=left, op=OP(), value=right, lineno=instr.lineno, col_offset=0) self.ast_stack.append(aug_assign)
def test_list_syncing(self): ast = self.ast mod = ast.Module([ast.Lt()]) raises(TypeError, compile, mod, "<string>", "exec") mod = self.get_ast("x = y = 3") assign = mod.body[0] assert len(assign.targets) == 2 assign.targets[1] = ast.Name("lemon", ast.Store(), lineno=0, col_offset=0) name = ast.Name("apple", ast.Store(), lineno=0, col_offset=0) mod.body.append(ast.Assign([name], ast.Num(4, lineno=0, col_offset=0), lineno=0, col_offset=0)) co = compile(mod, "<test>", "exec") ns = {} exec co in ns assert "y" not in ns assert ns["x"] == ns["lemon"] == 3 assert ns["apple"] == 4
def GetCtx(ctx_type): """Creates Load, Store, Del, and Param, used in the ctx kwarg.""" if ctx_type == CtxEnum.LOAD: return _ast.Load() elif ctx_type == CtxEnum.STORE: return _ast.Store() elif ctx_type == CtxEnum.DEL: return _ast.Del() elif ctx_type == CtxEnum.PARAM: return _ast.Param() raise InvalidCtx('ctx_type {} isn\'t a valid type'.format(ctx_type))
def STORE_SLICE_0(self, instr): 'obj[:] = expr' value = self.ast_stack.pop() expr = self.ast_stack.pop() kw = dict(lineno=instr.lineno, col_offset=0) slice = _ast.Slice(lower=None, step=None, upper=None, **kw) subscr = _ast.Subscript(value=value, slice=slice, ctx=_ast.Store(), **kw) assign = _ast.Assign(targets=[subscr], value=expr, **kw) self.ast_stack.append(assign)
def STORE_ATTR(self, instr): attrname = instr.arg node = self.ast_stack.pop() expr = self.ast_stack.pop() expr = self.process_ifexpr(expr) assattr = _ast.Attribute(value=node, attr=attrname, ctx=_ast.Store(), lineno=instr.lineno, col_offset=0) set_attr = _ast.Assign(targets=[assattr], value=expr, lineno=instr.lineno, col_offset=0) self.ast_stack.append(set_attr)
def INPLACE_OP(self, instr): right = self.pop_ast_item() left = self.pop_ast_item() left.ctx = _ast.Store() aug_assign = _ast.AugAssign(target=left, op=OP(), value=right, lineno=instr.lineno, col_offset=0) self.push_ast_item(aug_assign)
def make_store_delete_slice(i, bytecode, context=None): op = bytecode[i][2] is_delete = op in DELETE_SLICE_OPCODES if context is None: context = _ast.Store() if not is_delete else _ast.Del() lhs_expr = None if op in (STORE_SLICE_0, DELETE_SLICE_0): i, lhs_expr = Statement.make_expr(i - 1, bytecode, context=context) lhs_expr = _ast.Subscript(lhs_expr, _ast.Slice(None, None, None), _ast.Store()) elif op in (STORE_SLICE_1, STORE_SLICE_2, DELETE_SLICE_1, DELETE_SLICE_2): i, index_expr = Statement.make_expr(i - 1, bytecode) i, arr_expr = Statement.make_expr(i - 1, bytecode, context=context) args = [None] * 3 index_index = 0 if op in (STORE_SLICE_1, DELETE_SLICE_1) else 1 args[index_index] = index_expr lhs_expr = _ast.Subscript(arr_expr, _ast.Slice(*args), _ast.Store()) else: i, end_index_expr = Statement.make_expr(i - 1, bytecode) i, start_index_expr = Statement.make_expr(i - 1, bytecode) i, arr_expr = Statement.make_expr(i - 1, bytecode, context=context) lhs_expr = _ast.Subscript(arr_expr, _ast.Slice(start_index_expr, end_index_expr, None), _ast.Store()) if is_delete: return i, _ast.Delete([lhs_expr]) else: i, rhs_expr = Statement.make_expr(i - 1, bytecode) return i, _ast.Assign([lhs_expr], rhs_expr)
def make_assign_chained(i, bytecode): store_exprs = [] value_exprs = [] store_state, value_state = True, False while i >= 0: op, arg = bytecode[i][2], bytecode[i][3] if store_state: if op == DUP_TOP: prev_op = bytecode[i - 1][2] if i > 0 else -1 if prev_op not in STORE_OPCODES: value_state = True store_state = False elif op in STORE_OPCODES: i, store_stmt = Statement.make_expr(i, bytecode, context=_ast.Store()) store_exprs.insert(0, store_stmt) elif value_state: i, value_exprs = Statement.make_expr(i, bytecode) break i -= 1 store_exprs = _ast.Tuple(store_exprs, _ast.Store()) return i, _ast.Assign([store_exprs], value_exprs)
def STORE_SLICE_2(self, instr): 'obj[:upper] = expr' upper = self.pop_ast_item() value = self.pop_ast_item() expr = self.pop_ast_item() kw = dict(lineno=instr.lineno, col_offset=0) slice = _ast.Slice(lower=None, step=None, upper=upper, **kw) subscr = _ast.Subscript(value=value, slice=slice, ctx=_ast.Store(), **kw) assign = _ast.Assign(targets=[subscr], value=expr, **kw) self.push_ast_item(assign)
def visit_Expr(self, expression_node): value = expression_node.value if not isinstance(value, _ast.BinOp) or not (isinstance( value.op, _ast.RShift) or isinstance(value.op, _ast.LShift)): return expression_node mock_attribute = 'return_value' if isinstance(value.op, _ast.LShift): mock_attribute = 'side_effect' return _ast.Assign(targets=[ _ast.Attribute(attr=mock_attribute, ctx=_ast.Store(), value=value.left.func) ], value=value.right)
def _assign_list_form_variables(self, where_function_ast_node): copy_of_body = copy.deepcopy(where_function_ast_node.body) where_function_ast_node.body = [] for assignment_expression in copy_of_body: variable_name = assignment_expression.targets[0].id self.spec_metadata.add_feature_variable(self.feature_name, variable_name) variable_values = assignment_expression.value where_function_ast_node.body.append( _ast.Assign(targets=[ _ast.Subscript( value=_ast.Name(id='injectable_values', ctx=_ast.Load()), slice=_ast.Index(value=_ast.Str(s=variable_name)), ctx=_ast.Store()) ], value=variable_values))
def STORE_SUBSCR(self, instr): index = self.ast_stack.pop() value = self.ast_stack.pop() expr = self.ast_stack.pop() expr = self.process_ifexpr(expr) if isinstance(expr, _ast.AugAssign): self.ast_stack.append(expr) else: kw = dict(lineno=instr.lineno, col_offset=0) index = self.format_slice(index, kw) subscr = _ast.Subscript(value=value, slice=index, ctx=_ast.Store(), **kw) assign = _ast.Assign(targets=[subscr], value=expr, **kw) self.ast_stack.append(assign)
def _assign_matrix_form_variables(self, where_function_ast_node): copy_of_body = copy.deepcopy(where_function_ast_node.body) where_function_ast_node.body = [] variables_and_values = WhereBlockFunctions._get_variables_and_values(copy_of_body) # We might be screwing with line numbers here for variable_name, variable_values in variables_and_values.items(): self.spec_metadata.add_feature_variable(self.feature_name, variable_name) where_function_ast_node.body.append( _ast.Assign( targets=[ _ast.Subscript( value=_ast.Name(id='injectable_values', ctx=_ast.Load()), slice=_ast.Index(value=ast_proxy.ast_str(s=variable_name)), ctx=_ast.Store() ) ], value=_ast.List(elts=variable_values, ctx=_ast.Load()) ))
def test_bug_null_in_objspace_type(self): import ast code = ast.Expression( lineno=1, col_offset=1, body=ast.ListComp( lineno=1, col_offset=1, elt=ast.Call(lineno=1, col_offset=1, func=ast.Name(lineno=1, col_offset=1, id='str', ctx=ast.Load(lineno=1, col_offset=1)), args=[ ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Load(lineno=1, col_offset=1)) ], keywords=[]), generators=[ ast.comprehension( lineno=1, col_offset=1, target=ast.Name(lineno=1, col_offset=1, id='x', ctx=ast.Store(lineno=1, col_offset=1)), iter=ast.List( lineno=1, col_offset=1, elts=[ast.Num(lineno=1, col_offset=1, n=23)], ctx=ast.Load( lineno=1, col_offset=1, )), ifs=[]) ])) compile(code, '<template>', 'eval')
def STORE_SLICE_3(self, instr): 'obj[lower:upper] = expr' upper = self.ast_stack.pop() lower = self.ast_stack.pop() value = self.ast_stack.pop() expr = self.ast_stack.pop() kw = dict(lineno=instr.lineno, col_offset=0) slice = _ast.Slice(lower=lower, step=None, upper=upper, **kw) subscr = _ast.Subscript(value=value, slice=slice, ctx=_ast.Store(), **kw) if isinstance(expr, _ast.AugAssign): assign = expr result = cmp_ast(expr.target, subscr) assert result else: assign = _ast.Assign(targets=[subscr], value=expr, **kw) self.ast_stack.append(assign)
def _cfa(body, state, on_gen): assert isinstance(on_gen, list) for c in on_gen: assert callable(c) cfg = state.cfg def make_connect(cur_bid): assert isinstance(cur_bid, int) def inner(new_bid): assert isinstance(new_bid, int) state.connect([cur_bid], [new_bid]) return inner def push_block(block): block_id = len(cfg.blocks) assert block_id not in cfg.blocks cfg.blocks[block_id] = block for c in on_gen: c(block_id) return block_id cur_block = [] for b in body: if REDUCE_FORS_TO_WHILES and isinstance(b, _ast.For): if isinstance(b.iter, _ast.Call) and isinstance( b.iter.func, _ast.Name) and b.iter.func.id in ("range", "xrange"): if not b.iter.keywords and not b.iter.starargs and not b.iter.kwargs: end_var = "__wfend_%d_%d_" % (b.lineno, b.col_offset) iter_var = "__wfiter_%d_%d_" % (b.lineno, b.col_offset) if len(b.iter.args) in (1, 2): if len(b.iter.args) == 1: start = _ast.Num(0) end = b.iter.args[0] elif len(b.iter.args) == 2: start = b.iter.args[0] end = b.iter.args[1] else: start = b.iter.args[0] end = b.iter.args[1] cur_block.append( _ast.Assign([ _ast.Name( iter_var, _ast.Store(), not_real=True) ], start, lineno=b.lineno, col_offset=b.col_offset, not_real=True)) cur_block.append( _ast.Assign([ _ast.Name(end_var, _ast.Store(), not_real=True) ], end, lineno=b.lineno, col_offset=b.col_offset, not_real=True)) body = [ _ast.Assign([b.target], _ast.Name(iter_var, _ast.Load(), not_real=True), lineno=b.lineno, col_offset=b.col_offset, not_real=True), _ast.Assign([ _ast.Name( iter_var, _ast.Store(), not_real=True) ], _ast.BinOp( _ast.Name(iter_var, _ast.Load(), not_real=True), _ast.Add(), _ast.Num(1)), lineno=b.lineno, col_offset=b.col_offset, not_real=True) ] + b.body b = _ast.While(_ast.Compare( _ast.Name(iter_var, _ast.Load(), not_real=True), [_ast.Lt()], [_ast.Name(end_var, _ast.Load(), not_real=True)], lineno=b.lineno, col_offset=b.col_offset, not_real=True), body, b.orelse, not_real=True) if isinstance(b, ( _ast.Assign, _ast.AugAssign, _ast.ClassDef, _ast.Delete, _ast.Exec, _ast.Expr, _ast.FunctionDef, _ast.Global, _ast.Import, _ast.ImportFrom, _ast.Print, _ast.Pass, )): cur_block.append(b) elif isinstance(b, _ast.Assert): cur_block.append(b) if isinstance(b.test, _ast.Call) and isinstance( b.test.func, _ast.Name ) and b.test.func.id == "isinstance" and isinstance( b.test.args[0], _ast.Name) and isinstance( b.test.args[1], _ast.Name): varname = b.test.args[0].id cast = _ast.Call(_ast.Name( "__cast__", _ast.Load(), not_real=True, **pos(b)), [ _ast.Name( varname, _ast.Store(), not_real=True, **pos(b)), b.test.args[1] ], [], None, None, not_real=True, **pos(b)) assign = _ast.Assign([ _ast.Name(varname, _ast.Store(), not_real=True, **pos(b)) ], cast, not_real=True, lineno=b.lineno, col_offset=b.col_offset) cur_block.append(assign) elif isinstance(b, (_ast.Break, _ast.Continue)): f = state.add_break if isinstance( b, _ast.Break) else state.add_continue if cur_block: j = Jump() cur_block.append(j) block_id = push_block(cur_block) f(j.set_dest) f(make_connect(block_id)) else: for c in on_gen: f(c) return [] elif isinstance(b, _ast.If): br = Branch(b.test, lineno=b.lineno) cur_block.append(br) next_block = push_block(cur_block) on_gen = None # make sure this doesn't get used cur_block = [] gen_true = [br.set_true, make_connect(next_block)] gen_false = [br.set_false, make_connect(next_block)] if ENFORCE_NO_MULTIMULTI: on_gen = gen_true j1 = Jump() gen_true = [make_connect(push_block([j1])), j1.set_dest] on_gen = gen_false j2 = Jump() gen_false = [make_connect(push_block([j2])), j2.set_dest] on_gen = None assert b.body body = b.body if isinstance(b.test, _ast.Call) and isinstance( b.test.func, _ast.Name ) and b.test.func.id == "isinstance" and isinstance( b.test.args[0], _ast.Name) and isinstance( b.test.args[1], _ast.Name): varname = b.test.args[0].id cast = _ast.Call(_ast.Name( "__cast__", _ast.Load(), not_real=True, **pos(b)), [ _ast.Name( varname, _ast.Store(), not_real=True, **pos(b)), b.test.args[1] ], [], None, None, not_real=True, **pos(b)) assign = _ast.Assign([ _ast.Name(varname, _ast.Store(), not_real=True, **pos(b)) ], cast, not_real=True, lineno=b.lineno, col_offset=b.col_offset) body = [assign] + body if ADD_IF_ASSERTS: body = [_ast.Assert(b.test, None, not_real=True, **pos(b)) ] + body ending_gen = _cfa(body, state, gen_true) if b.orelse: ending_gen += _cfa(b.orelse, state, gen_false) else: ending_gen += gen_false on_gen = ending_gen if not on_gen and PRUNE_UNREACHABLE_BLOCKS: return [] elif isinstance(b, _ast.TryExcept): j = Jump() cur_block.append(j) next_block = push_block(cur_block) on_gen = [j.set_dest, make_connect(next_block)] cur_block = [] on_gen = _cfa(b.body, state, on_gen) # Set this to evaluate a string to try to defeat simple flow analysis br = Branch(_ast.Str("nonzero")) next_block = push_block([br]) on_except = [br.set_false, make_connect(next_block)] on_fine = [br.set_true, make_connect(next_block)] assert len(b.handlers) >= 1 # for handler in b.handlers: # on_except = _cfa(b.handlers[0].body, state, on_except) if b.orelse: on_fine = _cfa(b.orelse, state, on_fine) if ENFORCE_NO_MULTIMULTI: j = Jump() on_gen = on_fine next_block = push_block([j]) on_fine = [j.set_dest, make_connect(next_block)] j = Jump() on_gen = on_except next_block = push_block([j]) on_except = [j.set_dest, make_connect(next_block)] on_gen = on_fine + on_except cur_block = [] elif isinstance(b, _ast.TryFinally): j = Jump() cur_block.append(j) next_block = push_block(cur_block) on_gen = [j.set_dest, make_connect(next_block)] cur_block = [] on_gen = _cfa(b.body, state, on_gen) on_gen = _cfa(b.finalbody, state, on_gen) elif isinstance(b, _ast.While): # This could also be architected as having no extra block and having two jump statements, but I don't like that if cur_block: j = Jump() cur_block.append(j) on_gen = [make_connect(push_block(cur_block)), j.set_dest] cur_block = [] always_true = False always_false = False if isinstance(b.test, _ast.Name): if b.test.id == "True": always_true = True elif b.test.id == "False": always_false = True elif isinstance(b.test, _ast.Num): if b.test.n: always_true = True else: always_false = True if always_true: br = Jump() on_true = br.set_dest elif always_false: br = Jump() on_false = br.set_dest else: br = Branch(b.test) on_true = br.set_true on_false = br.set_false init_id = push_block([br]) on_gen = None assert cur_block == [] # just checking if not always_false: gen_true = [on_true, make_connect(init_id)] if not always_true: gen_false = [on_false, make_connect(init_id)] if ENFORCE_NO_MULTIMULTI: if not always_false: on_gen = gen_true j1 = Jump() gen_true = [make_connect(push_block([j1])), j1.set_dest] if not always_true: on_gen = gen_false j2 = Jump() gen_false = [make_connect(push_block([j2])), j2.set_dest] on_gen = None ending_gen = [] if not always_false: state.push_loop() assert b.body loop_ending_gen = _cfa(b.body, state, gen_true) loop_ending_gen += state.get_continues() for c in loop_ending_gen: c(init_id) ending_gen = state.get_breaks() state.pop_loop() if not always_true: if b.orelse: ending_gen += _cfa(b.orelse, state, gen_false) else: ending_gen += gen_false on_gen = ending_gen if not on_gen and PRUNE_UNREACHABLE_BLOCKS: return [] elif isinstance(b, _ast.For): iter_func = _ast.Attribute(b.iter, "__iter__", _ast.Load(), not_real=True, lineno=b.lineno, col_offset=b.col_offset) iter_call = _ast.Call(iter_func, [], [], None, None, not_real=True, lineno=b.lineno, col_offset=b.col_offset) # iter_var = _make_temp_name() iter_var = "__foriter_%d_%d_" % (b.lineno, b.col_offset) iter_assign = _ast.Assign( [_ast.Name(iter_var, _ast.Store(), not_real=True, **pos(b))], iter_call, not_real=True, lineno=b.lineno, col_offset=b.col_offset) cur_block.append(iter_assign) j = Jump() cur_block.append(j) on_gen = [make_connect(push_block(cur_block)), j.set_dest] cur_block = [] br = Branch( HasNext( _ast.Name(iter_var, _ast.Load(), not_real=True, **pos(b)), **pos(b)), **pos(b)) init_id = push_block([br]) on_gen = None assert cur_block == [] # just checking gen_true = [br.set_true, make_connect(init_id)] gen_false = [br.set_false, make_connect(init_id)] if ENFORCE_NO_MULTIMULTI: on_gen = gen_true j1 = Jump() gen_true = [make_connect(push_block([j1])), j1.set_dest] on_gen = gen_false j2 = Jump() gen_false = [make_connect(push_block([j2])), j2.set_dest] on_gen = None ending_gen = [] state.push_loop() next_func = _ast.Attribute(_ast.Name(iter_var, _ast.Load(), not_real=True, **pos(b)), "next", _ast.Load(), not_real=True, lineno=b.lineno, col_offset=b.col_offset) next = _ast.Call(next_func, [], [], None, None, not_real=True, lineno=b.lineno, col_offset=b.col_offset) next_assign = _ast.Assign([b.target], next, not_real=True, lineno=b.lineno, col_offset=b.col_offset) next_iter_gen = _cfa([next_assign] + b.body, state, gen_true) next_iter_gen += state.get_continues() for c in next_iter_gen: c(init_id) loop_done_gen = list(state.get_breaks()) state.pop_loop() if b.orelse: # if b.orelse and loop_ending_blocks: loop_done_gen += _cfa(b.orelse, state, gen_false) else: loop_done_gen += gen_false on_gen = loop_done_gen if not on_gen and PRUNE_UNREACHABLE_BLOCKS: return [] elif isinstance(b, (_ast.Return, _ast.Raise)): cur_block.append(b) block_id = push_block(cur_block) state.returns.append(make_connect(block_id)) return [] elif isinstance(b, _ast.With): # XXX totally ignores the functionality of with statements # Have to save the context manager because the expression might not be valid later mgr_name = "__mgr_%s_%s_" % (b.lineno, b.col_offset) save_mgr = _ast.Assign([ _ast.Name(mgr_name, _ast.Store(), lineno=b.lineno, col_offset=b.col_offset, not_real=True) ], b.context_expr, lineno=b.lineno, col_offset=b.col_offset, not_real=True) enter_func = _ast.Attribute(_ast.Name(mgr_name, _ast.Load(), lineno=b.lineno, col_offset=b.col_offset, not_real=True), "__enter__", _ast.Load(), lineno=b.lineno, col_offset=b.col_offset, not_real=True) bind = _ast.Call(enter_func, [], [], None, None, lineno=b.lineno, col_offset="__enter__()", not_real=True) if b.optional_vars: assert isinstance(b.optional_vars, _ast.AST) init = _ast.Assign([b.optional_vars], bind, lineno=b.lineno, col_offset=b.col_offset, not_real=True) else: init = _ast.Expr(bind, lineno=b.lineno, col_offset=b.col_offset, not_real=True) exit_func = _ast.Attribute(_ast.Name(mgr_name, _ast.Load(), lineno=b.lineno, col_offset=b.col_offset, not_real=True), "__exit__", _ast.Load(), lineno=b.lineno, col_offset=b.col_offset, not_real=True) if SIMPLE_WITH_EXIT: exit_call = _ast.Call(exit_func, [], [], None, None, lineno=b.lineno, col_offset=b.col_offset, not_real=True) else: none_ = _ast.Name("None", _ast.Load(), lineno=b.lineno, col_offset=b.col_offset, not_real=True) exit_call = _ast.Call(exit_func, [none_, none_, none_], [], None, None, lineno=b.lineno, col_offset=b.col_offset, not_real=True) exit = _ast.Expr(exit_call, lineno=b.lineno, col_offset="__exit__()", not_real=True) cur_block.extend([save_mgr, init]) j = Jump() cur_block.append(j) next_block = push_block(cur_block) on_gen = [j.set_dest, make_connect(next_block)] cur_block = [] body = b.body + [exit] next_gen = _cfa(body, state, on_gen) on_gen = next_gen else: raise Exception(b) if cur_block: j = Jump() cur_block.append(j) next = push_block(cur_block) return [j.set_dest, make_connect(next)] return on_gen return on_gen
def ast_wrap_in_assn(var_name, ast_expr): assn = _ast.Assign(targets=[_ast.Name(id=var_name, ctx=_ast.Store())], value=ast_expr.value) ast.fix_missing_locations(assn) return assn
def visit_GeneratorExp(self, node): """Rewrite the GeneratorExp visitor function to turn the generator expression into a iterator function. This is necessary to be able to correctly label any random functions that get called from within the generator expression. Basically, this function creates a function, and transforms the generator into a for loop that yields values from the. The function name is then returned, so that the parent node can handle the assignment properly. """ # make an identifier for the list self.newline(node) iden = self._gen_iden(node) argids = [] for gen in node.generators: argval = gen.iter argid = self._gen_iden(gen.iter) self.visit( _ast.Assign(targets=[_ast.Name(id=argid, ctx=_ast.Store())], value=argval)) argids.append(argid) elt = node.elt def parse_generator(nodes, ids): node = nodes[0] tempnode = _ast.For() tempnode.target = node.target tempnode.iter = _ast.Name(id=ids[0], ctx=_ast.Load()) if len(nodes) == 1: yield_node = _ast.Expr(value=_ast.Yield(value=elt)) body = [yield_node] else: body = [parse_generator(nodes[1:], ids[1:])] if len(node.ifs) == 1: ifnode = _ast.If(test=node.ifs[0], body=body, orelse=[]) tempnode.body = [ifnode] elif len(node.ifs) > 1: ifnode = _ast.If(test=_ast.BoolOp(op=_ast.And(), values=node.ifs), body=body, orelse=[]) tempnode.body = [ifnode] else: tempnode.body = body tempnode.orelse = None return tempnode node = _ast.FunctionDef( name=iden, args=_ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]), body=[parse_generator(node.generators, argids)], decorator_list=[]) self.visit(node) return iden
def STORE_NAME(self, instr): value = self.pop_ast_item() value = self.process_ifexpr(value) if isinstance(value, _ast.Import): if value.from_: assert isinstance(self._ast_stack[-1], _ast.ImportFrom) from_ = self.pop_ast_item() as_name = instr.arg name = from_.names[0].name if as_name != name: from_.names[0].asname = as_name self.push_ast_item(from_) else: as_name = instr.arg if value.names[0].asname is None: base_name = value.names[0].name.split('.')[0] if base_name != as_name: value.names[0].asname = as_name self.push_ast_item(value) elif isinstance(value, (_ast.Attribute)) and isinstance( value.value, (_ast.Import)): asname = instr.arg value = value.value value.names[0].asname = asname self.push_ast_item(value) elif isinstance(value, (_ast.ClassDef, _ast.FunctionDef)): as_name = instr.arg value.name = as_name self.push_ast_item(value) elif isinstance(value, _ast.AugAssign): self.push_ast_item(value) elif isinstance(value, _ast.Assign): _ = self.pop_ast_item() assname = _ast.Name(instr.arg, _ast.Store(), lineno=instr.lineno, col_offset=0) if _ is value.value or isinstance(_, _ast.Assign): value.targets.append(assname) else: if not isinstance(value.targets, _ast.Tuple): value.targets = [_ast.Tuple(value.targets, _ast.Store())] value.value = _ast.Tuple([value.value], _ast.Load()) value.targets[0].lineno = value.targets[0].elts[0].lineno value.targets[0].col_offset = value.targets[0].elts[ 0].col_offset value.value.lineno = value.value.elts[0].lineno value.value.col_offset = value.value.elts[0].col_offset value.targets[0].elts.append(assname) value.value.elts.append(_) self.push_ast_item(value) else: assname = _ast.Name(instr.arg, _ast.Store(), lineno=instr.lineno, col_offset=0) assign = _ast.Assign(targets=[assname], value=value, lineno=instr.lineno, col_offset=0) self.push_ast_item(assign)
def split_handlers(self, handlers_blocks): handlers = [] except_instrs = [] ends = [] while len(handlers_blocks): instr = handlers_blocks.pop(0) except_instrs.append(instr) if (instr.opname == 'COMPARE_OP') and (instr.arg == 'exception match'): jump = handlers_blocks.pop(0) assert jump.opname == 'POP_JUMP_IF_FALSE' next_handler = jump.oparg instr = handlers_blocks.pop(0) except_instrs.append(instr) instr = handlers_blocks.pop(0) except_instrs.append(instr) instr = handlers_blocks.pop(0) except_instrs.append(instr) assert except_instrs[0].opname == 'DUP_TOP' assert except_instrs[-3].opname == 'POP_TOP' assert except_instrs[-1].opname == 'POP_TOP' exec_stmnt = self.decompile_block(except_instrs[1:-4]).stmnt() assert len(exec_stmnt) == 1 exc_type = exec_stmnt[0] if except_instrs[-2].opname == 'STORE_NAME': exc_name = _ast.Name(id=except_instrs[-2].arg, ctx=_ast.Store(), lineno=except_instrs[-2].lineno, col_offset=0) else: assert except_instrs[-2].opname == 'POP_TOP' exc_name = None handler_body = [] while len(handlers_blocks): instr = handlers_blocks.pop(0) if instr.i == next_handler: handlers_blocks.insert(0, instr) break handler_body.append(instr) assert handler_body[-1].opname == 'JUMP_FORWARD' ends.append(handler_body[-1].arg) exc_body = self.decompile_block(handler_body[:-1]).stmnt() if not exc_body: exc_body.append(_ast.Pass(lineno=except_instrs[-2].lineno, col_offset=0)) # is this for python 3? if py3 and exc_name is not None: exc_name = exc_name.id handlers.append(_ast.ExceptHandler(type=exc_type, name=exc_name, body=exc_body, lineno=instr.lineno, col_offset=0)) except_instrs = [] assert except_instrs[-1].opname == 'END_FINALLY' if len(except_instrs) == 1: pass else: assert except_instrs[0].opname == 'POP_TOP' assert except_instrs[1].opname == 'POP_TOP' assert except_instrs[2].opname == 'POP_TOP' assert except_instrs[-2].opname in ['JUMP_FORWARD', 'JUMP_ABSOLUTE'], except_instrs[-2] ends.append(except_instrs[-2].arg) exc_body = self.decompile_block(except_instrs[3:-2]).stmnt() if not exc_body: exc_body.append(_ast.Pass(lineno=except_instrs[-2].lineno, col_offset=0)) handlers.append(_ast.ExceptHandler(type=None, name=None, body=exc_body, lineno=except_instrs[0].lineno, col_offset=0)) assert all(e == ends[0] for e in ends) end = ends[0] return end, handlers