def __cache_globals__(co, func_globals):
"Cache global objects in co_consts. See @cache_globals. Returns code object"
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL, LOAD_ATTR
# First, we want to crack open the function and look at it's bytecodes
code = Code.from_code(co)
# Look at each load global and replace with a load const if the
# global value is currently available. At the same time, if we
# find something like <const>.attr (and the attr is available),
# we keep folding
missing = object()
pc = 0
while pc < len(code.code):
op, arg = code.code[pc]
if op == LOAD_GLOBAL:
const = func_globals.get(arg, missing)
if const is not missing:
code.code[pc] = (LOAD_CONST, const)
else:
const = __builtins__.get(arg, missing)
if const is not missing:
code.code[pc] = (LOAD_CONST, const)
elif op == LOAD_ATTR:
prev_op, prev_arg = code.code[pc - 1]
const = getattr(prev_arg, arg, missing)
if const is not missing:
code.code[pc - 1:pc + 1] = [(LOAD_CONST, const)]
pc -= 1
pc += 1
return code.to_code()
def cache_globals(fn):
"""
Decorator to cache global lookups as constants.
"""
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL, LOAD_ATTR
code = Code.from_code(fn.func_code)
missing = object()
program_counter = 0
while program_counter < len(code.code):
opcode, arg = code.code[program_counter]
if opcode == LOAD_GLOBAL:
const = fn.func_globals.get(arg, missing)
if const is not missing:
code.code[program_counter] = (LOAD_CONST, const)
elif opcode == LOAD_ATTR:
prev_opcode, prev_arg = code.code[program_counter-1]
const = getattr(prev_arg, arg, missing)
if const is not missing:
code.code[program_counter-1:program_counter+1] = [(LOAD_CONST, const)]
program_counter -= 1
program_counter += 1
fn.func_code = code.to_code()
return fn
def __debuggable__(co):
"Apply DEBUG() calls in a code object. See @debuggable"
from byteplay import Code, LOAD_GLOBAL, CALL_FUNCTION, POP_TOP
# First, we want to crack open the function and look at it's bytecodes
code = Code.from_code(co)
pc = 0
while pc < len(code.code):
# Look for LOAD_GLOBAL,DEBUG
op, arg = code.code[pc]
if op != LOAD_GLOBAL or arg != 'DEBUG':
pc += 1
continue
expr_start = pc
pc += 1
# Figure out the "stack level" at each opcode
levels = __levels__(code.code)
start_level = levels[expr_start]
# Walk forward seeking a CALL_FUNCTION at the same level
n = start_level
for n in range(pc + 1, len(code.code)):
if levels[n] == start_level: break
# We should be at a CALL_FUNCTION. It's value should
# not be used. If it is, we don't remove it
if code.code[n][0] != CALL_FUNCTION: continue
if code.code[n + 1][0] != POP_TOP: continue
del code.code[expr_start:n + 2]
return code.to_code()
def __cache_globals__(co,func_globals):
"Cache global objects in co_consts. See @cache_globals. Returns code object"
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL, LOAD_ATTR
# First, we want to crack open the function and look at it's bytecodes
code = Code.from_code(co)
# Look at each load global and replace with a load const if the
# global value is currently available. At the same time, if we
# find something like <const>.attr (and the attr is available),
# we keep folding
missing = object()
pc = 0
while pc < len(code.code):
op,arg = code.code[pc]
if op == LOAD_GLOBAL:
const = func_globals.get(arg,missing)
if const is not missing:
code.code[pc] = (LOAD_CONST,const)
else:
const = __builtins__.get(arg,missing)
if const is not missing:
code.code[pc] = (LOAD_CONST,const)
elif op == LOAD_ATTR:
prev_op,prev_arg = code.code[pc-1]
const = getattr(prev_arg,arg,missing)
if const is not missing:
code.code[pc-1:pc+1] = [(LOAD_CONST,const)]
pc -= 1
pc += 1
return code.to_code()
def __debuggable__(co):
"Apply DEBUG() calls in a code object. See @debuggable"
from byteplay import Code, LOAD_GLOBAL, CALL_FUNCTION, POP_TOP
# First, we want to crack open the function and look at it's bytecodes
code = Code.from_code(co)
pc = 0
while pc < len(code.code):
# Look for LOAD_GLOBAL,DEBUG
op,arg = code.code[pc]
if op != LOAD_GLOBAL or arg != 'DEBUG':
pc += 1
continue
expr_start = pc
pc += 1
# Figure out the "stack level" at each opcode
levels = __levels__(code.code)
start_level = levels[expr_start]
# Walk forward seeking a CALL_FUNCTION at the same level
n = start_level
for n in range(pc+1,len(code.code)):
if levels[n] == start_level: break
# We should be at a CALL_FUNCTION. It's value should
# not be used. If it is, we don't remove it
if code.code[n][0] != CALL_FUNCTION: continue
if code.code[n+1][0] != POP_TOP: continue
del code.code[expr_start:n+2]
return code.to_code()
def cache_globals(fn):
"""
Decorator to cache global lookups as constants.
"""
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL, LOAD_ATTR
code = Code.from_code(fn.func_code)
missing = object()
program_counter = 0
while program_counter < len(code.code):
opcode, arg = code.code[program_counter]
if opcode == LOAD_GLOBAL:
const = fn.func_globals.get(arg, missing)
if const is not missing:
code.code[program_counter] = (LOAD_CONST, const)
elif opcode == LOAD_ATTR:
prev_opcode, prev_arg = code.code[program_counter - 1]
const = getattr(prev_arg, arg, missing)
if const is not missing:
code.code[program_counter - 1:program_counter + 1] = [
(LOAD_CONST, const)
]
program_counter -= 1
program_counter += 1
fn.func_code = code.to_code()
return fn
def transform(co):
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL
code = Code.from_code(co)
newcode = []
for pc, (op, arg) in enumerate(code.code):
if op == LOAD_GLOBAL and arg in what:
code.code[pc] = (LOAD_CONST, what[arg])
return code.to_code()
def _get_accumulated(func, usage_matcher, target_offset):
# what should we name this func??
accumulator = []
disassembled_code = Code.from_code(func.func_code).code
contained = _accumulate_contained(usage_matcher, disassembled_code)
for contained_index in contained:
accumulator.append(disassembled_code[contained_index + target_offset][1])
return accumulator
def transform(co):
from byteplay import Code, LOAD_CONST, LOAD_GLOBAL
code = Code.from_code(co)
newcode = []
for pc,(op,arg) in enumerate(code.code):
if op == LOAD_GLOBAL and arg in what:
code.code[pc] = (LOAD_CONST,what[arg])
return code.to_code()
def get_closure(func):
if isinstance(func, type):
methods = inspect.getmembers(func, predicate=inspect.ismethod)
return join(get_closure(meth.im_func) for _, meth in methods) or {}
code = Code.from_code(func.__code__)
names = _code_names(code)
return project(func.__globals__, names)
def inner(*args, **kws):
injected = func()
code = Code.from_code(f.func_code)
code.args = tuple(injected.keys() + list(code.args))
code.code = [(LOAD_FAST if (arg in injected and op == LOAD_GLOBAL) else op, arg) for op, arg in code.code]
f.func_code = code.to_code()
kws.update(injected)
return f(*args, **kws)
def _get_accumulated(func, usage_matcher, target_offset):
# what should we name this func??
accumulator = []
disassembled_code = Code.from_code(func.func_code).code
contained = _accumulate_contained(usage_matcher, disassembled_code)
for contained_index in contained:
accumulator.append(disassembled_code[contained_index +
target_offset][1])
return accumulator
def __unprint__(co):
"Apply unprint to code objects. See @unprint"
from byteplay import Code,getse, \
PRINT_ITEM,PRINT_NEWLINE, \
PRINT_ITEM_TO, PRINT_NEWLINE_TO, \
POP_TOP, ROT_TWO, DUP_TOP
code = Code.from_code(co)
instructions = code.code
# Now we kill every PRINT_NEWLINE and PRINT_ITEM
# (and associated value computations)
levels = __levels__(instructions)
kills = set()
def killback(pc):
kills.add(pc)
target_level = levels[pc]
pc -= 1
while pc >= 0 and levels[pc] > target_level:
kills.add(pc)
pc -= 1
return pc
for pc, (op, arg) in enumerate(instructions):
if pc in kills: continue
if op == PRINT_NEWLINE:
kills.add(pc)
elif op == PRINT_ITEM:
pc = killback(pc)
elif op == PRINT_ITEM_TO:
pc2 = killback(pc) # Kill the expression to print
pc3 = killback(pc2) # Kill the expression for out
# This chain of PRINT_ITEM_TO ends in one of
# two ways... with a PRINT_ITEM_TO/POP_TOP
# or a PRINT_ITEM_TO/PRINT_NEWLINE_TO
pc_end = pc
while pc_end < len(instructions):
if instructions[pc_end] == (PRINT_ITEM_TO, None):
if instructions[pc_end + 1][0] in (POP_TOP,
PRINT_NEWLINE_TO):
pc_end += 2
break
pc_end += 1
for i in xrange(pc, pc_end):
kills.add(i)
elif op == PRINT_NEWLINE_TO:
# You get this if you just have print >>out
killback(pc)
for x in reversed(sorted(kills)):
del instructions[x]
return code.to_code()
def __unprint__(co):
"Apply unprint to code objects. See @unprint"
from byteplay import Code,getse, \
PRINT_ITEM,PRINT_NEWLINE, \
PRINT_ITEM_TO, PRINT_NEWLINE_TO, \
POP_TOP, ROT_TWO, DUP_TOP
code = Code.from_code(co)
instructions = code.code
# Now we kill every PRINT_NEWLINE and PRINT_ITEM
# (and associated value computations)
levels = __levels__(instructions)
kills = set()
def killback(pc):
kills.add(pc)
target_level = levels[pc]
pc -= 1
while pc >= 0 and levels[pc] > target_level:
kills.add(pc)
pc -= 1
return pc
for pc,(op,arg) in enumerate(instructions):
if pc in kills: continue
if op == PRINT_NEWLINE:
kills.add(pc)
elif op == PRINT_ITEM:
pc = killback(pc)
elif op == PRINT_ITEM_TO:
pc2 = killback(pc) # Kill the expression to print
pc3 = killback(pc2) # Kill the expression for out
# This chain of PRINT_ITEM_TO ends in one of
# two ways... with a PRINT_ITEM_TO/POP_TOP
# or a PRINT_ITEM_TO/PRINT_NEWLINE_TO
pc_end = pc
while pc_end < len(instructions):
if instructions[pc_end] == (PRINT_ITEM_TO,None):
if instructions[pc_end+1][0] in (POP_TOP,PRINT_NEWLINE_TO):
pc_end += 2
break
pc_end += 1
for i in xrange(pc,pc_end): kills.add(i)
elif op == PRINT_NEWLINE_TO:
# You get this if you just have print >>out
killback(pc)
for x in reversed(sorted(kills)):
del instructions[x]
return code.to_code()
def __call__(self, func):
from byteplay import Code
self._func = func
self._c = Code.from_code(self._func.func_code)
c = self._c
# all return values must be initialized, for return (retvals)[:nargout]
# not necessary anymore, the parser has to take care of this
# check for maximum nargout, insert nargin nargout code
self.__add_narg()
# initialize novel variables, FIXME
self.__add_return()
self._func.func_code = self._c.to_code()
return self._func
def __call__(self, func):
from byteplay import Code
self._func = func
self._c = Code.from_code(self._func.func_code)
c = self._c
# all return values must be initialized, for return (retvals)[:nargout]
# not necessary anymore, the parser has to take care of this
# check for maximum nargout, insert nargin nargout code
self.__add_narg()
# initialize novel variables, FIXME
self.__add_return()
self._func.func_code = self._c.to_code()
return self._func
def validate_expression(expression, mode='eval'):
try:
code = compile(expression, '<dynamic>', mode)
except (SyntaxError, TypeError):
raise
used_codes = set(i[0] for i in Code.from_code(code).code
if isinstance(i[0], Opcode))
for opcode in used_codes:
if opcode not in allowed_op_codes:
raise ValueError("{} is not an allowed opcode".format(opcode))
return code
def validate_expression(expression, mode='eval'):
try:
code = compile(expression, '<dynamic>', mode)
except (SyntaxError, TypeError):
raise
used_codes = set(
i[0] for i in Code.from_code(code).code if isinstance(i[0], Opcode)
)
for opcode in used_codes:
if opcode not in allowed_op_codes:
raise ValueError("{} is not an allowed opcode".format(opcode))
return code
def shortgen(fnc):
code = Code.from_code(fnc.func_code)
pops = []
line_num = 0
source = inspect.getsourcelines(fnc)[0]
for num, line in enumerate(source):
if 'def' in line:
break
source = source[num + 1:]
vars_count = 0
last = []
for num, line in enumerate(code.code):
if SetLineno in line:
if '<<' in source[line_num]:
last = []
vars_count = len(
map(
str.strip,
source[line_num].split('<<')[0].split(','),
))
line_num += 1
elif vars_count:
last.append((num, line))
vars_count -= 1
if BINARY_LSHIFT in line:
code.code[num] = (YIELD_VALUE, None)
increaser = 2
if len(last) > 1:
code.code.insert(num + 3, (UNPACK_SEQUENCE, len(last)))
pops.append(num + 2)
increaser = 4
for store_num, store_line in enumerate(last):
code.code.insert(num + increaser + store_num,
(STORE_FAST, store_line[1][1]))
pops.append(num + 1)
for _num, _line in enumerate(code.code):
for store_line in last:
if _line == store_line[1]:
if _num >= num:
code.code[_num] = (LOAD_FAST, store_line[1][1])
else:
pops.append(_num)
for num, line in enumerate(sorted(pops)):
code.code.pop(line - num)
fnc.func_code = code.to_code()
return fnc
def AnonymousCodeBlock(function):
argSpec = inspect.getargspec(function)
if [i for x in argSpec if x is not None for i in x]:
raise TypeError("Function '%s' takes arguments" % function.func_name)
code = Code.from_code(function.func_code)
newBytecode = []
for opcode, arg in code.code:
if opcode in (LOAD_FAST, LOAD_DEREF, LOAD_GLOBAL):
opcode = LOAD_NAME
elif opcode in (STORE_FAST, STORE_NAME, STORE_DEREF, STORE_GLOBAL):
opcode = STORE_FAST
newBytecode.append((opcode, arg))
code.code = newBytecode
code.newlocals = False
code.freevars = ()
return code.to_code()
def shortgen(fnc):
code = Code.from_code(fnc.func_code)
pops = []
line_num = 0
source = inspect.getsourcelines(fnc)[0]
for num, line in enumerate(source):
if 'def' in line:
break
source = source[num + 1:]
vars_count = 0
last = []
for num, line in enumerate(code.code):
if SetLineno in line:
if '<<' in source[line_num]:
last = []
vars_count = len(map(str.strip,
source[line_num].split('<<')[0].split(','),
))
line_num += 1
elif vars_count:
last.append((num, line))
vars_count -= 1
if BINARY_LSHIFT in line:
code.code[num] = (YIELD_VALUE, None)
increaser = 2
if len(last) > 1:
code.code.insert(num + 3, (UNPACK_SEQUENCE, len(last)))
pops.append(num + 2)
increaser = 4
for store_num, store_line in enumerate(last):
code.code.insert(num + increaser + store_num,
(STORE_FAST, store_line[1][1]))
pops.append(num + 1)
for _num, _line in enumerate(code.code):
for store_line in last:
if _line == store_line[1]:
if _num >= num:
code.code[_num] = (LOAD_FAST, store_line[1][1])
else:
pops.append(_num)
for num, line in enumerate(sorted(pops)):
code.code.pop(line - num)
fnc.func_code = code.to_code()
return fnc
def block_anonymous_code(function):
argspec = inspect.getargspec(function)
args = argspec.args
if argspec.varargs is not None:
args += argspec.varargs,
if argspec.keywords is not None:
args += argspec.keywords,
code = Code.from_code(function.func_code)
newBytecode = []
for opcode, arg in code.code:
if opcode in REPLACES:
if not isinstance(arg, str) or not arg in args:
opcode = replace_opcode(opcode)
newBytecode.append((opcode, arg))
code.code = newBytecode
code.newlocals = False
code.freevars = ()
return code.to_code()
def __smartdebug__(co, func_globals):
"""Apply smartdebug to code objects, see @smartdebug"""
from byteplay import Code, SetLineno, Label, LOAD_GLOBAL, POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE, JUMP_FORWARD
code = Code.from_code(co)
instructions = code.code
# First, find all the "if DEBUG:" and "if not DEBUG"
# We collect in reverse order so that we can update
# in place more easily
debugs = []
for offset, op_arg in enumerate(instructions):
if op_arg == (LOAD_GLOBAL,
'DEBUG') and instructions[offset + 1][0] in (
POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE):
debugs.insert(0, offset)
# We want the bounds of the DEBUG true part and DEBUG false part for each
# most ifs look like
# LOAD_GLOBAL DEBUG
# POP_JUMP_IF_FALSE L1 (sense may be reversed with _TRUE)
# ...
# JUMP_FORWARD L2
# L1:
# ...
# L2:
# They look different at the ends of loops, but I'm skipping those
def back_one(x):
while x > 0:
opcode = instructions[x][0]
if opcode != SetLineno and not isinstance(opcode, Label):
break
x -= 1
return x
def offset_of(L):
for off, (op, _) in enumerate(instructions):
if op is L: return off
return None
def true_false(x):
pop_jump, L1 = instructions[x + 1]
O1 = offset_of(L1)
if O1 < x: return None # Jumping backward, Loop if
OJF = back_one(O1)
jf, L2 = instructions[OJF]
if jf != JUMP_FORWARD: return None # Not my pattern
O2 = offset_of(L2)
if pop_jump == POP_JUMP_IF_FALSE:
return ((x + 2, OJF), (OJF + 1, O2), (x, O2))
return ((OJF + 1, O2), (x + 2, OJF), (x, O2))
while debugs:
x = debugs[0]
del debugs[0]
bounds = true_false(x)
if not bounds: continue
(t0, t1), (f0, f1), (a, b) = bounds
if func_globals.get('DEBUG', False):
using = instructions[t0:t1]
else:
using = instructions[f0:f1]
instructions[a:b] = using
return code.to_code()
out = 12
return nargout > 0 and range(nargout) or None
# from http://www.mathworks.com/access/helpdesk/help/techdoc/ref/varargout.html
@mfunction_arguments("s", "varargout")
def mysize(x=None):
nout = max(nargout, 1) - 1
s = size(x)
for k in m_[1:nout]:
varargout(k).lvalue = mcellarray([s(k)])
from byteplay import Code
#c = Code.from_code(example1.func_code)
c = Code.from_code(mysize.func_code)
print c.code
example1()
example1(1)
example1(1, 2)
example1(1, 2, 4)
example1(1, 2, 4, 'a')
a = example1(1, 2, 4, 'a')
[a, b] = example1(1, 2, 4, 'a')
[a, b, c] = example1(1, 2, 4, 'a')
[a, b, c, d] = example1(1, 2, 4, 'a')
print 'OUT:', a
[s, rows, cols] = mysize(rand(4, 5))
def compileCode(code, gobj=globals()):
return FunctionType(
Code(code, tuple(), ('obj', ), False, False, True, "<code>",
"<auto_generated>", 0, "auto generated code").to_code(), gobj)
def __call__(self, mfunc):
from byteplay import Code, LOAD_GLOBAL, CALL_FUNCTION, POP_TOP, \
UNPACK_SEQUENCE, STORE_FAST, LOAD_FAST, \
LOAD_CONST, BUILD_LIST, BINARY_SUBTRACT, \
BINARY_MULTIPLY, BINARY_ADD, RETURN_VALUE, \
SLICE_2
# load the current function code
c = Code.from_code(mfunc.func_code)
# for nargin/nargout emulation
nargin = mfunc.func_code.co_argcount
if 'varargin' in self.names:
assert mfunc.func_code.co_flags & 4 == 4
nargin = -nargin - 1
mfunc.func_nargin = nargin
mfunc.func_nargout = len(self.names)
if 'varargout' in self.names:
mfunc.func_nargout = -mfunc.func_nargout
# prepare the bytecode changes
pre_code = [(LOAD_GLOBAL, '_get_narginout'), (CALL_FUNCTION, 0),
(UNPACK_SEQUENCE, 2), (STORE_FAST, 'nargin'),
(STORE_FAST, 'nargout')]
if 'varargin' in self.names:
# varargin = mcellarray(varargin)
pre_code += [(LOAD_GLOBAL, 'mcellarray'),
(LOAD_GLOBAL, 'varargin'), (CALL_FUNCTION, 1),
(STORE_FAST, 'varargin')]
if 'varargout' in self.names:
# varargout = mcellarray([None]*(nargout-1))
pre_code += [(LOAD_GLOBAL, 'mcellarray'), (LOAD_CONST, None),
(BUILD_LIST, 1), (LOAD_FAST, 'nargout'),
(LOAD_CONST, 1), (BINARY_SUBTRACT, None),
(BINARY_MULTIPLY, None), (CALL_FUNCTION, 1),
(STORE_FAST, 'varargout')]
# adjust the function preamble
c.code[:0] = pre_code
# replace LOAD_GLOBAL with LOAD_FAST for
for i, x in enumerate(c.code):
if x[0] == LOAD_GLOBAL and \
(x[1] == 'nargout' or x[1] == 'nargin' or \
x[1] == 'varargout' or x[1] == 'varargin'):
c.code[i] = (LOAD_FAST, x[1])
# return value
# first remove the original return
assert c.code[-1][0] == RETURN_VALUE
ret_code = [(POP_TOP, None)] # easier than deleting LOAD_CONST
# return what is supposed to be returned
if 'varargout' in self.names:
if len(self.names) > 1:
ret_code += [(LOAD_FAST, x) for x in self.names[:-1]] + \
[(BUILD_LIST, len(self.names)-1),
(LOAD_FAST , 'varargout'),
(BINARY_ADD, None)]
else:
ret_code += [(LOAD_FAST, 'varargout'), (BINARY_ADD, None)]
else:
ret_code += [(LOAD_FAST, x) for x in self.names] + \
[(BUILD_LIST, len(self.names))]
ret_code += [(LOAD_FAST, 'nargout'), (SLICE_2, None),
(RETURN_VALUE, None)]
c.code[-1:] = ret_code
# replace the original bytecode
mfunc.func_code = c.to_code()
return mfunc
def selfless(child):
code = Code.from_code(child.func_code)
code.args = tuple(['self'] + list(code.args))
code.code = [_transmute(op, arg) for op, arg in code.code]
function.func_code = code.to_code()
return function
def __enter__(self):
"""With block entry
On entry to the with, we grab the byte codes that make
up just the body of the block. We also get a copy of
all the local values take participate in the body.
On block exit (or if we call the timer() function),
we'll build a custom function from these parts and
run it to get the time.
We are expecting to be called in the form:
with LittleTimer() as T:
block
which looks like:
blah blah blah
CALL_FUNCTION x
SETUP_WITH
<something to store it or a pop_top>
code body
POP_BLOCK
LOAD_CONST None
WITH_CLEANUP
We don't support anything other than storing to a
variable or not storing at all
"""
# We pull in some useful bits
import inspect
from byteplay import Code,haslocal,SetLineno, \
SETUP_WITH,WITH_CLEANUP,\
STORE_FAST,STORE_NAME,STORE_GLOBAL,\
POP_TOP,POP_BLOCK
frame = inspect.currentframe(1)
self.__code = code = Code.from_code(frame.f_code)
self.__line = frame.f_lineno
self.__globals = frame.f_globals
# The SetLineno instructions get in the way here
# since I want to find the actual instruction
# by offset. I'll just strip them out
instructions = code.code
nolines = [x for x in instructions if x[0] != SetLineno]
instructions[:] = nolines
pc = __pc_to_byteplay_offset__(instructions).get(frame.f_lasti)
assert pc is not None, "Found invalid offset for with"
# Strip off everything through the SETUP_WITH
assert instructions[pc][
0] == SETUP_WITH, "LittleTimer must be invoked from a with statement"
end_label = instructions[pc][1]
del instructions[:pc + 1]
# which is followed by a STORE_NAME, STORE_LOCAL,
# STORE_GLOBAL, or POP_TOP
assert instructions[0][0] in (\
STORE_NAME,
STORE_FAST,
STORE_GLOBAL,
POP_TOP
),"Only simple assignment is supported, no more complex than LittleTimer() as T"
if instructions[0][0] == POP_TOP: self.__oneshot = True
del instructions[0]
# Find the closing WITH_CLEANUP
targets = [
offset for offset, (opcode, arg) in enumerate(instructions)
if opcode is end_label
]
assert targets, "This with-statement was not formed the way I expected"
pc = targets[0] + 1
assert instructions[pc][
0] == WITH_CLEANUP, "This with-statement was not formed the way I expected"
# Reverse until we find a POP_BLOCK
while pc >= 0:
opcode = instructions[pc][0]
if opcode == POP_BLOCK:
break
pc -= 1
del instructions[pc:]
self.__bytecodes = instructions
# We may have some local values that we need to set up
locals = set([x[1] for x in instructions if x[0] in haslocal])
self.__locals = dict(
(sym, frame.f_locals.get(sym, None)) for sym in locals)
return self
return self
def __smartdebug__(co,func_globals):
"""Apply smartdebug to code objects, see @smartdebug"""
from byteplay import Code,SetLineno,Label,LOAD_GLOBAL,POP_JUMP_IF_FALSE,POP_JUMP_IF_TRUE,JUMP_FORWARD
code = Code.from_code(co)
instructions = code.code
# First, find all the "if DEBUG:" and "if not DEBUG"
# We collect in reverse order so that we can update
# in place more easily
debugs = []
for offset,op_arg in enumerate(instructions):
if op_arg == (LOAD_GLOBAL,'DEBUG') and instructions[offset+1][0] in (POP_JUMP_IF_FALSE,POP_JUMP_IF_TRUE):
debugs.insert(0,offset)
# We want the bounds of the DEBUG true part and DEBUG false part for each
# most ifs look like
# LOAD_GLOBAL DEBUG
# POP_JUMP_IF_FALSE L1 (sense may be reversed with _TRUE)
# ...
# JUMP_FORWARD L2
# L1:
# ...
# L2:
# They look different at the ends of loops, but I'm skipping those
def back_one(x):
while x > 0:
opcode = instructions[x][0]
if opcode != SetLineno and not isinstance(opcode,Label):
break
x -= 1
return x
def offset_of(L):
for off,(op,_) in enumerate(instructions):
if op is L: return off
return None
def true_false(x):
pop_jump,L1 = instructions[x+1]
O1 = offset_of(L1)
if O1 < x: return None # Jumping backward, Loop if
OJF = back_one(O1)
jf,L2 = instructions[OJF]
if jf != JUMP_FORWARD: return None # Not my pattern
O2 = offset_of(L2)
if pop_jump == POP_JUMP_IF_FALSE:
return ((x+2,OJF),(OJF+1,O2),(x,O2))
return ((OJF+1,O2),(x+2,OJF),(x,O2))
while debugs:
x = debugs[0]
del debugs[0]
bounds = true_false(x)
if not bounds: continue
(t0,t1),(f0,f1),(a,b) = bounds
if func_globals.get('DEBUG',False):
using = instructions[t0:t1]
else:
using = instructions[f0:f1]
instructions[a:b] = using
return code.to_code()
def hecuba_filter(lambda_filter, iterable):
if hasattr(iterable, '_storage_father') and hasattr(iterable._storage_father, '_indexed_args') \
and iterable._storage_father._indexed_args is not None:
indexed_args = iterable._storage_father._indexed_args
father = iterable._storage_father
import inspect
from byteplay import Code
func = Code.from_code(lambda_filter.func_code)
if lambda_filter.func_closure is not None:
vars_to_vals = zip(
func.freevars,
map(lambda x: x.cell_contents, lambda_filter.func_closure))
inspected = inspect.findsource(lambda_filter)
inspected_function = '\n'.join(inspected[0][inspected[1]:]).replace(
'\n', '')
m = filter_reg.match(inspected_function)
if m is not None:
key_parameters, value_parameters, function_arguments = m.groups()
key_parameters = re.sub(r'\s+', '', key_parameters).split(',')
value_parameters = re.sub(r'\s+', '', value_parameters).split(',')
initial_index_arguments = []
m = random_reg.match(function_arguments)
precision = 1.0
precision_ind = -1
if m is not None:
params = m.groups()
for ind, param in enumerate(params):
if param == 'random()' or param == 'random.random()':
precision = float(params[ind + 1])
precision_ind = ind + 1
else:
if param != '' and 'random()' not in param and ind != precision_ind:
to_extend = param.split(' and ')
if '' in to_extend:
to_extend.remove('')
initial_index_arguments.extend(to_extend)
else:
initial_index_arguments = function_arguments.split(' and ')
stripped_index_arguments = []
for value in initial_index_arguments:
stripped_index_arguments.append(value.replace(" ", ""))
# for dict_name, props in iterable._persistent_props.iteritems():
index_arguments = set()
non_index_arguments = []
# if 'indexed_values' in props:
for value in stripped_index_arguments:
to_append = str(value)
if '<' in value:
splitval = value.split('<')
for pos in splitval:
if pos not in indexed_args:
try:
newval = eval(pos)
to_append = value.replace(str(pos), str(newval))
except Exception as e:
for tup in vars_to_vals:
if tup[0] == pos:
to_append = value.replace(
str(pos), str(tup[1]))
if splitval[0] in indexed_args or splitval[1] in indexed_args:
index_arguments.add(to_append)
else:
non_index_arguments.append(to_append)
elif '>' in value:
splitval = value.split('>')
for pos in splitval:
if pos not in indexed_args:
try:
newval = eval(pos)
to_append = value.replace(str(pos), str(newval))
except Exception as e:
for tup in vars_to_vals:
if tup[0] == pos:
to_append = value.replace(
str(pos), str(tup[1]))
if splitval[0] in indexed_args or splitval[1] in indexed_args:
index_arguments.add(to_append)
else:
non_index_arguments.append(to_append)
else:
non_index_arguments.append(value)
non_index_arguments = ' and '.join(non_index_arguments)
min_arguments = {}
max_arguments = {}
for argument in index_arguments:
if '<' in str(argument):
splitarg = (str(argument).replace(' ', '')).split('<')
if len(splitarg) == 2:
val = str(splitarg[0])
max_arguments[val] = float(splitarg[1])
elif len(splitarg) == 3:
val = str(splitarg[1])
min_arguments[val] = float(splitarg[0])
max_arguments[val] = float(splitarg[2])
else:
log.error("Malformed filtering predicate:" + str(argument))
if '>' in str(argument):
splitarg = (str(argument).replace(' ', '')).split('>')
if len(splitarg) == 2:
val = str(splitarg[0])
min_arguments[val] = float(splitarg[1])
elif len(splitarg) == 3:
val = str(splitarg[1])
min_arguments[val] = float(splitarg[2])
max_arguments[val] = float(splitarg[0])
else:
log.error("Malformed filtering predicate:" + str(argument))
from_p = []
to_p = []
for indexed_element in indexed_args:
from_p.append(min_arguments[indexed_element])
to_p.append(max_arguments[indexed_element])
from qbeast import QbeastMeta, QbeastIterator
qmeta = QbeastMeta(non_index_arguments, from_p, to_p, precision)
it = QbeastIterator(father._primary_keys, father._columns,
father._ksp + "." + father._table, qmeta)
return it
else:
filtered = python_filter(lambda_filter, iterable)
return filtered
def __enter__(self):
"""With block entry
On entry to the with, we grab the byte codes that make
up just the body of the block. We also get a copy of
all the local values take participate in the body.
On block exit (or if we call the timer() function),
we'll build a custom function from these parts and
run it to get the time.
We are expecting to be called in the form:
with LittleTimer() as T:
block
which looks like:
blah blah blah
CALL_FUNCTION x
SETUP_WITH
<something to store it or a pop_top>
code body
POP_BLOCK
LOAD_CONST None
WITH_CLEANUP
We don't support anything other than storing to a
variable or not storing at all
"""
# We pull in some useful bits
import inspect
from byteplay import Code,haslocal,SetLineno, \
SETUP_WITH,WITH_CLEANUP,\
STORE_FAST,STORE_NAME,STORE_GLOBAL,\
POP_TOP,POP_BLOCK
frame = inspect.currentframe(1)
self.__code = code = Code.from_code(frame.f_code)
self.__line = frame.f_lineno
self.__globals = frame.f_globals
# The SetLineno instructions get in the way here
# since I want to find the actual instruction
# by offset. I'll just strip them out
instructions = code.code
nolines = [x for x in instructions if x[0] != SetLineno]
instructions[:] = nolines
pc = __pc_to_byteplay_offset__(instructions).get(frame.f_lasti)
assert pc is not None,"Found invalid offset for with"
# Strip off everything through the SETUP_WITH
assert instructions[pc][0] == SETUP_WITH,"LittleTimer must be invoked from a with statement"
end_label = instructions[pc][1]
del instructions[:pc+1]
# which is followed by a STORE_NAME, STORE_LOCAL,
# STORE_GLOBAL, or POP_TOP
assert instructions[0][0] in (\
STORE_NAME,
STORE_FAST,
STORE_GLOBAL,
POP_TOP
),"Only simple assignment is supported, no more complex than LittleTimer() as T"
if instructions[0][0] == POP_TOP: self.__oneshot = True
del instructions[0]
# Find the closing WITH_CLEANUP
targets = [offset for offset,(opcode,arg) in enumerate(instructions)
if opcode is end_label]
assert targets,"This with-statement was not formed the way I expected"
pc = targets[0]+1
assert instructions[pc][0] == WITH_CLEANUP,"This with-statement was not formed the way I expected"
# Reverse until we find a POP_BLOCK
while pc >= 0:
opcode = instructions[pc][0]
if opcode == POP_BLOCK:
break
pc -= 1
del instructions[pc:]
self.__bytecodes = instructions
# We may have some local values that we need to set up
locals = set([x[1] for x in instructions if x[0] in haslocal])
self.__locals = dict( (sym,frame.f_locals.get(sym,None))
for sym in locals )
return self
return self
def test_python26_build_map():
import sys
if '.'.join(str(x) for x in sys.version_info[:2]) == '2.6':
from byteplay import Code
Code.from_code((lambda: {'a': 1}).func_code).to_code()
i += 2
#return co.co_names, co.co_freevars, co.co_cellvars, co.co_varnames
codestr = ''.join(map(chr,newcode))
return type(co)(co.co_argcount, len(names), co.co_stacksize, flags,
codestr, co.co_consts, tuple(names), tuple(varnames), co.co_filename, newname,
co.co_firstlineno, co.co_lnotab, (), ())
# TODO? flag to not modify builtins?
nlv = nonlocal_var()
from byteplay import Code
acb = AnonymousCodeBlock(nlv)
acb_code = Code.from_code(acb)
acb_cmp = acb_code.code
mbc = make_block_code(nlv.func_code)
#print repr(mbc.co_code)
#print repr(acb_code.to_code().co_code)
from bytely import dissco
dissco(mbc)
mbc_cmp = Code.from_code(mbc).code
print mbc.co_varnames
print mbc.co_names
print
print mbc.co_stacksize
print acb_code.to_code().co_stacksize
def test_python26_build_map():
import sys
if '.'.join(str(x) for x in sys.version_info[:2]) == '2.6':
from byteplay import Code
Code.from_code((lambda: {'a': 1}).func_code).to_code()
def __call__(self, mfunc):
from byteplay import Code, LOAD_GLOBAL, CALL_FUNCTION, POP_TOP, \
UNPACK_SEQUENCE, STORE_FAST, LOAD_FAST, \
LOAD_CONST, BUILD_LIST, BINARY_SUBTRACT, \
BINARY_MULTIPLY, BINARY_ADD, RETURN_VALUE, \
SLICE_2
# load the current function code
c = Code.from_code(mfunc.func_code)
# for nargin/nargout emulation
nargin = mfunc.func_code.co_argcount
if 'varargin' in self.names:
assert mfunc.func_code.co_flags&4 == 4
nargin = -nargin - 1
mfunc.func_nargin = nargin
mfunc.func_nargout = len(self.names)
if 'varargout' in self.names:
mfunc.func_nargout = -mfunc.func_nargout
# prepare the bytecode changes
pre_code = [(LOAD_GLOBAL,'_get_narginout'),
(CALL_FUNCTION,0),
(UNPACK_SEQUENCE, 2),
(STORE_FAST,'nargin'),
(STORE_FAST,'nargout')]
if 'varargin' in self.names:
# varargin = mcellarray(varargin)
pre_code += [(LOAD_GLOBAL, 'mcellarray'),
(LOAD_GLOBAL, 'varargin'),
(CALL_FUNCTION, 1),
(STORE_FAST, 'varargin')]
if 'varargout' in self.names:
# varargout = mcellarray([None]*(nargout-1))
pre_code += [(LOAD_GLOBAL, 'mcellarray'),
(LOAD_CONST, None),
(BUILD_LIST, 1),
(LOAD_FAST, 'nargout'),
(LOAD_CONST, 1),
(BINARY_SUBTRACT, None),
(BINARY_MULTIPLY, None),
(CALL_FUNCTION, 1),
(STORE_FAST, 'varargout')]
# adjust the function preamble
c.code[:0] = pre_code
# replace LOAD_GLOBAL with LOAD_FAST for
for i, x in enumerate(c.code):
if x[0] == LOAD_GLOBAL and \
(x[1] == 'nargout' or x[1] == 'nargin' or \
x[1] == 'varargout' or x[1] == 'varargin'):
c.code[i] = (LOAD_FAST,x[1])
# return value
# first remove the original return
assert c.code[-1][0] == RETURN_VALUE
ret_code = [(POP_TOP, None)] # easier than deleting LOAD_CONST
# return what is supposed to be returned
if 'varargout' in self.names:
if len(self.names) > 1:
ret_code += [(LOAD_FAST, x) for x in self.names[:-1]] + \
[(BUILD_LIST, len(self.names)-1),
(LOAD_FAST , 'varargout'),
(BINARY_ADD, None)]
else:
ret_code += [(LOAD_FAST , 'varargout'),
(BINARY_ADD, None)]
else:
ret_code += [(LOAD_FAST, x) for x in self.names] + \
[(BUILD_LIST, len(self.names))]
ret_code += [(LOAD_FAST, 'nargout'),
(SLICE_2, None),
(RETURN_VALUE, None)]
c.code[-1:] = ret_code
# replace the original bytecode
mfunc.func_code = c.to_code()
return mfunc
print "NIN: %d, NOUT: %d"%(nargin, nargout)
print "VARARGIN", varargin, "VARARGOUT:", varargout
out = 12
return nargout > 0 and range(nargout) or None
# from http://www.mathworks.com/access/helpdesk/help/techdoc/ref/varargout.html
@mfunction_arguments("s", "varargout")
def mysize(x=None):
nout = max(nargout, 1) - 1
s = size(x)
for k in m_[1:nout]:
varargout(k).lvalue = mcellarray([s(k)])
from byteplay import Code
#c = Code.from_code(example1.func_code)
c = Code.from_code(mysize.func_code)
print c.code
example1()
example1(1)
example1(1,2)
example1(1,2,4)
example1(1,2,4,'a')
a = example1(1,2,4,'a')
[a, b] = example1(1,2,4,'a')
[a, b, c] = example1(1,2,4,'a')
[a, b, c, d] = example1(1,2,4,'a')
print 'OUT:', a
[s,rows,cols] = mysize(rand(4,5))