def preprocess_array_subs(self):
"""
"""
while self.has_unprocessed_array_sub:
start_index_change = None
end_index_change = None
changed_items = None
for index, token in enumerate(self.oplist):
if token.py_op == pyop.STORE_SUBSCR and not token.array_processed:
token.array_processed = True
start_index_change = index - 3
end_index_change = index
item_to_sub = self.oplist[index - 3].args
array_to_sub = self.oplist[index - 2].args
index_to_sub_at = self.oplist[index - 1].args
changed_items = []
# load the array to set the item into
ld_op = PyToken(Opcode(pyop.LOAD_FAST),
token.line_no, args=array_to_sub)
changed_items.append(ld_op)
# create the setitem op
settoken = PyToken(Opcode(
pyop.SETITEM), token.line_no, args=index_to_sub_at, array_item=item_to_sub)
changed_items.append(settoken)
if start_index_change is not None and end_index_change is not None:
tstart = self.oplist[0:start_index_change]
tend = self.oplist[end_index_change + 2:]
self.oplist = tstart + changed_items + tend
def mark_as_end(self):
"""
"""
tstart = self.oplist[:-1]
tend = self.oplist[-1:]
newitems = [PyToken(Opcode(pyop.NOP), self.line),
# PyToken(pyop.DROP_BODY, self.line),
PyToken(Opcode(pyop.FROMALTSTACK), self.line),
PyToken(Opcode(pyop.DROP), self.line)]
self.oplist = tstart + newitems + tend
def process_list_comp_internal(self, list_comp_item_name):
# get rid of first op
"""
:param list_comp_item_name:
"""
print("OP 0 %s " % self.oplist[0].py_op)
if self.oplist[0].py_op == pyop.STORE_FAST:
argname = self.oplist[0].args
self.oplist[0] = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=0,
args=list_comp_item_name)
method_call = self.oplist[1]
for op in method_call.func_params:
print("op in method call:: %s %s " % (op, op.args))
if op.args == argname:
print("switch argname...")
op.args = list_comp_item_name
self.oplist = self.oplist[0:-3]
def preprocess_load_attr(self, method):
"""
:param method:
"""
while self.has_load_attr:
index_to_rep = -1
new_call = None
for index, token in enumerate(self.oplist):
if token.py_op == pyop.LOAD_ATTR:
what_to_load = 'Get%s' % token.args
call_func = PyToken(Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
index=-1,
args=what_to_load)
call_func.func_processed = True
call_func.func_name = what_to_load
call_func.func_params = [self.oplist[index - 1]]
index_to_rep = index
new_call = call_func
if index_to_rep > -1 and new_call is not None:
self.oplist[index_to_rep] = new_call
del self.oplist[index_to_rep - 1]
def process_iter_body(self, setup_block):
"""
:param setup_block:
"""
first_op = self.oplist[0]
#
# the following loads the iterated item into the block
#
# load the iterable collection
ld_load_iterable = PyToken(op=Opcode(
pyop.LOAD_FAST), lineno=first_op.line_no, index=-1, args=setup_block.iterable_item_name)
# load the counter var
ld_counter = PyToken(op=Opcode(pyop.LOAD_FAST), lineno=first_op.line_no,
index=-1, args=setup_block.iterable_loopcounter)
# binary subscript of the iterable collection
ld_subscript = PyToken(op=Opcode(pyop.BINARY_SUBSCR),
lineno=first_op.line_no, index=-1)
# now store the iterated item
st_iterable = PyToken(op=Opcode(
pyop.STORE_FAST), lineno=first_op.line_no, index=-1, args=setup_block.iterable_variable)
#
# the following load the forloop counter and increments it
#
# load the counter var
ld_counter_2 = PyToken(op=Opcode(
pyop.LOAD_FAST), lineno=first_op.line_no, index=-1, args=setup_block.iterable_loopcounter)
# load the constant 1
increment_const = PyToken(
op=Opcode(pyop.LOAD_CONST), lineno=first_op.line_no, index=-1, args=1)
# add it to the counter
increment_add = PyToken(
op=Opcode(pyop.INPLACE_ADD), lineno=first_op.line_no, index=-1)
# and store it again
increment_store = PyToken(op=Opcode(
pyop.STORE_FAST), lineno=first_op.line_no, index=-1, args=setup_block.iterable_loopcounter)
self.oplist = [
ld_load_iterable, ld_counter, ld_subscript, st_iterable,
ld_counter_2, increment_const, increment_add, increment_store
] + self.oplist
def __init__(self, item_list):
super(Definition, self).__init__(item_list)
# this is a simple definition like a = 3
if len(self.items) == 3:
self.value = PyToken(self.items[1], 1, args=self.items[1][1])
self.attr = PyToken(self.items[2], 1, args=self.items[2][1])
# a method based definition linke ctx = GetContext
elif len(self.items) == 4:
self.is_method_call = True
self.attr = PyToken(self.items[-1], 1, args=self.items[-1][1])
self.value = PyToken(Opcode(pyop.LOAD_CONST), 1, args=7)
self.convert_class_call_to_block()
elif len(self.items) == 5:
if self.items[-1][0] == pyop.RETURN_VALUE:
self.items = self.items[:3]
self.value = PyToken(self.items[1], 1, args=self.items[1][1])
self.attr = PyToken(self.items[2], 1, args=self.items[2][1])
else:
# self.is_method_call=True
self.attr = PyToken(self.items[-1], 1, args=self.items[-1][1])
def process_block_groups(self):
"""
Takes the current blocks ( similar to lines in a method ) and
processes them so they can be tokenized properly.
"""
iter_setup_block = None
for index, block in enumerate(self.blocks):
# if it is a return block
# we need to insert a jmp at the start of the block
# for the vm
if block.is_return:
# this jump needs to jump 3 bytes. why? stay tuned to find out
block_addr = b'\x03\x00'
ret_token = PyToken(Opcode(pyop.BR_S),
block.line,
args=block_addr)
ret_token.jump_label = block.oplist[0].jump_label
block.oplist[0].jump_label = None
block.oplist.insert(0, ret_token)
block.mark_as_end()
if block.has_load_attr:
block.preprocess_load_attr(self)
if block.has_store_attr:
block.preprocess_store_attr(self)
if block.has_make_function:
block.preprocess_make_function(self)
self.local_methods[
block.local_func_varname] = block.local_func_name
if block.has_unprocessed_array:
block.preprocess_arrays()
if block.has_unprocessed_array_sub:
block.preprocess_array_subs()
if block.has_unprocessed_method_calls:
ivars = block.preprocess_method_calls(self)
for key, val in ivars.items():
self.instance_vars[key] = val
if block.has_slice:
block.preprocess_slice()
if block.slice_item_length is not None:
length = len(self.local_stores)
self.local_stores[block.slice_item_length] = length
if iter_setup_block is not None:
block.process_iter_body(iter_setup_block)
iter_setup_block = None
if block.is_iter:
block.preprocess_iter()
for localvar in block.iterable_local_vars:
if localvar in self.local_stores.keys():
pass
else:
length = len(self.local_stores)
self.local_stores[localvar] = length
iter_setup_block = block
self.dynamic_iterator_count += 1
# print("ADDED BLOCK %s " % [str(op) for op in block.oplist])
alltokens = []
for block in self.blocks:
if block.has_make_function:
pass
else:
alltokens = alltokens + block.oplist
self.tokens = alltokens
for index, token in enumerate(self.tokens):
token.addr = index
def preprocess_iter(self):
# in a better world this would be done in a more efficient way
# for now this is kept to be as understandable as possible
"""
"""
loopsetup = self.oplist[0]
loopsetup.args = None
loopsetup.jump_label = None
# first we need to create a loop counter variable
self.iterable_loopcounter = 'forloop_counter_%s' % Block.forloop_counter
# load the value 0
loopcounter_start_ld_const = PyToken(
op=Opcode(pyop.LOAD_CONST), lineno=loopsetup.line_no, index=-1, args=0)
# now store the loop counter
loopcounter_store_fast = PyToken(op=Opcode(
pyop.STORE_FAST), lineno=loopsetup.line_no, index=-1, args=self.iterable_loopcounter)
# this loads the list that is going to be iterated over ( LOAD_FAST )
# this will be removed... its added into the call get length token function params
# unless this is a dynamic iteration, like for x in range(x,y)
dynamic_iterable_items = []
iterable_load = self.oplist[1]
self.iterable_item_name = iterable_load.args
if iterable_load.py_op == pyop.CALL_FUNCTION:
self.has_dynamic_iterator = True
self.iterable_item_name = 'forloop_dynamic_range_%s' % Block.forloop_counter
dynamic_iterator_store_fast = PyToken(op=Opcode(pyop.STORE_FAST), lineno=loopsetup.line_no, index=-1,
args=self.iterable_item_name)
# if we're calling a method in this for i in, like for i in range(x,y) then we need
# to call the function
dynamic_iterable_items = [
iterable_load, dynamic_iterator_store_fast]
# Now we need to get the length of that list, and store that as a local variable
call_get_length_token = PyToken(
op=Opcode(pyop.CALL_FUNCTION), lineno=loopsetup.line_no, args=1)
call_get_length_token.func_params = [iterable_load]
call_get_length_token.func_name = 'len'
# now we need a variable name to store the length of the array
self.iterable_looplength = 'forloop_length_%s' % Block.forloop_counter
# now store the variable which is the output of the len(items) call
looplength_store_op = PyToken(op=Opcode(
pyop.STORE_FAST), lineno=loopsetup.line_no, index=-1, args=self.iterable_looplength)
get_iter = self.oplist[2]
for_iter = self.oplist[3]
store_iterable_name = self.oplist[4]
# set the iterable variable name ( for example, i ) so that the loop body can use it
self.iterable_variable = store_iterable_name.args
ld_loopcounter = PyToken(op=Opcode(
pyop.LOAD_FAST), lineno=loopsetup.line_no, index=-1, args=self.iterable_loopcounter)
ld_loop_length = PyToken(op=Opcode(
pyop.LOAD_FAST), lineno=loopsetup.line_no, index=-1, args=self.iterable_looplength)
new__compare_op = PyToken(
op=Opcode(pyop.COMPARE_OP), lineno=loopsetup.line_no, index=-1, args='<')
new__popjump_op = PyToken(op=Opcode(
pyop.POP_JUMP_IF_FALSE), lineno=loopsetup.line_no, index=-1, args=for_iter.args)
for_iter.args = None
self.oplist = [
loopsetup, # SETUP_LOOP
get_iter, # GET_ITER, keep this in for now
# the following 4 ops set up the iterator
loopcounter_start_ld_const, # LOAD_CONST 0
loopcounter_store_fast, # STORE_FAST forloopcounter_X
# dynamic load loop stuff would go here
call_get_length_token, # CALL_FUNCTION 1
looplength_store_op, # STORE_FAST forloop_length_X
# these last 5 ops controls the operation of the loop
for_iter, # tihs is the jump target for the end of the loop execution block
ld_loopcounter, # load in the loop counter LOAD_FAST forloopcounter_X
ld_loop_length, # load in the loop length LOAD_FAST forloop_length_X
new__compare_op, # COMPARE_OP <, this will compare foorloop_counter_X < forloop_length_X
new__popjump_op # POP_JUMP_IF_FALSE jumps to the loop exit when counter == length
]
if len(dynamic_iterable_items):
self.oplist.insert(4, dynamic_iterable_items[0])
self.oplist.insert(5, dynamic_iterable_items[1])
Block.forloop_counter += 1
def preprocess_load_attr(self, method):
"""
:param method:
"""
while self.has_load_attr:
to_rep = {}
to_del = []
really_to_del = []
for index, token in enumerate(self.oplist):
index_to_rep = -1
new_call = None
if token.py_op == pyop.LOAD_ATTR:
from boa.code.items import Klass
to_load_from = self.oplist[index - 1]
varname = to_load_from.args
ivar_type = None
is_func_call = True
do_nothing = False
if varname in method.instance_vars.keys():
ivar_type = method.instance_vars[varname]
test_sysmodule = '%s.Get%s' % (ivar_type.module.module_path, token.args)
if NEO_SC_FRAMEWORK in test_sysmodule:
what_to_load = 'Get%s' % token.args
else:
what_to_load = '%s.%s' % (ivar_type.name, token.args)
token.instance_type = ivar_type
# if this is a class variable lookup, do this
if token.args in ivar_type.class_var_names:
is_func_call = False
what_to_load = token.args
# otherwise, it is a method call on an object, we don't want to do anything
else:
token.func_params = [self.oplist[index - 1]]
do_nothing = True
elif varname == 'self' and type(method.parent) is Klass:
ivar_type = method.parent
what_to_load = '%s.%s' % (str(ivar_type), token.args)
if token.args in ivar_type.class_var_names:
is_func_call = False
what_to_load = token.args
else:
what_to_load = 'Get%s' % token.args
if not do_nothing:
if is_func_call:
call_func = PyToken(
Opcode(pyop.CALL_FUNCTION), lineno=self.line, index=-1, args=what_to_load)
call_func.instance_type = ivar_type
call_func.func_processed = True
call_func.func_name = what_to_load
call_func.func_params = [self.oplist[index - 1]]
index_to_rep = index
new_call = call_func
else:
new_call = PyToken(Opcode(pyop.LOAD_CLASS_ATTR), lineno=self.line, args=what_to_load)
new_call.instance_type = ivar_type
new_call.instance_name = varname
new_call.func_processed = True
index_to_rep = index
if index_to_rep > 0:
to_rep[index_to_rep] = new_call
to_del.append(self.oplist[index_to_rep - 1])
for key, val in to_rep.items():
self.oplist[key] = val
for item in to_del:
# print("WILL DELET: %s %s" % (item, vars(item)))
if item in self.oplist:
self.oplist.remove(item)
else:
pdb.set_trace()
def preprocess_list_comprehension(self, method):
# i apologize for the following
# grab the list comprehestion code object and make a method out of it
# we will use it later
"""
:param method:
"""
code_obj = self.oplist[0].args
# now get rid of the first 3 ops for now
self.oplist = self.oplist[3:]
# setup a loop for the list comp
setup_loop = PyToken(op=Opcode(pyop.SETUP_LOOP),
lineno=self.line,
index=-1)
# first we need to create a loop counter variable
self.list_comp_iterable_loopcounter = 'list_comp_loop_counter_%s' % Block.forloop_counter
# load the value 0
loopcounter_start_ld_const = PyToken(op=Opcode(pyop.LOAD_CONST),
lineno=self.line,
index=-1,
args=0)
# now store the loop counter
loopcounter_store_fast = PyToken(
op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# this loads the list that is going to be iterated over ( LOAD_FAST )
# this will be removed... its added into the call get length token function params
# unless this is a dynamic iteration, like for x in range(x,y)
iterable_load = self.oplist[0]
self.list_comp_iterable_item_name = iterable_load.args
# the following is in the case that we're doing something like for i in range(x,y)
dynamic_iterable_items = []
if iterable_load.py_op == pyop.CALL_FUNCTION:
self.has_dynamic_iterator = True
self.iterable_item_name = 'forloop_dynamic_range_%s' % Block.forloop_counter
dynamic_iterator_store_fast = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.iterable_item_name)
# if we're calling a method in this for i in, like for i in range(x,y) then we need
# to call the function
dynamic_iterable_items = [
iterable_load, dynamic_iterator_store_fast
]
# Now we need to get the length of that list, and store that as a local variable
call_get_length_token = PyToken(op=Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
args=1)
call_get_length_token.func_processed = True
call_get_length_token.func_params = [iterable_load]
call_get_length_token.func_name = 'len'
# now we need a variable name to store the length of the array
self.list_comp_iterable_looplength = 'list_comp_loop_length_%s' % Block.forloop_counter
# now store the variable which is the output of the len(items) call
looplength_store_op = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_looplength)
get_iter = self.oplist[1]
for_iter_label = Label()
jmp_if_false_label = Label()
for_iter = PyToken(op=Opcode(pyop.FOR_ITER),
lineno=self.line,
index=-1)
for_iter.jump_label = for_iter_label
end_block = PyToken(op=Opcode(pyop.POP_BLOCK),
lineno=self.line,
index=-1)
end_block.jump_label = jmp_if_false_label
jmp_abs_back = PyToken(op=Opcode(pyop.JUMP_ABSOLUTE),
lineno=self.line,
index=-1,
args=for_iter_label)
self.list_comp_iterable_variable = 'list_comp_local_i_%s' % Block.forloop_counter
ld_loopcounter = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
ld_loop_length = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_looplength)
new__compare_op = PyToken(op=Opcode(pyop.COMPARE_OP),
lineno=self.line,
index=-1,
args='<')
new__popjump_op = PyToken(op=Opcode(pyop.POP_JUMP_IF_FALSE),
lineno=self.line,
index=-1,
args=jmp_if_false_label)
# ok now we do the loop block stuff here
#
# the following loads the iterated item into the block
#
# load the iterable collection
ld_load_iterable = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_item_name)
# load the counter var
ld_counter = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# binary subscript of the iterable collection
ld_subscript = PyToken(op=Opcode(pyop.BINARY_SUBSCR),
lineno=self.line,
index=-1)
# now store the iterated item
st_iterable = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_variable)
#
# the following load the forloop counter and increments it
#
# load the counter var
ld_counter_2 = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# load the constant 1
increment_const = PyToken(op=Opcode(pyop.LOAD_CONST),
lineno=self.line,
index=-1,
args=1)
# add it to the counter
increment_add = PyToken(op=Opcode(pyop.INPLACE_ADD),
lineno=self.line,
index=-1)
increment_add.func_processed = True
# and store it again
increment_store = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# and now we call the function of the list-comprehension
list_comp_call_func = PyToken(op=Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
index=-1,
args=1)
list_comp_call_func.func_name = self.local_func_name
list_comp_call_func.func_params = [self.list_comp_iterable_variable]
self.oplist = [
setup_loop, # SETUP_LOOP
get_iter, # GET_ITER, keep this in for now
# the following 4 ops set up the iterator
loopcounter_start_ld_const, # LOAD_CONST 0
loopcounter_store_fast, # STORE_FAST forloopcounter_X
# dynamic load loop stuff would go here
call_get_length_token, # CALL_FUNCTION 1
looplength_store_op, # STORE_FAST forloop_length_X
# these last 5 ops controls the operation of the loop
for_iter, # tihs is the jump target for the end of the loop execution block
ld_loopcounter, # load in the loop counter LOAD_FAST forloopcounter_X
ld_loop_length, # load in the loop length LOAD_FAST forloop_length_X
new__compare_op, # COMPARE_OP <, this will compare foorloop_counter_X < forloop_length_X
new__popjump_op, # POP_JUMP_IF_FALSE jumps to the loop exit when counter == length
# the following are the loop body items
ld_load_iterable,
ld_counter,
ld_subscript,
st_iterable,
ld_counter_2,
increment_const,
increment_add, # this is a hack... when the list_comp_call_func is processed, it
# takes out a few things from the block
# so we add it in twice (blerg...) so it gets put back in
increment_store,
# put call method of the list comp here...
# now pop back to for_iter
jmp_abs_back,
end_block,
]
# from boa.code.method import Method
# internal_method = Method(code_object=code_obj,
# parent=method.parent,
# make_func_name=self.local_func_name,
# is_list_comp_internal=True,
# list_comp_iterable_name=self.list_comp_iterable_variable)
# internal_ops = internal_method.blocks[0].oplist
# print("internal ops %s " % internal_ops)
# print(internal_method.tokenizer.to_s())
# self.oplist = self.oplist[:-2] + internal_ops + self.oplist[-2:]
# if len(dynamic_iterable_items):
# self.oplist.insert(4, dynamic_iterable_items[0])
# self.oplist.insert(5, dynamic_iterable_items[1])
Block.forloop_counter += 1
