def process_block_groups(self):
"""
Takes the current blocks ( similar to lines in a method ) and
processes them so they can be tokenized properly.
"""
iter_setup_block = None
for index, block in enumerate(self.blocks):
# if it is a return block
# we need to insert a jmp at the start of the block
# for the vm
if block.is_return:
# this jump needs to jump 3 bytes. why? stay tuned to find out
block_addr = b'\x03\x00'
ret_token = PyToken(Opcode(pyop.BR_S),
block.line,
args=block_addr)
ret_token.jump_label = block.oplist[0].jump_label
block.oplist[0].jump_label = None
block.oplist.insert(0, ret_token)
block.mark_as_end()
if block.has_load_attr:
block.preprocess_load_attr(self)
if block.has_store_attr:
block.preprocess_store_attr(self)
if block.has_make_function:
block.preprocess_make_function(self)
self.local_methods[
block.local_func_varname] = block.local_func_name
if block.has_unprocessed_array:
block.preprocess_arrays()
if block.has_unprocessed_array_sub:
block.preprocess_array_subs()
if block.has_unprocessed_method_calls:
ivars = block.preprocess_method_calls(self)
for key, val in ivars.items():
self.instance_vars[key] = val
if block.has_slice:
block.preprocess_slice()
if block.slice_item_length is not None:
length = len(self.local_stores)
self.local_stores[block.slice_item_length] = length
if iter_setup_block is not None:
block.process_iter_body(iter_setup_block)
iter_setup_block = None
if block.is_iter:
block.preprocess_iter()
for localvar in block.iterable_local_vars:
if localvar in self.local_stores.keys():
pass
else:
length = len(self.local_stores)
self.local_stores[localvar] = length
iter_setup_block = block
self.dynamic_iterator_count += 1
# print("ADDED BLOCK %s " % [str(op) for op in block.oplist])
alltokens = []
for block in self.blocks:
if block.has_make_function:
pass
else:
alltokens = alltokens + block.oplist
self.tokens = alltokens
for index, token in enumerate(self.tokens):
token.addr = index
def preprocess_load_attr(self, method):
"""
:param method:
"""
while self.has_load_attr:
to_rep = {}
to_del = []
is_sys_attr_lookup = False
for index, token in enumerate(self.oplist):
index_to_rep = -1
new_call = None
if token.py_op == pyop.LOAD_ATTR:
from boa.code.items import Klass
to_load_from = self.oplist[index - 1]
varname = to_load_from.args
ivar_type = None
is_func_call = True
do_nothing = False
if varname in method.instance_vars.keys():
ivar_type = method.instance_vars[varname]
test_sysmodule = '%s.Get%s' % (
ivar_type.module.module_path, token.args)
if NEO_SC_FRAMEWORK in test_sysmodule:
what_to_load = 'Get%s' % token.args
is_sys_attr_lookup = True
else:
what_to_load = '%s.%s' % (ivar_type.name,
token.args)
token.instance_type = ivar_type
# if this is a class variable lookup, do this
if token.args in ivar_type.class_var_names:
is_func_call = False
what_to_load = token.args
# otherwise, it is a method call on an object, we don't want to do anything
else:
token.func_params = [self.oplist[index - 1]]
do_nothing = True
elif varname == 'self' and type(method.parent) is Klass:
ivar_type = method.parent
what_to_load = '%s.%s' % (str(ivar_type), token.args)
if token.args in ivar_type.class_var_names:
is_func_call = False
what_to_load = token.args
else:
what_to_load = 'Get%s' % token.args
is_sys_attr_lookup = True
if not do_nothing:
if is_func_call:
call_func = PyToken(Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
index=-1,
args=what_to_load)
call_func.instance_type = ivar_type
call_func.func_processed = True
call_func.func_name = what_to_load
call_func.func_params = [self.oplist[index - 1]]
index_to_rep = index
new_call = call_func
else:
new_call = PyToken(Opcode(pyop.LOAD_CLASS_ATTR),
lineno=self.line,
args=what_to_load)
new_call.instance_type = ivar_type
new_call.instance_name = varname
new_call.func_processed = True
index_to_rep = index
if index_to_rep > 0:
td = self.oplist[index_to_rep - 1]
# print("TO DELETE: %s %s %s " % (td.py_op, td.jump_label, td.args))
if td.jump_label is not None and not is_sys_attr_lookup:
new_call.jump_label = td.jump_label
to_rep[index_to_rep] = new_call
to_del.append(self.oplist[index_to_rep - 1])
for key, val in to_rep.items():
self.oplist[key] = val
for item in to_del:
# print("WILL DELET: %s %s %s" % (item, vars(item), item.args))
if item in self.oplist:
self.oplist.remove(item)
else:
pdb.set_trace()
def read_initial_tokens(self):
"""
Take the initial set of tokens from the ``byteplay3`` code object and turn them into blocks.
"""
self.blocks = []
self.local_methods = collections.OrderedDict()
self.local_stores = collections.OrderedDict()
current_line_no = None
block_group = None
self.tokenizer = VMTokenizer(self)
current_label = None
current_loop_token = None
total_lines = 0
for i, (op, arg) in enumerate(self.code):
# print("[%s] %s -> %s " % (i, op, arg))
if type(op) is SetLinenoType:
current_line_no = arg
total_lines += 1
if self.start_line_no is None:
self.start_line_no = current_line_no
if block_group is not None:
self.blocks.append(Block(block_group))
block_group = []
elif type(op) is Label:
current_label = op
else:
instance_type = None
if op in [pyop.STORE_FAST, pyop.STORE_NAME, pyop.STORE_GLOBAL
] and arg not in self.local_stores.keys():
self._check_for_type(arg, total_lines)
length = len(self.local_stores)
self.local_stores[arg] = length
token = PyToken(op, current_line_no, i, arg)
if op == pyop.SETUP_LOOP:
token.args = None
current_loop_token = token
if op == pyop.BREAK_LOOP and current_loop_token is not None:
token.args = current_loop_token.args
current_loop_token = None
if current_label is not None:
token.jump_label = current_label
current_label = None
block_group.append(token)
if len(block_group):
self.blocks.append(Block(block_group))
def preprocess_list_comprehension(self, method):
# i apologize for the following
# grab the list comprehestion code object and make a method out of it
# we will use it later
"""
:param method:
"""
code_obj = self.oplist[0].args
# now get rid of the first 3 ops for now
self.oplist = self.oplist[3:]
# setup a loop for the list comp
setup_loop = PyToken(op=Opcode(pyop.SETUP_LOOP),
lineno=self.line,
index=-1)
# first we need to create a loop counter variable
self.list_comp_iterable_loopcounter = 'list_comp_loop_counter_%s' % Block.forloop_counter
# load the value 0
loopcounter_start_ld_const = PyToken(op=Opcode(pyop.LOAD_CONST),
lineno=self.line,
index=-1,
args=0)
# now store the loop counter
loopcounter_store_fast = PyToken(
op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# this loads the list that is going to be iterated over ( LOAD_FAST )
# this will be removed... its added into the call get length token function params
# unless this is a dynamic iteration, like for x in range(x,y)
iterable_load = self.oplist[0]
self.list_comp_iterable_item_name = iterable_load.args
# the following is in the case that we're doing something like for i in range(x,y)
dynamic_iterable_items = []
if iterable_load.py_op == pyop.CALL_FUNCTION:
self.has_dynamic_iterator = True
self.iterable_item_name = 'forloop_dynamic_range_%s' % Block.forloop_counter
dynamic_iterator_store_fast = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.iterable_item_name)
# if we're calling a method in this for i in, like for i in range(x,y) then we need
# to call the function
dynamic_iterable_items = [
iterable_load, dynamic_iterator_store_fast
]
# Now we need to get the length of that list, and store that as a local variable
call_get_length_token = PyToken(op=Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
args=1)
call_get_length_token.func_processed = True
call_get_length_token.func_params = [iterable_load]
call_get_length_token.func_name = 'len'
# now we need a variable name to store the length of the array
self.list_comp_iterable_looplength = 'list_comp_loop_length_%s' % Block.forloop_counter
# now store the variable which is the output of the len(items) call
looplength_store_op = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_looplength)
get_iter = self.oplist[1]
for_iter_label = Label()
jmp_if_false_label = Label()
for_iter = PyToken(op=Opcode(pyop.FOR_ITER),
lineno=self.line,
index=-1)
for_iter.jump_label = for_iter_label
end_block = PyToken(op=Opcode(pyop.POP_BLOCK),
lineno=self.line,
index=-1)
end_block.jump_label = jmp_if_false_label
jmp_abs_back = PyToken(op=Opcode(pyop.JUMP_ABSOLUTE),
lineno=self.line,
index=-1,
args=for_iter_label)
self.list_comp_iterable_variable = 'list_comp_local_i_%s' % Block.forloop_counter
ld_loopcounter = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
ld_loop_length = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_looplength)
new__compare_op = PyToken(op=Opcode(pyop.COMPARE_OP),
lineno=self.line,
index=-1,
args='<')
new__popjump_op = PyToken(op=Opcode(pyop.POP_JUMP_IF_FALSE),
lineno=self.line,
index=-1,
args=jmp_if_false_label)
# ok now we do the loop block stuff here
#
# the following loads the iterated item into the block
#
# load the iterable collection
ld_load_iterable = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_item_name)
# load the counter var
ld_counter = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# binary subscript of the iterable collection
ld_subscript = PyToken(op=Opcode(pyop.BINARY_SUBSCR),
lineno=self.line,
index=-1)
# now store the iterated item
st_iterable = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_variable)
#
# the following load the forloop counter and increments it
#
# load the counter var
ld_counter_2 = PyToken(op=Opcode(pyop.LOAD_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# load the constant 1
increment_const = PyToken(op=Opcode(pyop.LOAD_CONST),
lineno=self.line,
index=-1,
args=1)
# add it to the counter
increment_add = PyToken(op=Opcode(pyop.INPLACE_ADD),
lineno=self.line,
index=-1)
increment_add.func_processed = True
# and store it again
increment_store = PyToken(op=Opcode(pyop.STORE_FAST),
lineno=self.line,
index=-1,
args=self.list_comp_iterable_loopcounter)
# and now we call the function of the list-comprehension
list_comp_call_func = PyToken(op=Opcode(pyop.CALL_FUNCTION),
lineno=self.line,
index=-1,
args=1)
list_comp_call_func.func_name = self.local_func_name
list_comp_call_func.func_params = [self.list_comp_iterable_variable]
self.oplist = [
setup_loop, # SETUP_LOOP
get_iter, # GET_ITER, keep this in for now
# the following 4 ops set up the iterator
loopcounter_start_ld_const, # LOAD_CONST 0
loopcounter_store_fast, # STORE_FAST forloopcounter_X
# dynamic load loop stuff would go here
call_get_length_token, # CALL_FUNCTION 1
looplength_store_op, # STORE_FAST forloop_length_X
# these last 5 ops controls the operation of the loop
for_iter, # tihs is the jump target for the end of the loop execution block
ld_loopcounter, # load in the loop counter LOAD_FAST forloopcounter_X
ld_loop_length, # load in the loop length LOAD_FAST forloop_length_X
new__compare_op, # COMPARE_OP <, this will compare foorloop_counter_X < forloop_length_X
new__popjump_op, # POP_JUMP_IF_FALSE jumps to the loop exit when counter == length
# the following are the loop body items
ld_load_iterable,
ld_counter,
ld_subscript,
st_iterable,
ld_counter_2,
increment_const,
increment_add, # this is a hack... when the list_comp_call_func is processed, it
# takes out a few things from the block
# so we add it in twice (blerg...) so it gets put back in
increment_store,
# put call method of the list comp here...
# now pop back to for_iter
jmp_abs_back,
end_block,
]
# from boa.code.method import Method
# internal_method = Method(code_object=code_obj,
# parent=method.parent,
# make_func_name=self.local_func_name,
# is_list_comp_internal=True,
# list_comp_iterable_name=self.list_comp_iterable_variable)
# internal_ops = internal_method.blocks[0].oplist
# print("internal ops %s " % internal_ops)
# print(internal_method.tokenizer.to_s())
# self.oplist = self.oplist[:-2] + internal_ops + self.oplist[-2:]
# if len(dynamic_iterable_items):
# self.oplist.insert(4, dynamic_iterable_items[0])
# self.oplist.insert(5, dynamic_iterable_items[1])
Block.forloop_counter += 1