def ingest(self, co, classname=None, code_objects={}, show_asm=None):
"""
Pick out tokens from an uncompyle6 code object, and transform them,
returning a list of uncompyle6 Token's.
The transformations are made to assist the deparsing grammar.
Specificially:
- various types of LOAD_CONST's are categorized in terms of what they load
- COME_FROM instructions are added to assist parsing control structures
- MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
- some EXTENDED_ARGS instructions are removed
Also, when we encounter certain tokens, we add them to a set which will cause custom
grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
cause specific rules for the specific number of arguments they take.
"""
if not show_asm:
show_asm = self.show_asm
bytecode = self.build_instructions(co)
# show_asm = 'both'
if show_asm in ('both', 'before'):
for instr in bytecode.get_instructions(co):
print(instr.disassemble())
# list of tokens/instructions
tokens = []
# "customize" is in the process of going away here
customize = {}
if self.is_pypy:
customize['PyPy'] = 0
# Scan for assertions. Later we will
# turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
# 'LOAD_ASSERT' is used in assert statements.
self.load_asserts = set()
n = len(self.insts)
for i, inst in enumerate(self.insts):
# We need to detect the difference between:
# raise AssertionError
# and
# assert ...
# If we have a JUMP_FORWARD after the
# RAISE_VARARGS then we have a "raise" statement
# else we have an "assert" statement.
if self.version == 3.0:
# There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
assert_can_follow = inst.opname == 'POP_TOP' and i+1 < n
else:
assert_can_follow = inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n
if assert_can_follow:
next_inst = self.insts[i+1]
if (next_inst.opname == 'LOAD_GLOBAL' and
next_inst.argval == 'AssertionError'):
if (i + 2 < n and self.insts[i+2].opname.startswith('RAISE_VARARGS')):
self.load_asserts.add(next_inst.offset)
pass
pass
# Get jump targets
# Format: {target offset: [jump offsets]}
jump_targets = self.find_jump_targets(show_asm)
# print("XXX2", jump_targets)
last_op_was_break = False
for i, inst in enumerate(self.insts):
argval = inst.argval
op = inst.opcode
if inst.opname == 'EXTENDED_ARG':
# FIXME: The EXTENDED_ARG is used to signal annotation
# parameters
if (i+1 < n and
self.insts[i+1].opcode != self.opc.MAKE_FUNCTION):
continue
if inst.offset in jump_targets:
jump_idx = 0
# We want to process COME_FROMs to the same offset to be in *descending*
# offset order so we have the larger range or biggest instruction interval
# last. (I think they are sorted in increasing order, but for safety
# we sort them). That way, specific COME_FROM tags will match up
# properly. For example, a "loop" with an "if" nested in it should have the
# "loop" tag last so the grammar rule matches that properly.
for jump_offset in sorted(jump_targets[inst.offset], reverse=True):
come_from_name = 'COME_FROM'
opname = self.opname_for_offset(jump_offset)
if opname == 'EXTENDED_ARG':
j = xdis.next_offset(op, self.opc, jump_offset)
opname = self.opname_for_offset(j)
if opname.startswith('SETUP_'):
come_from_type = opname[len('SETUP_'):]
come_from_name = 'COME_FROM_%s' % come_from_type
pass
elif inst.offset in self.except_targets:
come_from_name = 'COME_FROM_EXCEPT_CLAUSE'
tokens.append(Token(come_from_name,
jump_offset, repr(jump_offset),
offset='%s_%s' % (inst.offset, jump_idx),
has_arg = True, opc=self.opc))
jump_idx += 1
pass
pass
elif inst.offset in self.else_start:
end_offset = self.else_start[inst.offset]
tokens.append(Token('ELSE',
None, repr(end_offset),
offset='%s' % (inst.offset),
has_arg = True, opc=self.opc))
pass
pattr = inst.argrepr
opname = inst.opname
if op in self.opc.CONST_OPS:
const = argval
if iscode(const):
if const.co_name == '<lambda>':
assert opname == 'LOAD_CONST'
opname = 'LOAD_LAMBDA'
elif const.co_name == '<genexpr>':
opname = 'LOAD_GENEXPR'
elif const.co_name == '<dictcomp>':
opname = 'LOAD_DICTCOMP'
elif const.co_name == '<setcomp>':
opname = 'LOAD_SETCOMP'
elif const.co_name == '<listcomp>':
opname = 'LOAD_LISTCOMP'
# verify() uses 'pattr' for comparison, since 'attr'
# now holds Code(const) and thus can not be used
# for comparison (todo: think about changing this)
# pattr = 'code_object @ 0x%x %s->%s' %\
# (id(const), const.co_filename, const.co_name)
pattr = '<code_object ' + const.co_name + '>'
else:
if isinstance(inst.arg, int) and inst.arg < len(co.co_consts):
argval, _ = _get_const_info(inst.arg, co.co_consts)
# Why don't we use _ above for "pattr" rather than "const"?
# This *is* a little hoaky, but we have to coordinate with
# other parts like n_LOAD_CONST in pysource.py for example.
pattr = const
pass
elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):
if self.version >= 3.6:
# 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_FUNCTION'
flags = argval
opname = 'MAKE_FUNCTION_%d' % (flags)
attr = []
for flag in self.MAKE_FUNCTION_FLAGS:
bit = flags & 1
attr.append(bit)
flags >>= 1
attr = attr[:4] # remove last value: attr[5] == False
else:
pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
pattr = ("%d positional, %d keyword pair, %d annotated" %
(pos_args, name_pair_args, annotate_args))
if name_pair_args > 0:
opname = '%s_N%d' % (opname, name_pair_args)
pass
if annotate_args > 0:
opname = '%s_A_%d' % (opname, annotate_args)
pass
opname = '%s_%d' % (opname, pos_args)
attr = (pos_args, name_pair_args, annotate_args)
tokens.append(
Token(
opname = opname,
attr = attr,
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = inst.has_arg,
opc = self.opc
)
)
continue
elif op in self.varargs_ops:
pos_args = argval
if self.is_pypy and not pos_args and opname == 'BUILD_MAP':
opname = 'BUILD_MAP_n'
else:
opname = '%s_%d' % (opname, pos_args)
elif self.is_pypy and opname == 'JUMP_IF_NOT_DEBUG':
# The value in the dict is in special cases in semantic actions, such
# as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put
# in arbitrary value 0.
customize[opname] = 0
elif opname == 'UNPACK_EX':
# FIXME: try with scanner and parser by
# changing argval
before_args = argval & 0xFF
after_args = (argval >> 8) & 0xff
pattr = "%d before vararg, %d after" % (before_args, after_args)
argval = (before_args, after_args)
opname = '%s_%d+%d' % (opname, before_args, after_args)
elif op == self.opc.JUMP_ABSOLUTE:
# Further classify JUMP_ABSOLUTE into backward jumps
# which are used in loops, and "CONTINUE" jumps which
# may appear in a "continue" statement. The loop-type
# and continue-type jumps will help us classify loop
# boundaries The continue-type jumps help us get
# "continue" statements with would otherwise be turned
# into a "pass" statement because JUMPs are sometimes
# ignored in rules as just boundary overhead. In
# comprehensions we might sometimes classify JUMP_BACK
# as CONTINUE, but that's okay since we add a grammar
# rule for that.
pattr = argval
target = self.get_target(inst.offset)
if target <= inst.offset:
next_opname = self.insts[i+1].opname
# 'Continue's include jumps to loops that are not
# and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP.
# If the JUMP_ABSOLUTE is to a FOR_ITER and it is followed by another JUMP_FORWARD
# then we'll take it as a "continue".
is_continue = (self.insts[self.offset2inst_index[target]]
.opname == 'FOR_ITER'
and self.insts[i+1].opname == 'JUMP_FORWARD')
if (is_continue or
(inst.offset in self.stmts and (inst.starts_line and
next_opname not in self.not_continue_follow))):
opname = 'CONTINUE'
else:
opname = 'JUMP_BACK'
# FIXME: this is a hack to catch stuff like:
# if x: continue
# the "continue" is not on a new line.
# There are other situations where we don't catch
# CONTINUE as well.
if tokens[-1].kind == 'JUMP_BACK' and tokens[-1].attr <= argval:
if tokens[-2].kind == 'BREAK_LOOP':
del tokens[-1]
else:
# intern is used because we are changing the *previous* token
tokens[-1].kind = intern('CONTINUE')
if last_op_was_break and opname == 'CONTINUE':
last_op_was_break = False
continue
# FIXME: go over for Python 3.6+. This is sometimes wrong
elif op == self.opc.RETURN_VALUE:
if inst.offset in self.return_end_ifs:
opname = 'RETURN_END_IF'
elif inst.offset in self.load_asserts:
opname = 'LOAD_ASSERT'
last_op_was_break = opname == 'BREAK_LOOP'
tokens.append(
Token(
opname = opname,
attr = argval,
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = inst.has_arg,
opc = self.opc
)
)
pass
if show_asm in ('both', 'after'):
for t in tokens:
print(t.format(line_prefix='L.'))
print()
return tokens, customize
def ingest(self, co, classname=None, code_objects={}, show_asm=None):
"""
Pick out tokens from an uncompyle6 code object, and transform them,
returning a list of uncompyle6 'Token's.
The transformations are made to assist the deparsing grammar.
Specificially:
- various types of LOAD_CONST's are categorized in terms of what they load
- COME_FROM instructions are added to assist parsing control structures
- MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
Also, when we encounter certain tokens, we add them to a set which will cause custom
grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
cause specific rules for the specific number of arguments they take.
"""
show_asm = self.show_asm if not show_asm else show_asm
# show_asm = 'after'
if show_asm in ('both', 'before'):
bytecode = Bytecode(co, self.opc)
for instr in bytecode.get_instructions(co):
print(instr._disassemble())
# Container for tokens
tokens = []
customize = {}
if self.is_pypy:
customize['PyPy'] = 1
self.code = array('B', co.co_code)
self.build_lines_data(co)
self.build_prev_op()
bytecode = Bytecode(co, self.opc)
# FIXME: put as its own method?
# Scan for assertions. Later we will
# turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
# 'LOAD_ASSERT' is used in assert statements.
self.load_asserts = set()
bs = list(bytecode)
n = len(bs)
for i in range(n):
inst = bs[i]
# We need to detect the difference between
# "raise AssertionError" and "assert"
# If we have a JUMP_FORWARD after the
# RAISE_VARARGS then we have a "raise" statement
# else we have an "assert" statement.
if inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n:
next_inst = bs[i+1]
if (next_inst.opname == 'LOAD_GLOBAL' and
next_inst.argval == 'AssertionError'):
for j in range(i+2, n):
raise_inst = bs[j]
if raise_inst.opname.startswith('RAISE_VARARGS'):
if j+1 >= n or bs[j+1].opname != 'JUMP_FORWARD':
self.load_asserts.add(next_inst.offset)
pass
break
pass
pass
# Get jump targets
# Format: {target offset: [jump offsets]}
jump_targets = self.find_jump_targets(show_asm)
for inst in bytecode:
argval = inst.argval
if inst.offset in jump_targets:
jump_idx = 0
# We want to process COME_FROMs to the same offset to be in *descending*
# offset order so we have the larger range or biggest instruction interval
# last. (I think they are sorted in increasing order, but for safety
# we sort them). That way, specific COME_FROM tags will match up
# properly. For example, a "loop" with an "if" nested in it should have the
# "loop" tag last so the grammar rule matches that properly.
for jump_offset in sorted(jump_targets[inst.offset], reverse=True):
come_from_name = 'COME_FROM'
opname = self.opName(jump_offset)
if opname.startswith('SETUP_'):
come_from_type = opname[len('SETUP_'):]
come_from_name = 'COME_FROM_%s' % come_from_type
pass
tokens.append(Token(come_from_name,
None, repr(jump_offset),
offset='%s_%s' % (inst.offset, jump_idx),
has_arg = True, opc=self.opc))
jump_idx += 1
pass
pass
elif inst.offset in self.else_start:
end_offset = self.else_start[inst.offset]
tokens.append(Token('ELSE',
None, repr(end_offset),
offset='%s' % (inst.offset),
has_arg = True, opc=self.opc))
pass
pattr = inst.argrepr
opname = inst.opname
op = inst.opcode
if opname in ['LOAD_CONST']:
const = inst.argval
if iscode(const):
if const.co_name == '<lambda>':
opname = 'LOAD_LAMBDA'
elif const.co_name == '<genexpr>':
opname = 'LOAD_GENEXPR'
elif const.co_name == '<dictcomp>':
opname = 'LOAD_DICTCOMP'
elif const.co_name == '<setcomp>':
opname = 'LOAD_SETCOMP'
elif const.co_name == '<listcomp>':
opname = 'LOAD_LISTCOMP'
# verify() uses 'pattr' for comparison, since 'attr'
# now holds Code(const) and thus can not be used
# for comparison (todo: think about changing this)
# pattr = 'code_object @ 0x%x %s->%s' %\
# (id(const), const.co_filename, const.co_name)
pattr = '<code_object ' + const.co_name + '>'
else:
pattr = const
pass
elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):
pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
if name_pair_args > 0:
opname = '%s_N%d' % (opname, name_pair_args)
pass
if annotate_args > 0:
opname = '%s_A_%d' % (opname, annotate_args)
pass
opname = '%s_%d' % (opname, pos_args)
pattr = ("%d positional, %d keyword pair, %d annotated" %
(pos_args, name_pair_args, annotate_args))
tokens.append(
Token(
type_ = opname,
attr = (pos_args, name_pair_args, annotate_args),
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = op_has_argument(op, op3),
opc = self.opc
)
)
continue
elif op in self.varargs_ops:
pos_args = inst.argval
if self.is_pypy and not pos_args and opname == 'BUILD_MAP':
opname = 'BUILD_MAP_n'
else:
opname = '%s_%d' % (opname, pos_args)
elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):
# The value in the dict is in special cases in semantic actions, such
# as CALL_FUNCTION. The value is not used in these cases, so we put
# in arbitrary value 0.
customize[opname] = 0
elif opname == 'UNPACK_EX':
# FIXME: try with scanner and parser by
# changing inst.argval
before_args = inst.argval & 0xFF
after_args = (inst.argval >> 8) & 0xff
pattr = "%d before vararg, %d after" % (before_args, after_args)
argval = (before_args, after_args)
opname = '%s_%d+%d' % (opname, before_args, after_args)
elif op == self.opc.JUMP_ABSOLUTE:
# Further classify JUMP_ABSOLUTE into backward jumps
# which are used in loops, and "CONTINUE" jumps which
# may appear in a "continue" statement. The loop-type
# and continue-type jumps will help us classify loop
# boundaries The continue-type jumps help us get
# "continue" statements with would otherwise be turned
# into a "pass" statement because JUMPs are sometimes
# ignored in rules as just boundary overhead. In
# comprehensions we might sometimes classify JUMP_BACK
# as CONTINUE, but that's okay since we add a grammar
# rule for that.
pattr = inst.argval
target = self.get_target(inst.offset)
if target <= inst.offset:
next_opname = self.opname[self.code[inst.offset+3]]
if (inst.offset in self.stmts and
next_opname not in ('END_FINALLY', 'POP_BLOCK',
# Python 3.0 only uses POP_TOP
'POP_TOP')
and inst.offset not in self.not_continue):
opname = 'CONTINUE'
else:
opname = 'JUMP_BACK'
# FIXME: this is a hack to catch stuff like:
# if x: continue
# the "continue" is not on a new line.
# There are other situations where we don't catch
# CONTINUE as well.
if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
# intern is used because we are changing the *previous* token
tokens[-1].type = intern('CONTINUE')
elif op == self.opc.RETURN_VALUE:
if inst.offset in self.return_end_ifs:
opname = 'RETURN_END_IF'
elif inst.offset in self.load_asserts:
opname = 'LOAD_ASSERT'
tokens.append(
Token(
type_ = opname,
attr = argval,
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = (op >= op3.HAVE_ARGUMENT),
opc = self.opc
)
)
pass
if show_asm in ('both', 'after'):
for t in tokens:
print(t)
print()
return tokens, customize
def disassemble(self, co, classname=None, code_objects={}, show_asm=None):
"""
Pick out tokens from an uncompyle6 code object, and transform them,
returning a list of uncompyle6 'Token's.
The tranformations are made to assist the deparsing grammar.
Specificially:
- various types of LOAD_CONST's are categorized in terms of what they load
- COME_FROM instructions are added to assist parsing control structures
- MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
Also, when we encounter certain tokens, we add them to a set which will cause custom
grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
cause specific rules for the specific number of arguments they take.
"""
show_asm = self.show_asm if not show_asm else show_asm
# show_asm = 'both'
if show_asm in ('both', 'before'):
bytecode = Bytecode(co, self.opc)
for instr in bytecode.get_instructions(co):
print(instr._disassemble())
# Container for tokens
tokens = []
customize = {}
if self.is_pypy:
customize['PyPy'] = 1;
self.code = array('B', co.co_code)
self.build_lines_data(co)
self.build_prev_op()
bytecode = Bytecode(co, self.opc)
# FIXME: put as its own method?
# Scan for assertions. Later we will
# turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
# 'LOAD_ASSERT' is used in assert statements.
self.load_asserts = set()
bs = list(bytecode)
n = len(bs)
for i in range(n):
inst = bs[i]
# We need to detect the difference between
# "raise AssertionError" and "assert"
# If we have a JUMP_FORWARD after the
# RAISE_VARARGS then we have a "raise" statement
# else we have an "assert" statement.
if inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n:
next_inst = bs[i+1]
if (next_inst.opname == 'LOAD_GLOBAL' and
next_inst.argval == 'AssertionError'):
for j in range(i+2, n):
raise_inst = bs[j]
if raise_inst.opname.startswith('RAISE_VARARGS'):
if j+1 >= n or bs[j+1].opname != 'JUMP_FORWARD':
self.load_asserts.add(next_inst.offset)
pass
break
pass
pass
# Get jump targets
# Format: {target offset: [jump offsets]}
jump_targets = self.find_jump_targets()
for inst in bytecode:
argval = inst.argval
if inst.offset in jump_targets:
jump_idx = 0
for jump_offset in jump_targets[inst.offset]:
tokens.append(Token('COME_FROM', None, repr(jump_offset),
offset='%s_%s' % (inst.offset, jump_idx),
has_arg = True, opc=self.opc))
jump_idx += 1
pass
pass
pattr = inst.argrepr
opname = inst.opname
op = inst.opcode
if opname in ['LOAD_CONST']:
const = inst.argval
if iscode(const):
if const.co_name == '<lambda>':
opname = 'LOAD_LAMBDA'
elif const.co_name == '<genexpr>':
opname = 'LOAD_GENEXPR'
elif const.co_name == '<dictcomp>':
opname = 'LOAD_DICTCOMP'
elif const.co_name == '<setcomp>':
opname = 'LOAD_SETCOMP'
elif const.co_name == '<listcomp>':
opname = 'LOAD_LISTCOMP'
# verify() uses 'pattr' for comparison, since 'attr'
# now holds Code(const) and thus can not be used
# for comparison (todo: think about changing this)
# pattr = 'code_object @ 0x%x %s->%s' %\
# (id(const), const.co_filename, const.co_name)
pattr = '<code_object ' + const.co_name + '>'
else:
pattr = const
pass
elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):
pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
if name_pair_args > 0:
opname = '%s_N%d' % (opname, name_pair_args)
pass
if annotate_args > 0:
opname = '%s_A_%d' % [opname, annotate_args]
pass
opname = '%s_%d' % (opname, pos_args)
pattr = ("%d positional, %d keyword pair, %d annotated" %
(pos_args, name_pair_args, annotate_args))
tokens.append(
Token(
type_ = opname,
attr = (pos_args, name_pair_args, annotate_args),
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = op_has_argument(op, op3),
opc = self.opc
)
)
continue
elif op in self.varargs_ops:
pos_args = inst.argval
if self.is_pypy and not pos_args and opname == 'BUILD_MAP':
opname = 'BUILD_MAP_n'
else:
opname = '%s_%d' % (opname, pos_args)
elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):
# The value in the dict is in special cases in semantic actions, such
# as CALL_FUNCTION. The value is not used in these cases, so we put
# in arbitrary value 0.
customize[opname] = 0
elif opname == 'UNPACK_EX':
# FIXME: try with scanner and parser by
# changing inst.argval
before_args = inst.argval & 0xFF
after_args = (inst.argval >> 8) & 0xff
pattr = "%d before vararg, %d after" % (before_args, after_args)
argval = (before_args, after_args)
opname = '%s_%d+%d' % (opname, before_args, after_args)
elif op == self.opc.JUMP_ABSOLUTE:
# Further classify JUMP_ABSOLUTE into backward jumps
# which are used in loops, and "CONTINUE" jumps which
# may appear in a "continue" statement. The loop-type
# and continue-type jumps will help us classify loop
# boundaries The continue-type jumps help us get
# "continue" statements with would otherwise be turned
# into a "pass" statement because JUMPs are sometimes
# ignored in rules as just boundary overhead. In
# comprehensions we might sometimes classify JUMP_BACK
# as CONTINUE, but that's okay since we add a grammar
# rule for that.
pattr = inst.argval
target = self.get_target(inst.offset)
if target <= inst.offset:
next_opname = self.opname[self.code[inst.offset+3]]
if (inst.offset in self.stmts and
next_opname not in ('END_FINALLY', 'POP_BLOCK')
and inst.offset not in self.not_continue):
opname = 'CONTINUE'
else:
opname = 'JUMP_BACK'
# FIXME: this is a hack to catch stuff like:
# if x: continue
# the "continue" is not on a new line.
# There are other situations were we don't catch
# CONTINUE as well.
if tokens[-1].type == 'JUMP_BACK':
tokens[-1].type = intern('CONTINUE')
elif op == self.opc.RETURN_VALUE:
if inst.offset in self.return_end_ifs:
opname = 'RETURN_END_IF'
elif inst.offset in self.load_asserts:
opname = 'LOAD_ASSERT'
tokens.append(
Token(
type_ = opname,
attr = argval,
pattr = pattr,
offset = inst.offset,
linestart = inst.starts_line,
op = op,
has_arg = (op >= op3.HAVE_ARGUMENT),
opc = self.opc
)
)
pass
if show_asm in ('both', 'after'):
for t in tokens:
print(t)
print()
return tokens, customize