def test_folding_of_unaryops_on_constants(self):
for line, elem in (
('-0.5', -0.5), # unary negative
('-0.0', -0.0), # -0.0
('-(1.0-1.0)', -0.0), # -0.0 after folding
('-0', 0), # -0
('~-2', 1), # unary invert
('+1', 1), # unary positive
):
code = compile(line, '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', elem)
for instr in dis.get_instructions(code):
self.assertFalse(instr.opname.startswith('UNARY_'))
# Check that -0.0 works after marshaling
def negzero():
return -(1.0-1.0)
for instr in dis.get_instructions(code):
self.assertFalse(instr.opname.startswith('UNARY_'))
# Verify that unfoldables are skipped
for line, elem, opname in (
('-"abc"', 'abc', 'UNARY_NEGATIVE'),
('~"abc"', 'abc', 'UNARY_INVERT'),
):
code = compile(line, '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', elem)
self.assertInBytecode(code, opname)
def test_elim_jump_after_return2(self):
# Eliminate dead code: jumps immediately after returns can't be reached
def f(cond1, cond2):
while 1:
if cond1: return 4
self.assertNotInBytecode(f, 'JUMP_FORWARD')
# There should be one jump for the while loop.
returns = [instr for instr in dis.get_instructions(f)
if instr.opname == 'JUMP_ABSOLUTE']
self.assertEqual(len(returns), 1)
returns = [instr for instr in dis.get_instructions(f)
if instr.opname == 'RETURN_VALUE']
self.assertEqual(len(returns), 2)
def _check_async_thread_block(code, offset, _cache={}):
'''
Analyze a given async_thread() context manager block to make sure it doesn't
contain any await operations
'''
if (code, offset) in _cache:
return _cache[code, offset]
import dis
instr = dis.get_instructions(code)
level = 0
result = True
for op in instr:
if op.offset < offset:
continue
if op.opname == 'SETUP_ASYNC_WITH':
level += 1
if op.opname in { 'YIELD_FROM', 'YIELD_VALUE' } and level > 0:
result = False
break
if op.opname == 'WITH_CLEANUP_START':
level -= 1
if level == 0:
break
_cache[code, offset] = result
return result
def test_folding_of_binops_on_constants(self):
for line, elem in (
('a = 2+3+4', 9), # chained fold
('"@"*4', '@@@@'), # check string ops
('a="abc" + "def"', 'abcdef'), # check string ops
('a = 3**4', 81), # binary power
('a = 3*4', 12), # binary multiply
('a = 13//4', 3), # binary floor divide
('a = 14%4', 2), # binary modulo
('a = 2+3', 5), # binary add
('a = 13-4', 9), # binary subtract
('a = (12,13)[1]', 13), # binary subscr
('a = 13 << 2', 52), # binary lshift
('a = 13 >> 2', 3), # binary rshift
('a = 13 & 7', 5), # binary and
('a = 13 ^ 7', 10), # binary xor
('a = 13 | 7', 15), # binary or
):
code = compile(line, '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', elem)
for instr in dis.get_instructions(code):
self.assertFalse(instr.opname.startswith('BINARY_'))
# Verify that unfoldables are skipped
code = compile('a=2+"b"', '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', 2)
self.assertInBytecode(code, 'LOAD_CONST', 'b')
# Verify that large sequences do not result from folding
code = compile('a="x"*1000', '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', 1000)
def get_assigned_name(frame):
"""
Checks the bytecode of *frame* to find the name of the variable a result is
being assigned to and returns that name. Returns the full left operand of the
assignment. Raises a #ValueError if the variable name could not be retrieved
from the bytecode (eg. if an unpack sequence is on the left side of the
assignment).
> **Known Limitations**: The expression in the *frame* from which this
> function is called must be the first part of that expression. For
> example, `foo = [get_assigned_name(get_frame())] + [42]` works,
> but `foo = [42, get_assigned_name(get_frame())]` does not!
```python
>>> var = get_assigned_name(sys._getframe())
>>> assert var == 'var'
```
__Available in Python 3.4, 3.5__
"""
SEARCHING, MATCHED = 1, 2
state = SEARCHING
result = ''
stacksize = 0
for op in dis.get_instructions(frame.f_code):
if state == SEARCHING and op.offset == frame.f_lasti:
if not op.opname.startswith('CALL_FUNCTION'):
raise RuntimeError('get_assigned_name() requires entry at CALL_FUNCTION')
state = MATCHED
# For a top-level expression, the stack-size should be 1 after
# the function at which we entered was executed.
stacksize = 1
elif state == MATCHED:
# Update the would-be size of the stack after this instruction.
# If we're at zero, we found the last instruction of the expression.
try:
stacksize += get_stackdelta(op)
except KeyError:
raise RuntimeError('could not determined assigned name, instruction '
'{} is not supported'.format(op.opname))
if stacksize == 0:
if op.opname not in ('STORE_NAME', 'STORE_ATTR', 'STORE_GLOBAL', 'STORE_FAST'):
raise ValueError('expression is not assigned or branch is not first part of the expression')
return result + op.argval
elif stacksize < 0:
raise ValueError('not a top-level expression')
if op.opname.startswith('CALL_FUNCTION'):
# Chained or nested function call.
raise ValueError('inside a chained or nested function call')
elif op.opname == 'LOAD_ATTR':
result += op.argval + '.'
if not result:
raise RuntimeError('last frame instruction not found')
assert False
def test_elim_extra_return(self):
# RETURN LOAD_CONST None RETURN --> RETURN
def f(x):
return x
self.assertNotInBytecode(f, 'LOAD_CONST', None)
returns = [instr for instr in dis.get_instructions(f)
if instr.opname == 'RETURN_VALUE']
self.assertEqual(len(returns), 1)
def get_load_const(self, tree):
# Compile to bytecode, disassemble and get parameter of LOAD_CONST
# instructions
co = compile(tree, '<string>', 'exec')
consts = []
for instr in dis.get_instructions(co):
if instr.opname == 'LOAD_CONST':
consts.append(instr.argval)
return consts
def _walk_global_ops(code):
"""
Yield (opcode, argument number) tuples for all
global-referencing instructions in *code*.
"""
for instr in dis.get_instructions(code):
op = instr.opcode
if op in GLOBAL_OPS:
yield op, instr.arg
def test_elim_jump_to_return(self):
# JUMP_FORWARD to RETURN --> RETURN
def f(cond, true_value, false_value):
return true_value if cond else false_value
self.assertNotInBytecode(f, 'JUMP_FORWARD')
self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
returns = [instr for instr in dis.get_instructions(f)
if instr.opname == 'RETURN_VALUE']
self.assertEqual(len(returns), 2)
def __init__(cls, name, bases, clsdict):
if not UnitRegistry.enabled:
super(UnitRegistry, cls).__init__(name, bases, clsdict)
return
yours = set(cls.mro())
mine = set(Distributable.mro())
left = yours - mine
if len(left) > 1:
if (not clsdict.get('hide_from_registry', False) and
not getattr(cls, 'hide_from_registry_all', False)):
UnitRegistry.units.add(cls)
else:
UnitRegistry.hidden_units.add(cls)
if "DISABLE_KWARGS_CHECK" in clsdict:
super(UnitRegistry, cls).__init__(name, bases, clsdict)
return
kwattrs = set(getattr(cls, "KWATTRS", set()))
for base in cls.__mro__:
try:
kw_var = inspect.getargspec(base.__init__).keywords
except TypeError:
continue
if USE_DIS:
try:
instrs = get_instructions(base.__init__)
except TypeError:
continue
loading_fast_kwargs = False
for inst in instrs:
# https://hg.python.org/cpython/file/b3f0d7f50544/Include/opcode.h # nopep8
# 124 = LOAD_FAST
# 136 = LOAD_DEREF
# 106 = LOAD_ATTR
# 100 = LOAD_CONST
if inst.opcode in (124, 136) and inst.argval == kw_var:
loading_fast_kwargs = True
elif loading_fast_kwargs and inst.opcode == 106:
continue
elif loading_fast_kwargs and inst.opcode == 100:
kwattrs.add(inst.argval)
loading_fast_kwargs = False
else:
loading_fast_kwargs = False
else:
try:
src, _ = inspect.getsourcelines(base.__init__)
except TypeError:
continue
kwarg_re = re.compile(
r"%(kwargs)s\.(get|pop)\(([^\s,\)]+)|%(kwargs)s\[([^\]]+)"
% {"kwargs": kw_var})
src = "".join((l.strip() for l in src)).replace('\n', '')
for match in kwarg_re.finditer(src):
kwattrs.add((match.group(2) or match.group(3))[1:-1])
cls.KWATTRS = kwattrs
super(UnitRegistry, cls).__init__(name, bases, clsdict)
def _get_opcodes(codeobj):
"""_get_opcodes(codeobj) -> [opcodes]
Extract the actual opcodes as an iterator from a code object
>>> c = compile("[1 + 2, (1,2)]", "", "eval")
>>> list(_get_opcodes(c))
[100, 100, 23, 100, 100, 102, 103, 83]
"""
return (i.opcode for i in dis.get_instructions(codeobj))
def assertNotInBytecode(self, x, opname, argval=_UNSPECIFIED):
"""Throws AssertionError if op is found"""
for instr in dis.get_instructions(x):
if instr.opname == opname:
disassembly = self.get_disassembly_as_string(co)
if opargval is _UNSPECIFIED:
msg = '%s occurs in bytecode:\n%s' % (opname, disassembly)
elif instr.argval == argval:
msg = '(%s,%r) occurs in bytecode:\n%s'
msg = msg % (opname, argval, disassembly)
self.fail(msg)
def get_function_instructions(funcname):
# Recompile with appropriate optimization setting
code = compile(source=code_string,
filename="<string>",
mode="exec",
optimize=self.optimize_python)
for c in code.co_consts:
if isinstance(c, types.CodeType) and c.co_name == funcname:
return dis.get_instructions(c)
return []
def is_safe_generator(agen):
'''
Examine the code of an async generator to see if it appears
unsafe with respect to async finalization. A generator
is unsafe if it utilizes any of the following constructs:
1. Use of async-code in a finally block
try:
yield v
finally:
await coro()
2. Use of yield inside an async context manager
async with m:
...
yield v
...
3. Use of async-code in try-except
try:
yield v
except Exception:
await coro()
'''
if agen.ag_code in _safe_async_generators:
return True
def _is_unsafe_block(instr, end_offset=-1):
is_generator = False
in_final = False
is_unsafe = False
for op in instr:
if op.offset == end_offset:
in_final = True
if op.opname == 'YIELD_VALUE':
is_generator = True
if op.opname == 'END_FINALLY':
return (is_generator, is_unsafe)
if op.opname in {'SETUP_FINALLY', 'SETUP_EXCEPT', 'SETUP_ASYNC_WITH'}:
is_g, is_u = _is_unsafe_block(instr, op.argval)
is_generator |= is_g
is_unsafe |= is_u
if op.opname == 'YIELD_FROM' and is_generator and in_final:
is_unsafe = True
return (is_generator, is_unsafe)
if not _is_unsafe_block(dis.get_instructions(agen.ag_code))[1]:
_safe_async_generators[agen.ag_code] = True
return True
else:
return False
def count_instr_recursively(f, opname):
count = 0
for instr in dis.get_instructions(f):
if instr.opname == opname:
count += 1
if hasattr(f, '__code__'):
f = f.__code__
for c in f.co_consts:
if hasattr(c, 'co_code'):
count += count_instr_recursively(c, opname)
return count
def test_peephole_opt_unreachable_code_array_access_in_bounds(self):
"""Regression test for issue35193 when run under clang msan."""
def unused_code_at_end():
return 3
raise RuntimeError("unreachable")
# The above function definition will trigger the out of bounds
# bug in the peephole optimizer as it scans opcodes past the
# RETURN_VALUE opcode. This does not always crash an interpreter.
# When you build with the clang memory sanitizer it reliably aborts.
self.assertEqual(
'RETURN_VALUE',
list(dis.get_instructions(unused_code_at_end))[-1].opname)
def get_globals(func):
result = set()
for inst in dis.get_instructions(func):
if inst.opname == 'LOAD_GLOBAL':
result.add(inst.argval)
ignore_globals = getattr(func, '_ignore_globals', set())
if ignore_globals is _all_globals:
return []
return [i for i in result if i not in ignore_globals]
def assertBytecodeExactlyMatches(self, x, expected, *, line_offset=0):
"""Throws AssertionError if any discrepancy is found in bytecode
*x* is the object to be introspected
*expected* is a list of dis.Instruction objects
Set *line_offset* as appropriate to adjust for the location of the
object to be disassembled within the test file. If the expected list
assumes the first line is line 1, then an appropriate offset would be
``1 - f.__code__.co_firstlineno``.
"""
actual = dis.get_instructions(x, line_offset=line_offset)
self.assertEqual(list(actual), expected)
def assertInBytecode(self, x, opname, argval=_UNSPECIFIED):
"""Returns instr if op is found, otherwise throws AssertionError"""
for instr in dis.get_instructions(x):
if instr.opname == opname:
if argval is _UNSPECIFIED or instr.argval == argval:
return instr
disassembly = self.get_disassembly_as_string(x)
if argval is _UNSPECIFIED:
msg = '%s not found in bytecode:\n%s' % (opname, disassembly)
else:
msg = '(%s,%r) not found in bytecode:\n%s'
msg = msg % (opname, argval, disassembly)
self.fail(msg)
def test_elim_jump_after_return1(self):
# Eliminate dead code: jumps immediately after returns can't be reached
def f(cond1, cond2):
if cond1: return 1
if cond2: return 2
while 1:
return 3
while 1:
if cond1: return 4
return 5
return 6
self.assertNotInBytecode(f, 'JUMP_FORWARD')
self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
returns = [instr for instr in dis.get_instructions(f)
if instr.opname == 'RETURN_VALUE']
self.assertEqual(len(returns), 6)
def get_imports(code):
imp = []
instrs = dis.get_instructions(code)
for instr in instrs:
if instr.opname in ('LOAD_GLOBAL', 'LOAD_NAME'):
name = code.co_names[instr.arg]
imp.append(name)
for instr_ in instrs:
if instr_.opname == 'LOAD_ATTR':
name += '.' + code.co_names[instr_.arg]
imp.append(name)
elif instr_.opname in ('LOAD_GLOBAL', 'LOAD_NAME'):
name = code.co_names[instr_.arg]
imp.append(name)
else:
break
return imp
def accessed_attributes_of_local(f, local_name):
"""
Get a list of attributes of ``local_name`` accessed by ``f``.
The analysis performed by this function is conservative, meaning that
it's not guaranteed to find **all** attributes used.
"""
used = set()
# Find sequences of the form: LOAD_FAST(local_name), LOAD_ATTR(<name>).
# This will find all usages of the form ``local_name.<name>``.
#
# It will **NOT** find usages in which ``local_name`` is aliased to
# another name.
for first, second in sliding_window(dis.get_instructions(f), 2):
if first.opname == 'LOAD_FAST' and first.argval == local_name:
if second.opname in ('LOAD_ATTR', 'LOAD_METHOD', 'STORE_ATTR'):
used.add(second.argval)
return used
def translate(func, ceval_snippets):
start_offset = 0
code_obj = getattr(func, '__code__', None)
if code_obj and os.path.exists(code_obj.co_filename):
start_offset = code_obj.co_firstlineno
with open(code_obj.co_filename) as f:
code_line_at = {
i: line.strip()
for i, line in enumerate(f, 1)
if line.strip()
}.get
else:
code_line_at = lambda _: None
for instr in get_instructions(func):
code_line = code_line_at(instr.starts_line)
line_no = (instr.starts_line or start_offset) - start_offset
yield line_no, code_line, instr, ceval_snippets.get(instr.opname)
def caller_assignment_name(depth=0):
frame = sys._getframe(depth + 2)
dis.disassemble(frame.f_code, frame.f_lasti)
# from pprint import pprint
# pprint(list(dis.get_instructions(frame.f_code)))
for op in list(dis.get_instructions(frame.f_code)):
if op.offset < frame.f_lasti:
continue
if op.starts_line is not None:
# We hit a new source line.
break
if op.opname == "STORE_NAME":
return op.argval
raise RuntimeError("Return value of caller must be assigned to a variable")
def typecheckFunction(func, scopeTypes = {}, infer = typeInferrer(), logFile = None):
symbols = []
# __builtins__ is sometimes a module and sometimes a dict?
builtins = __builtins__ if type(__builtins__) == dict else __builtins__.__dict__
if func.__closure__:
closure = {val.__name__: val for val in (cell.cell_contents for cell in func.__closure__)}
else:
closure = {}
context = Context(
scopeVals = extend(
func.__globals__,
builtins,
closure,
{func.__name__: func}),
# todo: scopeTypes should also include types from scopeVals that have _type
# todo: actually should just be scopeSymbols instead of scopeVals and scopeTypes
scopeTypes = extend(
defaultdict(lambda: Object),
scopeTypes,
func.__annotations__,
{func.__name__: infer(func)}),
signature = infer(func))
for path in executionPaths(list(dis.get_instructions(func))):
for inst in path:
handler = instructionHandlers[inst.opname]
if logFile:
pprint(symbols, stream=logFile)
print(file=logFile) # newline
print(inst, file=logFile)
errors = handler(context, symbols, inst, infer)
if errors:
if logFile:
print(errors, file=logFile)
return errors
return []
def test_constant_folding(self):
# Issue #11244: aggressive constant folding.
exprs = [
'3 * -5',
'-3 * 5',
'2 * (3 * 4)',
'(2 * 3) * 4',
'(-1, 2, 3)',
'(1, -2, 3)',
'(1, 2, -3)',
'(1, 2, -3) * 6',
'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}',
]
for e in exprs:
code = compile(e, '', 'single')
for instr in dis.get_instructions(code):
self.assertFalse(instr.opname.startswith('UNARY_'))
self.assertFalse(instr.opname.startswith('BINARY_'))
self.assertFalse(instr.opname.startswith('BUILD_'))
def get_assigned_name(frame):
''' Checks the bytecode of *frame* to find the name of the variable
a result is being assigned to and returns that name. Returns the full
left operand of the assignment. Raises a `ValueError` if the variable
name could not be retrieved from the bytecode (eg. if an unpack sequence
is on the left side of the assignment).
>>> var = get_assigned_frame(sys._getframe())
>>> assert var == 'var'
'''
SEARCHING, MATCHED = 1, 2
state = SEARCHING
result = ''
for op in dis.get_instructions(frame.f_code):
if state == SEARCHING and op.offset == frame.f_lasti:
state = MATCHED
elif state == MATCHED:
if result:
if op.opname == 'LOAD_ATTR':
result += op.argval + '.'
elif op.opname == 'STORE_ATTR':
result += op.argval
break
else:
raise ValueError('expected {LOAD_ATTR, STORE_ATTR}', op.opname)
else:
if op.opname in ('LOAD_NAME', 'LOAD_FAST'):
result += op.argval + '.'
elif op.opname in ('STORE_NAME', 'STORE_FAST'):
result = op.argval
break
else:
message = 'expected {LOAD_NAME, LOAD_FAST, STORE_NAME, STORE_FAST}'
raise ValueError(message, op.opname)
if not result:
raise RuntimeError('last frame instruction not found')
return result
def test_folding_of_tuples_of_constants(self):
for line, elem in (
('a = 1,2,3', (1, 2, 3)),
('("a","b","c")', ('a', 'b', 'c')),
('a,b,c = 1,2,3', (1, 2, 3)),
('(None, 1, None)', (None, 1, None)),
('((1, 2), 3, 4)', ((1, 2), 3, 4)),
):
code = compile(line,'','single')
self.assertInBytecode(code, 'LOAD_CONST', elem)
self.assertNotInBytecode(code, 'BUILD_TUPLE')
# Long tuples should be folded too.
code = compile(repr(tuple(range(10000))),'','single')
self.assertNotInBytecode(code, 'BUILD_TUPLE')
# One LOAD_CONST for the tuple, one for the None return value
load_consts = [instr for instr in dis.get_instructions(code)
if instr.opname == 'LOAD_CONST']
self.assertEqual(len(load_consts), 2)
# Bug 1053819: Tuple of constants misidentified when presented with:
# . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . .
# The following would segfault upon compilation
def crater():
(~[
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
],)
def is_simple(fun):
"""A heuristic to find out if a function is simple enough."""
seen_load_fast_0 = False
seen_load_response = False
seen_call_fun = False
for instruction in dis.get_instructions(fun):
if instruction.opname == 'LOAD_FAST' and instruction.arg == 0:
seen_load_fast_0 = True
continue
if instruction.opname == 'LOAD_ATTR' \
and instruction.argval == 'Response':
seen_load_response = True
continue
if instruction.opname.startswith('CALL_FUNCTION'):
if seen_call_fun:
return False
seen_call_fun = True
continue
return seen_call_fun and seen_load_fast_0 and seen_load_response
def _walk_global_ops(code):
for instr in dis.get_instructions(code):
op = instr.opcode
if op in GLOBAL_OPS:
yield op, instr.arg
def __init__(self, myfn):
def lstadd(hmap, key, val):
if key not in hmap:
hmap[key] = [val]
else:
hmap[key].append(val)
enter = CFGNode(
dis.Instruction('NOP',
opcode=dis.opmap['NOP'],
arg=0,
argval=0,
argrepr=0,
offset=0,
starts_line=0,
is_jump_target=False), 0)
last = enter
self.jump_to = {}
self.opcodes = {}
for i, ins in enumerate(dis.get_instructions(myfn)):
byte = i * 2
node = CFGNode(ins, byte)
self.opcodes[byte] = node
print(i, ins)
if ins.opname in [
'LOAD_CONST', 'LOAD_FAST', 'STORE_FAST', 'COMPARE_OP',
'INPLACE_ADD', 'INPLACE_SUBTRACT', 'RETURN_VALUE',
'BINARY_MODULO', 'POP_BLOCK'
]:
last.add_child(node)
last = node
elif ins.opname == 'POP_JUMP_IF_FALSE':
print("will jump to", ins.arg)
lstadd(self.jump_to, ins.arg, node)
node.props['jmp'] = True
last.add_child(node)
last = node
elif ins.opname == 'JUMP_FORWARD':
node.props['jmp'] = True
lstadd(self.jump_to, (i + 1) * 2 + ins.arg, node)
print("will jump to", (i + 1) * 2 + ins.arg)
last.add_child(node)
last = node
elif ins.opname == 'SETUP_LOOP':
print("setuploop: ", byte, ins.arg)
last.add_child(node)
last = node
elif ins.opname == 'JUMP_ABSOLUTE':
print("will jump to", ins.arg)
lstadd(self.jump_to, ins.arg, node)
node.props['jmp'] = True
last.add_child(node)
last = node
else:
assert False
for byte in self.opcodes:
if byte in self.jump_to:
node = self.opcodes[byte]
assert node.i.is_jump_target
for b in self.jump_to[byte]:
b.add_child(node)
def docopt(
docstring: Optional[str] = None,
argv: Optional[Union[List[str], str]] = None,
default_help: bool = True,
version: Any = None,
options_first: bool = False,
more_magic: bool = False,
) -> ParsedOptions:
"""Parse `argv` based on command-line interface described in `doc`.
`docopt` creates your command-line interface based on its
description that you pass as `docstring`. Such description can contain
--options, <positional-argument>, commands, which could be
[optional], (required), (mutually | exclusive) or repeated...
Parameters
----------
docstring : str (default: first __doc__ in parent scope)
Description of your command-line interface.
argv : list of str, optional
Argument vector to be parsed. sys.argv[1:] is used if not
provided.
default_help : bool (default: True)
Set to False to disable automatic help on -h or --help
options.
version : any object
If passed, the object will be printed if --version is in
`argv`.
options_first : bool (default: False)
Set to True to require options precede positional arguments,
i.e. to forbid options and positional arguments intermix.
more_magic : bool (default: False)
Try to be extra-helpful; pull results into globals() of caller as 'arguments',
offer advanced pattern-matching and spellcheck.
Also activates if `docopt` aliased to a name containing 'magic'.
Returns
-------
arguments: dict-like
A dictionary, where keys are names of command-line elements
such as e.g. "--verbose" and "<path>", and values are the
parsed values of those elements. Also supports dot acccess.
Example
-------
>>> from docopt import docopt
>>> doc = '''
... Usage:
... my_program tcp <host> <port> [--timeout=<seconds>]
... my_program serial <port> [--baud=<n>] [--timeout=<seconds>]
... my_program (-h | --help | --version)
...
... Options:
... -h, --help Show this screen and exit.
... --baud=<n> Baudrate [default: 9600]
... '''
>>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30']
>>> docopt(doc, argv)
{'--baud': '9600',
'--help': False,
'--timeout': '30',
'--version': False,
'<host>': '127.0.0.1',
'<port>': '80',
'serial': False,
'tcp': True}
"""
argv = sys.argv[1:] if argv is None else argv
maybe_frame = inspect.currentframe()
if maybe_frame:
parent_frame = doc_parent_frame = magic_parent_frame = maybe_frame.f_back
if not more_magic: # make sure 'magic' isn't in the calling name
while not more_magic and magic_parent_frame:
imported_as = {v: k for k, v in magic_parent_frame.f_globals.items() if hasattr(v, "__name__") and v.__name__ == docopt.__name__}.get(docopt)
if imported_as and "magic" in imported_as:
more_magic = True
else:
magic_parent_frame = magic_parent_frame.f_back
if not docstring: # go look for one, if none exists, raise Exception
while not docstring and doc_parent_frame:
docstring = doc_parent_frame.f_locals.get("__doc__")
if not docstring:
doc_parent_frame = doc_parent_frame.f_back
if not docstring:
raise DocoptLanguageError("Either __doc__ must be defined in the scope of a parent or passed as the first argument.")
output_value_assigned = False
if more_magic and parent_frame:
import dis
instrs = dis.get_instructions(parent_frame.f_code)
for instr in instrs:
if instr.offset == parent_frame.f_lasti:
break
assert instr.opname.startswith("CALL_")
MAYBE_STORE = next(instrs)
if MAYBE_STORE and (MAYBE_STORE.opname.startswith("STORE") or MAYBE_STORE.opname.startswith("RETURN")):
output_value_assigned = True
usage_sections = parse_section("usage:", docstring)
if len(usage_sections) == 0:
raise DocoptLanguageError('"usage:" section (case-insensitive) not found. Perhaps missing indentation?')
if len(usage_sections) > 1:
raise DocoptLanguageError('More than one "usage:" (case-insensitive).')
options_pattern = re.compile(r"\n\s*?options:", re.IGNORECASE)
if options_pattern.search(usage_sections[0]):
raise DocoptExit("Warning: options (case-insensitive) was found in usage." "Use a blank line between each section..")
DocoptExit.usage = usage_sections[0]
options = parse_defaults(docstring)
pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
pattern_options = set(pattern.flat(Option))
for options_shortcut in pattern.flat(OptionsShortcut):
doc_options = parse_defaults(docstring)
options_shortcut.children = [opt for opt in doc_options if opt not in pattern_options]
parsed_arg_vector = parse_argv(Tokens(argv), list(options), options_first, more_magic)
extras(default_help, version, parsed_arg_vector, docstring)
matched, left, collected = pattern.fix().match(parsed_arg_vector)
if matched and left == []:
output_obj = ParsedOptions((a.name, a.value) for a in (pattern.flat() + collected))
target_parent_frame = parent_frame or magic_parent_frame or doc_parent_frame
if more_magic and target_parent_frame and not output_value_assigned:
if not target_parent_frame.f_globals.get("arguments"):
target_parent_frame.f_globals["arguments"] = output_obj
return output_obj
if left:
raise DocoptExit(f"Warning: found unmatched (duplicate?) arguments {left}")
raise DocoptExit(collected=collected, left=left)
res = {}
for repo_dir in os.listdir(sys.argv[1]):
repo_imports = []
full_repo_path = os.path.join(sys.argv[1], repo_dir)
for root, dirs, files in os.walk(full_repo_path):
for filename in files:
if not filename.endswith('.py'):
continue
full_path = os.path.join(root, filename)
try:
with open(full_path) as handle:
statements = handle.read()
instructions = dis.get_instructions(statements)
imports = [
__ for __ in instructions if 'IMPORT' in __.opname
]
grouped = defaultdict(list)
for instr in imports:
grouped[instr.opname].append(instr.argval)
repo_imports.extend(grouped['IMPORT_NAME'])
except:
pass
print(json.dumps(repo_imports))
def is_call_function(code: CodeType, bytei: ByteCodeIndex) -> bool:
"""Returns true iff the bytecode at the given index is a function call."""
for ins in dis.get_instructions(code):
if ins.offset == bytei and ins.opcode in Scalene.__call_opcodes:
return True
return False
def from_pycode(cls, co):
"""Create a Code object from a python code object.
Parameters
----------
co : CodeType
The python code object.
Returns
-------
code : Code
The codetransformer Code object.
"""
# Make it sparse to instrs[n] is the instruction at bytecode[n]
sparse_instrs = tuple(
_sparse_args(
Instruction.from_opcode(
b.opcode,
Instruction._no_arg if b.arg is None else _RawArg(b.arg),
) for b in get_instructions(co)), )
for idx, instr in enumerate(sparse_instrs):
if instr is None:
# The sparse value
continue
if instr.absjmp:
instr.arg = sparse_instrs[instr.arg]
elif instr.reljmp:
instr.arg = sparse_instrs[instr.arg + idx + argsize + 1]
elif isinstance(instr, LOAD_CONST):
instr.arg = co.co_consts[instr.arg]
elif instr.uses_name:
instr.arg = co.co_names[instr.arg]
elif instr.uses_varname:
instr.arg = co.co_varnames[instr.arg]
elif instr.uses_free:
instr.arg = _freevar_argname(
instr.arg,
co.co_freevars,
co.co_cellvars,
)
elif instr.have_arg and isinstance(instr.arg, _RawArg):
instr.arg = int(instr.arg)
flags = Flag.unpack(co.co_flags)
has_vargs = flags['CO_VARARGS']
has_kwargs = flags['CO_VARKEYWORDS']
# Here we convert the varnames format into our argnames format.
paramnames = co.co_varnames[:(co.co_argcount +
getattr(co, 'co_kwonlyargcount', 0) +
has_vargs + has_kwargs)]
# We start with the positional arguments.
new_paramnames = list(paramnames[:co.co_argcount])
# Add *args next.
if has_vargs:
new_paramnames.append('*' + paramnames[-1 - has_kwargs])
# Add positional only arguments next.
new_paramnames.extend(paramnames[co.co_argcount:co.co_argcount +
getattr(co, 'co_kwonlyargcount', 0)])
# Add **kwargs last.
if has_kwargs:
new_paramnames.append('**' + paramnames[-1])
return cls(
filter(bool, sparse_instrs),
argnames=new_paramnames,
cellvars=co.co_cellvars,
freevars=co.co_freevars,
name=co.co_name,
filename=co.co_filename,
firstlineno=co.co_firstlineno,
# lnotab={
# lno: sparse_instrs[off] for off, lno in findlinestarts(co)
# },
flags=flags,
)
def test_first_line_set_to_None(self):
actual = dis.get_instructions(simple, first_line=None)
self.assertEqual(list(actual), expected_opinfo_simple)
def test_default_first_line(self):
actual = dis.get_instructions(simple)
self.assertEqual(list(actual), expected_opinfo_simple)
def print_inst(obj):
lines = dis.get_instructions(obj)
for line in lines:
log.debug(line)
def run_code(self, code):
if isinstance(code, types.CodeType):
code_obj = code
elif isinstance(code, str):
code_obj = compile(code, '<test>', 'exec')
else:
raise TypeError
instructions = list(dis.get_instructions(code_obj))
step = 0
while step < len(instructions):
instruction = instructions[step]
# Load and store consts, names, etc
if instruction.opname == "LOAD_CONST":
self.stack.append(instruction.argval)
elif instruction.opname == "LOAD_NAME":
name = instruction.argval
value, namespace = self.find_instance_by_name(name)
self.stack.append(value)
elif instruction.opname == "LOAD_GLOBAL":
name = instruction.argval
value, namespace = self.find_instance_by_name(name)
self.stack.append(value)
elif instruction.opname == "LOAD_FAST":
args, kwargs, optional_args = self.co_varnames[-1]
if instruction.argval in kwargs:
arg = kwargs[instruction.argval]
elif instruction.arg < len(args):
arg = args[instruction.arg]
else:
arg = optional_args[instruction.argval]
# print("ARGUMENT", instruction.argval, "=", arg) # TODO: remove line
self.stack.append(arg)
elif instruction.opname == "STORE_NAME":
name = instruction.argval
value = self.stack.pop()
self.locals[name] = value
# Functions
elif instruction.opname == "CALL_FUNCTION":
args = []
for _ in range(instruction.arg):
args.insert(0, self.stack.pop())
func = self.stack.pop()
retval = func(*args)
self.stack.append(retval)
elif instruction.opname == "CALL_FUNCTION_KW":
args = []
kwargs = {}
key_words = self.stack.pop()
for key_word in reversed(key_words):
kwargs[key_word] = self.stack.pop()
for _ in range(instruction.arg - len(kwargs)):
args.insert(0, self.stack.pop())
func = self.stack.pop()
# print(func, args, kwargs)
retval = func(*args, **kwargs)
self.stack.append(retval)
elif instruction.opname == "CALL_FUNCTION_EX":
if instruction.arg == 0:
kwargs = {}
args = self.stack.pop()
elif instruction.arg == 1:
kwargs = self.stack.pop()
args = self.stack.pop()
else:
raise Exception(
"Unknown instruction.arg in CALL_FUNCTION_EX")
func = self.stack.pop()
retval = func(*args, **kwargs)
self.stack.append(retval)
elif instruction.opname == "MAKE_FUNCTION":
flags = instruction.arg
func = self.Function(self, flags)
self.stack.append(func)
elif instruction.opname == "RETURN_VALUE":
# TODO: implement, if complex logic is necessary
pass
# Stack operations
elif instruction.opname == "POP_TOP":
self.stack.pop()
elif instruction.opname == "DUP_TOP":
value = self.stack[-1]
self.stack.append(value)
elif instruction.opname == "UNPACK_SEQUENCE":
# count = instruction.arg
values = self.stack.pop()
for value in reversed(values):
self.stack.append(value)
# Building containers
elif instruction.opname.startswith("BUILD_"):
count = instruction.arg
if instruction.opname.endswith(("_LIST", "_TUPLE", "_SET")):
result = []
for _ in range(count):
result.insert(0, self.stack.pop())
if instruction.opname == "BUILD_LIST":
self.stack.append(result)
elif instruction.opname == "BUILD_TUPLE":
self.stack.append(tuple(result))
elif instruction.opname == "BUILD_SET":
self.stack.append(set(result))
else:
raise Exception("Unknown type of container")
# elif instruction.opname == "BUILD_MAP":
# result = {}
# for _ in range(count):
# key = self.stack.pop()
# value = self.stack.pop()
# result[key] = value
# self.stack.append(result)
elif instruction.opname == "BUILD_CONST_KEY_MAP":
items = []
keys = self.stack.pop()
for key in reversed(keys):
value = self.stack.pop()
items.insert(0, (key, value))
result = dict(items)
self.stack.append(result)
elif instruction.opname == "BUILD_SLICE":
arg1 = self.stack.pop()
arg2 = self.stack.pop()
if instruction.arg == 2:
self.stack.append(slice(arg2, arg1))
elif instruction.arg == 3:
arg3 = self.stack.pop()
self.stack.append(slice(arg3, arg2, arg1))
else:
raise Exception(
"Unknown instruction.arg in BUILD_SLICE")
else:
raise Exception("Unknown type of container")
# Comparison operators
elif instruction.opname == "COMPARE_OP":
arg2 = self.stack.pop()
arg1 = self.stack.pop()
if instruction.argval == '==':
result = arg1 == arg2
elif instruction.argval == '!=':
result = arg1 != arg2
elif instruction.argval == '>':
result = arg1 > arg2
elif instruction.argval == '<':
result = arg1 < arg2
elif instruction.argval == '>=':
result = arg1 >= arg2
elif instruction.argval == '<=':
result = arg1 <= arg2
elif instruction.argval == 'in':
result = arg1 in arg2
elif instruction.argval == 'not in':
result = arg1 not in arg2
elif instruction.argval == 'is':
result = arg1 is arg2
elif instruction.argval == 'is not':
result = arg1 is not arg2
else:
raise Exception("Unsupported comparison operator")
self.stack.append(result)
# Unary operators
elif instruction.opname.startswith("UNARY_"):
arg = self.stack.pop()
if instruction.opname == "UNARY_POSITIVE":
result = +arg
elif instruction.opname == "UNARY_NEGATIVE":
result = -arg
elif instruction.opname == "UNARY_NOT":
result = not arg
elif instruction.opname == "UNARY_INVERT":
result = ~arg
else:
raise Exception("Unsupported unary operator")
self.stack.append(result)
elif instruction.opname == "GET_ITER":
arg = self.stack.pop()
result = iter(arg)
self.stack.append(result)
# Binary and inplace operators
elif instruction.opname.startswith(("BINARY_", "INPLACE_")):
arg2 = self.stack.pop()
arg1 = self.stack.pop()
if instruction.opname.endswith("_ADD"):
result = arg1 + arg2
elif instruction.opname.endswith("_SUBTRACT"):
result = arg1 - arg2
elif instruction.opname.endswith("_MULTIPLY"):
result = arg1 * arg2
elif instruction.opname.endswith("_POWER"):
result = arg1**arg2
elif instruction.opname.endswith("_FLOOR_DIVIDE"):
result = arg1 // arg2
elif instruction.opname.endswith("_TRUE_DIVIDE"):
result = arg1 / arg2
elif instruction.opname.endswith("_MODULO"):
result = arg1 % arg2
elif instruction.opname.endswith("_SUBSCR"):
result = arg1[arg2]
elif instruction.opname.endswith("_LSHIFT"):
result = arg1 << arg2
elif instruction.opname.endswith("_RSHIFT"):
result = arg1 >> arg2
elif instruction.opname.endswith("_AND"):
result = arg1 & arg2
elif instruction.opname.endswith("_XOR"):
result = arg1 ^ arg2
elif instruction.opname.endswith("_OR"):
result = arg1 | arg2
elif instruction.opname.endswith("_MATRIX_MULTIPLY"):
result = arg1 @ arg2
else:
raise Exception("Unsupported binary (or inplace) operator")
self.stack.append(result)
# Operations with jumps (logical, cycles, etc)
elif instruction.opname == "JUMP_IF_TRUE_OR_POP":
top = self.stack.pop()
if top:
step = self.get_step_by_argval(instruction.argval)
self.stack.append(top)
elif instruction.opname == "JUMP_IF_FALSE_OR_POP":
top = self.stack.pop()
if not top:
step = self.get_step_by_argval(instruction.argval)
self.stack.append(top)
elif instruction.opname == "POP_JUMP_IF_TRUE":
top = self.stack.pop()
if top:
step = self.get_step_by_argval(instruction.argval)
elif instruction.opname == "POP_JUMP_IF_FALSE":
top = self.stack.pop()
if not top:
step = self.get_step_by_argval(instruction.argval)
elif instruction.opname == "JUMP_FORWARD":
step = self.get_step_by_argval(instruction.argval)
elif instruction.opname == "JUMP_ABSOLUTE":
step = self.get_step_by_argval(instruction.argval)
elif instruction.opname == "FOR_ITER":
top = self.stack[-1]
try:
elem = top.__next__()
self.stack.append(elem)
except StopIteration:
self.stack.pop()
step = self.get_step_by_argval(instruction.argval)
# Loops
elif instruction.opname == "SETUP_LOOP":
self.block_stack.append(instruction)
elif instruction.opname == "POP_BLOCK":
self.block_stack.pop()
elif instruction.opname == "BREAK_LOOP":
block = self.block_stack.pop()
step = self.get_step_by_argval(block.argval)
else:
raise Exception("Unknown instruction " + instruction.opname)
step += 1
return None
def _parse_lambda(function: types.FunctionType, tables: Dict[FEID, FeSummaryTable],
ret_ref: Flag) -> Optional[Dict[str, Any]]:
"""Convert a lambda function into its argument-based representation.
The `function` is expected to be a lambda expression, which means that the set of bytecode instructions is limited
compared to examining any possible function.
Args:
function: A lambda function to be inspected.
tables: A collection of tables representing objects which are used by the current stack of inputs.
ret_ref: A flag to indicate that _trace_value is returning a reference (this is used to figure out whether
functions can be in-lined or deserve their own tables).
Returns:
The arguments being used to invoke `function`, or None if parsing fails.
"""
code = function.__code__
instructions = [x for x in dis.get_instructions(code)]
closure_vars = inspect.getclosurevars(function) # The variables defining the current scope.
conditions = []
args = []
idx = 0
while idx < len(instructions):
instruction = instructions[idx]
if instruction.opname == 'RETURN_VALUE':
# Lambda functions don't support the return keyword, instead values are returned implicitly
if conditions:
current_condition = conditions.pop()
arg = args.pop()
instructions.pop(idx - 1)
idx -= 1
# In lambda functions, conditions always fill in the order: condition -> left -> right
if current_condition.left is None:
conditions.append(_Condition(left=arg, condition=current_condition.condition, right=None))
instructions.pop(idx - 1)
idx -= 1
else:
args.append(
_Condition(left=current_condition.left, condition=current_condition.condition, right=arg))
if conditions:
# The return value can be used to satisfy a condition slot
idx -= 1
else:
break
elif instruction.opname == 'LOAD_CONST':
# It's a constant value
args.append(instruction.argval)
elif instruction.opname == 'LOAD_FAST':
# It's a variable from a lambda expression
args.append(_VarWrap(instruction.argval))
elif instruction.opname in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
# It's an iterable
n_args = instruction.argval
arg = deque()
for i in range(n_args):
arg.appendleft(args.pop())
instructions.pop(idx - 1)
idx -= 1
arg = list(arg)
if instruction.opname == 'BUILD_TUPLE':
arg = tuple(arg)
elif instruction.opname == 'BUILD_SET':
arg = set(arg)
args.append(arg)
elif instruction.opname == "BUILD_MAP":
# It's a map
n_keys = instruction.argval
arg = {}
for i in range(n_keys):
v = args.pop()
k = args.pop()
instructions.pop(idx - 1)
idx -= 1
instructions.pop(idx - 1)
idx -= 1
arg[k] = v
args.append(arg)
elif instruction.opname == "BUILD_CONST_KEY_MAP":
# It's a map that had constant keys
keys = args.pop()
instructions.pop(idx - 1)
idx -= 1
vals = deque()
for i in range(instruction.argval):
vals.appendleft(args.pop())
instructions.pop(idx - 1)
idx -= 1
args.append({key: val for key, val in zip(keys, vals)})
elif instruction.opname == 'LOAD_DEREF' and not _deref_is_callable(
instruction, closure_vars) and not instructions[idx + 1].opname in ('LOAD_METHOD', 'LOAD_ATTR'):
# It's a reference to a variable that's not being used to invoke some other function
args.append(
closure_vars.nonlocals.get(
instruction.argval,
closure_vars.globals.get(instruction.argval, closure_vars.builtins.get(instruction.argval, None))))
elif instruction.opname in ('LOAD_METHOD', 'LOAD_ATTR', 'LOAD_GLOBAL', 'LOAD_DEREF'):
# We're setting up a function call, which may or may not be invoked
# Look ahead to combine all of the function pieces together into 1 variable
name = instructions[idx].argval
func_pair = _Function(
closure_vars.nonlocals.get(name, closure_vars.globals.get(name, closure_vars.builtins.get(name, None))),
name=name)
if func_pair.func is None:
# This function can't be found for some reason
return _parse_lambda_fallback(function, tables, ret_ref)
while idx + 1 < len(instructions):
if instructions[idx + 1].opname in ('LOAD_METHOD', 'LOAD_ATTR'):
name = instructions[idx + 1].argval
func_pair = _Function(getattr(func_pair.func, name), name=func_pair.name + f".{name}")
instructions.pop(idx + 1)
else:
break
args.append(func_pair)
elif instruction.opname in ('CALL_METHOD', 'CALL_FUNCTION', 'CALL_FUNCTION_KW'):
kwargs = {}
kwarg_names = []
if instruction.opname == 'CALL_FUNCTION_KW':
# Gather the keywords, which were added with a LOAD_CONST call
kwarg_names = args.pop()
instructions.pop(idx - 1)
idx -= 1
# Gather the args
n_args = instruction.argval
fn_args = deque()
for i in range(n_args):
fn_args.appendleft(args.pop())
instructions.pop(idx - 1)
idx -= 1
for name in reversed(kwarg_names):
kwargs[name] = fn_args.pop()
# Gather the fn
func_pair = args.pop()
instructions.pop(idx - 1) # Remove the method def from the stack
idx -= 1
# Bind the fn
if not callable(func_pair.func):
# This shouldn't ever happen, but just in case...
return _parse_lambda_fallback(function, tables, ret_ref)
try:
bound_args = inspect.signature(func_pair.func).bind(*fn_args, **kwargs)
bound_args.apply_defaults()
except ValueError:
# Some functions (C bindings) don't have convenient signature lookup
bound_args = _PartialBind(tuple(fn_args), kwargs)
args.append(_BoundFn(func_pair, bound_args))
elif instruction.opname.startswith('BINARY_') or instruction.opname.startswith(
'INPLACE_') or instruction.opname == 'COMPARE_OP':
# Capture actual inline function stuff like: 0.5 + x
command = strip_prefix(strip_prefix(instruction.opname, 'BINARY_'), 'INPLACE_')
if instruction.opname == 'COMPARE_OP':
command = instruction.argval
if command not in _CommandTable:
return _parse_lambda_fallback(function, tables, ret_ref)
right = args.pop()
instructions.pop(idx - 1)
idx -= 1
left = args.pop()
instructions.pop(idx - 1)
idx -= 1
args.append(_Command(left, right, _CommandTable[command]))
elif instruction.opname == 'POP_JUMP_IF_FALSE':
# a if a < b else b ||| <left> if <condition> else <right>
conditions.append(_Condition(left=None, right=None, condition=args.pop()))
instructions.pop(idx - 1)
idx -= 1
else:
# TODO - to be fully rigorous we need the rest: https://docs.python.org/3.7/library/dis.html#bytecodes
# TODO - LIST_APPEND, SET_ADD, MAP_ADD, BUILD_STRING, CALL_FUNCTION_EX, BUILD_TUPLE_UNPACK, etc.
# Note that this function is only ever used to examine lambda functions, which helps to restrict the set of
# possible commands
return _parse_lambda_fallback(function, tables, ret_ref)
idx += 1
# Return the bound args
if conditions or len(args) != 1:
return _parse_lambda_fallback(function, tables, ret_ref)
return {"function": _trace_value(args[0], tables, ret_ref=ret_ref, wrap_str=True)}
def test_folding_of_tuples_of_constants(self):
for line, elem in (
('a = 1,2,3', (1, 2, 3)),
('("a","b","c")', ('a', 'b', 'c')),
('a,b,c = 1,2,3', (1, 2, 3)),
('(None, 1, None)', (None, 1, None)),
('((1, 2), 3, 4)', ((1, 2), 3, 4)),
):
code = compile(line, '', 'single')
self.assertInBytecode(code, 'LOAD_CONST', elem)
self.assertNotInBytecode(code, 'BUILD_TUPLE')
# Long tuples should be folded too.
code = compile(repr(tuple(range(10000))), '', 'single')
self.assertNotInBytecode(code, 'BUILD_TUPLE')
# One LOAD_CONST for the tuple, one for the None return value
load_consts = [
instr for instr in dis.get_instructions(code)
if instr.opname == 'LOAD_CONST'
]
self.assertEqual(len(load_consts), 2)
# Bug 1053819: Tuple of constants misidentified when presented with:
# . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . .
# The following would segfault upon compilation
def crater():
(~ [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
], )
def test_outer(self):
actual = dis.get_instructions(outer, first_line=expected_outer_line)
self.assertEqual(list(actual), expected_opinfo_outer)
def interpreter(model, debug=0):
closure = getclosurevars(model).nonlocals
if 'pro_tip' in closure:
tips = closure['pro_tip']
tip_type = tips[0]
if tip_type == 'STAN_NUTS':
param = tips[1]
if not param[0] in handled:
pro_tips.append(closure['pro_tip'])
handled.extend(param)
elif tip_type == 'Block':
pro_tips.append(closure['pro_tip'])
else:
print('Unknown tip', tip_type)
exit(0)
if debug >= 1:
print('Start: ', model.__name__, ' with path ', closure['path'])
# Local evaluation stack, contains both symbolic and real values
stack = StackList([])
# Local variables binding
varmap = {}
ret = ''
code = []
for ins in dis.get_instructions(model):
code.append(ins)
code_pointer = 0
while (True):
ins = code[code_pointer]
op = ins.opname
arg = ins.argval
if debug >= 2:
print('')
print(op, ' ', arg)
if (op == 'LOAD_DEREF'):
v = closure[arg]
stack.push(v)
elif (op == 'LOAD_GLOBAL'):
stack.push(arg)
elif (op == 'CALL_FUNCTION'):
l = []
l_obj = []
for i in range(arg):
a = stack.pop()
l.append(str(a))
l_obj.append(a)
l.reverse()
l_obj.reverse()
a = stack.pop()
if arg != 0:
# Symbolic call
if a == 'sample':
# Pyro sample, execute RV path
name = l[0]
dist = l_obj[1]
rv = RV(name, dist)
stack.push(rv)
elif (a in fmap):
if len(l) == 2:
#savepath = l[0]
stack.push(fmap[a](l[0], eval(l[1])))
elif len(l) == 3:
#savepath = l[1]
stack.push(fmap[a](l[0], l[1], eval(l[2])))
else:
print(
'Not working with more than 3 arguments yet (0)'
)
exit(0)
elif (type(a) == types.FunctionType):
if len(l) == 2:
#savepath = l[0]
stack.push(a(l[0], eval(l[1])))
elif len(l) == 3:
#savepath = l[1]
stack.push(a(l[0], l[1], eval(l[2])))
else:
print(
'Not working with more than 3 arguments yet (1)',
len(l))
exit(0)
else:
dist = make_dist(str(a), l)
stack.push(dist)
else:
# Actual model execution
if debug >= 2:
print('To execute: ', a)
(ret, _) = interpreter(a, 0)
stack.push(ret)
elif (op == 'UNPACK_SEQUENCE'):
a = stack.pop()
for i in range(arg):
stack.push(a[i])
elif (op == 'STORE_FAST'):
a = stack.pop()
# Handling loops. That seems crazy but that's not me!
if type(a) == LOOPMARKER:
code_pointer = a.ptr - 1
else:
# Normal case
varmap[arg] = a
if type(a) == RV:
rvmap[a.name] = a
elif (op == 'LOAD_FAST'):
if type(varmap[arg]) == RV:
stack.push(varmap[arg].name)
else:
stack.push(varmap[arg])
elif (op == 'LOAD_CONST'):
stack.push(arg)
elif (op == 'BINARY_ADD'):
a1 = stack.pop()
a2 = stack.pop()
stack.push(a2 + a1)
elif (op == 'BINARY_MULTIPLY'):
a1 = stack.pop()
a2 = stack.pop()
stack.push(str(a2) + ' * ' + str(a1))
elif (op == 'BINARY_SUBSCR'):
a1 = stack.pop()
a2 = stack.pop()
if (type(a2) == str):
stack.push(str(a2) + '[' + str(a1) + ']')
else:
stack.push(a2[a1])
elif (op == 'BUILD_LIST'):
l = []
for i in range(arg):
a = stack.pop()
l.append(a)
stack.push(l)
elif (op == 'SETUP_LOOP'):
stack.push(LOOPMARKER(arg // 2))
elif (op == 'LOAD_ATTR'):
pass
elif (op == 'CALL_FUNCTION_KW'):
pass
elif (op == 'GET_ITER'):
a = stack.pop()
stack.push(iter(a))
elif (op == 'FOR_ITER'):
a = stack.pop()
for i in a:
stack.push(i)
elif (op == 'POP_TOP'):
stack.pop()
elif (op == 'JUMP_ABSOLUTE'):
code_pointer = (arg // 2)
elif (op == 'POP_BLOCK'):
pass
elif (op == ''):
pass
elif (op == 'RETURN_VALUE'):
if debug >= 1:
print('End: ', model.__name__)
a = stack.pop()
assert (stack == [])
if a == None:
return ([], varmap)
else:
l = []
for rv in a:
l.append(rvmap[rv])
return (l, varmap)
else:
print('Not implemented: ', op)
exit(0)
code_pointer += 1
if debug >= 2:
print('stack: ', stack)
print('map: ', varmap)
print('rvs', rvmap)
def __init__(self, clsname, bases, clsdict):
# clsname - экземпляр метакласса - Server
# bases - кортеж базовых классов - ()
# clsdict - словарь атрибутов и методов экземпляра метакласса
# {'__module__': '__main__',
# '__qualname__': 'Server',
# 'port': <descrptrs.Port object at 0x000000DACC8F5748>,
# '__init__': <function Server.__init__ at 0x000000DACCE3E378>,
# 'init_socket': <function Server.init_socket at 0x000000DACCE3E400>,
# 'main_loop': <function Server.main_loop at 0x000000DACCE3E488>,
# 'process_message': <function Server.process_message at 0x000000DACCE3E510>,
# 'process_client_message': <function Server.process_client_message at 0x000000DACCE3E598>}
# Список методов, которые используются в функциях класса:
methods = []
# Атрибуты, используемые в функциях классов
attrs = []
# перебираем ключи
for func in clsdict:
# Пробуем
try:
# Возвращает итератор по инструкциям в предоставленной функции
# , методе, строке исходного кода или объекте кода.
ret = dis.get_instructions(clsdict[func])
# ret - <generator object _get_instructions_bytes at 0x00000062EAEAD7C8>
# ret - <generator object _get_instructions_bytes at 0x00000062EAEADF48>
# ...
# Если не функция то ловим исключение
# (если порт)
except TypeError:
pass
else:
# Раз функция разбираем код, получая используемые методы и
# атрибуты.
for i in ret:
#print(i)
# i - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=9, argval='send_message',
# argrepr='send_message', offset=308, starts_line=201, is_jump_target=False)
# opname - имя для операции
if i.opname == 'LOAD_GLOBAL':
if i.argval not in methods:
# заполняем список методами, использующимися в
# функциях класса
methods.append(i.argval)
elif i.opname == 'LOAD_ATTR':
if i.argval not in attrs:
# заполняем список атрибутами, использующимися в
# функциях класса
attrs.append(i.argval)
#print(methods)
# Если обнаружено использование недопустимого метода connect, бросаем
# исключение:
if 'connect' in methods:
raise TypeError(
'Использование метода connect недопустимо в серверном классе')
# Если сокет не инициализировался константами SOCK_STREAM(TCP)
# AF_INET(IPv4), тоже исключение.
if not ('SOCK_STREAM' in attrs and 'AF_INET' in attrs):
raise TypeError('Некорректная инициализация сокета.')
# Обязательно вызываем конструктор предка:
super().__init__(clsname, bases, clsdict)
def decompile(code_object):
"""
Taken from
http://thermalnoise.wordpress.com/2007/12/30/exploring-python-bytecode/
Extracts dissasembly information from the byte code and stores it in
a list for further use.
Call signature(s):
instructions=decompile(f.f_code)
Required arguments:
========= =====================================================================
code_object A bytecode object extracted with inspect.currentframe()
or any other mechanism that returns byte code.
Optional keyword arguments: NONE
Outputs:
========= =====================================================================
instructions a list of offsets, op_codes, names, arguments,
argument_value which can be deconstructed to find out various things
about a function call.
Example:
# Two frames back so that we get the callers' caller
f = inspect.currentframe().f_back.f_back
i = f.f_lasti # index of the last attempted instruction in byte code
ins = decompile(f.f_code)
"""
instructions = []
if PY3:
for ins in dis.get_instructions(code_object):
instructions.append(
(ins.offset, ins.opcode, ins.opname, ins.arg, ins.argval))
else:
code = code_object.co_code
variables = code_object.co_cellvars + code_object.co_freevars
n = len(code)
i = 0
e = 0
while i < n:
i_offset = i
i_opcode = ord(code[i])
i = i + 1
if i_opcode >= opcode.HAVE_ARGUMENT:
i_argument = ord(code[i]) + (ord(code[i + 1]) << (4 * 2)) + e
i = i + 2
if i_opcode == opcode.EXTENDED_ARG:
e = i_argument << 16
else:
e = 0
if i_opcode in opcode.hasconst:
i_arg_value = repr(code_object.co_consts[i_argument])
elif i_opcode in opcode.hasname:
i_arg_value = code_object.co_names[i_argument]
elif i_opcode in opcode.hasjrel:
i_arg_value = repr(i + i_argument)
elif i_opcode in opcode.haslocal:
i_arg_value = code_object.co_varnames[i_argument]
elif i_opcode in opcode.hascompare:
i_arg_value = opcode.cmp_op[i_argument]
elif i_opcode in opcode.hasfree:
i_arg_value = variables[i_argument]
else:
i_arg_value = i_argument
else:
i_argument = None
i_arg_value = None
instructions.append((i_offset, i_opcode, opcode.opname[i_opcode],
i_argument, i_arg_value))
return instructions
def _generate_opcode_new(code_object):
import dis
for ins in dis.get_instructions(code_object):
yield (ins.opcode, ins.arg)
def get_imports(self, source: str) -> defaultdict:
self.instructions = dis.get_instructions(source)
for instruction in self.instructions:
if "IMPORT_NAME" in instruction.opname or "IMPORT_FROM" in instruction.opname:
self.imports[instruction.opname].append(instruction.argval)
return self.imports
def preview(code):
decompiler = Decompiler()
decompiler.build_graph(list(dis.get_instructions(code)))
dis.dis(code)
display_graph(decompiler)
def test_nested(self):
with captured_stdout():
f = outer()
actual = dis.get_instructions(f, first_line=expected_f_line)
self.assertEqual(list(actual), expected_opinfo_f)
def infer_return_type_func(f, input_types, debug=False, depth=0):
"""Analyses a function to deduce its return type.
Args:
f: A Python function object to infer the return type of.
input_types: A sequence of inputs corresponding to the input types.
debug: Whether to print verbose debugging information.
depth: Maximum inspection depth during type inference.
Returns:
A TypeConstraint that that the return value of this function will (likely)
satisfy given the specified inputs.
Raises:
TypeInferenceError: if no type can be inferred.
"""
if debug:
print()
print(f, id(f), input_types)
dis.dis(f)
from . import opcodes
simple_ops = dict((k.upper(), v) for k, v in opcodes.__dict__.items())
co = f.__code__
code = co.co_code
end = len(code)
pc = 0
extended_arg = 0 # Python 2 only.
free = None
yields = set()
returns = set()
# TODO(robertwb): Default args via inspect module.
local_vars = list(input_types) + [typehints.Union[
()]] * (len(co.co_varnames) - len(input_types))
state = FrameState(f, local_vars)
states = collections.defaultdict(lambda: None)
jumps = collections.defaultdict(int)
# In Python 3, use dis library functions to disassemble bytecode and handle
# EXTENDED_ARGs.
is_py3 = sys.version_info[0] == 3
if is_py3:
ofs_table = {} # offset -> instruction
for instruction in dis.get_instructions(f):
ofs_table[instruction.offset] = instruction
# Python 2 - 3.5: 1 byte opcode + optional 2 byte arg (1 or 3 bytes).
# Python 3.6+: 1 byte opcode + 1 byte arg (2 bytes, arg may be ignored).
if sys.version_info >= (3, 6):
inst_size = 2
opt_arg_size = 0
else:
inst_size = 1
opt_arg_size = 2
last_pc = -1
while pc < end: # pylint: disable=too-many-nested-blocks
start = pc
if is_py3:
instruction = ofs_table[pc]
op = instruction.opcode
else:
op = ord(code[pc])
if debug:
print('-->' if pc == last_pc else ' ', end=' ')
print(repr(pc).rjust(4), end=' ')
print(dis.opname[op].ljust(20), end=' ')
pc += inst_size
if op >= dis.HAVE_ARGUMENT:
if is_py3:
arg = instruction.arg
else:
arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
extended_arg = 0
pc += opt_arg_size
if op == dis.EXTENDED_ARG:
extended_arg = arg * 65536
if debug:
print(str(arg).rjust(5), end=' ')
if op in dis.hasconst:
print('(' + repr(co.co_consts[arg]) + ')', end=' ')
elif op in dis.hasname:
print('(' + co.co_names[arg] + ')', end=' ')
elif op in dis.hasjrel:
print('(to ' + repr(pc + arg) + ')', end=' ')
elif op in dis.haslocal:
print('(' + co.co_varnames[arg] + ')', end=' ')
elif op in dis.hascompare:
print('(' + dis.cmp_op[arg] + ')', end=' ')
elif op in dis.hasfree:
if free is None:
free = co.co_cellvars + co.co_freevars
print('(' + free[arg] + ')', end=' ')
# Actually emulate the op.
if state is None and states[start] is None:
# No control reaches here (yet).
if debug:
print()
continue
state |= states[start]
opname = dis.opname[op]
jmp = jmp_state = None
if opname.startswith('CALL_FUNCTION'):
if sys.version_info < (3, 6):
# Each keyword takes up two arguments on the stack (name and value).
standard_args = (arg & 0xFF) + 2 * (arg >> 8)
var_args = 'VAR' in opname
kw_args = 'KW' in opname
pop_count = standard_args + var_args + kw_args + 1
if depth <= 0:
return_type = Any
elif arg >> 8:
if not var_args and not kw_args and not arg & 0xFF:
# Keywords only, maybe it's a call to Row.
if isinstance(state.stack[-pop_count], Const):
from apache_beam.pvalue import Row
if state.stack[-pop_count].value == Row:
fields = state.stack[-pop_count + 1::2]
types = state.stack[-pop_count + 2::2]
return_type = row_type.RowTypeConstraint(
zip([fld.value for fld in fields],
Const.unwrap_all(types)))
else:
return_type = Any
else:
# TODO(robertwb): Handle this case.
return_type = Any
elif isinstance(state.stack[-pop_count], Const):
# TODO(robertwb): Handle this better.
if var_args or kw_args:
state.stack[-1] = Any
state.stack[-var_args - kw_args] = Any
return_type = infer_return_type(
state.stack[-pop_count].value,
state.stack[1 - pop_count:],
debug=debug,
depth=depth - 1)
else:
return_type = Any
state.stack[-pop_count:] = [return_type]
else: # Python 3.6+
if opname == 'CALL_FUNCTION':
pop_count = arg + 1
if depth <= 0:
return_type = Any
elif isinstance(state.stack[-pop_count], Const):
return_type = infer_return_type(
state.stack[-pop_count].value,
state.stack[1 - pop_count:],
debug=debug,
depth=depth - 1)
else:
return_type = Any
elif opname == 'CALL_FUNCTION_KW':
# TODO(udim): Handle keyword arguments. Requires passing them by name
# to infer_return_type.
pop_count = arg + 2
if isinstance(state.stack[-pop_count], Const):
from apache_beam.pvalue import Row
if state.stack[-pop_count].value == Row:
fields = state.stack[-1].value
return_type = row_type.RowTypeConstraint(
zip(
fields,
Const.unwrap_all(state.stack[-pop_count +
1:-1])))
else:
return_type = Any
else:
return_type = Any
elif opname == 'CALL_FUNCTION_EX':
# stack[-has_kwargs]: Map of keyword args.
# stack[-1 - has_kwargs]: Iterable of positional args.
# stack[-2 - has_kwargs]: Function to call.
has_kwargs = arg & 1 # type: int
pop_count = has_kwargs + 2
if has_kwargs:
# TODO(udim): Unimplemented. Requires same functionality as a
# CALL_FUNCTION_KW implementation.
return_type = Any
else:
args = state.stack[-1]
_callable = state.stack[-2]
if isinstance(args, typehints.ListConstraint):
# Case where there's a single var_arg argument.
args = [args]
elif isinstance(args, typehints.TupleConstraint):
args = list(args._inner_types())
return_type = infer_return_type(_callable.value,
args,
debug=debug,
depth=depth - 1)
else:
raise TypeInferenceError('unable to handle %s' % opname)
state.stack[-pop_count:] = [return_type]
elif opname == 'CALL_METHOD':
pop_count = 1 + arg
# LOAD_METHOD will return a non-Const (Any) if loading from an Any.
if isinstance(state.stack[-pop_count], Const) and depth > 0:
return_type = infer_return_type(state.stack[-pop_count].value,
state.stack[1 - pop_count:],
debug=debug,
depth=depth - 1)
else:
return_type = typehints.Any
state.stack[-pop_count:] = [return_type]
elif opname in simple_ops:
if debug:
print("Executing simple op " + opname)
simple_ops[opname](state, arg)
elif opname == 'RETURN_VALUE':
returns.add(state.stack[-1])
state = None
elif opname == 'YIELD_VALUE':
yields.add(state.stack[-1])
elif opname == 'JUMP_FORWARD':
jmp = pc + arg
jmp_state = state
state = None
elif opname == 'JUMP_ABSOLUTE':
jmp = arg
jmp_state = state
state = None
elif opname in ('POP_JUMP_IF_TRUE', 'POP_JUMP_IF_FALSE'):
state.stack.pop()
jmp = arg
jmp_state = state.copy()
elif opname in ('JUMP_IF_TRUE_OR_POP', 'JUMP_IF_FALSE_OR_POP'):
jmp = arg
jmp_state = state.copy()
state.stack.pop()
elif opname == 'FOR_ITER':
jmp = pc + arg
jmp_state = state.copy()
jmp_state.stack.pop()
state.stack.append(element_type(state.stack[-1]))
else:
raise TypeInferenceError('unable to handle %s' % opname)
if jmp is not None:
# TODO(robertwb): Is this guaranteed to converge?
new_state = states[jmp] | jmp_state
if jmp < pc and new_state != states[jmp] and jumps[pc] < 5:
jumps[pc] += 1
pc = jmp
states[jmp] = new_state
if debug:
print()
print(state)
pprint.pprint(dict(item for item in states.items() if item[1]))
if yields:
result = typehints.Iterable[reduce(union, Const.unwrap_all(yields))]
else:
result = reduce(union, Const.unwrap_all(returns))
finalize_hints(result)
if debug:
print(f, id(f), input_types, '->', result)
return result
def sql_template(function: types.FunctionType) -> types.FunctionType:
signature = inspect.signature(function)
instructions = list(dis.get_instructions(function))
@functools.wraps(function)
def build(*args: typing.Any, **kwargs: typing.Any) -> typing.Any:
bound_arguments = signature.bind(*args, **kwargs)
bound_arguments.apply_defaults()
global_dict: typing.Dict[str, typing.Any] = {
# supported queries:
# with ... select ...
# select ...
# insert into ... select ...
# create table ... engine = ... as select ...
# create view ... as select ...
# create materialized view ... as select ...
'with_':
ast.Initial('with'),
'select':
ast.Initial('select'),
'select_distinct':
ast.Initial('select_distinct'),
'insert':
ast.Initial('insert'),
'insert_into':
ast.Initial('insert_into'),
'create':
ast.Initial('create'),
'create_table':
ast.Initial('create_table'),
'create_table_if_not_exists':
ast.Initial('create_table_if_not_exists'),
'create_view':
ast.Initial('create_view'),
'create_or_replace_view':
ast.Initial('create_or_replace_view'),
'create_view_if_not_exists':
ast.Initial('create_view_if_not_exists'),
'create_materialized_view':
ast.Initial('create_materialized_view'),
'create_materialized_view_if_not_exists':
ast.Initial('create_materialized_view_if_not_exists'),
}
local_dict: typing.Dict[str, typing.Any] = {
**bound_arguments.arguments,
}
# TODO: use types.CellType in type annotation
cells: typing.Tuple[typing.Any, ...] = (
*(function.__closure__ or ()),
*(
types.CellType() # type: ignore[attr-defined]
for _ in function.__code__.co_cellvars or ()),
)
stack: typing.List[typing.Any] = []
# notice: see dis.opmap
for instruction in instructions:
done = _run(global_dict, local_dict, cells, stack,
instruction.opname, instruction.arg,
instruction.argval)
if done:
assert len(stack) == 1
return stack.pop()
return None
# TODO
return typing.cast(types.FunctionType, build)
def test_doubly_nested(self):
with captured_stdout():
inner = outer()()
actual = dis.get_instructions(inner, first_line=expected_inner_line)
self.assertEqual(list(actual), expected_opinfo_inner)
def _get_globals_py3(func):
for o in dis.get_instructions(func):
if o.opname == 'LOAD_GLOBAL':
yield o.argval
def test_jumpy(self):
actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
self.assertEqual(list(actual), expected_opinfo_jumpy)
def find_forbidden_methods_call(func, method_names):
for instr in dis.get_instructions(func):
if instr.opname == 'LOAD_METHOD' and instr.argval in method_names:
return instr.argval
def test_iteration(self):
for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:
with self.subTest(obj=obj):
via_object = list(dis.Bytecode(obj))
via_generator = list(dis.get_instructions(obj))
self.assertEqual(via_object, via_generator)
def get_referenced_objects(code, context, suppress_warnings=False):
"""
Attempts to return all objects referenced externally by a code object. In
some cases, these objects are:
- replaced with ReferenceProxy class: when we can't find a variable or when
the reference is an attribute of the result of a function call, or
- omitted: when the reference is accessed using exec or is the result of a
function call.
An externally referenced object is any object used by a piece of code but not
defined in that code. These objects can include classes, functions, modules,
or any other variables. They can be referenced from various scopes: global,
"free" (when the variable is used in a code block, but is not defined there
and is not global), etc.; or as attributes of other referenced objects.
Note that this function does not actually run any code. This includes
calling any functions used by the input code, because doing so can be
expensive and have unintended consequences. Due to this, any references that
are attributes of the result of a function call won’t be detected. This means
that if an inner function returns a module, any attributes of the module
won't be detected. In this case, we return the name of the attribute as a
proxy for the object itself. So for the function below, the returned
references would be ``[get_my_class, "call"]``.
.. code-block:: python
def x():
my_cls = get_my_class("MyClass") # Returns class MyClass
my_cls.call()
This function uses a CodeContext object to look up variables defined outside
the local scope and to track local variables created while running the logic
to find references.
"""
# We mutate context while finding references. Let's make a copy of
# context to not change the original context. The original context
# can be shared between different code objects, like between an
# outer and an inner function.
context = context.copy()
# Top of the stack.
tos = None
lineno = None
refs = []
def set_tos(t):
nonlocal tos
if tos is not None:
# If the top of stack item already exists, that means we
# have gone through all the instructions that use the item,
# and it is a reference object.
refs.append(tos)
tos = t
# Our goal is to find referenced objects. The problem is that co_names
# does not have fully qualified names in it. So if you access `foo.bar`,
# co_names has `foo` and `bar` in it but it doesn't tell us that the
# code reads `bar` of `foo`. We are going over the bytecode to resolve
# from which object an attribute is requested.
# Read more about bytecode at https://docs.python.org/3/library/dis.html
for op in dis.get_instructions(code):
try:
if op.opname not in SUPPORTED_INSTRUCTIONS and not suppress_warnings:
if sys.version_info < (3, 6) or sys.version_info > (3, 8):
message = """
You are using an unsupported Python version for Bionic. This
can result in Bionic missing some code changes to invalidate
cache. Consider using a supported Python version to avoid any
caching issues.
"""
else:
message = f"""
Bionic does not recognize {op.opname} Bytecode operation.
This should be impossible and is most likely a bug in Bionic.
Please raise a new issue at
https://github.com/square/bionic/issues to let us know.
"""
message += """
You can also suppress this warning by removing the
`suppress_bytecode_warnings` override from the
`@version` decorator on the corresponding function.
"""
warnings.warn(oneline(message))
# Sometimes starts_line is None, in which case let's just remember the
# previous start_line (if any). This way when there's an exception we at
# least can point users somewhat near the line where the error stems from.
if op.starts_line is not None:
lineno = op.starts_line
if op.opname in ["LOAD_GLOBAL", "LOAD_NAME"]:
if op.argval in context.globals:
set_tos(context.globals[op.argval])
else:
# This can happen if the variable does not exist, or if LOAD_NAME
# is trying to access a local frame argument. If we cannot find the
# variable, we return its name instead.
set_tos(ReferenceProxy(op.argval))
elif op.opname in ["LOAD_DEREF", "LOAD_CLOSURE"]:
if op.argval in context.cells:
set_tos(context.cells[op.argval])
else:
# This can happen when we have nested functions. The second
# level or further nested functions won't have free variables
# from any preceding function except for the top level
# function. This is because the code context that gives us
# free variables is created from the function variable, which
# we only have for the top level function. We get only the
# code object for any inner functions. Since we can't get the
# code context for inner functions, we use the code context
# of the top level function.
set_tos(ReferenceProxy(op.argval))
elif op.opname == "IMPORT_NAME":
# This instruction only appears if the code object imports a
# module using the import statement. If a user is importing
# modules inside a function, they probably don't want to import
# the module until the function execution time.
message = f"""
Entity function in file {code.co_filename} imports the
'{op.argval}' module at line {lineno};
Bionic will not be able to automatically detect any changes to this
module.
To enable automatic detection of changes, import the module at the
global level (outside the function) instead.
To suppress this warning, remove the `suppress_bytecode_warnings`
override from the `@version` decorator on the corresponding function.
f"""
warnings.warn(oneline(message))
set_tos(None)
elif op.opname in ["LOAD_METHOD", "LOAD_ATTR"]:
if isinstance(tos, ReferenceProxy):
tos.val += "." + op.argval
elif inspect.ismodule(tos) and hasattr(tos, op.argval):
tos = getattr(tos, op.argval)
else:
set_tos(ReferenceProxy(op.argval))
elif op.opname == "STORE_FAST" and tos:
context.varnames[op.argval] = tos
set_tos(None)
elif op.opname == "LOAD_FAST" and op.argval in context.varnames:
set_tos(context.varnames[op.argval])
else:
# For all other instructions, add the current TOS as a
# reference.
set_tos(None)
except Exception as e:
message = oneline(f"""
Bionic found a code reference in file ${code.co_filename}
at line ${lineno} that it cannot hash when hashing
${code.co_name}. This should be impossible and is most
likely a bug in Bionic. Please raise a new issue at
https://github.com/square/bionic/issues to let us know.
In the meantime, you can disable bytecode analysis for
the corresponding function by setting `ignore_bytecode`
on its `@version` decorator. Please note that Bionic won't
automatically detect changes in this function; you'll need
to manually update the version yourself.
""")
raise AssertionError(message) from e
return refs
cell = []
for s in range(len(c.co_cellvars) + len(c.co_freevars)):
l = builder.alloca(ppyobj_type,1)
tup = builder.load(builder.gep(func.args[2],[int32(0),int32(1)]))
builder.store(builder.load(builder.gep(tup,[int32(0),int32(2),int32(s)])),l)
cell.append(l)
blocks = [[0,0,block,builder]]
blocks_by_ofs = {0: block}
block_num=0
ins_idx=0
nxt=False
for ins in dis.get_instructions(c):
if ins.is_jump_target or nxt:
b = func.append_basic_block(name="block" + str(block_num+1))
blocks.append([ins_idx,block_num+1,b,ir.IRBuilder(b)])
blocks_by_ofs[ins.offset] = b
block_num += 1
nxt=False
if ins.opname=='POP_JUMP_IF_FALSE' or ins.opname=='POP_JUMP_IF_TRUE' or ins.opname=='SETUP_EXCEPT' or ins.opname=='JUMP_FORWARD' or ins.opname=='JUMP_ABSOLUTE' or ins.opname=='SETUP_LOOP' or ins.opname=='BREAK_LOOP':
nxt=True
ins_idx +=1
ins_idxs = [r[0] for r in blocks]
stack_ptr = 0
ins_idx=0
branch_stack = {}
except_stack = []
def _have_nested_for_statement(generator):
matched = [True for instruction in dis.get_instructions(generator)
if instruction.opname == "FOR_ITER"]
return True if len(matched) > 1 else False
def get_referenced_objects(code, context):
# Top of the stack
tos = None # type: Any
lineno = None
refs = []
def set_tos(t):
nonlocal tos
if tos is not None:
# Hash tos so we support reading multiple objects
refs.append(tos)
tos = t
# Our goal is to find referenced objects. The problem is that co_names
# does not have full qualified names in it. So if you access `foo.bar`,
# co_names has `foo` and `bar` in it but it doesn't tell us that the
# code reads `bar` of `foo`. We are going over the bytecode to resolve
# from which object an attribute is requested.
# Read more about bytecode at https://docs.python.org/3/library/dis.html
for op in dis.get_instructions(code):
try:
# Sometimes starts_line is None, in which case let's just remember the
# previous start_line (if any). This way when there's an exception we at
# least can point users somewhat near the line where the error stems from.
if op.starts_line is not None:
lineno = op.starts_line
if op.opname in ["LOAD_GLOBAL", "LOAD_NAME"]:
if op.argval in context.globals:
set_tos(context.globals[op.argval])
else:
set_tos(op.argval)
elif op.opname in ["LOAD_DEREF", "LOAD_CLOSURE"]:
set_tos(context.cells[op.argval])
elif op.opname == "IMPORT_NAME":
try:
set_tos(importlib.import_module(op.argval))
except ImportError:
set_tos(op.argval)
elif op.opname in ["LOAD_METHOD", "LOAD_ATTR", "IMPORT_FROM"]:
if tos is None:
refs.append(op.argval)
elif isinstance(tos, str):
tos += "." + op.argval
else:
tos = getattr(tos, op.argval)
elif op.opname == "DELETE_FAST" and tos:
del context.varnames[op.argval]
tos = None
elif op.opname == "STORE_FAST" and tos:
context.varnames[op.argval] = tos
tos = None
elif op.opname == "LOAD_FAST" and op.argval in context.varnames:
set_tos(context.varnames[op.argval])
else:
# For all other instructions, hash the current TOS.
if tos is not None:
refs.append(tos)
tos = None
except Exception as e:
raise UserHashError(e, code, lineno=lineno)
return refs