Exemplo n.º 1
0
    def test_folding_of_unaryops_on_constants(self):
        for line, elem in (
            ('-0.5', -0.5),                     # unary negative
            ('-0.0', -0.0),                     # -0.0
            ('-(1.0-1.0)', -0.0),               # -0.0 after folding
            ('-0', 0),                          # -0
            ('~-2', 1),                         # unary invert
            ('+1', 1),                          # unary positive
        ):
            code = compile(line, '', 'single')
            self.assertInBytecode(code, 'LOAD_CONST', elem)
            for instr in dis.get_instructions(code):
                self.assertFalse(instr.opname.startswith('UNARY_'))

        # Check that -0.0 works after marshaling
        def negzero():
            return -(1.0-1.0)

        for instr in dis.get_instructions(code):
            self.assertFalse(instr.opname.startswith('UNARY_'))

        # Verify that unfoldables are skipped
        for line, elem, opname in (
            ('-"abc"', 'abc', 'UNARY_NEGATIVE'),
            ('~"abc"', 'abc', 'UNARY_INVERT'),
        ):
            code = compile(line, '', 'single')
            self.assertInBytecode(code, 'LOAD_CONST', elem)
            self.assertInBytecode(code, opname)
Exemplo n.º 2
0
 def test_elim_jump_after_return2(self):
     # Eliminate dead code: jumps immediately after returns can't be reached
     def f(cond1, cond2):
         while 1:
             if cond1: return 4
     self.assertNotInBytecode(f, 'JUMP_FORWARD')
     # There should be one jump for the while loop.
     returns = [instr for instr in dis.get_instructions(f)
                       if instr.opname == 'JUMP_ABSOLUTE']
     self.assertEqual(len(returns), 1)
     returns = [instr for instr in dis.get_instructions(f)
                       if instr.opname == 'RETURN_VALUE']
     self.assertEqual(len(returns), 2)
Exemplo n.º 3
0
def _check_async_thread_block(code, offset, _cache={}):
    '''
    Analyze a given async_thread() context manager block to make sure it doesn't
    contain any await operations
    '''
    if (code, offset) in _cache:
        return _cache[code, offset]

    import dis
    instr = dis.get_instructions(code)
    level = 0
    result = True
    for op in instr:
        if op.offset < offset:
            continue
        if op.opname == 'SETUP_ASYNC_WITH':
            level += 1
        if op.opname in { 'YIELD_FROM', 'YIELD_VALUE' } and level > 0:
            result = False
            break
        if op.opname == 'WITH_CLEANUP_START':
            level -= 1
            if level == 0:
                break
            
    _cache[code, offset] = result
    return result
Exemplo n.º 4
0
    def test_folding_of_binops_on_constants(self):
        for line, elem in (
            ('a = 2+3+4', 9),                   # chained fold
            ('"@"*4', '@@@@'),                  # check string ops
            ('a="abc" + "def"', 'abcdef'),      # check string ops
            ('a = 3**4', 81),                   # binary power
            ('a = 3*4', 12),                    # binary multiply
            ('a = 13//4', 3),                   # binary floor divide
            ('a = 14%4', 2),                    # binary modulo
            ('a = 2+3', 5),                     # binary add
            ('a = 13-4', 9),                    # binary subtract
            ('a = (12,13)[1]', 13),             # binary subscr
            ('a = 13 << 2', 52),                # binary lshift
            ('a = 13 >> 2', 3),                 # binary rshift
            ('a = 13 & 7', 5),                  # binary and
            ('a = 13 ^ 7', 10),                 # binary xor
            ('a = 13 | 7', 15),                 # binary or
            ):
            code = compile(line, '', 'single')
            self.assertInBytecode(code, 'LOAD_CONST', elem)
            for instr in dis.get_instructions(code):
                self.assertFalse(instr.opname.startswith('BINARY_'))

        # Verify that unfoldables are skipped
        code = compile('a=2+"b"', '', 'single')
        self.assertInBytecode(code, 'LOAD_CONST', 2)
        self.assertInBytecode(code, 'LOAD_CONST', 'b')

        # Verify that large sequences do not result from folding
        code = compile('a="x"*1000', '', 'single')
        self.assertInBytecode(code, 'LOAD_CONST', 1000)
Exemplo n.º 5
0
def get_assigned_name(frame):
  """
  Checks the bytecode of *frame* to find the name of the variable a result is
  being assigned to and returns that name. Returns the full left operand of the
  assignment. Raises a #ValueError if the variable name could not be retrieved
  from the bytecode (eg. if an unpack sequence is on the left side of the
  assignment).

  > **Known Limitations**:  The expression in the *frame* from which this
  > function is called must be the first part of that expression. For
  > example, `foo = [get_assigned_name(get_frame())] + [42]` works,
  > but `foo = [42, get_assigned_name(get_frame())]` does not!

  ```python
  >>> var = get_assigned_name(sys._getframe())
  >>> assert var == 'var'
  ```

  __Available in Python 3.4, 3.5__
  """

  SEARCHING, MATCHED = 1, 2
  state = SEARCHING
  result = ''
  stacksize = 0

  for op in dis.get_instructions(frame.f_code):
    if state == SEARCHING and op.offset == frame.f_lasti:
      if not op.opname.startswith('CALL_FUNCTION'):
        raise RuntimeError('get_assigned_name() requires entry at CALL_FUNCTION')
      state = MATCHED

      # For a top-level expression, the stack-size should be 1 after
      # the function at which we entered was executed.
      stacksize = 1
    elif state == MATCHED:
      # Update the would-be size of the stack after this instruction.
      # If we're at zero, we found the last instruction of the expression.
      try:
        stacksize += get_stackdelta(op)
      except KeyError:
        raise RuntimeError('could not determined assigned name, instruction '
            '{} is not supported'.format(op.opname))
      if stacksize == 0:
        if op.opname not in ('STORE_NAME', 'STORE_ATTR', 'STORE_GLOBAL', 'STORE_FAST'):
          raise ValueError('expression is not assigned or branch is not first part of the expression')
        return result + op.argval
      elif stacksize < 0:
        raise ValueError('not a top-level expression')

      if op.opname.startswith('CALL_FUNCTION'):
        # Chained or nested function call.
        raise ValueError('inside a chained or nested function call')
      elif op.opname == 'LOAD_ATTR':
        result += op.argval + '.'

  if not result:
    raise RuntimeError('last frame instruction not found')
  assert False
Exemplo n.º 6
0
 def test_elim_extra_return(self):
     # RETURN LOAD_CONST None RETURN  -->  RETURN
     def f(x):
         return x
     self.assertNotInBytecode(f, 'LOAD_CONST', None)
     returns = [instr for instr in dis.get_instructions(f)
                       if instr.opname == 'RETURN_VALUE']
     self.assertEqual(len(returns), 1)
Exemplo n.º 7
0
 def get_load_const(self, tree):
     # Compile to bytecode, disassemble and get parameter of LOAD_CONST
     # instructions
     co = compile(tree, '<string>', 'exec')
     consts = []
     for instr in dis.get_instructions(co):
         if instr.opname == 'LOAD_CONST':
             consts.append(instr.argval)
     return consts
Exemplo n.º 8
0
 def _walk_global_ops(code):
     """
     Yield (opcode, argument number) tuples for all
     global-referencing instructions in *code*.
     """
     for instr in dis.get_instructions(code):
         op = instr.opcode
         if op in GLOBAL_OPS:
             yield op, instr.arg
Exemplo n.º 9
0
 def test_elim_jump_to_return(self):
     # JUMP_FORWARD to RETURN -->  RETURN
     def f(cond, true_value, false_value):
         return true_value if cond else false_value
     self.assertNotInBytecode(f, 'JUMP_FORWARD')
     self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
     returns = [instr for instr in dis.get_instructions(f)
                       if instr.opname == 'RETURN_VALUE']
     self.assertEqual(len(returns), 2)
Exemplo n.º 10
0
 def __init__(cls, name, bases, clsdict):
     if not UnitRegistry.enabled:
         super(UnitRegistry, cls).__init__(name, bases, clsdict)
         return
     yours = set(cls.mro())
     mine = set(Distributable.mro())
     left = yours - mine
     if len(left) > 1:
         if (not clsdict.get('hide_from_registry', False) and
                 not getattr(cls, 'hide_from_registry_all', False)):
             UnitRegistry.units.add(cls)
         else:
             UnitRegistry.hidden_units.add(cls)
     if "DISABLE_KWARGS_CHECK" in clsdict:
         super(UnitRegistry, cls).__init__(name, bases, clsdict)
         return
     kwattrs = set(getattr(cls, "KWATTRS", set()))
     for base in cls.__mro__:
         try:
             kw_var = inspect.getargspec(base.__init__).keywords
         except TypeError:
             continue
         if USE_DIS:
             try:
                 instrs = get_instructions(base.__init__)
             except TypeError:
                 continue
             loading_fast_kwargs = False
             for inst in instrs:
                 # https://hg.python.org/cpython/file/b3f0d7f50544/Include/opcode.h  # nopep8
                 # 124 = LOAD_FAST
                 # 136 = LOAD_DEREF
                 # 106 = LOAD_ATTR
                 # 100 = LOAD_CONST
                 if inst.opcode in (124, 136) and inst.argval == kw_var:
                     loading_fast_kwargs = True
                 elif loading_fast_kwargs and inst.opcode == 106:
                     continue
                 elif loading_fast_kwargs and inst.opcode == 100:
                     kwattrs.add(inst.argval)
                     loading_fast_kwargs = False
                 else:
                     loading_fast_kwargs = False
         else:
             try:
                 src, _ = inspect.getsourcelines(base.__init__)
             except TypeError:
                 continue
             kwarg_re = re.compile(
                 r"%(kwargs)s\.(get|pop)\(([^\s,\)]+)|%(kwargs)s\[([^\]]+)"
                 % {"kwargs": kw_var})
             src = "".join((l.strip() for l in src)).replace('\n', '')
             for match in kwarg_re.finditer(src):
                 kwattrs.add((match.group(2) or match.group(3))[1:-1])
     cls.KWATTRS = kwattrs
     super(UnitRegistry, cls).__init__(name, bases, clsdict)
Exemplo n.º 11
0
    def _get_opcodes(codeobj):
        """_get_opcodes(codeobj) -> [opcodes]

        Extract the actual opcodes as an iterator from a code object

        >>> c = compile("[1 + 2, (1,2)]", "", "eval")
        >>> list(_get_opcodes(c))
        [100, 100, 23, 100, 100, 102, 103, 83]
        """
        return (i.opcode for i in dis.get_instructions(codeobj))
Exemplo n.º 12
0
 def assertNotInBytecode(self, x, opname, argval=_UNSPECIFIED):
     """Throws AssertionError if op is found"""
     for instr in dis.get_instructions(x):
         if instr.opname == opname:
             disassembly = self.get_disassembly_as_string(co)
             if opargval is _UNSPECIFIED:
                 msg = '%s occurs in bytecode:\n%s' % (opname, disassembly)
             elif instr.argval == argval:
                 msg = '(%s,%r) occurs in bytecode:\n%s'
                 msg = msg % (opname, argval, disassembly)
             self.fail(msg)
Exemplo n.º 13
0
        def get_function_instructions(funcname):
            # Recompile with appropriate optimization setting
            code = compile(source=code_string,
                           filename="<string>",
                           mode="exec",
                           optimize=self.optimize_python)

            for c in code.co_consts:
                if isinstance(c, types.CodeType) and c.co_name == funcname:
                    return dis.get_instructions(c)
            return []
Exemplo n.º 14
0
def is_safe_generator(agen):
    '''
    Examine the code of an async generator to see if it appears
    unsafe with respect to async finalization.  A generator
    is unsafe if it utilizes any of the following constructs:

    1. Use of async-code in a finally block

       try:
           yield v
       finally:
           await coro()

    2. Use of yield inside an async context manager

       async with m:
           ...
           yield v
           ...

    3. Use of async-code in try-except

       try:
           yield v
       except Exception:
           await coro()
    '''
    if agen.ag_code in _safe_async_generators:
        return True

    def _is_unsafe_block(instr, end_offset=-1):
        is_generator = False
        in_final = False
        is_unsafe = False
        for op in instr:
            if op.offset == end_offset:
                in_final = True
            if op.opname == 'YIELD_VALUE':
                is_generator = True
            if op.opname == 'END_FINALLY':
                return (is_generator, is_unsafe)
            if op.opname in {'SETUP_FINALLY', 'SETUP_EXCEPT', 'SETUP_ASYNC_WITH'}:
                is_g, is_u = _is_unsafe_block(instr, op.argval)
                is_generator |= is_g
                is_unsafe |= is_u
            if op.opname == 'YIELD_FROM' and is_generator and in_final:
                is_unsafe = True
        return (is_generator, is_unsafe)

    if not _is_unsafe_block(dis.get_instructions(agen.ag_code))[1]:
        _safe_async_generators[agen.ag_code] = True
        return True
    else:
        return False
Exemplo n.º 15
0
def count_instr_recursively(f, opname):
    count = 0
    for instr in dis.get_instructions(f):
        if instr.opname == opname:
            count += 1
    if hasattr(f, '__code__'):
        f = f.__code__
    for c in f.co_consts:
        if hasattr(c, 'co_code'):
            count += count_instr_recursively(c, opname)
    return count
Exemplo n.º 16
0
 def test_peephole_opt_unreachable_code_array_access_in_bounds(self):
     """Regression test for issue35193 when run under clang msan."""
     def unused_code_at_end():
         return 3
         raise RuntimeError("unreachable")
     # The above function definition will trigger the out of bounds
     # bug in the peephole optimizer as it scans opcodes past the
     # RETURN_VALUE opcode.  This does not always crash an interpreter.
     # When you build with the clang memory sanitizer it reliably aborts.
     self.assertEqual(
         'RETURN_VALUE',
         list(dis.get_instructions(unused_code_at_end))[-1].opname)
Exemplo n.º 17
0
def get_globals(func):
    result = set()

    for inst in dis.get_instructions(func):
        if inst.opname == 'LOAD_GLOBAL':
            result.add(inst.argval)

    ignore_globals = getattr(func, '_ignore_globals', set())

    if ignore_globals is _all_globals:
        return []

    return [i for i in result if i not in ignore_globals]
Exemplo n.º 18
0
    def assertBytecodeExactlyMatches(self, x, expected, *, line_offset=0):
        """Throws AssertionError if any discrepancy is found in bytecode

        *x* is the object to be introspected
        *expected* is a list of dis.Instruction objects

        Set *line_offset* as appropriate to adjust for the location of the
        object to be disassembled within the test file. If the expected list
        assumes the first line is line 1, then an appropriate offset would be
        ``1 - f.__code__.co_firstlineno``.
        """
        actual = dis.get_instructions(x, line_offset=line_offset)
        self.assertEqual(list(actual), expected)
Exemplo n.º 19
0
 def assertInBytecode(self, x, opname, argval=_UNSPECIFIED):
     """Returns instr if op is found, otherwise throws AssertionError"""
     for instr in dis.get_instructions(x):
         if instr.opname == opname:
             if argval is _UNSPECIFIED or instr.argval == argval:
                 return instr
     disassembly = self.get_disassembly_as_string(x)
     if argval is _UNSPECIFIED:
         msg = '%s not found in bytecode:\n%s' % (opname, disassembly)
     else:
         msg = '(%s,%r) not found in bytecode:\n%s'
         msg = msg % (opname, argval, disassembly)
     self.fail(msg)
Exemplo n.º 20
0
 def test_elim_jump_after_return1(self):
     # Eliminate dead code: jumps immediately after returns can't be reached
     def f(cond1, cond2):
         if cond1: return 1
         if cond2: return 2
         while 1:
             return 3
         while 1:
             if cond1: return 4
             return 5
         return 6
     self.assertNotInBytecode(f, 'JUMP_FORWARD')
     self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
     returns = [instr for instr in dis.get_instructions(f)
                       if instr.opname == 'RETURN_VALUE']
     self.assertEqual(len(returns), 6)
Exemplo n.º 21
0
Arquivo: main.py Projeto: lowks/spy
def get_imports(code):
    imp = []
    instrs = dis.get_instructions(code)
    for instr in instrs:
        if instr.opname in ('LOAD_GLOBAL', 'LOAD_NAME'):
            name = code.co_names[instr.arg]
            imp.append(name)
            for instr_ in instrs:
                if instr_.opname == 'LOAD_ATTR':
                    name += '.' + code.co_names[instr_.arg]
                    imp.append(name)
                elif instr_.opname in ('LOAD_GLOBAL', 'LOAD_NAME'):
                    name = code.co_names[instr_.arg]
                    imp.append(name)
                else:
                    break
    return imp
Exemplo n.º 22
0
    def accessed_attributes_of_local(f, local_name):
        """
        Get a list of attributes of ``local_name`` accessed by ``f``.

        The analysis performed by this function is conservative, meaning that
        it's not guaranteed to find **all** attributes used.
        """
        used = set()
        # Find sequences of the form: LOAD_FAST(local_name), LOAD_ATTR(<name>).
        # This will find all usages of the form ``local_name.<name>``.
        #
        # It will **NOT** find usages in which ``local_name`` is aliased to
        # another name.
        for first, second in sliding_window(dis.get_instructions(f), 2):
            if first.opname == 'LOAD_FAST' and first.argval == local_name:
                if second.opname in ('LOAD_ATTR', 'LOAD_METHOD', 'STORE_ATTR'):
                    used.add(second.argval)
        return used
Exemplo n.º 23
0
def translate(func, ceval_snippets):
    start_offset = 0
    code_obj = getattr(func, '__code__', None)
    if code_obj and os.path.exists(code_obj.co_filename):
        start_offset = code_obj.co_firstlineno
        with open(code_obj.co_filename) as f:
            code_line_at = {
                i: line.strip()
                for i, line in enumerate(f, 1)
                if line.strip()
            }.get
    else:
        code_line_at = lambda _: None

    for instr in get_instructions(func):
        code_line = code_line_at(instr.starts_line)
        line_no = (instr.starts_line or start_offset) - start_offset
        yield line_no, code_line, instr, ceval_snippets.get(instr.opname)
Exemplo n.º 24
0
def caller_assignment_name(depth=0):
    frame = sys._getframe(depth + 2)

    dis.disassemble(frame.f_code, frame.f_lasti)

    # from pprint import pprint
    # pprint(list(dis.get_instructions(frame.f_code)))

    for op in list(dis.get_instructions(frame.f_code)):
        if op.offset < frame.f_lasti:
            continue
        if op.starts_line is not None:
            # We hit a new source line.
            break
        if op.opname == "STORE_NAME":
            return op.argval

    raise RuntimeError("Return value of caller must be assigned to a variable")
Exemplo n.º 25
0
def typecheckFunction(func, scopeTypes = {}, infer = typeInferrer(), logFile = None):
    symbols = []

    # __builtins__ is sometimes a module and sometimes a dict?
    builtins = __builtins__ if type(__builtins__) == dict else __builtins__.__dict__

    if func.__closure__:
        closure = {val.__name__: val for val in (cell.cell_contents for cell in func.__closure__)}
    else:
        closure = {}

    context = Context(
        scopeVals = extend(
            func.__globals__,
            builtins,
            closure,
            {func.__name__: func}),
        # todo: scopeTypes should also include types from scopeVals that have _type
        # todo: actually should just be scopeSymbols instead of scopeVals and scopeTypes
        scopeTypes = extend(
            defaultdict(lambda: Object),
            scopeTypes,
            func.__annotations__,
            {func.__name__: infer(func)}),
        signature = infer(func))

    for path in executionPaths(list(dis.get_instructions(func))):
        for inst in path:
            handler = instructionHandlers[inst.opname]

            if logFile:
                pprint(symbols, stream=logFile)
                print(file=logFile) # newline
                print(inst, file=logFile)

            errors = handler(context, symbols, inst, infer)

            if errors:
                if logFile:
                    print(errors, file=logFile)

                return errors

    return []
Exemplo n.º 26
0
 def test_constant_folding(self):
     # Issue #11244: aggressive constant folding.
     exprs = [
         '3 * -5',
         '-3 * 5',
         '2 * (3 * 4)',
         '(2 * 3) * 4',
         '(-1, 2, 3)',
         '(1, -2, 3)',
         '(1, 2, -3)',
         '(1, 2, -3) * 6',
         'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}',
     ]
     for e in exprs:
         code = compile(e, '', 'single')
         for instr in dis.get_instructions(code):
             self.assertFalse(instr.opname.startswith('UNARY_'))
             self.assertFalse(instr.opname.startswith('BINARY_'))
             self.assertFalse(instr.opname.startswith('BUILD_'))
Exemplo n.º 27
0
def get_assigned_name(frame):
  ''' Checks the bytecode of *frame* to find the name of the variable
  a result is being assigned to and returns that name. Returns the full
  left operand of the assignment. Raises a `ValueError` if the variable
  name could not be retrieved from the bytecode (eg. if an unpack sequence
  is on the left side of the assignment).

      >>> var = get_assigned_frame(sys._getframe())
      >>> assert var == 'var'
  '''

  SEARCHING, MATCHED = 1, 2
  state = SEARCHING
  result = ''
  for op in dis.get_instructions(frame.f_code):
    if state == SEARCHING and op.offset == frame.f_lasti:
      state = MATCHED
    elif state == MATCHED:
      if result:
        if op.opname == 'LOAD_ATTR':
          result += op.argval + '.'
        elif op.opname == 'STORE_ATTR':
          result += op.argval
          break
        else:
          raise ValueError('expected {LOAD_ATTR, STORE_ATTR}', op.opname)
      else:
        if op.opname in ('LOAD_NAME', 'LOAD_FAST'):
          result += op.argval + '.'
        elif op.opname in ('STORE_NAME', 'STORE_FAST'):
          result = op.argval
          break
        else:
          message = 'expected {LOAD_NAME, LOAD_FAST, STORE_NAME, STORE_FAST}'
          raise ValueError(message, op.opname)

  if not result:
    raise RuntimeError('last frame instruction not found')
  return result
Exemplo n.º 28
0
    def test_folding_of_tuples_of_constants(self):
        for line, elem in (
            ('a = 1,2,3', (1, 2, 3)),
            ('("a","b","c")', ('a', 'b', 'c')),
            ('a,b,c = 1,2,3', (1, 2, 3)),
            ('(None, 1, None)', (None, 1, None)),
            ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
            ):
            code = compile(line,'','single')
            self.assertInBytecode(code, 'LOAD_CONST', elem)
            self.assertNotInBytecode(code, 'BUILD_TUPLE')

        # Long tuples should be folded too.
        code = compile(repr(tuple(range(10000))),'','single')
        self.assertNotInBytecode(code, 'BUILD_TUPLE')
        # One LOAD_CONST for the tuple, one for the None return value
        load_consts = [instr for instr in dis.get_instructions(code)
                              if instr.opname == 'LOAD_CONST']
        self.assertEqual(len(load_consts), 2)

        # Bug 1053819:  Tuple of constants misidentified when presented with:
        # . . . opcode_with_arg 100   unary_opcode   BUILD_TUPLE 1  . . .
        # The following would segfault upon compilation
        def crater():
            (~[
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
                0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
            ],)
Exemplo n.º 29
0
def is_simple(fun):
    """A heuristic to find out if a function is simple enough."""
    seen_load_fast_0 = False
    seen_load_response = False
    seen_call_fun = False

    for instruction in dis.get_instructions(fun):
        if instruction.opname == 'LOAD_FAST' and instruction.arg == 0:
            seen_load_fast_0 = True
            continue

        if instruction.opname == 'LOAD_ATTR' \
           and instruction.argval == 'Response':
            seen_load_response = True
            continue

        if instruction.opname.startswith('CALL_FUNCTION'):
            if seen_call_fun:
                return False

            seen_call_fun = True
            continue

    return seen_call_fun and seen_load_fast_0 and seen_load_response
Exemplo n.º 30
0
def _walk_global_ops(code):
    for instr in dis.get_instructions(code):
        op = instr.opcode
        if op in GLOBAL_OPS:
            yield op, instr.arg
Exemplo n.º 31
0
    def __init__(self, myfn):
        def lstadd(hmap, key, val):
            if key not in hmap:
                hmap[key] = [val]
            else:
                hmap[key].append(val)

        enter = CFGNode(
            dis.Instruction('NOP',
                            opcode=dis.opmap['NOP'],
                            arg=0,
                            argval=0,
                            argrepr=0,
                            offset=0,
                            starts_line=0,
                            is_jump_target=False), 0)
        last = enter
        self.jump_to = {}
        self.opcodes = {}
        for i, ins in enumerate(dis.get_instructions(myfn)):
            byte = i * 2
            node = CFGNode(ins, byte)
            self.opcodes[byte] = node
            print(i, ins)
            if ins.opname in [
                    'LOAD_CONST', 'LOAD_FAST', 'STORE_FAST', 'COMPARE_OP',
                    'INPLACE_ADD', 'INPLACE_SUBTRACT', 'RETURN_VALUE',
                    'BINARY_MODULO', 'POP_BLOCK'
            ]:
                last.add_child(node)
                last = node
            elif ins.opname == 'POP_JUMP_IF_FALSE':
                print("will jump to", ins.arg)
                lstadd(self.jump_to, ins.arg, node)
                node.props['jmp'] = True
                last.add_child(node)
                last = node
            elif ins.opname == 'JUMP_FORWARD':
                node.props['jmp'] = True
                lstadd(self.jump_to, (i + 1) * 2 + ins.arg, node)
                print("will jump to", (i + 1) * 2 + ins.arg)
                last.add_child(node)
                last = node
            elif ins.opname == 'SETUP_LOOP':
                print("setuploop: ", byte, ins.arg)
                last.add_child(node)
                last = node
            elif ins.opname == 'JUMP_ABSOLUTE':
                print("will jump to", ins.arg)
                lstadd(self.jump_to, ins.arg, node)
                node.props['jmp'] = True
                last.add_child(node)
                last = node
            else:
                assert False
        for byte in self.opcodes:
            if byte in self.jump_to:
                node = self.opcodes[byte]
                assert node.i.is_jump_target
                for b in self.jump_to[byte]:
                    b.add_child(node)
Exemplo n.º 32
0
def docopt(
    docstring: Optional[str] = None,
    argv: Optional[Union[List[str], str]] = None,
    default_help: bool = True,
    version: Any = None,
    options_first: bool = False,
    more_magic: bool = False,
) -> ParsedOptions:
    """Parse `argv` based on command-line interface described in `doc`.

    `docopt` creates your command-line interface based on its
    description that you pass as `docstring`. Such description can contain
    --options, <positional-argument>, commands, which could be
    [optional], (required), (mutually | exclusive) or repeated...

    Parameters
    ----------
    docstring : str (default: first __doc__ in parent scope)
        Description of your command-line interface.
    argv : list of str, optional
        Argument vector to be parsed. sys.argv[1:] is used if not
        provided.
    default_help : bool (default: True)
        Set to False to disable automatic help on -h or --help
        options.
    version : any object
        If passed, the object will be printed if --version is in
        `argv`.
    options_first : bool (default: False)
        Set to True to require options precede positional arguments,
        i.e. to forbid options and positional arguments intermix.
    more_magic : bool (default: False)
        Try to be extra-helpful; pull results into globals() of caller as 'arguments',
        offer advanced pattern-matching and spellcheck.
        Also activates if `docopt` aliased to a name containing 'magic'.

    Returns
    -------
    arguments: dict-like
        A dictionary, where keys are names of command-line elements
        such as e.g. "--verbose" and "<path>", and values are the
        parsed values of those elements. Also supports dot acccess.

    Example
    -------
    >>> from docopt import docopt
    >>> doc = '''
    ... Usage:
    ...     my_program tcp <host> <port> [--timeout=<seconds>]
    ...     my_program serial <port> [--baud=<n>] [--timeout=<seconds>]
    ...     my_program (-h | --help | --version)
    ...
    ... Options:
    ...     -h, --help  Show this screen and exit.
    ...     --baud=<n>  Baudrate [default: 9600]
    ... '''
    >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30']
    >>> docopt(doc, argv)
    {'--baud': '9600',
     '--help': False,
     '--timeout': '30',
     '--version': False,
     '<host>': '127.0.0.1',
     '<port>': '80',
     'serial': False,
     'tcp': True}

    """
    argv = sys.argv[1:] if argv is None else argv
    maybe_frame = inspect.currentframe()
    if maybe_frame:
        parent_frame = doc_parent_frame = magic_parent_frame = maybe_frame.f_back
    if not more_magic:  # make sure 'magic' isn't in the calling name
        while not more_magic and magic_parent_frame:
            imported_as = {v: k for k, v in magic_parent_frame.f_globals.items() if hasattr(v, "__name__") and v.__name__ == docopt.__name__}.get(docopt)
            if imported_as and "magic" in imported_as:
                more_magic = True
            else:
                magic_parent_frame = magic_parent_frame.f_back
    if not docstring:  # go look for one, if none exists, raise Exception
        while not docstring and doc_parent_frame:
            docstring = doc_parent_frame.f_locals.get("__doc__")
            if not docstring:
                doc_parent_frame = doc_parent_frame.f_back
        if not docstring:
            raise DocoptLanguageError("Either __doc__ must be defined in the scope of a parent or passed as the first argument.")
    output_value_assigned = False
    if more_magic and parent_frame:
        import dis

        instrs = dis.get_instructions(parent_frame.f_code)
        for instr in instrs:
            if instr.offset == parent_frame.f_lasti:
                break
        assert instr.opname.startswith("CALL_")
        MAYBE_STORE = next(instrs)
        if MAYBE_STORE and (MAYBE_STORE.opname.startswith("STORE") or MAYBE_STORE.opname.startswith("RETURN")):
            output_value_assigned = True
    usage_sections = parse_section("usage:", docstring)
    if len(usage_sections) == 0:
        raise DocoptLanguageError('"usage:" section (case-insensitive) not found. Perhaps missing indentation?')
    if len(usage_sections) > 1:
        raise DocoptLanguageError('More than one "usage:" (case-insensitive).')
    options_pattern = re.compile(r"\n\s*?options:", re.IGNORECASE)
    if options_pattern.search(usage_sections[0]):
        raise DocoptExit("Warning: options (case-insensitive) was found in usage." "Use a blank line between each section..")
    DocoptExit.usage = usage_sections[0]
    options = parse_defaults(docstring)
    pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
    pattern_options = set(pattern.flat(Option))
    for options_shortcut in pattern.flat(OptionsShortcut):
        doc_options = parse_defaults(docstring)
        options_shortcut.children = [opt for opt in doc_options if opt not in pattern_options]
    parsed_arg_vector = parse_argv(Tokens(argv), list(options), options_first, more_magic)
    extras(default_help, version, parsed_arg_vector, docstring)
    matched, left, collected = pattern.fix().match(parsed_arg_vector)
    if matched and left == []:
        output_obj = ParsedOptions((a.name, a.value) for a in (pattern.flat() + collected))
        target_parent_frame = parent_frame or magic_parent_frame or doc_parent_frame
        if more_magic and target_parent_frame and not output_value_assigned:
            if not target_parent_frame.f_globals.get("arguments"):
                target_parent_frame.f_globals["arguments"] = output_obj
        return output_obj
    if left:
        raise DocoptExit(f"Warning: found unmatched (duplicate?) arguments {left}")
    raise DocoptExit(collected=collected, left=left)
res = {}

for repo_dir in os.listdir(sys.argv[1]):
    repo_imports = []
    full_repo_path = os.path.join(sys.argv[1], repo_dir)
    for root, dirs, files in os.walk(full_repo_path):
        for filename in files:
            if not filename.endswith('.py'):
                continue

            full_path = os.path.join(root, filename)

            try:
                with open(full_path) as handle:

                    statements = handle.read()
                    instructions = dis.get_instructions(statements)
                    imports = [
                        __ for __ in instructions if 'IMPORT' in __.opname
                    ]
                    grouped = defaultdict(list)
                    for instr in imports:
                        grouped[instr.opname].append(instr.argval)

                    repo_imports.extend(grouped['IMPORT_NAME'])
            except:
                pass

    print(json.dumps(repo_imports))
Exemplo n.º 34
0
 def is_call_function(code: CodeType, bytei: ByteCodeIndex) -> bool:
     """Returns true iff the bytecode at the given index is a function call."""
     for ins in dis.get_instructions(code):
         if ins.offset == bytei and ins.opcode in Scalene.__call_opcodes:
             return True
     return False
Exemplo n.º 35
0
    def from_pycode(cls, co):
        """Create a Code object from a python code object.

        Parameters
        ----------
        co : CodeType
            The python code object.

        Returns
        -------
        code : Code
            The codetransformer Code object.
        """
        # Make it sparse to instrs[n] is the instruction at bytecode[n]
        sparse_instrs = tuple(
            _sparse_args(
                Instruction.from_opcode(
                    b.opcode,
                    Instruction._no_arg if b.arg is None else _RawArg(b.arg),
                ) for b in get_instructions(co)), )
        for idx, instr in enumerate(sparse_instrs):
            if instr is None:
                # The sparse value
                continue
            if instr.absjmp:
                instr.arg = sparse_instrs[instr.arg]
            elif instr.reljmp:
                instr.arg = sparse_instrs[instr.arg + idx + argsize + 1]
            elif isinstance(instr, LOAD_CONST):
                instr.arg = co.co_consts[instr.arg]
            elif instr.uses_name:
                instr.arg = co.co_names[instr.arg]
            elif instr.uses_varname:
                instr.arg = co.co_varnames[instr.arg]
            elif instr.uses_free:
                instr.arg = _freevar_argname(
                    instr.arg,
                    co.co_freevars,
                    co.co_cellvars,
                )
            elif instr.have_arg and isinstance(instr.arg, _RawArg):
                instr.arg = int(instr.arg)

        flags = Flag.unpack(co.co_flags)
        has_vargs = flags['CO_VARARGS']
        has_kwargs = flags['CO_VARKEYWORDS']

        # Here we convert the varnames format into our argnames format.
        paramnames = co.co_varnames[:(co.co_argcount +
                                      getattr(co, 'co_kwonlyargcount', 0) +
                                      has_vargs + has_kwargs)]
        # We start with the positional arguments.
        new_paramnames = list(paramnames[:co.co_argcount])
        # Add *args next.
        if has_vargs:
            new_paramnames.append('*' + paramnames[-1 - has_kwargs])
        # Add positional only arguments next.
        new_paramnames.extend(paramnames[co.co_argcount:co.co_argcount +
                                         getattr(co, 'co_kwonlyargcount', 0)])
        # Add **kwargs last.
        if has_kwargs:
            new_paramnames.append('**' + paramnames[-1])

        return cls(
            filter(bool, sparse_instrs),
            argnames=new_paramnames,
            cellvars=co.co_cellvars,
            freevars=co.co_freevars,
            name=co.co_name,
            filename=co.co_filename,
            firstlineno=co.co_firstlineno,
            # lnotab={
            #     lno: sparse_instrs[off] for off, lno in findlinestarts(co)
            # },
            flags=flags,
        )
Exemplo n.º 36
0
 def test_first_line_set_to_None(self):
     actual = dis.get_instructions(simple, first_line=None)
     self.assertEqual(list(actual), expected_opinfo_simple)
Exemplo n.º 37
0
 def test_default_first_line(self):
     actual = dis.get_instructions(simple)
     self.assertEqual(list(actual), expected_opinfo_simple)
Exemplo n.º 38
0
def print_inst(obj):
    lines = dis.get_instructions(obj)
    for line in lines:
        log.debug(line)
    def run_code(self, code):
        if isinstance(code, types.CodeType):
            code_obj = code
        elif isinstance(code, str):
            code_obj = compile(code, '<test>', 'exec')
        else:
            raise TypeError

        instructions = list(dis.get_instructions(code_obj))
        step = 0
        while step < len(instructions):
            instruction = instructions[step]

            #  Load and store consts, names, etc
            if instruction.opname == "LOAD_CONST":
                self.stack.append(instruction.argval)

            elif instruction.opname == "LOAD_NAME":
                name = instruction.argval
                value, namespace = self.find_instance_by_name(name)
                self.stack.append(value)

            elif instruction.opname == "LOAD_GLOBAL":
                name = instruction.argval
                value, namespace = self.find_instance_by_name(name)
                self.stack.append(value)

            elif instruction.opname == "LOAD_FAST":
                args, kwargs, optional_args = self.co_varnames[-1]
                if instruction.argval in kwargs:
                    arg = kwargs[instruction.argval]
                elif instruction.arg < len(args):
                    arg = args[instruction.arg]
                else:
                    arg = optional_args[instruction.argval]
                # print("ARGUMENT", instruction.argval, "=", arg)  # TODO: remove line
                self.stack.append(arg)

            elif instruction.opname == "STORE_NAME":
                name = instruction.argval
                value = self.stack.pop()
                self.locals[name] = value

            #  Functions
            elif instruction.opname == "CALL_FUNCTION":
                args = []
                for _ in range(instruction.arg):
                    args.insert(0, self.stack.pop())
                func = self.stack.pop()
                retval = func(*args)
                self.stack.append(retval)

            elif instruction.opname == "CALL_FUNCTION_KW":
                args = []
                kwargs = {}
                key_words = self.stack.pop()
                for key_word in reversed(key_words):
                    kwargs[key_word] = self.stack.pop()
                for _ in range(instruction.arg - len(kwargs)):
                    args.insert(0, self.stack.pop())
                func = self.stack.pop()
                # print(func, args, kwargs)
                retval = func(*args, **kwargs)
                self.stack.append(retval)

            elif instruction.opname == "CALL_FUNCTION_EX":
                if instruction.arg == 0:
                    kwargs = {}
                    args = self.stack.pop()
                elif instruction.arg == 1:
                    kwargs = self.stack.pop()
                    args = self.stack.pop()
                else:
                    raise Exception(
                        "Unknown instruction.arg in CALL_FUNCTION_EX")
                func = self.stack.pop()
                retval = func(*args, **kwargs)
                self.stack.append(retval)

            elif instruction.opname == "MAKE_FUNCTION":
                flags = instruction.arg
                func = self.Function(self, flags)
                self.stack.append(func)

            elif instruction.opname == "RETURN_VALUE":
                # TODO: implement, if complex logic is necessary
                pass

            # Stack operations
            elif instruction.opname == "POP_TOP":
                self.stack.pop()

            elif instruction.opname == "DUP_TOP":
                value = self.stack[-1]
                self.stack.append(value)

            elif instruction.opname == "UNPACK_SEQUENCE":
                # count = instruction.arg
                values = self.stack.pop()
                for value in reversed(values):
                    self.stack.append(value)

            #  Building containers
            elif instruction.opname.startswith("BUILD_"):
                count = instruction.arg
                if instruction.opname.endswith(("_LIST", "_TUPLE", "_SET")):
                    result = []
                    for _ in range(count):
                        result.insert(0, self.stack.pop())
                    if instruction.opname == "BUILD_LIST":
                        self.stack.append(result)
                    elif instruction.opname == "BUILD_TUPLE":
                        self.stack.append(tuple(result))
                    elif instruction.opname == "BUILD_SET":
                        self.stack.append(set(result))
                    else:
                        raise Exception("Unknown type of container")
                # elif instruction.opname == "BUILD_MAP":
                #     result = {}
                #     for _ in range(count):
                #         key = self.stack.pop()
                #         value = self.stack.pop()
                #         result[key] = value
                #     self.stack.append(result)
                elif instruction.opname == "BUILD_CONST_KEY_MAP":
                    items = []
                    keys = self.stack.pop()
                    for key in reversed(keys):
                        value = self.stack.pop()
                        items.insert(0, (key, value))
                    result = dict(items)
                    self.stack.append(result)
                elif instruction.opname == "BUILD_SLICE":
                    arg1 = self.stack.pop()
                    arg2 = self.stack.pop()
                    if instruction.arg == 2:
                        self.stack.append(slice(arg2, arg1))
                    elif instruction.arg == 3:
                        arg3 = self.stack.pop()
                        self.stack.append(slice(arg3, arg2, arg1))
                    else:
                        raise Exception(
                            "Unknown instruction.arg in BUILD_SLICE")
                else:
                    raise Exception("Unknown type of container")

            #  Comparison operators
            elif instruction.opname == "COMPARE_OP":
                arg2 = self.stack.pop()
                arg1 = self.stack.pop()
                if instruction.argval == '==':
                    result = arg1 == arg2
                elif instruction.argval == '!=':
                    result = arg1 != arg2
                elif instruction.argval == '>':
                    result = arg1 > arg2
                elif instruction.argval == '<':
                    result = arg1 < arg2
                elif instruction.argval == '>=':
                    result = arg1 >= arg2
                elif instruction.argval == '<=':
                    result = arg1 <= arg2
                elif instruction.argval == 'in':
                    result = arg1 in arg2
                elif instruction.argval == 'not in':
                    result = arg1 not in arg2
                elif instruction.argval == 'is':
                    result = arg1 is arg2
                elif instruction.argval == 'is not':
                    result = arg1 is not arg2
                else:
                    raise Exception("Unsupported comparison operator")
                self.stack.append(result)

            #  Unary operators
            elif instruction.opname.startswith("UNARY_"):
                arg = self.stack.pop()
                if instruction.opname == "UNARY_POSITIVE":
                    result = +arg
                elif instruction.opname == "UNARY_NEGATIVE":
                    result = -arg
                elif instruction.opname == "UNARY_NOT":
                    result = not arg
                elif instruction.opname == "UNARY_INVERT":
                    result = ~arg
                else:
                    raise Exception("Unsupported unary operator")
                self.stack.append(result)

            elif instruction.opname == "GET_ITER":
                arg = self.stack.pop()
                result = iter(arg)
                self.stack.append(result)

            #  Binary and inplace operators
            elif instruction.opname.startswith(("BINARY_", "INPLACE_")):
                arg2 = self.stack.pop()
                arg1 = self.stack.pop()
                if instruction.opname.endswith("_ADD"):
                    result = arg1 + arg2
                elif instruction.opname.endswith("_SUBTRACT"):
                    result = arg1 - arg2
                elif instruction.opname.endswith("_MULTIPLY"):
                    result = arg1 * arg2
                elif instruction.opname.endswith("_POWER"):
                    result = arg1**arg2
                elif instruction.opname.endswith("_FLOOR_DIVIDE"):
                    result = arg1 // arg2
                elif instruction.opname.endswith("_TRUE_DIVIDE"):
                    result = arg1 / arg2
                elif instruction.opname.endswith("_MODULO"):
                    result = arg1 % arg2
                elif instruction.opname.endswith("_SUBSCR"):
                    result = arg1[arg2]
                elif instruction.opname.endswith("_LSHIFT"):
                    result = arg1 << arg2
                elif instruction.opname.endswith("_RSHIFT"):
                    result = arg1 >> arg2
                elif instruction.opname.endswith("_AND"):
                    result = arg1 & arg2
                elif instruction.opname.endswith("_XOR"):
                    result = arg1 ^ arg2
                elif instruction.opname.endswith("_OR"):
                    result = arg1 | arg2
                elif instruction.opname.endswith("_MATRIX_MULTIPLY"):
                    result = arg1 @ arg2
                else:
                    raise Exception("Unsupported binary (or inplace) operator")
                self.stack.append(result)

            #  Operations with jumps (logical, cycles, etc)
            elif instruction.opname == "JUMP_IF_TRUE_OR_POP":
                top = self.stack.pop()
                if top:
                    step = self.get_step_by_argval(instruction.argval)
                    self.stack.append(top)

            elif instruction.opname == "JUMP_IF_FALSE_OR_POP":
                top = self.stack.pop()
                if not top:
                    step = self.get_step_by_argval(instruction.argval)
                    self.stack.append(top)

            elif instruction.opname == "POP_JUMP_IF_TRUE":
                top = self.stack.pop()
                if top:
                    step = self.get_step_by_argval(instruction.argval)

            elif instruction.opname == "POP_JUMP_IF_FALSE":
                top = self.stack.pop()
                if not top:
                    step = self.get_step_by_argval(instruction.argval)

            elif instruction.opname == "JUMP_FORWARD":
                step = self.get_step_by_argval(instruction.argval)

            elif instruction.opname == "JUMP_ABSOLUTE":
                step = self.get_step_by_argval(instruction.argval)

            elif instruction.opname == "FOR_ITER":
                top = self.stack[-1]
                try:
                    elem = top.__next__()
                    self.stack.append(elem)
                except StopIteration:
                    self.stack.pop()
                    step = self.get_step_by_argval(instruction.argval)

            #  Loops
            elif instruction.opname == "SETUP_LOOP":
                self.block_stack.append(instruction)

            elif instruction.opname == "POP_BLOCK":
                self.block_stack.pop()

            elif instruction.opname == "BREAK_LOOP":
                block = self.block_stack.pop()
                step = self.get_step_by_argval(block.argval)

            else:
                raise Exception("Unknown instruction " + instruction.opname)

            step += 1

        return None
Exemplo n.º 40
0
def _parse_lambda(function: types.FunctionType, tables: Dict[FEID, FeSummaryTable],
                  ret_ref: Flag) -> Optional[Dict[str, Any]]:
    """Convert a lambda function into its argument-based representation.

    The `function` is expected to be a lambda expression, which means that the set of bytecode instructions is limited
    compared to examining any possible function.

    Args:
        function: A lambda function to be inspected.
        tables: A collection of tables representing objects which are used by the current stack of inputs.
        ret_ref: A flag to indicate that _trace_value is returning a reference (this is used to figure out whether
            functions can be in-lined or deserve their own tables).

    Returns:
        The arguments being used to invoke `function`, or None if parsing fails.
    """
    code = function.__code__
    instructions = [x for x in dis.get_instructions(code)]
    closure_vars = inspect.getclosurevars(function)  # The variables defining the current scope.
    conditions = []
    args = []
    idx = 0
    while idx < len(instructions):
        instruction = instructions[idx]
        if instruction.opname == 'RETURN_VALUE':
            # Lambda functions don't support the return keyword, instead values are returned implicitly
            if conditions:
                current_condition = conditions.pop()
                arg = args.pop()
                instructions.pop(idx - 1)
                idx -= 1
                # In lambda functions, conditions always fill in the order: condition -> left -> right
                if current_condition.left is None:
                    conditions.append(_Condition(left=arg, condition=current_condition.condition, right=None))
                    instructions.pop(idx - 1)
                    idx -= 1
                else:
                    args.append(
                        _Condition(left=current_condition.left, condition=current_condition.condition, right=arg))
                    if conditions:
                        # The return value can be used to satisfy a condition slot
                        idx -= 1
            else:
                break
        elif instruction.opname == 'LOAD_CONST':
            # It's a constant value
            args.append(instruction.argval)
        elif instruction.opname == 'LOAD_FAST':
            # It's a variable from a lambda expression
            args.append(_VarWrap(instruction.argval))
        elif instruction.opname in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
            # It's an iterable
            n_args = instruction.argval
            arg = deque()
            for i in range(n_args):
                arg.appendleft(args.pop())
                instructions.pop(idx - 1)
                idx -= 1
            arg = list(arg)
            if instruction.opname == 'BUILD_TUPLE':
                arg = tuple(arg)
            elif instruction.opname == 'BUILD_SET':
                arg = set(arg)
            args.append(arg)
        elif instruction.opname == "BUILD_MAP":
            # It's a map
            n_keys = instruction.argval
            arg = {}
            for i in range(n_keys):
                v = args.pop()
                k = args.pop()
                instructions.pop(idx - 1)
                idx -= 1
                instructions.pop(idx - 1)
                idx -= 1
                arg[k] = v
            args.append(arg)
        elif instruction.opname == "BUILD_CONST_KEY_MAP":
            # It's a map that had constant keys
            keys = args.pop()
            instructions.pop(idx - 1)
            idx -= 1
            vals = deque()
            for i in range(instruction.argval):
                vals.appendleft(args.pop())
                instructions.pop(idx - 1)
                idx -= 1
            args.append({key: val for key, val in zip(keys, vals)})
        elif instruction.opname == 'LOAD_DEREF' and not _deref_is_callable(
                instruction, closure_vars) and not instructions[idx + 1].opname in ('LOAD_METHOD', 'LOAD_ATTR'):
            # It's a reference to a variable that's not being used to invoke some other function
            args.append(
                closure_vars.nonlocals.get(
                    instruction.argval,
                    closure_vars.globals.get(instruction.argval, closure_vars.builtins.get(instruction.argval, None))))
        elif instruction.opname in ('LOAD_METHOD', 'LOAD_ATTR', 'LOAD_GLOBAL', 'LOAD_DEREF'):
            # We're setting up a function call, which may or may not be invoked
            # Look ahead to combine all of the function pieces together into 1 variable
            name = instructions[idx].argval
            func_pair = _Function(
                closure_vars.nonlocals.get(name, closure_vars.globals.get(name, closure_vars.builtins.get(name, None))),
                name=name)
            if func_pair.func is None:
                # This function can't be found for some reason
                return _parse_lambda_fallback(function, tables, ret_ref)
            while idx + 1 < len(instructions):
                if instructions[idx + 1].opname in ('LOAD_METHOD', 'LOAD_ATTR'):
                    name = instructions[idx + 1].argval
                    func_pair = _Function(getattr(func_pair.func, name), name=func_pair.name + f".{name}")
                    instructions.pop(idx + 1)
                else:
                    break
            args.append(func_pair)
        elif instruction.opname in ('CALL_METHOD', 'CALL_FUNCTION', 'CALL_FUNCTION_KW'):
            kwargs = {}
            kwarg_names = []
            if instruction.opname == 'CALL_FUNCTION_KW':
                # Gather the keywords, which were added with a LOAD_CONST call
                kwarg_names = args.pop()
                instructions.pop(idx - 1)
                idx -= 1
            # Gather the args
            n_args = instruction.argval
            fn_args = deque()
            for i in range(n_args):
                fn_args.appendleft(args.pop())
                instructions.pop(idx - 1)
                idx -= 1
            for name in reversed(kwarg_names):
                kwargs[name] = fn_args.pop()
            # Gather the fn
            func_pair = args.pop()
            instructions.pop(idx - 1)  # Remove the method def from the stack
            idx -= 1
            # Bind the fn
            if not callable(func_pair.func):
                # This shouldn't ever happen, but just in case...
                return _parse_lambda_fallback(function, tables, ret_ref)
            try:
                bound_args = inspect.signature(func_pair.func).bind(*fn_args, **kwargs)
                bound_args.apply_defaults()
            except ValueError:
                # Some functions (C bindings) don't have convenient signature lookup
                bound_args = _PartialBind(tuple(fn_args), kwargs)
            args.append(_BoundFn(func_pair, bound_args))
        elif instruction.opname.startswith('BINARY_') or instruction.opname.startswith(
                'INPLACE_') or instruction.opname == 'COMPARE_OP':
            # Capture actual inline function stuff like: 0.5 + x
            command = strip_prefix(strip_prefix(instruction.opname, 'BINARY_'), 'INPLACE_')
            if instruction.opname == 'COMPARE_OP':
                command = instruction.argval
            if command not in _CommandTable:
                return _parse_lambda_fallback(function, tables, ret_ref)
            right = args.pop()
            instructions.pop(idx - 1)
            idx -= 1
            left = args.pop()
            instructions.pop(idx - 1)
            idx -= 1
            args.append(_Command(left, right, _CommandTable[command]))
        elif instruction.opname == 'POP_JUMP_IF_FALSE':
            # a if a < b else b     |||     <left> if <condition> else <right>
            conditions.append(_Condition(left=None, right=None, condition=args.pop()))
            instructions.pop(idx - 1)
            idx -= 1
        else:
            # TODO - to be fully rigorous we need the rest: https://docs.python.org/3.7/library/dis.html#bytecodes
            # TODO - LIST_APPEND, SET_ADD, MAP_ADD, BUILD_STRING, CALL_FUNCTION_EX, BUILD_TUPLE_UNPACK, etc.
            # Note that this function is only ever used to examine lambda functions, which helps to restrict the set of
            # possible commands
            return _parse_lambda_fallback(function, tables, ret_ref)
        idx += 1
    # Return the bound args
    if conditions or len(args) != 1:
        return _parse_lambda_fallback(function, tables, ret_ref)
    return {"function": _trace_value(args[0], tables, ret_ref=ret_ref, wrap_str=True)}
Exemplo n.º 41
0
    def test_folding_of_tuples_of_constants(self):
        for line, elem in (
            ('a = 1,2,3', (1, 2, 3)),
            ('("a","b","c")', ('a', 'b', 'c')),
            ('a,b,c = 1,2,3', (1, 2, 3)),
            ('(None, 1, None)', (None, 1, None)),
            ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
        ):
            code = compile(line, '', 'single')
            self.assertInBytecode(code, 'LOAD_CONST', elem)
            self.assertNotInBytecode(code, 'BUILD_TUPLE')

        # Long tuples should be folded too.
        code = compile(repr(tuple(range(10000))), '', 'single')
        self.assertNotInBytecode(code, 'BUILD_TUPLE')
        # One LOAD_CONST for the tuple, one for the None return value
        load_consts = [
            instr for instr in dis.get_instructions(code)
            if instr.opname == 'LOAD_CONST'
        ]
        self.assertEqual(len(load_consts), 2)

        # Bug 1053819:  Tuple of constants misidentified when presented with:
        # . . . opcode_with_arg 100   unary_opcode   BUILD_TUPLE 1  . . .
        # The following would segfault upon compilation
        def crater():
            (~ [
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
                0,
                1,
                2,
                3,
                4,
                5,
                6,
                7,
                8,
                9,
            ], )
Exemplo n.º 42
0
 def test_outer(self):
     actual = dis.get_instructions(outer, first_line=expected_outer_line)
     self.assertEqual(list(actual), expected_opinfo_outer)
Exemplo n.º 43
0
    def interpreter(model, debug=0):

        closure = getclosurevars(model).nonlocals

        if 'pro_tip' in closure:
            tips = closure['pro_tip']
            tip_type = tips[0]
            if tip_type == 'STAN_NUTS':
                param = tips[1]
                if not param[0] in handled:
                    pro_tips.append(closure['pro_tip'])
                    handled.extend(param)
            elif tip_type == 'Block':
                pro_tips.append(closure['pro_tip'])
            else:
                print('Unknown tip', tip_type)
                exit(0)

        if debug >= 1:
            print('Start: ', model.__name__, ' with path ', closure['path'])

        # Local evaluation stack, contains both symbolic and real values
        stack = StackList([])
        # Local variables binding
        varmap = {}

        ret = ''

        code = []
        for ins in dis.get_instructions(model):
            code.append(ins)

        code_pointer = 0

        while (True):

            ins = code[code_pointer]
            op = ins.opname
            arg = ins.argval

            if debug >= 2:
                print('')
                print(op, ' ', arg)

            if (op == 'LOAD_DEREF'):
                v = closure[arg]
                stack.push(v)
            elif (op == 'LOAD_GLOBAL'):
                stack.push(arg)
            elif (op == 'CALL_FUNCTION'):
                l = []
                l_obj = []
                for i in range(arg):
                    a = stack.pop()
                    l.append(str(a))
                    l_obj.append(a)
                l.reverse()
                l_obj.reverse()
                a = stack.pop()
                if arg != 0:
                    # Symbolic call
                    if a == 'sample':
                        # Pyro sample, execute RV path
                        name = l[0]
                        dist = l_obj[1]
                        rv = RV(name, dist)
                        stack.push(rv)
                    elif (a in fmap):
                        if len(l) == 2:
                            #savepath = l[0]
                            stack.push(fmap[a](l[0], eval(l[1])))
                        elif len(l) == 3:
                            #savepath = l[1]
                            stack.push(fmap[a](l[0], l[1], eval(l[2])))
                        else:
                            print(
                                'Not working with more than 3 arguments yet (0)'
                            )
                            exit(0)
                    elif (type(a) == types.FunctionType):
                        if len(l) == 2:
                            #savepath = l[0]
                            stack.push(a(l[0], eval(l[1])))
                        elif len(l) == 3:
                            #savepath = l[1]
                            stack.push(a(l[0], l[1], eval(l[2])))
                        else:
                            print(
                                'Not working with more than 3 arguments yet (1)',
                                len(l))
                            exit(0)
                    else:
                        dist = make_dist(str(a), l)
                        stack.push(dist)
                else:
                    # Actual model execution
                    if debug >= 2:
                        print('To execute: ', a)
                    (ret, _) = interpreter(a, 0)
                    stack.push(ret)
            elif (op == 'UNPACK_SEQUENCE'):
                a = stack.pop()
                for i in range(arg):
                    stack.push(a[i])
            elif (op == 'STORE_FAST'):
                a = stack.pop()
                # Handling loops. That seems crazy but that's not me!
                if type(a) == LOOPMARKER:
                    code_pointer = a.ptr - 1
                else:
                    # Normal case
                    varmap[arg] = a
                    if type(a) == RV:
                        rvmap[a.name] = a
            elif (op == 'LOAD_FAST'):
                if type(varmap[arg]) == RV:
                    stack.push(varmap[arg].name)
                else:
                    stack.push(varmap[arg])
            elif (op == 'LOAD_CONST'):
                stack.push(arg)
            elif (op == 'BINARY_ADD'):
                a1 = stack.pop()
                a2 = stack.pop()
                stack.push(a2 + a1)
            elif (op == 'BINARY_MULTIPLY'):
                a1 = stack.pop()
                a2 = stack.pop()
                stack.push(str(a2) + ' * ' + str(a1))
            elif (op == 'BINARY_SUBSCR'):
                a1 = stack.pop()
                a2 = stack.pop()
                if (type(a2) == str):
                    stack.push(str(a2) + '[' + str(a1) + ']')
                else:
                    stack.push(a2[a1])
            elif (op == 'BUILD_LIST'):
                l = []
                for i in range(arg):
                    a = stack.pop()
                    l.append(a)
                stack.push(l)
            elif (op == 'SETUP_LOOP'):
                stack.push(LOOPMARKER(arg // 2))
            elif (op == 'LOAD_ATTR'):
                pass
            elif (op == 'CALL_FUNCTION_KW'):
                pass
            elif (op == 'GET_ITER'):
                a = stack.pop()
                stack.push(iter(a))
            elif (op == 'FOR_ITER'):
                a = stack.pop()
                for i in a:
                    stack.push(i)
            elif (op == 'POP_TOP'):
                stack.pop()
            elif (op == 'JUMP_ABSOLUTE'):
                code_pointer = (arg // 2)
            elif (op == 'POP_BLOCK'):
                pass
            elif (op == ''):
                pass
            elif (op == 'RETURN_VALUE'):
                if debug >= 1:
                    print('End: ', model.__name__)
                a = stack.pop()
                assert (stack == [])
                if a == None:
                    return ([], varmap)
                else:
                    l = []
                    for rv in a:
                        l.append(rvmap[rv])
                    return (l, varmap)
            else:
                print('Not implemented: ', op)
                exit(0)

            code_pointer += 1

            if debug >= 2:
                print('stack: ', stack)
                print('map: ', varmap)
                print('rvs', rvmap)
Exemplo n.º 44
0
    def __init__(self, clsname, bases, clsdict):
        # clsname - экземпляр метакласса - Server
        # bases - кортеж базовых классов - ()
        # clsdict - словарь атрибутов и методов экземпляра метакласса
        # {'__module__': '__main__',
        # '__qualname__': 'Server',
        # 'port': <descrptrs.Port object at 0x000000DACC8F5748>,
        # '__init__': <function Server.__init__ at 0x000000DACCE3E378>,
        # 'init_socket': <function Server.init_socket at 0x000000DACCE3E400>,
        # 'main_loop': <function Server.main_loop at 0x000000DACCE3E488>,
        # 'process_message': <function Server.process_message at 0x000000DACCE3E510>,
        # 'process_client_message': <function Server.process_client_message at 0x000000DACCE3E598>}

        # Список методов, которые используются в функциях класса:
        methods = []
        # Атрибуты, используемые в функциях классов
        attrs = []
        # перебираем ключи
        for func in clsdict:
            # Пробуем
            try:
                # Возвращает итератор по инструкциям в предоставленной функции
                # , методе, строке исходного кода или объекте кода.
                ret = dis.get_instructions(clsdict[func])
                # ret - <generator object _get_instructions_bytes at 0x00000062EAEAD7C8>
                # ret - <generator object _get_instructions_bytes at 0x00000062EAEADF48>
                # ...
                # Если не функция то ловим исключение
                # (если порт)
            except TypeError:
                pass
            else:
                # Раз функция разбираем код, получая используемые методы и
                # атрибуты.
                for i in ret:
                    #print(i)
                    # i - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=9, argval='send_message',
                    # argrepr='send_message', offset=308, starts_line=201, is_jump_target=False)
                    # opname - имя для операции
                    if i.opname == 'LOAD_GLOBAL':
                        if i.argval not in methods:
                            # заполняем список методами, использующимися в
                            # функциях класса
                            methods.append(i.argval)
                    elif i.opname == 'LOAD_ATTR':
                        if i.argval not in attrs:
                            # заполняем список атрибутами, использующимися в
                            # функциях класса
                            attrs.append(i.argval)
        #print(methods)
        # Если обнаружено использование недопустимого метода connect, бросаем
        # исключение:
        if 'connect' in methods:
            raise TypeError(
                'Использование метода connect недопустимо в серверном классе')
        # Если сокет не инициализировался константами SOCK_STREAM(TCP)
        # AF_INET(IPv4), тоже исключение.
        if not ('SOCK_STREAM' in attrs and 'AF_INET' in attrs):
            raise TypeError('Некорректная инициализация сокета.')
        # Обязательно вызываем конструктор предка:
        super().__init__(clsname, bases, clsdict)
Exemplo n.º 45
0
def decompile(code_object):
    """
    Taken from
    http://thermalnoise.wordpress.com/2007/12/30/exploring-python-bytecode/

    Extracts dissasembly information from the byte code and stores it in
    a list for further use.

    Call signature(s):
        instructions=decompile(f.f_code)

    Required      arguments:
    =========     =====================================================================
    code_object   A bytecode object extracted with inspect.currentframe()
                  or any other mechanism that returns byte code.

    Optional keyword arguments: NONE

    Outputs:
    =========     =====================================================================
    instructions  a list of offsets, op_codes, names, arguments,
                  argument_value which can be deconstructed to find out various things
                  about a function call.

    Example:
    # Two frames back so that we get the callers' caller
    f = inspect.currentframe().f_back.f_back
    i = f.f_lasti  # index of the last attempted instruction in byte code
    ins = decompile(f.f_code)
    """
    instructions = []

    if PY3:
        for ins in dis.get_instructions(code_object):
            instructions.append(
                (ins.offset, ins.opcode, ins.opname, ins.arg, ins.argval))
    else:
        code = code_object.co_code
        variables = code_object.co_cellvars + code_object.co_freevars
        n = len(code)
        i = 0
        e = 0
        while i < n:
            i_offset = i
            i_opcode = ord(code[i])
            i = i + 1
            if i_opcode >= opcode.HAVE_ARGUMENT:
                i_argument = ord(code[i]) + (ord(code[i + 1]) << (4 * 2)) + e
                i = i + 2
                if i_opcode == opcode.EXTENDED_ARG:
                    e = i_argument << 16
                else:
                    e = 0
                if i_opcode in opcode.hasconst:
                    i_arg_value = repr(code_object.co_consts[i_argument])
                elif i_opcode in opcode.hasname:
                    i_arg_value = code_object.co_names[i_argument]
                elif i_opcode in opcode.hasjrel:
                    i_arg_value = repr(i + i_argument)
                elif i_opcode in opcode.haslocal:
                    i_arg_value = code_object.co_varnames[i_argument]
                elif i_opcode in opcode.hascompare:
                    i_arg_value = opcode.cmp_op[i_argument]
                elif i_opcode in opcode.hasfree:
                    i_arg_value = variables[i_argument]
                else:
                    i_arg_value = i_argument
            else:
                i_argument = None
                i_arg_value = None
            instructions.append((i_offset, i_opcode, opcode.opname[i_opcode],
                                 i_argument, i_arg_value))
    return instructions
Exemplo n.º 46
0
def _generate_opcode_new(code_object):
    import dis
    for ins in dis.get_instructions(code_object):
        yield (ins.opcode, ins.arg)
Exemplo n.º 47
0
 def get_imports(self, source: str) -> defaultdict:
     self.instructions = dis.get_instructions(source)
     for instruction in self.instructions:
         if "IMPORT_NAME" in instruction.opname or "IMPORT_FROM" in instruction.opname:
             self.imports[instruction.opname].append(instruction.argval)
     return self.imports
Exemplo n.º 48
0
def preview(code):
    decompiler = Decompiler()
    decompiler.build_graph(list(dis.get_instructions(code)))

    dis.dis(code)
    display_graph(decompiler)
Exemplo n.º 49
0
 def test_nested(self):
     with captured_stdout():
         f = outer()
     actual = dis.get_instructions(f, first_line=expected_f_line)
     self.assertEqual(list(actual), expected_opinfo_f)
def infer_return_type_func(f, input_types, debug=False, depth=0):
    """Analyses a function to deduce its return type.

  Args:
    f: A Python function object to infer the return type of.
    input_types: A sequence of inputs corresponding to the input types.
    debug: Whether to print verbose debugging information.
    depth: Maximum inspection depth during type inference.

  Returns:
    A TypeConstraint that that the return value of this function will (likely)
    satisfy given the specified inputs.

  Raises:
    TypeInferenceError: if no type can be inferred.
  """
    if debug:
        print()
        print(f, id(f), input_types)
        dis.dis(f)
    from . import opcodes
    simple_ops = dict((k.upper(), v) for k, v in opcodes.__dict__.items())

    co = f.__code__
    code = co.co_code
    end = len(code)
    pc = 0
    extended_arg = 0  # Python 2 only.
    free = None

    yields = set()
    returns = set()
    # TODO(robertwb): Default args via inspect module.
    local_vars = list(input_types) + [typehints.Union[
        ()]] * (len(co.co_varnames) - len(input_types))
    state = FrameState(f, local_vars)
    states = collections.defaultdict(lambda: None)
    jumps = collections.defaultdict(int)

    # In Python 3, use dis library functions to disassemble bytecode and handle
    # EXTENDED_ARGs.
    is_py3 = sys.version_info[0] == 3
    if is_py3:
        ofs_table = {}  # offset -> instruction
        for instruction in dis.get_instructions(f):
            ofs_table[instruction.offset] = instruction

    # Python 2 - 3.5: 1 byte opcode + optional 2 byte arg (1 or 3 bytes).
    # Python 3.6+: 1 byte opcode + 1 byte arg (2 bytes, arg may be ignored).
    if sys.version_info >= (3, 6):
        inst_size = 2
        opt_arg_size = 0
    else:
        inst_size = 1
        opt_arg_size = 2

    last_pc = -1
    while pc < end:  # pylint: disable=too-many-nested-blocks
        start = pc
        if is_py3:
            instruction = ofs_table[pc]
            op = instruction.opcode
        else:
            op = ord(code[pc])
        if debug:
            print('-->' if pc == last_pc else '    ', end=' ')
            print(repr(pc).rjust(4), end=' ')
            print(dis.opname[op].ljust(20), end=' ')

        pc += inst_size
        if op >= dis.HAVE_ARGUMENT:
            if is_py3:
                arg = instruction.arg
            else:
                arg = ord(code[pc]) + ord(code[pc + 1]) * 256 + extended_arg
            extended_arg = 0
            pc += opt_arg_size
            if op == dis.EXTENDED_ARG:
                extended_arg = arg * 65536
            if debug:
                print(str(arg).rjust(5), end=' ')
                if op in dis.hasconst:
                    print('(' + repr(co.co_consts[arg]) + ')', end=' ')
                elif op in dis.hasname:
                    print('(' + co.co_names[arg] + ')', end=' ')
                elif op in dis.hasjrel:
                    print('(to ' + repr(pc + arg) + ')', end=' ')
                elif op in dis.haslocal:
                    print('(' + co.co_varnames[arg] + ')', end=' ')
                elif op in dis.hascompare:
                    print('(' + dis.cmp_op[arg] + ')', end=' ')
                elif op in dis.hasfree:
                    if free is None:
                        free = co.co_cellvars + co.co_freevars
                    print('(' + free[arg] + ')', end=' ')

        # Actually emulate the op.
        if state is None and states[start] is None:
            # No control reaches here (yet).
            if debug:
                print()
            continue
        state |= states[start]

        opname = dis.opname[op]
        jmp = jmp_state = None
        if opname.startswith('CALL_FUNCTION'):
            if sys.version_info < (3, 6):
                # Each keyword takes up two arguments on the stack (name and value).
                standard_args = (arg & 0xFF) + 2 * (arg >> 8)
                var_args = 'VAR' in opname
                kw_args = 'KW' in opname
                pop_count = standard_args + var_args + kw_args + 1
                if depth <= 0:
                    return_type = Any
                elif arg >> 8:
                    if not var_args and not kw_args and not arg & 0xFF:
                        # Keywords only, maybe it's a call to Row.
                        if isinstance(state.stack[-pop_count], Const):
                            from apache_beam.pvalue import Row
                            if state.stack[-pop_count].value == Row:
                                fields = state.stack[-pop_count + 1::2]
                                types = state.stack[-pop_count + 2::2]
                                return_type = row_type.RowTypeConstraint(
                                    zip([fld.value for fld in fields],
                                        Const.unwrap_all(types)))
                            else:
                                return_type = Any
                    else:
                        # TODO(robertwb): Handle this case.
                        return_type = Any
                elif isinstance(state.stack[-pop_count], Const):
                    # TODO(robertwb): Handle this better.
                    if var_args or kw_args:
                        state.stack[-1] = Any
                        state.stack[-var_args - kw_args] = Any
                    return_type = infer_return_type(
                        state.stack[-pop_count].value,
                        state.stack[1 - pop_count:],
                        debug=debug,
                        depth=depth - 1)
                else:
                    return_type = Any
                state.stack[-pop_count:] = [return_type]
            else:  # Python 3.6+
                if opname == 'CALL_FUNCTION':
                    pop_count = arg + 1
                    if depth <= 0:
                        return_type = Any
                    elif isinstance(state.stack[-pop_count], Const):
                        return_type = infer_return_type(
                            state.stack[-pop_count].value,
                            state.stack[1 - pop_count:],
                            debug=debug,
                            depth=depth - 1)
                    else:
                        return_type = Any
                elif opname == 'CALL_FUNCTION_KW':
                    # TODO(udim): Handle keyword arguments. Requires passing them by name
                    #   to infer_return_type.
                    pop_count = arg + 2
                    if isinstance(state.stack[-pop_count], Const):
                        from apache_beam.pvalue import Row
                        if state.stack[-pop_count].value == Row:
                            fields = state.stack[-1].value
                            return_type = row_type.RowTypeConstraint(
                                zip(
                                    fields,
                                    Const.unwrap_all(state.stack[-pop_count +
                                                                 1:-1])))
                        else:
                            return_type = Any
                    else:
                        return_type = Any
                elif opname == 'CALL_FUNCTION_EX':
                    # stack[-has_kwargs]: Map of keyword args.
                    # stack[-1 - has_kwargs]: Iterable of positional args.
                    # stack[-2 - has_kwargs]: Function to call.
                    has_kwargs = arg & 1  # type: int
                    pop_count = has_kwargs + 2
                    if has_kwargs:
                        # TODO(udim): Unimplemented. Requires same functionality as a
                        #   CALL_FUNCTION_KW implementation.
                        return_type = Any
                    else:
                        args = state.stack[-1]
                        _callable = state.stack[-2]
                        if isinstance(args, typehints.ListConstraint):
                            # Case where there's a single var_arg argument.
                            args = [args]
                        elif isinstance(args, typehints.TupleConstraint):
                            args = list(args._inner_types())
                        return_type = infer_return_type(_callable.value,
                                                        args,
                                                        debug=debug,
                                                        depth=depth - 1)
                else:
                    raise TypeInferenceError('unable to handle %s' % opname)
                state.stack[-pop_count:] = [return_type]
        elif opname == 'CALL_METHOD':
            pop_count = 1 + arg
            # LOAD_METHOD will return a non-Const (Any) if loading from an Any.
            if isinstance(state.stack[-pop_count], Const) and depth > 0:
                return_type = infer_return_type(state.stack[-pop_count].value,
                                                state.stack[1 - pop_count:],
                                                debug=debug,
                                                depth=depth - 1)
            else:
                return_type = typehints.Any
            state.stack[-pop_count:] = [return_type]
        elif opname in simple_ops:
            if debug:
                print("Executing simple op " + opname)
            simple_ops[opname](state, arg)
        elif opname == 'RETURN_VALUE':
            returns.add(state.stack[-1])
            state = None
        elif opname == 'YIELD_VALUE':
            yields.add(state.stack[-1])
        elif opname == 'JUMP_FORWARD':
            jmp = pc + arg
            jmp_state = state
            state = None
        elif opname == 'JUMP_ABSOLUTE':
            jmp = arg
            jmp_state = state
            state = None
        elif opname in ('POP_JUMP_IF_TRUE', 'POP_JUMP_IF_FALSE'):
            state.stack.pop()
            jmp = arg
            jmp_state = state.copy()
        elif opname in ('JUMP_IF_TRUE_OR_POP', 'JUMP_IF_FALSE_OR_POP'):
            jmp = arg
            jmp_state = state.copy()
            state.stack.pop()
        elif opname == 'FOR_ITER':
            jmp = pc + arg
            jmp_state = state.copy()
            jmp_state.stack.pop()
            state.stack.append(element_type(state.stack[-1]))
        else:
            raise TypeInferenceError('unable to handle %s' % opname)

        if jmp is not None:
            # TODO(robertwb): Is this guaranteed to converge?
            new_state = states[jmp] | jmp_state
            if jmp < pc and new_state != states[jmp] and jumps[pc] < 5:
                jumps[pc] += 1
                pc = jmp
            states[jmp] = new_state

        if debug:
            print()
            print(state)
            pprint.pprint(dict(item for item in states.items() if item[1]))

    if yields:
        result = typehints.Iterable[reduce(union, Const.unwrap_all(yields))]
    else:
        result = reduce(union, Const.unwrap_all(returns))
    finalize_hints(result)

    if debug:
        print(f, id(f), input_types, '->', result)
    return result
Exemplo n.º 51
0
def sql_template(function: types.FunctionType) -> types.FunctionType:
    signature = inspect.signature(function)
    instructions = list(dis.get_instructions(function))

    @functools.wraps(function)
    def build(*args: typing.Any, **kwargs: typing.Any) -> typing.Any:
        bound_arguments = signature.bind(*args, **kwargs)
        bound_arguments.apply_defaults()

        global_dict: typing.Dict[str, typing.Any] = {
            # supported queries:
            #     with ... select ...
            #     select ...
            #     insert into ... select ...
            #     create table ... engine = ... as select ...
            #     create view ... as select ...
            #     create materialized view ... as select ...
            'with_':
            ast.Initial('with'),
            'select':
            ast.Initial('select'),
            'select_distinct':
            ast.Initial('select_distinct'),
            'insert':
            ast.Initial('insert'),
            'insert_into':
            ast.Initial('insert_into'),
            'create':
            ast.Initial('create'),
            'create_table':
            ast.Initial('create_table'),
            'create_table_if_not_exists':
            ast.Initial('create_table_if_not_exists'),
            'create_view':
            ast.Initial('create_view'),
            'create_or_replace_view':
            ast.Initial('create_or_replace_view'),
            'create_view_if_not_exists':
            ast.Initial('create_view_if_not_exists'),
            'create_materialized_view':
            ast.Initial('create_materialized_view'),
            'create_materialized_view_if_not_exists':
            ast.Initial('create_materialized_view_if_not_exists'),
        }

        local_dict: typing.Dict[str, typing.Any] = {
            **bound_arguments.arguments,
        }

        # TODO: use types.CellType in type annotation
        cells: typing.Tuple[typing.Any, ...] = (
            *(function.__closure__ or ()),
            *(
                types.CellType()  # type: ignore[attr-defined]
                for _ in function.__code__.co_cellvars or ()),
        )

        stack: typing.List[typing.Any] = []

        # notice: see dis.opmap
        for instruction in instructions:
            done = _run(global_dict, local_dict, cells, stack,
                        instruction.opname, instruction.arg,
                        instruction.argval)

            if done:
                assert len(stack) == 1

                return stack.pop()

        return None

    # TODO
    return typing.cast(types.FunctionType, build)
Exemplo n.º 52
0
 def test_doubly_nested(self):
     with captured_stdout():
         inner = outer()()
     actual = dis.get_instructions(inner, first_line=expected_inner_line)
     self.assertEqual(list(actual), expected_opinfo_inner)
Exemplo n.º 53
0
def _get_globals_py3(func):
    for o in dis.get_instructions(func):
        if o.opname == 'LOAD_GLOBAL':
            yield o.argval
Exemplo n.º 54
0
 def test_jumpy(self):
     actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
     self.assertEqual(list(actual), expected_opinfo_jumpy)
Exemplo n.º 55
0
def find_forbidden_methods_call(func, method_names):
    for instr in dis.get_instructions(func):
        if instr.opname == 'LOAD_METHOD' and instr.argval in method_names:
            return instr.argval
Exemplo n.º 56
0
 def test_iteration(self):
     for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:
         with self.subTest(obj=obj):
             via_object = list(dis.Bytecode(obj))
             via_generator = list(dis.get_instructions(obj))
             self.assertEqual(via_object, via_generator)
Exemplo n.º 57
0
def get_referenced_objects(code, context, suppress_warnings=False):
    """
    Attempts to return all objects referenced externally by a code object. In
    some cases, these objects are:
    - replaced with ReferenceProxy class: when we can't find a variable or when
      the reference is an attribute of the result of a function call, or
    - omitted: when the reference is accessed using exec or is the result of a
      function call.

    An externally referenced object is any object used by a piece of code but not
    defined in that code. These objects can include classes, functions, modules,
    or any other variables. They can be referenced from various scopes: global,
    "free" (when the variable is used in a code block, but is not defined there
    and is not global), etc.; or as attributes of other referenced objects.

    Note that this function does not actually run any code. This includes
    calling any functions used by the input code, because doing so can be
    expensive and have unintended consequences. Due to this, any references that
    are attributes of the result of a function call won’t be detected. This means
    that if an inner function returns a module, any attributes of the module
    won't be detected. In this case, we return the name of the attribute as a
    proxy for the object itself. So for the function below, the returned
    references would be ``[get_my_class, "call"]``.

    .. code-block:: python

        def x():
            my_cls = get_my_class("MyClass") # Returns class MyClass
            my_cls.call()

    This function uses a CodeContext object to look up variables defined outside
    the local scope and to track local variables created while running the logic
    to find references.
    """

    # We mutate context while finding references. Let's make a copy of
    # context to not change the original context. The original context
    # can be shared between different code objects, like between an
    # outer and an inner function.
    context = context.copy()

    # Top of the stack.
    tos = None
    lineno = None
    refs = []

    def set_tos(t):
        nonlocal tos
        if tos is not None:
            # If the top of stack item already exists, that means we
            # have gone through all the instructions that use the item,
            # and it is a reference object.
            refs.append(tos)

        tos = t

    # Our goal is to find referenced objects. The problem is that co_names
    # does not have fully qualified names in it. So if you access `foo.bar`,
    # co_names has `foo` and `bar` in it but it doesn't tell us that the
    # code reads `bar` of `foo`. We are going over the bytecode to resolve
    # from which object an attribute is requested.
    # Read more about bytecode at https://docs.python.org/3/library/dis.html

    for op in dis.get_instructions(code):
        try:
            if op.opname not in SUPPORTED_INSTRUCTIONS and not suppress_warnings:
                if sys.version_info < (3, 6) or sys.version_info > (3, 8):
                    message = """
                    You are using an unsupported Python version for Bionic. This
                    can result in Bionic missing some code changes to invalidate
                    cache. Consider using a supported Python version to avoid any
                    caching issues.
                    """
                else:
                    message = f"""
                    Bionic does not recognize {op.opname} Bytecode operation.
                    This should be impossible and is most likely a bug in Bionic.
                    Please raise a new issue at
                    https://github.com/square/bionic/issues to let us know.
                    """
                message += """
                You can also suppress this warning by removing the
                `suppress_bytecode_warnings` override from the
                `@version` decorator on the corresponding function.
                """
                warnings.warn(oneline(message))
            # Sometimes starts_line is None, in which case let's just remember the
            # previous start_line (if any). This way when there's an exception we at
            # least can point users somewhat near the line where the error stems from.
            if op.starts_line is not None:
                lineno = op.starts_line

            if op.opname in ["LOAD_GLOBAL", "LOAD_NAME"]:
                if op.argval in context.globals:
                    set_tos(context.globals[op.argval])
                else:
                    # This can happen if the variable does not exist, or if LOAD_NAME
                    # is trying to access a local frame argument. If we cannot find the
                    # variable, we return its name instead.
                    set_tos(ReferenceProxy(op.argval))
            elif op.opname in ["LOAD_DEREF", "LOAD_CLOSURE"]:
                if op.argval in context.cells:
                    set_tos(context.cells[op.argval])
                else:
                    # This can happen when we have nested functions. The second
                    # level or further nested functions won't have free variables
                    # from any preceding function except for the top level
                    # function. This is because the code context that gives us
                    # free variables is created from the function variable, which
                    # we only have for the top level function. We get only the
                    # code object for any inner functions. Since we can't get the
                    # code context for inner functions, we use the code context
                    # of the top level function.
                    set_tos(ReferenceProxy(op.argval))
            elif op.opname == "IMPORT_NAME":
                # This instruction only appears if the code object imports a
                # module using the import statement. If a user is importing
                # modules inside a function, they probably don't want to import
                # the module until the function execution time.
                message = f"""
                Entity function in file {code.co_filename} imports the
                '{op.argval}' module at line {lineno};
                Bionic will not be able to automatically detect any changes to this
                module.
                To enable automatic detection of changes, import the module at the
                global level (outside the function) instead.

                To suppress this warning, remove the `suppress_bytecode_warnings`
                override from the `@version` decorator on the corresponding function.
                f"""
                warnings.warn(oneline(message))
                set_tos(None)
            elif op.opname in ["LOAD_METHOD", "LOAD_ATTR"]:
                if isinstance(tos, ReferenceProxy):
                    tos.val += "." + op.argval
                elif inspect.ismodule(tos) and hasattr(tos, op.argval):
                    tos = getattr(tos, op.argval)
                else:
                    set_tos(ReferenceProxy(op.argval))
            elif op.opname == "STORE_FAST" and tos:
                context.varnames[op.argval] = tos
                set_tos(None)
            elif op.opname == "LOAD_FAST" and op.argval in context.varnames:
                set_tos(context.varnames[op.argval])
            else:
                # For all other instructions, add the current TOS as a
                # reference.
                set_tos(None)
        except Exception as e:
            message = oneline(f"""
            Bionic found a code reference in file ${code.co_filename}
            at line ${lineno} that it cannot hash when hashing
            ${code.co_name}. This should be impossible and is most
            likely a bug in Bionic. Please raise a new issue at
            https://github.com/square/bionic/issues to let us know.

            In the meantime, you can disable bytecode analysis for
            the corresponding function by setting `ignore_bytecode`
            on its `@version` decorator. Please note that Bionic won't
            automatically detect changes in this function; you'll need
            to manually update the version yourself.
            """)
            raise AssertionError(message) from e

    return refs
Exemplo n.º 58
0
   cell = []
   for s in range(len(c.co_cellvars) + len(c.co_freevars)):
      l = builder.alloca(ppyobj_type,1)
      tup = builder.load(builder.gep(func.args[2],[int32(0),int32(1)]))
      
      builder.store(builder.load(builder.gep(tup,[int32(0),int32(2),int32(s)])),l)
      cell.append(l)


   blocks = [[0,0,block,builder]]
   blocks_by_ofs = {0: block}
   block_num=0
   ins_idx=0
   nxt=False
   for ins in dis.get_instructions(c):
      if ins.is_jump_target or nxt:
        b = func.append_basic_block(name="block" + str(block_num+1))
        blocks.append([ins_idx,block_num+1,b,ir.IRBuilder(b)])
        blocks_by_ofs[ins.offset] = b
        block_num += 1
        nxt=False
      if ins.opname=='POP_JUMP_IF_FALSE' or ins.opname=='POP_JUMP_IF_TRUE' or ins.opname=='SETUP_EXCEPT' or ins.opname=='JUMP_FORWARD' or ins.opname=='JUMP_ABSOLUTE' or ins.opname=='SETUP_LOOP' or ins.opname=='BREAK_LOOP':
        nxt=True
      ins_idx +=1

   ins_idxs = [r[0] for r in blocks] 
   stack_ptr = 0
   ins_idx=0
   branch_stack = {}
   except_stack = []
Exemplo n.º 59
0
def _have_nested_for_statement(generator):
    matched = [True for instruction in dis.get_instructions(generator)
               if instruction.opname == "FOR_ITER"]
    return True if len(matched) > 1 else False
Exemplo n.º 60
0
def get_referenced_objects(code, context):
    # Top of the stack
    tos = None  # type: Any
    lineno = None
    refs = []

    def set_tos(t):
        nonlocal tos
        if tos is not None:
            # Hash tos so we support reading multiple objects
            refs.append(tos)
        tos = t

    # Our goal is to find referenced objects. The problem is that co_names
    # does not have full qualified names in it. So if you access `foo.bar`,
    # co_names has `foo` and `bar` in it but it doesn't tell us that the
    # code reads `bar` of `foo`. We are going over the bytecode to resolve
    # from which object an attribute is requested.
    # Read more about bytecode at https://docs.python.org/3/library/dis.html

    for op in dis.get_instructions(code):
        try:
            # Sometimes starts_line is None, in which case let's just remember the
            # previous start_line (if any). This way when there's an exception we at
            # least can point users somewhat near the line where the error stems from.
            if op.starts_line is not None:
                lineno = op.starts_line

            if op.opname in ["LOAD_GLOBAL", "LOAD_NAME"]:
                if op.argval in context.globals:
                    set_tos(context.globals[op.argval])
                else:
                    set_tos(op.argval)
            elif op.opname in ["LOAD_DEREF", "LOAD_CLOSURE"]:
                set_tos(context.cells[op.argval])
            elif op.opname == "IMPORT_NAME":
                try:
                    set_tos(importlib.import_module(op.argval))
                except ImportError:
                    set_tos(op.argval)
            elif op.opname in ["LOAD_METHOD", "LOAD_ATTR", "IMPORT_FROM"]:
                if tos is None:
                    refs.append(op.argval)
                elif isinstance(tos, str):
                    tos += "." + op.argval
                else:
                    tos = getattr(tos, op.argval)
            elif op.opname == "DELETE_FAST" and tos:
                del context.varnames[op.argval]
                tos = None
            elif op.opname == "STORE_FAST" and tos:
                context.varnames[op.argval] = tos
                tos = None
            elif op.opname == "LOAD_FAST" and op.argval in context.varnames:
                set_tos(context.varnames[op.argval])
            else:
                # For all other instructions, hash the current TOS.
                if tos is not None:
                    refs.append(tos)
                    tos = None
        except Exception as e:
            raise UserHashError(e, code, lineno=lineno)

    return refs