def _make_ll_dictnext(kind):
# make three versions of the following function: keys, values, items
@jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter)
and (iter.dict is None or
jit.isvirtual(iter.dict)))
@jit.oopspec("dictiter.next%s(iter)" % kind)
def ll_dictnext(RETURNTYPE, iter):
# note that RETURNTYPE is None for keys and values
dict = iter.dict
if dict:
entries = dict.entries
index = iter.index
assert index >= 0
entries_len = len(entries)
while index < entries_len:
entry = entries[index]
is_valid = entries.valid(index)
index = index + 1
if is_valid:
iter.index = index
if RETURNTYPE is lltype.Void:
return None
elif kind == 'items':
r = lltype.malloc(RETURNTYPE.TO)
r.item0 = recast(RETURNTYPE.TO.item0, entry.key)
r.item1 = recast(RETURNTYPE.TO.item1, entry.value)
return r
elif kind == 'keys':
return entry.key
elif kind == 'values':
return entry.value
# clear the reference to the dict and prevent restarts
iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO)
raise StopIteration
return ll_dictnext
def scan_continuation(curr, prompt_tag, look_for=None):
"""
Segment a continuation based on a given continuation-prompt-tag.
The head of the continuation, up to and including the desired continuation
prompt is reversed (in place), and the tail is returned un-altered.
The hint value |look_for| is used to determine when the continuation being
installed is a prefix of the extant continuation.
In this case, installing the continuation is much simpler, as the expensive
merge operation needed to find common substructure is the two continuation is
not needed.
"""
handlers = False
xs = []
while isinstance(curr, Cont):
if curr is look_for:
return None, handlers
handlers |= isinstance(curr, DynamicWindValueCont)
xs.append(curr)
if isinstance(curr, Prompt) and curr.tag is prompt_tag:
break
curr = curr.prev
if not jit.isvirtual(curr):
return _scan_continuation(curr, prompt_tag, look_for, xs, handlers)
return xs, handlers
def w_dict_unrolling_heuristic(w_dct):
"""In which cases iterating over dict items can be unrolled.
Note that w_dct is an instance of W_DictMultiObject, not necesarilly
an actual dict
"""
return jit.isvirtual(w_dct) or (jit.isconstant(w_dct) and
w_dct.length() <= UNROLL_CUTOFF)
def scan_continuation(curr, prompt_tag, look_for=None, escape=False):
"""
Segment a continuation based on a given continuation-prompt-tag.
The head of the continuation, up to and including the desired continuation
prompt is reversed (in place), and the tail is returned un-altered.
The hint value |look_for| is used to determine when the continuation being
installed is a prefix of the extant continuation.
In this case, installing the continuation is much simpler, as the expensive
merge operation needed to find common substructure is the two continuation is
not needed.
"""
handlers = False
xs = []
while isinstance(curr, Cont):
if curr is look_for:
return None, handlers
handlers |= isinstance(curr, DynamicWindValueCont)
xs.append(curr)
if isinstance(curr, Prompt) and curr.tag is prompt_tag:
break
curr = curr.prev
if not escape and not jit.isvirtual(curr):
return _scan_continuation(curr, prompt_tag, look_for, xs, handlers)
return xs, handlers
def listeq_unroll_case(l1, l2, eqfn):
if jit.isvirtual(l1) and l1.ll_length() < 10:
return True
if jit.isvirtual(l2) and l2.ll_length() < 10:
return True
return False
def ll_pop(func, l, index):
length = l.ll_length()
if index < 0:
index += length
if func is dum_checkidx:
if index < 0 or index >= length:
raise IndexError
else:
ll_assert(index >= 0, "negative list pop index out of bound")
ll_assert(index < length, "list pop index out of bound")
res = l.ll_getitem_fast(index)
ll_delitem_nonneg(dum_nocheck, l, index)
return res
@jit.look_inside_iff(lambda l: jit.isvirtual(l))
def ll_reverse(l):
length = l.ll_length()
i = 0
length_1_i = length - 1 - i
while i < length_1_i:
tmp = l.ll_getitem_fast(i)
l.ll_setitem_fast(i, l.ll_getitem_fast(length_1_i))
l.ll_setitem_fast(length_1_i, tmp)
i += 1
length_1_i -= 1
def ll_getitem_nonneg(func, basegetitem, l, index):
ll_assert(index >= 0, "unexpectedly negative list getitem index")
if func is dum_checkidx:
class PyFrame(W_Root):
"""Represents a frame for a regular Python function
that needs to be interpreted.
Public fields:
* 'space' is the object space this frame is running in
* 'code' is the PyCode object this frame runs
* 'w_locals' is the locals dictionary to use, if needed, stored on a
debug object
* 'w_globals' is the attached globals dictionary
* 'builtin' is the attached built-in module
* 'valuestack_w', 'blockstack', control the interpretation
Cell Vars:
my local variables that are exposed to my inner functions
Free Vars:
variables coming from a parent function in which i'm nested
'closure' is a list of Cell instances: the received free vars.
"""
__metaclass__ = extendabletype
frame_finished_execution = False
f_generator_wref = rweakref.dead_ref # for generators/coroutines
f_generator_nowref = None # (only one of the two attrs)
w_yielding_from = None
last_instr = -1
f_backref = jit.vref_None
escaped = False # see mark_as_escaped()
debugdata = None
pycode = None # code object executed by that frame
locals_cells_stack_w = None # the list of all locals, cells and the valuestack
valuestackdepth = 0 # number of items on valuestack
lastblock = None
# other fields:
# builtin - builtin cache, only if honor__builtins__ is True
# defaults to False
# there is also self.space which is removed by the annotator
# additionally JIT uses vable_token field that is representing
# frame current virtualizable state as seen by the JIT
def __init__(self, space, code, w_globals, outer_func):
if not we_are_translated():
assert type(self) == space.FrameClass, (
"use space.FrameClass(), not directly PyFrame()")
self = hint(self, access_directly=True, fresh_virtualizable=True)
assert isinstance(code, pycode.PyCode)
self.space = space
self.pycode = code
if code.frame_stores_global(w_globals):
self.getorcreatedebug().w_globals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
size = code.co_nlocals + ncellvars + nfreevars + code.co_stacksize
# the layout of this list is as follows:
# | local vars | cells | stack |
self.locals_cells_stack_w = [None] * size
self.valuestackdepth = code.co_nlocals + ncellvars + nfreevars
make_sure_not_resized(self.locals_cells_stack_w)
check_nonneg(self.valuestackdepth)
#
if space.config.objspace.honor__builtins__:
self.builtin = space.builtin.pick_builtin(w_globals)
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
self.initialize_frame_scopes(outer_func, code)
def getdebug(self):
return self.debugdata
def getorcreatedebug(self):
if self.debugdata is None:
self.debugdata = FrameDebugData(self.pycode)
return self.debugdata
def get_w_globals(self):
debugdata = self.getdebug()
if debugdata is not None:
return debugdata.w_globals
return jit.promote(self.pycode).w_globals
def get_w_f_trace(self):
d = self.getdebug()
if d is None:
return None
return d.w_f_trace
def get_is_being_profiled(self):
d = self.getdebug()
if d is None:
return False
return d.is_being_profiled
def get_w_locals(self):
d = self.getdebug()
if d is None:
return None
return d.w_locals
def get_f_trace_lines(self):
d = self.getdebug()
if d is None:
return True
return d.f_trace_lines
def get_f_trace_opcodes(self):
d = self.getdebug()
if d is None:
return False
return d.f_trace_opcodes
@not_rpython
def __repr__(self):
# useful in tracebacks
return "<%s.%s executing %s at line %s" % (
self.__class__.__module__, self.__class__.__name__, self.pycode,
self.get_last_lineno())
def _getcell(self, varindex):
cell = self.locals_cells_stack_w[varindex + self.pycode.co_nlocals]
assert isinstance(cell, Cell)
return cell
def mark_as_escaped(self):
"""
Must be called on frames that are exposed to applevel, e.g. by
sys._getframe(). This ensures that the virtualref holding the frame
is properly forced by ec.leave(), and thus the frame will be still
accessible even after the corresponding C stack died.
"""
self.escaped = True
def append_block(self, block):
assert block.previous is self.lastblock
self.lastblock = block
def pop_block(self):
block = self.lastblock
self.lastblock = block.previous
return block
def blockstack_non_empty(self):
return self.lastblock is not None
def get_blocklist(self):
"""Returns a list containing all the blocks in the frame"""
lst = []
block = self.lastblock
while block is not None:
lst.append(block)
block = block.previous
return lst
def set_blocklist(self, lst):
self.lastblock = None
i = len(lst) - 1
while i >= 0:
block = lst[i]
i -= 1
block.previous = self.lastblock
self.lastblock = block
def get_builtin(self):
if self.space.config.objspace.honor__builtins__:
return self.builtin
else:
return self.space.builtin
@jit.unroll_safe
def initialize_frame_scopes(self, outer_func, code):
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
# CO_NEWLOCALS: make a locals dict unless optimized is also set
# CO_OPTIMIZED: no locals dict needed at all
flags = code.co_flags
if not (flags & pycode.CO_OPTIMIZED):
if flags & pycode.CO_NEWLOCALS:
self.getorcreatedebug().w_locals = self.space.newdict(
module=True)
else:
w_globals = self.get_w_globals()
assert w_globals is not None
self.getorcreatedebug().w_locals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
if not nfreevars:
if not ncellvars:
return # no cells needed - fast path
elif outer_func is None:
space = self.space
raise oefmt(
space.w_TypeError,
"directly executed code object may not contain free "
"variables")
if outer_func and outer_func.closure:
closure_size = len(outer_func.closure)
else:
closure_size = 0
if closure_size != nfreevars:
raise ValueError("code object received a closure with "
"an unexpected number of free variables")
index = code.co_nlocals
for i in range(ncellvars):
self.locals_cells_stack_w[index] = Cell(
None, self.pycode.cell_families[i])
index += 1
for i in range(nfreevars):
self.locals_cells_stack_w[index] = outer_func.closure[i]
index += 1
def _is_generator_or_coroutine(self):
return (self.getcode().co_flags &
(pycode.CO_COROUTINE | pycode.CO_GENERATOR
| pycode.CO_ASYNC_GENERATOR)) != 0
def run(self, name=None, qualname=None):
"""Start this frame's execution."""
if self._is_generator_or_coroutine():
return self.initialize_as_generator(name, qualname)
else:
return self.execute_frame()
run._always_inline_ = True
def initialize_as_generator(self, name, qualname):
space = self.space
flags = self.getcode().co_flags
if flags & pycode.CO_COROUTINE:
from pypy.interpreter.generator import Coroutine
gen = Coroutine(self, name, qualname)
ec = space.getexecutioncontext()
w_wrapper = ec.w_coroutine_wrapper_fn
gen.capture_origin(ec)
elif flags & pycode.CO_ASYNC_GENERATOR:
from pypy.interpreter.generator import AsyncGenerator
gen = AsyncGenerator(self, name, qualname)
ec = None
w_wrapper = None
elif flags & pycode.CO_GENERATOR:
from pypy.interpreter.generator import GeneratorIterator
gen = GeneratorIterator(self, name, qualname)
ec = None
w_wrapper = None
else:
raise AssertionError("bad co_flags")
if space.config.translation.rweakref:
self.f_generator_wref = rweakref.ref(gen)
else:
self.f_generator_nowref = gen
w_gen = gen
if w_wrapper is not None:
if ec.in_coroutine_wrapper:
raise oefmt(
space.w_RuntimeError, "coroutine wrapper %R attempted "
"to recursively wrap %R", w_wrapper, self.getcode())
ec.in_coroutine_wrapper = True
try:
w_gen = space.call_function(w_wrapper, w_gen)
finally:
ec.in_coroutine_wrapper = False
return w_gen
def resume_execute_frame(self, w_arg_or_err):
# Called from execute_frame() just before resuming the bytecode
# interpretation.
from pypy.interpreter.pyopcode import SApplicationException
space = self.space
w_yf = self.w_yielding_from
if w_yf is not None:
self.w_yielding_from = None
try:
self.next_yield_from(w_yf, w_arg_or_err)
except OperationError as operr:
operr.record_context(space, space.getexecutioncontext())
return self.handle_generator_error(operr)
# Normal case: the call above raises Yield.
# We reach this point if the iterable is exhausted.
last_instr = jit.promote(self.last_instr)
assert last_instr & 1 == 0
assert last_instr >= 0
return r_uint(last_instr + 2)
if isinstance(w_arg_or_err, SApplicationException):
return self.handle_generator_error(w_arg_or_err.operr)
last_instr = jit.promote(self.last_instr)
if last_instr != -1:
assert last_instr & 1 == 0
self.pushvalue(w_arg_or_err)
return r_uint(last_instr + 2)
else:
return r_uint(0)
def execute_frame(self, w_arg_or_err=None):
"""Execute this frame. Main entry point to the interpreter.
'w_arg_or_err' is non-None iff we are starting or resuming
a generator or coroutine frame; in that case, w_arg_or_err
is the input argument -or- an SApplicationException instance.
"""
from pypy.interpreter import pyopcode
# the following 'assert' is an annotation hint: it hides from
# the annotator all methods that are defined in PyFrame but
# overridden in the {,Host}FrameClass subclasses of PyFrame.
assert (isinstance(self, self.space.FrameClass)
or not self.space.config.translating)
executioncontext = self.space.getexecutioncontext()
executioncontext.enter(self)
got_exception = True
w_exitvalue = self.space.w_None
try:
executioncontext.call_trace(self)
#
# Execution starts just after the last_instr. Initially,
# last_instr is -1. After a generator suspends it points to
# the YIELD_VALUE/YIELD_FROM instruction.
try:
try:
if w_arg_or_err is None:
assert self.last_instr == -1
next_instr = r_uint(0)
else:
next_instr = self.resume_execute_frame(w_arg_or_err)
except pyopcode.Yield:
w_exitvalue = self.popvalue()
else:
w_exitvalue = self.dispatch(self.pycode, next_instr,
executioncontext)
except OperationError:
raise
except Exception as e: # general fall-back
raise self._convert_unexpected_exception(e)
finally:
executioncontext.return_trace(self, w_exitvalue)
got_exception = False
finally:
executioncontext.leave(self, w_exitvalue, got_exception)
return w_exitvalue
execute_frame.insert_stack_check_here = True
# stack manipulation helpers
def pushvalue(self, w_object):
depth = self.valuestackdepth
self.locals_cells_stack_w[depth] = ll_assert_not_none(w_object)
self.valuestackdepth = depth + 1
def pushvalue_none(self):
depth = self.valuestackdepth
# the entry is already None, and remains None
assert self.locals_cells_stack_w[depth] is None
self.valuestackdepth = depth + 1
def assert_stack_index(self, index):
if we_are_translated():
return
assert self._check_stack_index(index)
def _check_stack_index(self, index):
code = self.pycode
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
stackstart = code.co_nlocals + ncellvars + nfreevars
return index >= stackstart
def popvalue(self):
return ll_assert_not_none(self.popvalue_maybe_none())
def popvalue_maybe_none(self):
depth = self.valuestackdepth - 1
self.assert_stack_index(depth)
assert depth >= 0
w_object = self.locals_cells_stack_w[depth]
self.locals_cells_stack_w[depth] = None
self.valuestackdepth = depth
return w_object
# we need two popvalues that return different data types:
# one in case we want list another in case of tuple
def _new_popvalues():
@jit.unroll_safe
def popvalues(self, n):
values_w = [None] * n
while True:
n -= 1
if n < 0:
break
values_w[n] = self.popvalue()
return values_w
return popvalues
popvalues = _new_popvalues()
popvalues_mutable = _new_popvalues()
del _new_popvalues
@jit.unroll_safe
def peekvalues(self, n):
values_w = [None] * n
base = self.valuestackdepth - n
self.assert_stack_index(base)
assert base >= 0
while True:
n -= 1
if n < 0:
break
values_w[n] = self.locals_cells_stack_w[base + n]
return values_w
@jit.unroll_safe
def dropvalues(self, n):
n = hint(n, promote=True)
finaldepth = self.valuestackdepth - n
self.assert_stack_index(finaldepth)
assert finaldepth >= 0
while True:
n -= 1
if n < 0:
break
self.locals_cells_stack_w[finaldepth + n] = None
self.valuestackdepth = finaldepth
@jit.unroll_safe
def pushrevvalues(self, n, values_w): # n should be len(values_w)
make_sure_not_resized(values_w)
while True:
n -= 1
if n < 0:
break
self.pushvalue(values_w[n])
@jit.unroll_safe
def dupvalues(self, n):
delta = n - 1
while True:
n -= 1
if n < 0:
break
w_value = self.peekvalue(delta)
self.pushvalue(w_value)
def peekvalue(self, index_from_top=0):
# NOTE: top of the stack is peekvalue(0).
# Contrast this with CPython where it's PEEK(-1).
return ll_assert_not_none(self.peekvalue_maybe_none(index_from_top))
def peekvalue_maybe_none(self, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
self.assert_stack_index(index)
assert index >= 0
return self.locals_cells_stack_w[index]
def settopvalue(self, w_object, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
self.assert_stack_index(index)
assert index >= 0
self.locals_cells_stack_w[index] = ll_assert_not_none(w_object)
@jit.unroll_safe
def dropvaluesuntil(self, finaldepth):
depth = self.valuestackdepth - 1
finaldepth = hint(finaldepth, promote=True)
assert finaldepth >= 0
while depth >= finaldepth:
self.locals_cells_stack_w[depth] = None
depth -= 1
self.valuestackdepth = finaldepth
def _guess_function_name_parens(self, fnname=None, w_function=None):
""" Returns 'funcname()' from either a function name fnname or a
wrapped callable w_function. If it's not a function or a method, returns
'Classname object'"""
# XXX this is super annoying to compute every time we do a function call!
# CPython has a similar function, PyEval_GetFuncName
from pypy.interpreter.function import Function, Method
if fnname is not None:
return fnname + '()'
if w_function is None:
return None
if isinstance(w_function, Function):
return w_function.name + '()'
if isinstance(w_function, Method):
return self._guess_function_name_parens(None,
w_function.w_function)
return self.space.type(w_function).getname(self.space) + ' object'
def make_arguments(self,
nargs,
methodcall=False,
w_function=None,
fnname=None):
fnname_parens = self._guess_function_name_parens(fnname, w_function)
return Arguments(self.space,
self.peekvalues(nargs),
methodcall=methodcall,
fnname_parens=fnname_parens)
def argument_factory(self,
arguments,
keywords,
keywords_w,
w_star,
w_starstar,
methodcall=False,
w_function=None,
fnname=None):
fnname_parens = self._guess_function_name_parens(fnname, w_function)
return Arguments(self.space,
arguments,
keywords,
keywords_w,
w_star,
w_starstar,
methodcall=methodcall,
fnname_parens=fnname_parens)
def hide(self):
return self.pycode.hidden_applevel
def getcode(self):
return hint(self.pycode, promote=True)
@jit.look_inside_iff(lambda self, scope_w: jit.isvirtual(scope_w))
def setfastscope(self, scope_w):
"""Initialize the fast locals from a list of values,
where the order is according to self.pycode.signature()."""
scope_len = len(scope_w)
if scope_len > self.pycode.co_nlocals:
raise ValueError("new fastscope is longer than the allocated area")
# don't assign directly to 'locals_cells_stack_w[:scope_len]' to be
# virtualizable-friendly
for i in range(scope_len):
self.locals_cells_stack_w[i] = scope_w[i]
self.init_cells()
def getdictscope(self):
"""
Get the locals as a dictionary
"""
self.fast2locals()
return self.debugdata.w_locals
def setdictscope(self, w_locals):
"""
Initialize the locals from a dictionary.
"""
self.getorcreatedebug().w_locals = w_locals
self.locals2fast()
@jit.unroll_safe
def fast2locals(self):
# Copy values from the fastlocals to self.w_locals
d = self.getorcreatedebug()
if d.w_locals is None:
d.w_locals = self.space.newdict(module=True)
varnames = self.getcode().getvarnames()
for i in range(min(len(varnames), self.getcode().co_nlocals)):
name = varnames[i]
w_value = self.locals_cells_stack_w[i]
if w_value is not None:
self.space.setitem_str(d.w_locals, name, w_value)
else:
w_name = self.space.newtext(name)
try:
self.space.delitem(d.w_locals, w_name)
except OperationError as e:
if not e.match(self.space, self.space.w_KeyError):
raise
# cellvars are values exported to inner scopes
# freevars are values coming from outer scopes
# (see locals2fast for why CO_OPTIMIZED)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
try:
w_value = cell.get()
except ValueError:
pass
else:
self.space.setitem_str(d.w_locals, name, w_value)
@jit.unroll_safe
def locals2fast(self):
# Copy values from self.w_locals to the fastlocals
w_locals = self.getorcreatedebug().w_locals
assert w_locals is not None
varnames = self.getcode().getvarnames()
numlocals = self.getcode().co_nlocals
new_fastlocals_w = [None] * numlocals
for i in range(min(len(varnames), numlocals)):
name = varnames[i]
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
new_fastlocals_w[i] = w_value
self.setfastscope(new_fastlocals_w)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
# If the namespace is unoptimized, then one of the
# following cases applies:
# 1. It does not contain free variables, because it
# uses import * or is a top-level namespace.
# 2. It is a class namespace.
# We don't want to accidentally copy free variables
# into the locals dict used by the class.
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
cell.set(w_value)
@jit.unroll_safe
def init_cells(self):
"""
Initialize cellvars from self.locals_cells_stack_w.
"""
args_to_copy = self.pycode._args_as_cellvars
index = self.pycode.co_nlocals
for i in range(len(args_to_copy)):
argnum = args_to_copy[i]
if argnum >= 0:
cell = self.locals_cells_stack_w[index]
assert isinstance(cell, Cell)
cell.set(self.locals_cells_stack_w[argnum])
index += 1
def getclosure(self):
return None
def fget_code(self, space):
return self.getcode()
def fget_getdictscope(self, space):
return self.getdictscope()
def fget_w_globals(self, space):
# bit silly, but GetSetProperty passes a space
return self.get_w_globals()
### line numbers ###
def fget_f_lineno(self, space):
"Returns the line number of the instruction currently being executed."
if self.get_w_f_trace() is None:
return space.newint(self.get_last_lineno())
else:
return space.newint(self.getorcreatedebug().f_lineno)
def fset_f_lineno(self, space, w_new_lineno):
"Change the line number of the instruction currently being executed."
try:
new_lineno = space.int_w(w_new_lineno)
except OperationError:
raise oefmt(space.w_ValueError, "lineno must be an integer")
# You can only do this from within a trace function, not via
# _getframe or similar hackery.
if space.int_w(self.fget_f_lasti(space)) == -1:
raise oefmt(
space.w_ValueError,
"can't jump from the 'call' trace event of a new frame")
if self.get_w_f_trace() is None:
raise oefmt(space.w_ValueError,
"f_lineno can only be set by a trace function")
code = self.pycode.co_code
if ord(code[self.last_instr]) == YIELD_VALUE:
raise oefmt(space.w_ValueError,
"can't jump from a yield statement")
# Only allow jumps when we're tracing a line event.
d = self.getorcreatedebug()
if not d.is_in_line_tracing:
raise oefmt(space.w_ValueError,
"can only jump from a 'line' trace event")
line = self.pycode.co_firstlineno
if new_lineno < line:
raise oefmt(space.w_ValueError,
"line %d comes before the current code block",
new_lineno)
elif new_lineno == line:
new_lasti = 0
else:
# Find the bytecode offset for the start of the given
# line, or the first code-owning line after it.
lnotab = self.pycode.co_lnotab
addr = 0
new_lasti = -1
for offset in xrange(0, len(lnotab), 2):
addr += ord(lnotab[offset])
line_offset = ord(lnotab[offset + 1])
if line_offset >= 0x80:
line_offset -= 0x100
line += line_offset
if line >= new_lineno:
new_lasti = addr
new_lineno = line
break
# If we didn't reach the requested line, return an error.
if new_lasti == -1:
raise oefmt(space.w_ValueError,
"line %d comes after the current code block",
new_lineno)
min_addr = min(new_lasti, self.last_instr)
max_addr = max(new_lasti, self.last_instr)
# You can't jump onto a line with an 'except' statement on it -
# they expect to have an exception on the top of the stack, which
# won't be true if you jump to them. They always start with code
# that either pops the exception using POP_TOP (plain 'except:'
# lines do this) or duplicates the exception on the stack using
# DUP_TOP (if there's an exception type specified). See compile.c,
# 'com_try_except' for the full details. There aren't any other
# cases (AFAIK) where a line's code can start with DUP_TOP or
# POP_TOP, but if any ever appear, they'll be subject to the same
# restriction (but with a different error message).
if ord(code[new_lasti]) in (DUP_TOP, POP_TOP):
raise oefmt(space.w_ValueError,
"can't jump to 'except' line as there's no exception")
# You can't jump into or out of a 'finally' block because the 'try'
# block leaves something on the stack for the END_FINALLY to clean
# up. So we walk the bytecode, maintaining a simulated blockstack.
# When we reach the old or new address and it's in a 'finally' block
# we note the address of the corresponding SETUP_FINALLY. The jump
# is only legal if neither address is in a 'finally' block or
# they're both in the same one. 'blockstack' is a stack of the
# bytecode addresses of the SETUP_X opcodes, and 'in_finally' tracks
# whether we're in a 'finally' block at each blockstack level.
# PYPY NOTE: CPython (at least 3.5.2+) doesn't check except blocks,
# but that results in crashes. But for now I'm going to follow the
# logic of CPython 3.6.9 exactly anyway, which is quite messy already.
f_lasti_setup_addr = -1
new_lasti_setup_addr = -1
blockstack = [] # current blockstack (addresses of SETUP_*)
in_finally = [] # list of flags "is this a finally block?"
addr = 0
while addr < len(code):
assert addr & 1 == 0
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
SETUP_ASYNC_WITH):
blockstack.append(addr)
in_finally.append(False)
elif op == POP_BLOCK:
if len(blockstack) == 0:
raise oefmt(
space.w_SystemError,
"POP_BLOCK not properly nested in this bytecode")
setup_op = ord(code[blockstack[-1]])
if setup_op in (SETUP_FINALLY, SETUP_WITH, SETUP_ASYNC_WITH):
in_finally[-1] = True
else:
del blockstack[-1]
elif op == END_FINALLY:
# Ignore END_FINALLYs for SETUP_EXCEPTs - they exist
# in the bytecode but don't correspond to an actual
# 'finally' block. (If len(blockstack) is 0, we must
# be seeing such an END_FINALLY.)
if len(blockstack) > 0:
setup_op = ord(code[blockstack[-1]])
if setup_op in (SETUP_FINALLY, SETUP_WITH,
SETUP_ASYNC_WITH):
del blockstack[-1]
# For the addresses we're interested in, see whether they're
# within a 'finally' block and if so, remember the address
# of the SETUP_FINALLY.
if addr == new_lasti or addr == self.last_instr:
setup_addr = -1
for i in xrange(len(blockstack) - 1, -1, -1):
if in_finally[i]:
setup_addr = blockstack[i]
break
if setup_addr != -1:
if addr == new_lasti:
new_lasti_setup_addr = setup_addr
if addr == self.last_instr:
f_lasti_setup_addr = setup_addr
addr += 2
# Verify that the blockstack tracking code didn't get lost.
if len(blockstack) != 0:
raise oefmt(space.w_SystemError,
"blocks not properly nested in this bytecode")
# After all that, are we jumping into / out of a 'finally' block?
if new_lasti_setup_addr != f_lasti_setup_addr:
raise oefmt(
space.w_ValueError,
"can't jump into or out of a 'finally' block "
"(%d -> %d)", f_lasti_setup_addr, new_lasti_setup_addr)
# now we know we're not jumping into or out of a place which
# needs a SysExcInfoRestorer. Check that we're not jumping
# *into* a block, but only (potentially) out of some blocks.
# Police block-jumping (you can't jump into the middle of a block)
# and ensure that the blockstack finishes up in a sensible state (by
# popping any blocks we're jumping out of). We look at all the
# blockstack operations between the current position and the new
# one, and keep track of how many blocks we drop out of on the way.
# By also keeping track of the lowest blockstack position we see, we
# can tell whether the jump goes into any blocks without coming out
# again - in that case we raise an exception below.
delta_iblock = min_delta_iblock = 0
addr = min_addr
while addr < max_addr:
assert addr & 1 == 0
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
SETUP_ASYNC_WITH):
delta_iblock += 1
elif op == POP_BLOCK:
delta_iblock -= 1
min_delta_iblock = min(min_delta_iblock, delta_iblock)
addr += 2
# 'min_delta_iblock' is <= 0; its absolute value is the number of
# blocks we exit. 'go_iblock' is the delta number of blocks
# between the last_instr and the new_lasti, in this order.
if new_lasti > self.last_instr:
go_iblock = delta_iblock # Forwards jump.
else:
go_iblock = -delta_iblock # Backwards jump.
if go_iblock > min_delta_iblock:
raise oefmt(space.w_ValueError,
"can't jump into the middle of a block")
assert go_iblock <= 0
# Pop any blocks that we're jumping out of.
from pypy.interpreter.pyopcode import FinallyBlock
for ii in range(-go_iblock):
block = self.pop_block()
block.cleanupstack(self)
if (isinstance(block, FinallyBlock) and ord(
code[block.handlerposition]) == WITH_CLEANUP_START):
self.popvalue() # Pop the exit function.
d.f_lineno = new_lineno
assert new_lasti & 1 == 0
self.last_instr = new_lasti
def get_last_lineno(self):
"Returns the line number of the instruction currently being executed."
return pytraceback.offset2lineno(self.pycode, self.last_instr)
def fget_f_builtins(self, space):
return self.get_builtin().getdict(space)
def get_f_back(self):
return ExecutionContext.getnextframe_nohidden(self)
def fget_f_back(self, space):
return self.get_f_back()
def fget_f_lasti(self, space):
return self.space.newint(self.last_instr)
def fget_f_trace(self, space):
return self.get_w_f_trace()
def fset_f_trace(self, space, w_trace):
if space.is_w(w_trace, space.w_None):
self.getorcreatedebug().w_f_trace = None
else:
d = self.getorcreatedebug()
d.w_f_trace = w_trace
d.f_lineno = self.get_last_lineno()
def fdel_f_trace(self, space):
self.getorcreatedebug().w_f_trace = None
def fget_f_trace_lines(self, space):
return space.newbool(self.get_f_trace_lines())
def fset_f_trace_lines(self, space, w_trace):
self.getorcreatedebug().f_trace_lines = space.is_true(w_trace)
def fget_f_trace_opcodes(self, space):
return space.newbool(self.get_f_trace_opcodes())
def fset_f_trace_opcodes(self, space, w_trace):
self.getorcreatedebug().f_trace_opcodes = space.is_true(w_trace)
def get_generator(self):
if self.space.config.translation.rweakref:
return self.f_generator_wref()
else:
return self.f_generator_nowref
def descr_clear(self, space):
"""F.clear(): clear most references held by the frame"""
# Clears a random subset of the attributes: the local variables
# and the w_locals. Note that CPython doesn't clear f_locals
# (which can create leaks) but it's hard to notice because
# the next Python-level read of 'frame.f_locals' will clear it.
if not self.frame_finished_execution:
if not self._is_generator_or_coroutine():
raise oefmt(space.w_RuntimeError,
"cannot clear an executing frame")
gen = self.get_generator()
if gen is not None:
if gen.running:
raise oefmt(space.w_RuntimeError,
"cannot clear an executing frame")
# xxx CPython raises the RuntimeWarning "coroutine was never
# awaited" in this case too. Does it make any sense?
gen.descr_close()
debug = self.getdebug()
if debug is not None:
debug.w_f_trace = None
if debug.w_locals is not None:
debug.w_locals = space.newdict()
# clear the locals, including the cell/free vars, and the stack
for i in range(len(self.locals_cells_stack_w)):
w_oldvalue = self.locals_cells_stack_w[i]
if isinstance(w_oldvalue, Cell):
w_newvalue = Cell(None, w_oldvalue.family)
else:
w_newvalue = None
self.locals_cells_stack_w[i] = w_newvalue
self.valuestackdepth = 0
self.lastblock = None # the FrameBlock chained list
def _convert_unexpected_exception(self, e):
from pypy.interpreter import error
operr = error.get_converted_unexpected_exception(self.space, e)
pytraceback.record_application_traceback(self.space, operr, self,
self.last_instr)
raise operr
def descr_repr(self, space):
code = self.pycode
moreinfo = ", file '%s', line %s, code %s" % (
code.co_filename, self.get_last_lineno(), code.co_name)
return self.getrepr(space, "frame", moreinfo)
# ------------------------------------------------------------
# builder.getlength()
@always_inline
def ll_getlength(ll_builder):
num_chars_missing_from_last_piece = (ll_builder.current_end -
ll_builder.current_pos)
return ll_builder.total_size - num_chars_missing_from_last_piece
# ------------------------------------------------------------
# builder.build()
@jit.look_inside_iff(lambda ll_builder: jit.isvirtual(ll_builder))
def ll_build(ll_builder):
# NB. usually the JIT doesn't look inside this function; it does
# so only in the simplest example where it could virtualize everything
if ll_builder.extra_pieces:
ll_fold_pieces(ll_builder)
elif ll_builder.current_pos != ll_builder.total_size:
ll_shrink_final(ll_builder)
return ll_builder.current_buf
def ll_shrink_final(ll_builder):
final_size = ll_builder.current_pos
ll_assert(final_size <= ll_builder.total_size,
"final_size > ll_builder.total_size?")
buf = rgc.ll_shrink_array(ll_builder.current_buf, final_size)
class abstract_callable(Callable):
_immutable_fields_ = ["val_%d" % x for x in arg_iter]
def __init__(self, term_name, args, signature):
raise NotImplementedError
def _init_values(self, args):
if args is None:
return
for x in arg_iter:
setattr(self, 'val_%d' % x, args[x])
def _make_new(self, name, signature):
raise NotImplementedError("abstract base class")
def arguments(self):
result = [None] * n_args
for x in arg_iter:
result[x] = getattr(self, 'val_%d' % x)
return result
def argument_at(self, i):
for x in arg_iter:
if x == i:
return getattr(self, 'val_%d' % x)
raise IndexError
def argument_count(self):
return n_args
def quick_unify_check(self, other, similarity=None):
other = other.dereference(None)
if isinstance(other, Var):
return True
if not isinstance(other, Callable):
return False
if similarity and similarity.get_score_signatures(self.signature(), other.signature()) < similarity.threshold:
return False
if not similarity and not self.signature().eq(other.signature()):
return False
if not isinstance(other, abstract_callable):
return Callable.quick_unify_check(self, other)
for x in arg_iter:
a = getattr(self, 'val_%d' % x)
b = getattr(other, 'val_%d' % x)
if not a.quick_unify_check(b):
return False
return True
def nonvar_unify_and_standardize_apart(self, other, heap, env, similarity=None):
if not isinstance(other, abstract_callable):
raise UnificationFailed
if self.signature().eq(other.signature()):
for x in arg_iter:
a = getattr(self, 'val_%d' % x)
b = getattr(other, 'val_%d' % x)
a.unify_and_standardize_apart(b, heap, env)
else:
raise UnificationFailed
def copy_standardize_apart(self, heap, env):
result = self._make_new(self.name(), self.signature())
reuse = True
i = 0
for i in arg_iter:
arg = getattr(self, 'val_%d' % i)
cloned, arg_is_reused = arg.copy_standardize_apart(heap, env)
reuse = reuse & arg_is_reused
setattr(result, 'val_%d' % i, cloned)
i += 1
if reuse:
return self, True
else:
# XXX what about the variable shunting in Callable.build
return result, False
@specialize.arg(3)
def nonvar_unify(self, other, heap, occurs_check, similarity=None):
if not isinstance(other, abstract_callable):
raise UnificationFailed
if self.signature().eq(other.signature()):
return self._nonvar_unify_jit(other, heap, occurs_check)
else:
raise UnificationFailed
@specialize.arg(3)
@jit.look_inside_iff(lambda self, other, heap, occurs_check:
jit.isvirtual(self) or jit.isvirtual(other) or
jit.isconstant(self) or jit.isconstant(other))
def _nonvar_unify_jit(self, other, heap, occurs_check):
for x in arg_iter:
a = getattr(self, 'val_%d' % x)
b = getattr(other, 'val_%d' % x)
a.unify(b, heap, occurs_check)
@specialize.arg(1)
def _copy_term(self, copy_individual, heap, *extraargs):
result = self._make_new(self.name(), self.signature())
newinstance = False
i = 0
for i in arg_iter:
arg = getattr(self, 'val_%d' % i)
cloned = copy_individual(arg, i, heap, *extraargs)
newinstance = newinstance | (cloned is not arg)
setattr(result, 'val_%d' % i, cloned)
i += 1
if newinstance:
# XXX what about the variable shunting in Callable.build
return result
else:
return self
# struct dicttable {
# int num_live_items;
# int num_ever_used_items;
# int resize_counter;
# {byte, short, int, long} *indexes;
# dictentry *entries;
# lookup_function_no; # one of the four possible functions for different
# # size dicts; the rest of the word is a counter for how
# # many 'entries' at the start are known to be deleted
# (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
# (Function DICTKEY -> int) *fnkeyhash;
# }
#
#
@jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
@jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
def ll_call_lookup_function(d, key, hash, flag):
fun = d.lookup_function_no & FUNC_MASK
# This likely() here forces gcc to compile the check for fun == FUNC_BYTE
# first. Otherwise, this is a regular switch and gcc (at least 4.7)
# compiles this as a series of checks, with the FUNC_BYTE case last.
# It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
if likely(fun == FUNC_BYTE):
return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
elif fun == FUNC_SHORT:
return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
elif IS_64BIT and fun == FUNC_INT:
return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
elif fun == FUNC_LONG:
return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
class LLOrderedDict(object):
INIT_SIZE = 8
HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
MASK = intmask(HIGHEST_BIT - 1)
PERTURB_SHIFT = 5
@staticmethod
def ll_valid_from_flag(entries, i):
return entries[i].valid
@staticmethod
def ll_everused_from_flag(entries, i):
return entries[i].everused
@staticmethod
def ll_mark_deleted_in_flag(entries, i):
entries[i].valid = False
@staticmethod
def ll_hashkey_custom(d, key):
DICT = lltype.typeOf(d).TO
return hlinvoke(DICT.r_hashkey, d.hashkey_func, key)
@staticmethod
def ll_keyeq_custom(d, key1, key2):
DICT = lltype.typeOf(d).TO
return hlinvoke(DICT.r_keyeq, d.keyeq_func, key1, key2)
@staticmethod
def ll_hash_recompute(entries, i):
ENTRIES = lltype.typeOf(entries).TO
return ENTRIES.fast_hash_func(entries[i].key)
@staticmethod
def ll_hash_from_cache(entries, i):
return entries[i].hash
@staticmethod
def recast(P, v):
if isinstance(P, lltype.Ptr):
return lltype.cast_pointer(P, v)
else:
return v
@staticmethod
def ll_newdict(DICT):
d = lltype.malloc(DICT)
d.entries = lltype.malloc(DICT.entries.TO, LLOrderedDict.INIT_SIZE, zero=True)
d.num_items = 0
d.first_entry = -1
d.last_entry = -1
d.resize_counter = LLOrderedDict.INIT_SIZE * 2
return d
@staticmethod
def ll_len(d):
return d.num_items
@staticmethod
@jit.look_inside_iff(lambda d, key, hash:
jit.isvirtual(d) and jit.isconstant(key))
def ll_lookup(d, key, hash):
entries = d.entries
mask = len(entries) - 1
i = hash & mask
if entries.valid(i):
checkingkey = entries[i].key
if checkingkey == key:
return i
if d.keyeq is not None and entries.hash(i) == hash:
found = d.keyeq(checkingkey, key)
if d.paranoia:
if (entries != d.entries or
not entries.valid(i) or entries[i].key != checkingkey):
return LLOrderedDict.ll_lookup(d, key, hash)
if found:
return i
freeslot = -1
elif entries.everused(i):
freeslot = i
else:
return i | LLOrderedDict.HIGHEST_BIT
perturb = r_uint(hash)
while True:
i = r_uint(i)
i = (i << 2) + i + perturb + 1
i = intmask(i) & mask
if not entries.everused(i):
if freeslot == -1:
freeslot = i
return freeslot | LLOrderedDict.HIGHEST_BIT
elif entries.valid(i):
checkingkey = entries[i].key
if checkingkey == key:
return i
if d.keyeq is not None and entries.hash(i) == hash:
found = d.keyeq(checkingkey, key)
if d.paranoia:
if (entries != d.entries or
not entries.valid(i) or entries[i].key != checkingkey):
return LLOrderedDict.ll_lookup(d, key, hash)
if found:
return i
elif freeslot == -1:
freeslot = i
perturb >>= LLOrderedDict.PERTURB_SHIFT
@staticmethod
def ll_setitem(d, key, value):
hash = d.hashkey(key)
i = LLOrderedDict.ll_lookup(d, key, hash)
LLOrderedDict.ll_setitem_lookup_done(d, key, value, hash, i)
@staticmethod
def ll_setitem_lookup_done(d, key, value, hash, i):
valid = (i & LLOrderedDict.HIGHEST_BIT) == 0
i &= LLOrderedDict.MASK
everused = d.entries.everused(i)
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
entry.value = value
if valid:
return
entry.key = key
if hasattr(ENTRY, "hash"):
entry.hash = hash
if hasattr(ENTRY, "valid"):
entry.valid = True
d.num_items += 1
if d.first_entry == -1:
d.first_entry = i
else:
d.entries[d.last_entry].next = i
entry.prev = d.last_entry
d.last_entry = i
entry.next = -1
if not everused:
if hasattr(ENTRY, "everused"):
entry.everused = True
d.resize_counter -= 3
if d.resize_counter <= 0:
LLOrderedDict.ll_resize(d)
@staticmethod
def ll_getitem(d, key):
i = LLOrderedDict.ll_lookup(d, key, d.hashkey(key))
if not i & LLOrderedDict.HIGHEST_BIT:
return d.entries[i].value
else:
raise KeyError
@staticmethod
def ll_delitem(d, key):
i = LLOrderedDict.ll_lookup(d, key, d.hashkey(key))
if i & LLOrderedDict.HIGHEST_BIT:
raise KeyError
LLOrderedDict._ll_del(d, i)
@staticmethod
def _ll_del(d, i):
d.entries.mark_deleted(i)
d.num_items -= 1
entry = d.entries[i]
if entry.prev == -1:
d.first_entry = entry.next
else:
d.entries[entry.prev].next = entry.next
if entry.next == -1:
d.last_entry = entry.prev
else:
d.entries[entry.next].prev = entry.prev
ENTRIES = lltype.typeOf(d.entries).TO
ENTRY = ENTRIES.OF
if ENTRIES.must_clear_key:
entry.key = lltype.nullptr(ENTRY.key.TO)
if ENTRIES.must_clear_value:
entry.value = lltype.nullptr(ENTRY.value.TO)
@staticmethod
def ll_contains(d, key):
i = LLOrderedDict.ll_lookup(d, key, d.hashkey(key))
return not bool(i & LLOrderedDict.HIGHEST_BIT)
@staticmethod
def ll_resize(d):
old_entries = d.entries
if d.num_items > 50000:
new_estimate = d.num_items * 2
else:
new_estimate = d.num_items * 4
new_size = LLOrderedDict.INIT_SIZE
while new_size <= new_estimate:
new_size *= 2
d.entries = lltype.malloc(lltype.typeOf(old_entries).TO, new_size, zero=True)
d.num_items = 0
d.resize_counter = new_size * 2
i = d.first_entry
d.first_entry = -1
d.last_entry = -1
while i != -1:
hash = old_entries.hash(i)
entry = old_entries[i]
LLOrderedDict.ll_insert_clean(d, entry.key, entry.value, hash)
i = entry.next
@staticmethod
def ll_insert_clean(d, key, value, hash):
i = LLOrderedDict.ll_lookup_clean(d, hash)
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
entry.value = value
entry.key = key
if hasattr(ENTRY, "hash"):
entry.hash = hash
if hasattr(ENTRY, "valid"):
entry.valid = True
if hasattr(ENTRY, "everused"):
entry.everused = True
d.num_items += 1
if d.first_entry == -1:
d.first_entry = i
else:
d.entries[d.last_entry].next = i
entry.prev = d.last_entry
d.last_entry = i
entry.next = -1
d.resize_counter -= 3
@staticmethod
def ll_lookup_clean(d, hash):
entries = d.entries
mask = len(entries) - 1
i = hash & mask
perturb = r_uint(hash)
while entries.everused(i):
i = r_uint(i)
i = (i << 2) + i + perturb + 1
i = intmask(i) & mask
perturb >>= LLOrderedDict.PERTURB_SHIFT
return i
@staticmethod
def ll_keys(LIST, d):
res = LIST.ll_newlist(d.num_items)
ELEM = lltype.typeOf(res.ll_items()).TO.OF
i = 0
idx = d.first_entry
while idx != -1:
res.ll_items()[i] = LLOrderedDict.recast(ELEM, d.entries[idx].key)
idx = d.entries[idx].next
i += 1
return res
@staticmethod
def ll_values(LIST, d):
res = LIST.ll_newlist(d.num_items)
ELEM = lltype.typeOf(res.ll_items()).TO.OF
i = 0
idx = d.first_entry
while idx != -1:
res.ll_items()[i] = LLOrderedDict.recast(ELEM, d.entries[idx].value)
idx = d.entries[idx].next
i += 1
return res
@staticmethod
def ll_get(d, key, default):
i = LLOrderedDict.ll_lookup(d, key, d.hashkey(key))
if not i & LLOrderedDict.HIGHEST_BIT:
return d.entries[i].value
else:
return default
@staticmethod
def ll_pop(d, key):
i = LLOrderedDict.ll_lookup(d, key, d.hashkey(key))
if not i & LLOrderedDict.HIGHEST_BIT:
value = d.entries[i].value
LLOrderedDict._ll_del(d, i)
return value
else:
raise KeyError
@staticmethod
def ll_pop_default(d, key, default):
try:
return LLOrderedDict.ll_pop(d, key)
except KeyError:
return default
@staticmethod
def ll_popitem(RESTYPE, d):
if not d.num_items:
raise KeyError
entry = d.entries[d.first_entry]
r = lltype.malloc(RESTYPE.TO)
r.item0 = LLOrderedDict.recast(RESTYPE.TO.item0, entry.key)
r.item1 = LLOrderedDict.recast(RESTYPE.TO.item1, entry.value)
LLOrderedDict._ll_del(d, d.first_entry)
return r
@staticmethod
def ll_update(d, other):
idx = other.first_entry
while idx != -1:
entry = other.entries[idx]
i = LLOrderedDict.ll_lookup(d, entry.key, other.entries.hash(idx))
LLOrderedDict.ll_setitem_lookup_done(d, entry.key, entry.value, other.entries.hash(idx), i)
idx = entry.next
@staticmethod
def ll_clear(d):
if d.num_items == 0:
return
d.entries = lltype.malloc(lltype.typeOf(d.entries).TO, LLOrderedDict.INIT_SIZE, zero=True)
d.num_items = 0
d.first_entry = -1
d.last_entry = -1
d.resize_counter = LLOrderedDict.INIT_SIZE * 2
@staticmethod
def ll_copy(d):
DICT = lltype.typeOf(d).TO
new_d = lltype.malloc(DICT)
new_d.entries = lltype.malloc(DICT.entries.TO, len(d.entries), zero=True)
new_d.num_items = d.num_items
new_d.resize_counter = d.resize_counter
new_d.first_entry = d.first_entry
new_d.last_entry = d.last_entry
if hasattr(DICT, "hashkey_func"):
new_d.hashkey_func = d.hashkey_func
if hasattr(DICT, "keyeq_func"):
new_d.keyeq_func = d.keyeq_func
for i in xrange(len(d.entries)):
entry = d.entries[i]
new_entry = new_d.entries[i]
new_entry.key = entry.key
new_entry.value = entry.value
new_entry.next = entry.next
new_entry.prev = entry.prev
new_entry.everused = entry.everused
new_entry.valid = entry.valid
if hasattr(DICT.entries.TO.OF, "hash"):
new_entry.hash = entry.hash
return new_d
@staticmethod
def ll_newdictiter(ITER, d):
it = lltype.malloc(ITER)
it.d = d
it.index = d.first_entry
return it
@staticmethod
def ll_dictiternext(RESTYPE, it):
if it.index == -1:
raise StopIteration
r = lltype.malloc(RESTYPE.TO)
entry = it.d.entries[it.index]
r.item0 = LLOrderedDict.recast(RESTYPE.TO.item0, entry.key)
r.item1 = LLOrderedDict.recast(RESTYPE.TO.item1, entry.value)
it.index = entry.next
return r
class PyFrame(W_Root):
"""Represents a frame for a regular Python function
that needs to be interpreted.
Public fields:
* 'space' is the object space this frame is running in
* 'code' is the PyCode object this frame runs
* 'w_locals' is the locals dictionary to use, if needed, stored on a
debug object
* 'w_globals' is the attached globals dictionary
* 'builtin' is the attached built-in module
* 'valuestack_w', 'blockstack', control the interpretation
Cell Vars:
my local variables that are exposed to my inner functions
Free Vars:
variables coming from a parent function in which i'm nested
'closure' is a list of Cell instances: the received free vars.
"""
__metaclass__ = extendabletype
frame_finished_execution = False
last_instr = -1
last_exception = None
f_backref = jit.vref_None
escaped = False # see mark_as_escaped()
debugdata = None
pycode = None # code object executed by that frame
locals_cells_stack_w = None # the list of all locals, cells and the valuestack
valuestackdepth = 0 # number of items on valuestack
lastblock = None
# other fields:
# builtin - builtin cache, only if honor__builtins__ is True
# defaults to False
# there is also self.space which is removed by the annotator
# additionally JIT uses vable_token field that is representing
# frame current virtualizable state as seen by the JIT
def __init__(self, space, code, w_globals, outer_func):
if not we_are_translated():
assert type(self) == space.FrameClass, (
"use space.FrameClass(), not directly PyFrame()")
self = hint(self, access_directly=True, fresh_virtualizable=True)
assert isinstance(code, pycode.PyCode)
self.space = space
self.pycode = code
if code.frame_stores_global(w_globals):
self.getorcreatedebug().w_globals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
size = code.co_nlocals + ncellvars + nfreevars + code.co_stacksize
# the layout of this list is as follows:
# | local vars | cells | stack |
self.locals_cells_stack_w = [None] * size
self.valuestackdepth = code.co_nlocals + ncellvars + nfreevars
make_sure_not_resized(self.locals_cells_stack_w)
check_nonneg(self.valuestackdepth)
#
if space.config.objspace.honor__builtins__:
self.builtin = space.builtin.pick_builtin(w_globals)
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
self.initialize_frame_scopes(outer_func, code)
def getdebug(self):
return self.debugdata
def getorcreatedebug(self):
if self.debugdata is None:
self.debugdata = FrameDebugData(self.pycode)
return self.debugdata
def get_w_globals(self):
debugdata = self.getdebug()
if debugdata is not None:
return debugdata.w_globals
return jit.promote(self.pycode).w_globals
def get_w_f_trace(self):
d = self.getdebug()
if d is None:
return None
return d.w_f_trace
def get_is_being_profiled(self):
d = self.getdebug()
if d is None:
return False
return d.is_being_profiled
def get_w_locals(self):
d = self.getdebug()
if d is None:
return None
return d.w_locals
@not_rpython
def __repr__(self):
# useful in tracebacks
return "<%s.%s executing %s at line %s" % (
self.__class__.__module__, self.__class__.__name__, self.pycode,
self.get_last_lineno())
def _getcell(self, varindex):
cell = self.locals_cells_stack_w[varindex + self.pycode.co_nlocals]
assert isinstance(cell, Cell)
return cell
def mark_as_escaped(self):
"""
Must be called on frames that are exposed to applevel, e.g. by
sys._getframe(). This ensures that the virtualref holding the frame
is properly forced by ec.leave(), and thus the frame will be still
accessible even after the corresponding C stack died.
"""
self.escaped = True
def append_block(self, block):
assert block.previous is self.lastblock
self.lastblock = block
def pop_block(self):
block = self.lastblock
self.lastblock = block.previous
return block
def blockstack_non_empty(self):
return self.lastblock is not None
def get_blocklist(self):
"""Returns a list containing all the blocks in the frame"""
lst = []
block = self.lastblock
while block is not None:
lst.append(block)
block = block.previous
return lst
def set_blocklist(self, lst):
self.lastblock = None
i = len(lst) - 1
while i >= 0:
block = lst[i]
i -= 1
block.previous = self.lastblock
self.lastblock = block
def get_builtin(self):
if self.space.config.objspace.honor__builtins__:
return self.builtin
else:
return self.space.builtin
@jit.unroll_safe
def initialize_frame_scopes(self, outer_func, code):
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
# CO_NEWLOCALS: make a locals dict unless optimized is also set
# CO_OPTIMIZED: no locals dict needed at all
flags = code.co_flags
if not (flags & pycode.CO_OPTIMIZED):
if flags & pycode.CO_NEWLOCALS:
self.getorcreatedebug().w_locals = self.space.newdict(
module=True)
else:
w_globals = self.get_w_globals()
assert w_globals is not None
self.getorcreatedebug().w_locals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
if not nfreevars:
if not ncellvars:
return # no cells needed - fast path
elif outer_func is None:
space = self.space
raise oefmt(
space.w_TypeError,
"directly executed code object may not contain free "
"variables")
if outer_func and outer_func.closure:
closure_size = len(outer_func.closure)
else:
closure_size = 0
if closure_size != nfreevars:
raise ValueError("code object received a closure with "
"an unexpected number of free variables")
index = code.co_nlocals
for i in range(ncellvars):
self.locals_cells_stack_w[index] = Cell(
None, self.pycode.cell_families[i])
index += 1
for i in range(nfreevars):
self.locals_cells_stack_w[index] = outer_func.closure[i]
index += 1
def run(self):
"""Start this frame's execution."""
if self.getcode().co_flags & pycode.CO_GENERATOR:
from pypy.interpreter.generator import GeneratorIterator
return GeneratorIterator(self)
else:
return self.execute_frame()
def execute_frame(self, w_inputvalue=None, operr=None):
"""Execute this frame. Main entry point to the interpreter.
The optional arguments are there to handle a generator's frame:
w_inputvalue is for generator.send() and operr is for
generator.throw().
"""
# the following 'assert' is an annotation hint: it hides from
# the annotator all methods that are defined in PyFrame but
# overridden in the {,Host}FrameClass subclasses of PyFrame.
assert (isinstance(self, self.space.FrameClass)
or not self.space.config.translating)
executioncontext = self.space.getexecutioncontext()
executioncontext.enter(self)
got_exception = True
w_exitvalue = self.space.w_None
try:
executioncontext.call_trace(self)
#
try:
if operr is not None:
next_instr = self.handle_operation_error(
executioncontext, operr)
self.last_instr = intmask(next_instr - 1)
else:
# Execution starts just after the last_instr. Initially,
# last_instr is -1. After a generator suspends it points to
# the YIELD_VALUE instruction.
next_instr = r_uint(self.last_instr + 1)
if next_instr != 0:
self.pushvalue(w_inputvalue)
w_exitvalue = self.dispatch(self.pycode, next_instr,
executioncontext)
except OperationError:
raise
except Exception as e: # general fall-back
raise self._convert_unexpected_exception(e)
finally:
executioncontext.return_trace(self, w_exitvalue)
# it used to say self.last_exception = None
# this is now done by the code in pypyjit module
# since we don't want to invalidate the virtualizable
# for no good reason
got_exception = False
finally:
executioncontext.leave(self, w_exitvalue, got_exception)
return w_exitvalue
execute_frame.insert_stack_check_here = True
# stack manipulation helpers
def pushvalue(self, w_object):
depth = self.valuestackdepth
self.locals_cells_stack_w[depth] = ll_assert_not_none(w_object)
self.valuestackdepth = depth + 1
def pushvalue_none(self):
depth = self.valuestackdepth
# the entry is already None, and remains None
assert self.locals_cells_stack_w[depth] is None
self.valuestackdepth = depth + 1
def assert_stack_index(self, index):
if we_are_translated():
return
assert self._check_stack_index(index)
def _check_stack_index(self, index):
code = self.pycode
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
stackstart = code.co_nlocals + ncellvars + nfreevars
return index >= stackstart
def popvalue(self):
return ll_assert_not_none(self.popvalue_maybe_none())
def popvalue_maybe_none(self):
depth = self.valuestackdepth - 1
self.assert_stack_index(depth)
assert depth >= 0
w_object = self.locals_cells_stack_w[depth]
self.locals_cells_stack_w[depth] = None
self.valuestackdepth = depth
return w_object
# we need two popvalues that return different data types:
# one in case we want list another in case of tuple
def _new_popvalues():
@jit.unroll_safe
def popvalues(self, n):
values_w = [None] * n
while True:
n -= 1
if n < 0:
break
values_w[n] = self.popvalue()
return values_w
return popvalues
popvalues = _new_popvalues()
popvalues_mutable = _new_popvalues()
del _new_popvalues
@jit.unroll_safe
def peekvalues(self, n):
values_w = [None] * n
base = self.valuestackdepth - n
self.assert_stack_index(base)
assert base >= 0
while True:
n -= 1
if n < 0:
break
values_w[n] = self.locals_cells_stack_w[base + n]
return values_w
@jit.unroll_safe
def dropvalues(self, n):
n = hint(n, promote=True)
finaldepth = self.valuestackdepth - n
self.assert_stack_index(finaldepth)
assert finaldepth >= 0
while True:
n -= 1
if n < 0:
break
self.locals_cells_stack_w[finaldepth + n] = None
self.valuestackdepth = finaldepth
@jit.unroll_safe
def pushrevvalues(self, n, values_w): # n should be len(values_w)
make_sure_not_resized(values_w)
while True:
n -= 1
if n < 0:
break
self.pushvalue(values_w[n])
@jit.unroll_safe
def dupvalues(self, n):
delta = n - 1
while True:
n -= 1
if n < 0:
break
w_value = self.peekvalue(delta)
self.pushvalue(w_value)
def peekvalue(self, index_from_top=0):
# NOTE: top of the stack is peekvalue(0).
# Contrast this with CPython where it's PEEK(-1).
return ll_assert_not_none(self.peekvalue_maybe_none(index_from_top))
def peekvalue_maybe_none(self, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
self.assert_stack_index(index)
assert index >= 0
return self.locals_cells_stack_w[index]
def settopvalue(self, w_object, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
self.assert_stack_index(index)
assert index >= 0
self.locals_cells_stack_w[index] = ll_assert_not_none(w_object)
@jit.unroll_safe
def dropvaluesuntil(self, finaldepth):
depth = self.valuestackdepth - 1
finaldepth = hint(finaldepth, promote=True)
assert finaldepth >= 0
while depth >= finaldepth:
self.locals_cells_stack_w[depth] = None
depth -= 1
self.valuestackdepth = finaldepth
def make_arguments(self, nargs, methodcall=False):
return Arguments(self.space,
self.peekvalues(nargs),
methodcall=methodcall)
def argument_factory(self,
arguments,
keywords,
keywords_w,
w_star,
w_starstar,
methodcall=False):
return Arguments(self.space,
arguments,
keywords,
keywords_w,
w_star,
w_starstar,
methodcall=methodcall)
@jit.dont_look_inside
def descr__reduce__(self, space):
from pypy.interpreter.mixedmodule import MixedModule
w_mod = space.getbuiltinmodule('_pickle_support')
mod = space.interp_w(MixedModule, w_mod)
new_inst = mod.get('frame_new')
w_tup_state = self._reduce_state(space)
nt = space.newtuple
return nt([new_inst, nt([]), w_tup_state])
@jit.dont_look_inside
def _reduce_state(self, space):
from pypy.module._pickle_support import maker # helper fns
nt = space.newtuple
if self.get_w_f_trace() is None:
f_lineno = self.get_last_lineno()
else:
f_lineno = self.getorcreatedebug().f_lineno
nlocals = self.pycode.co_nlocals
values_w = self.locals_cells_stack_w
w_locals_cells_stack = maker.slp_into_tuple_with_nulls(space, values_w)
w_blockstack = nt(
[block._get_state_(space) for block in self.get_blocklist()])
if self.last_exception is None:
w_exc_value = space.w_None
w_tb = space.w_None
else:
w_exc_value = self.last_exception.get_w_value(space)
w_tb = self.last_exception.get_w_traceback(space)
d = self.getorcreatedebug()
w_backref = self.f_backref()
if w_backref is None:
w_backref = space.w_None
tup_state = [
w_backref,
self.get_builtin(),
self.pycode,
w_locals_cells_stack,
w_blockstack,
w_exc_value, # last_exception
w_tb, #
self.get_w_globals(),
space.newint(self.last_instr),
space.newbool(self.frame_finished_execution),
space.newint(f_lineno),
space.w_None, #XXX placeholder for f_locals
#f_restricted requires no additional data!
space.w_None,
space.newint(d.instr_lb),
space.newint(d.instr_ub),
space.newint(d.instr_prev_plus_one),
space.newint(self.valuestackdepth),
]
return nt(tup_state)
@jit.dont_look_inside
def descr__setstate__(self, space, w_args):
from pypy.module._pickle_support import maker # helper fns
from pypy.interpreter.pycode import PyCode
from pypy.interpreter.module import Module
args_w = space.unpackiterable(w_args, 17)
w_f_back, w_builtin, w_pycode, w_locals_cells_stack, w_blockstack, w_exc_value, w_tb,\
w_globals, w_last_instr, w_finished, w_f_lineno, w_f_locals, \
w_f_trace, w_instr_lb, w_instr_ub, w_instr_prev_plus_one, w_stackdepth = args_w
new_frame = self
pycode = space.interp_w(PyCode, w_pycode)
values_w = maker.slp_from_tuple_with_nulls(space, w_locals_cells_stack)
nfreevars = len(pycode.co_freevars)
closure = None
if nfreevars:
base = pycode.co_nlocals + len(pycode.co_cellvars)
closure = values_w[base:base + nfreevars]
# do not use the instance's __init__ but the base's, because we set
# everything like cells from here
# XXX hack
from pypy.interpreter.function import Function
outer_func = Function(space, None, closure=closure, forcename="fake")
PyFrame.__init__(self, space, pycode, w_globals, outer_func)
f_back = space.interp_w(PyFrame, w_f_back, can_be_None=True)
new_frame.f_backref = jit.non_virtual_ref(f_back)
if space.config.objspace.honor__builtins__:
new_frame.builtin = space.interp_w(Module, w_builtin)
else:
assert space.interp_w(Module, w_builtin) is space.builtin
new_frame.set_blocklist([
unpickle_block(space, w_blk)
for w_blk in space.unpackiterable(w_blockstack)
])
self.locals_cells_stack_w = values_w[:]
valuestackdepth = space.int_w(w_stackdepth)
if not self._check_stack_index(valuestackdepth):
raise oefmt(space.w_ValueError, "invalid stackdepth")
assert valuestackdepth >= 0
self.valuestackdepth = valuestackdepth
if space.is_w(w_exc_value, space.w_None):
new_frame.last_exception = None
else:
from pypy.interpreter.pytraceback import PyTraceback
tb = space.interp_w(PyTraceback, w_tb)
new_frame.last_exception = OperationError(space.type(w_exc_value),
w_exc_value, tb)
new_frame.last_instr = space.int_w(w_last_instr)
new_frame.frame_finished_execution = space.is_true(w_finished)
d = new_frame.getorcreatedebug()
d.f_lineno = space.int_w(w_f_lineno)
if space.is_w(w_f_trace, space.w_None):
d.w_f_trace = None
else:
d.w_f_trace = w_f_trace
d.instr_lb = space.int_w(w_instr_lb) #the three for tracing
d.instr_ub = space.int_w(w_instr_ub)
d.instr_prev_plus_one = space.int_w(w_instr_prev_plus_one)
def hide(self):
return self.pycode.hidden_applevel
def getcode(self):
return hint(self.pycode, promote=True)
@jit.look_inside_iff(lambda self, scope_w: jit.isvirtual(scope_w))
def setfastscope(self, scope_w):
"""Initialize the fast locals from a list of values,
where the order is according to self.pycode.signature()."""
scope_len = len(scope_w)
if scope_len > self.pycode.co_nlocals:
raise ValueError("new fastscope is longer than the allocated area")
# don't assign directly to 'locals_cells_stack_w[:scope_len]' to be
# virtualizable-friendly
for i in range(scope_len):
self.locals_cells_stack_w[i] = scope_w[i]
self.init_cells()
def getdictscope(self):
"""
Get the locals as a dictionary
"""
self.fast2locals()
return self.debugdata.w_locals
def setdictscope(self, w_locals):
"""
Initialize the locals from a dictionary.
"""
self.getorcreatedebug().w_locals = w_locals
self.locals2fast()
@jit.unroll_safe
def fast2locals(self):
# Copy values from the fastlocals to self.w_locals
d = self.getorcreatedebug()
if d.w_locals is None:
d.w_locals = self.space.newdict()
varnames = self.getcode().getvarnames()
for i in range(min(len(varnames), self.getcode().co_nlocals)):
name = varnames[i]
w_value = self.locals_cells_stack_w[i]
if w_value is not None:
self.space.setitem_str(d.w_locals, name, w_value)
else:
w_name = self.space.newtext(name)
try:
self.space.delitem(d.w_locals, w_name)
except OperationError as e:
if not e.match(self.space, self.space.w_KeyError):
raise
# cellvars are values exported to inner scopes
# freevars are values coming from outer scopes
# (see locals2fast for why CO_OPTIMIZED)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
try:
w_value = cell.get()
except ValueError:
pass
else:
self.space.setitem_str(d.w_locals, name, w_value)
@jit.unroll_safe
def locals2fast(self):
# Copy values from self.w_locals to the fastlocals
w_locals = self.getorcreatedebug().w_locals
assert w_locals is not None
varnames = self.getcode().getvarnames()
numlocals = self.getcode().co_nlocals
new_fastlocals_w = [None] * numlocals
for i in range(min(len(varnames), numlocals)):
name = varnames[i]
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
new_fastlocals_w[i] = w_value
self.setfastscope(new_fastlocals_w)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
# If the namespace is unoptimized, then one of the
# following cases applies:
# 1. It does not contain free variables, because it
# uses import * or is a top-level namespace.
# 2. It is a class namespace.
# We don't want to accidentally copy free variables
# into the locals dict used by the class.
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
cell.set(w_value)
@jit.unroll_safe
def init_cells(self):
"""
Initialize cellvars from self.locals_cells_stack_w.
"""
args_to_copy = self.pycode._args_as_cellvars
index = self.pycode.co_nlocals
for i in range(len(args_to_copy)):
argnum = args_to_copy[i]
if argnum >= 0:
cell = self.locals_cells_stack_w[index]
assert isinstance(cell, Cell)
cell.set(self.locals_cells_stack_w[argnum])
index += 1
def getclosure(self):
return None
def fget_code(self, space):
return self.getcode()
def fget_getdictscope(self, space):
return self.getdictscope()
def fget_w_globals(self, space):
# bit silly, but GetSetProperty passes a space
return self.get_w_globals()
### line numbers ###
def fget_f_lineno(self, space):
"Returns the line number of the instruction currently being executed."
if self.get_w_f_trace() is None:
return space.newint(self.get_last_lineno())
else:
return space.newint(self.getorcreatedebug().f_lineno)
def fset_f_lineno(self, space, w_new_lineno):
"Returns the line number of the instruction currently being executed."
try:
new_lineno = space.int_w(w_new_lineno)
except OperationError:
raise oefmt(space.w_ValueError, "lineno must be an integer")
# You can only do this from within a trace function, not via
# _getframe or similar hackery.
if space.int_w(self.fget_f_lasti(space)) == -1:
raise oefmt(
space.w_ValueError,
"can't jump from the 'call' trace event of a new frame")
if self.get_w_f_trace() is None:
raise oefmt(space.w_ValueError,
"f_lineno can only be set by a trace function.")
code = self.pycode.co_code
if ord(code[self.last_instr]) == YIELD_VALUE:
raise oefmt(space.w_ValueError,
"can't jump from a yield statement")
# Only allow jumps when we're tracing a line event.
d = self.getorcreatedebug()
if not d.is_in_line_tracing:
raise oefmt(space.w_ValueError,
"can only jump from a 'line' trace event")
line = self.pycode.co_firstlineno
if new_lineno < line:
raise oefmt(space.w_ValueError,
"line %d comes before the current code.", new_lineno)
elif new_lineno == line:
new_lasti = 0
else:
new_lasti = -1
addr = 0
lnotab = self.pycode.co_lnotab
for offset in xrange(0, len(lnotab), 2):
addr += ord(lnotab[offset])
line += ord(lnotab[offset + 1])
if line >= new_lineno:
new_lasti = addr
new_lineno = line
break
if new_lasti == -1:
raise oefmt(space.w_ValueError,
"line %d comes after the current code.", new_lineno)
# Don't jump to a line with an except in it.
if ord(code[new_lasti]) in (DUP_TOP, POP_TOP):
raise oefmt(space.w_ValueError,
"can't jump to 'except' line as there's no exception")
# Don't jump into or out of a finally block.
f_lasti_setup_addr = -1
new_lasti_setup_addr = -1
blockstack = []
addr = 0
while addr < len(code):
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
blockstack.append([addr, False])
elif op == POP_BLOCK:
setup_op = ord(code[blockstack[-1][0]])
if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
blockstack[-1][1] = True
else:
blockstack.pop()
elif op == END_FINALLY:
if len(blockstack) > 0:
setup_op = ord(code[blockstack[-1][0]])
if setup_op == SETUP_FINALLY or setup_op == SETUP_WITH:
blockstack.pop()
if addr == new_lasti or addr == self.last_instr:
for ii in range(len(blockstack)):
setup_addr, in_finally = blockstack[~ii]
if in_finally:
if addr == new_lasti:
new_lasti_setup_addr = setup_addr
if addr == self.last_instr:
f_lasti_setup_addr = setup_addr
break
if op >= HAVE_ARGUMENT:
addr += 3
else:
addr += 1
assert len(blockstack) == 0
if new_lasti_setup_addr != f_lasti_setup_addr:
raise oefmt(
space.w_ValueError,
"can't jump into or out of a 'finally' block %d -> %d",
f_lasti_setup_addr, new_lasti_setup_addr)
if new_lasti < self.last_instr:
min_addr = new_lasti
max_addr = self.last_instr
else:
min_addr = self.last_instr
max_addr = new_lasti
delta_iblock = min_delta_iblock = 0
addr = min_addr
while addr < max_addr:
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH):
delta_iblock += 1
elif op == POP_BLOCK:
delta_iblock -= 1
if delta_iblock < min_delta_iblock:
min_delta_iblock = delta_iblock
if op >= stdlib_opcode.HAVE_ARGUMENT:
addr += 3
else:
addr += 1
f_iblock = 0
block = self.lastblock
while block:
f_iblock += 1
block = block.previous
min_iblock = f_iblock + min_delta_iblock
if new_lasti > self.last_instr:
new_iblock = f_iblock + delta_iblock
else:
new_iblock = f_iblock - delta_iblock
if new_iblock > min_iblock:
raise oefmt(space.w_ValueError,
"can't jump into the middle of a block")
# Pop any blocks that we're jumping out of.
from pypy.interpreter.pyopcode import FinallyBlock
while f_iblock > new_iblock:
block = self.pop_block()
block.cleanup(self)
f_iblock -= 1
if (isinstance(block, FinallyBlock)
and ord(code[block.handlerposition]) == WITH_CLEANUP):
self.popvalue() # Pop the exit function.
d.f_lineno = new_lineno
self.last_instr = new_lasti
def get_last_lineno(self):
"Returns the line number of the instruction currently being executed."
return pytraceback.offset2lineno(self.pycode, self.last_instr)
def fget_f_builtins(self, space):
return self.get_builtin().getdict(space)
def get_f_back(self):
return ExecutionContext.getnextframe_nohidden(self)
def fget_f_back(self, space):
return self.get_f_back()
def fget_f_lasti(self, space):
return self.space.newint(self.last_instr)
def fget_f_trace(self, space):
return self.get_w_f_trace()
def fset_f_trace(self, space, w_trace):
if space.is_w(w_trace, space.w_None):
self.getorcreatedebug().w_f_trace = None
else:
d = self.getorcreatedebug()
d.w_f_trace = w_trace
d.f_lineno = self.get_last_lineno()
def fdel_f_trace(self, space):
self.getorcreatedebug().w_f_trace = None
def fget_f_exc_type(self, space):
if self.last_exception is not None:
f = self.f_backref()
while f is not None and f.last_exception is None:
f = f.f_backref()
if f is not None:
return f.last_exception.w_type
return space.w_None
def fget_f_exc_value(self, space):
if self.last_exception is not None:
f = self.f_backref()
while f is not None and f.last_exception is None:
f = f.f_backref()
if f is not None:
return f.last_exception.get_w_value(space)
return space.w_None
def fget_f_exc_traceback(self, space):
if self.last_exception is not None:
f = self.f_backref()
while f is not None and f.last_exception is None:
f = f.f_backref()
if f is not None:
return f.last_exception.get_w_traceback(space)
return space.w_None
def fget_f_restricted(self, space):
if space.config.objspace.honor__builtins__:
return space.newbool(self.builtin is not space.builtin)
return space.w_False
@jit.unroll_safe
@specialize.arg(2)
def _exc_info_unroll(self, space, for_hidden=False):
"""Return the most recent OperationError being handled in the
call stack
"""
frame = self
while frame:
last = frame.last_exception
if last is not None:
if last is get_cleared_operation_error(self.space):
break
if for_hidden or not frame.hide():
return last
frame = frame.f_backref()
return None
def _convert_unexpected_exception(self, e):
from pypy.interpreter import error
operr = error.get_converted_unexpected_exception(self.space, e)
pytraceback.record_application_traceback(self.space, operr, self,
self.last_instr)
raise operr
def unroll_pred(lst, idx, unroll_to=0):
if not jit.we_are_jitted():
return False
return not jit.isvirtual(lst) and idx > unroll_to
def ll_pop(func, l, index):
length = l.ll_length()
if index < 0:
index += length
if func is dum_checkidx:
if index < 0 or index >= length:
raise IndexError
else:
ll_assert(index >= 0, "negative list pop index out of bound")
ll_assert(index < length, "list pop index out of bound")
res = l.ll_getitem_fast(index)
ll_delitem_nonneg(dum_nocheck, l, index)
return res
@jit.look_inside_iff(lambda l: jit.isvirtual(l))
def ll_reverse(l):
length = l.ll_length()
i = 0
length_1_i = length-1-i
while i < length_1_i:
tmp = l.ll_getitem_fast(i)
l.ll_setitem_fast(i, l.ll_getitem_fast(length_1_i))
l.ll_setitem_fast(length_1_i, tmp)
i += 1
length_1_i -= 1
def ll_getitem_nonneg(func, basegetitem, l, index):
ll_assert(index >= 0, "unexpectedly negative list getitem index")
if func is dum_checkidx:
if index >= l.ll_length():
def ll_dict_getitem(d, key):
i = ll_dict_lookup(d, key, d.keyhash(key))
if not i & HIGHEST_BIT:
return ll_get_value(d, i)
else:
raise KeyError
def ll_dict_setitem(d, key, value):
hash = d.keyhash(key)
i = ll_dict_lookup(d, key, hash)
return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
# It may be safe to look inside always, it has a few branches though, and their
# frequencies needs to be investigated.
@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
valid = (i & HIGHEST_BIT) == 0
i = i & MASK
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
if not d.entries.everused(i):
# a new entry that was never used before
ll_assert(not valid, "valid but not everused")
rc = d.resize_counter - 3
if rc <= 0: # if needed, resize the dict -- before the insertion
ll_dict_resize(d)
i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key'
entry = d.entries[i]
rc = d.resize_counter - 3
ll_assert(rc > 0, "ll_dict_resize failed?")
class W_ArrayObject(W_Object):
"""Abstract base class. Concrete subclasses use various strategies.
This base class defines the general methods that can be implemented
without needing to call (too often) the arraylen(), _getitem_str()
and _getitem_int() methods.
"""
@staticmethod
def new_array_from_list(space, lst_w):
return W_ListArrayObject(space, lst_w)
@staticmethod
def new_array_from_rdict(space, dct_w):
return W_RDictArrayObject(space, dct_w, compute_next_idx(dct_w))
@staticmethod
@jit.look_inside_iff(
lambda space, pairs_ww, allow_bogus: jit.isvirtual(pairs_ww))
def new_array_from_pairs(space, pairs_ww, allow_bogus):
rdct_w = new_rdict()
next_idx = 0
for w_key, w_value in pairs_ww:
if w_key is not None:
try:
as_int, as_str = convert_to_index(space,
w_key,
allow_bogus=allow_bogus)
except CannotConvertToIndex:
continue
if as_str is not None:
# it was string, maybe it can be integer
try:
as_int = try_convert_str_to_int(as_str)
# yes it can
as_str = None
except ValueError:
pass
# no key, just increment next_idx
else:
as_int, as_str = next_idx, None
if as_str is None:
if as_int >= next_idx:
next_idx = as_int + 1
as_str = str(as_int)
rdct_w[as_str] = w_value
return W_RDictArrayObject(space, rdct_w, next_idx=next_idx)
def copy_item(self):
return self.copy()
def is_true(self, space):
return self.arraylen() > 0
def int_w(self, space):
return int(self.is_true(space))
def as_int_arg(self, space):
raise ConvertError('cannot use array as integer argument')
def as_number(self, space):
return space.wrap(self.is_true(space))
def abs(self, space):
return space.w_False
def _lookup_item_ref(self, space, w_arg):
"""Return a possibly virtual reference to the item,
or None if the lookup fails
"""
try:
as_int, as_str = convert_to_index(space, w_arg)
except CannotConvertToIndex:
space.ec.warn("Illegal offset type")
return None
if as_str is None:
r_item = self._getitem_int(as_int)
if r_item is None:
return None
else:
r_item = self._getitem_str(as_str)
if r_item is None:
return None
return r_item
def getitem(self, space, w_arg, give_notice=False):
try:
as_int, as_str = convert_to_index(space, w_arg)
except CannotConvertToIndex:
space.ec.warn("Illegal offset type")
return space.w_Null
if as_str is None:
r_item = self._getitem_int(as_int)
if r_item is None:
if give_notice:
space.ec.notice("Undefined offset: %d" % as_int)
return space.w_Null
else:
r_item = self._getitem_str(as_str)
if r_item is None:
if give_notice:
space.ec.notice("Undefined index: %s" % as_str)
return space.w_Null
if isinstance(r_item, VirtualReference):
return r_item.deref()
else:
return r_item
def setitem2_maybe_inplace(self, space, w_arg, w_value, unique_item=False):
try:
as_int, as_str = convert_to_index(space, w_arg)
except CannotConvertToIndex:
space.ec.warn("Illegal offset type")
return self, space.w_Null
if as_str is None:
w_arr = self._setitem_int(as_int, w_value, False, unique_item)
else:
w_arr = self._setitem_str(as_str, w_value, False, unique_item)
return w_arr, w_value
def _setitem_ref(self, space, w_arg, w_ref):
try:
as_int, as_str = convert_to_index(space, w_arg)
except CannotConvertToIndex:
space.ec.warn("Illegal offset type")
return self
if as_str is None:
return self._setitem_int(as_int, w_ref, True)
else:
return self._setitem_str(as_str, w_ref, True)
def appenditem_inplace(self, space, w_item, as_ref=False):
# For now this always succeeds in appending the item in-place.
# It may need to be reconsidered if we add more strategies.
self._appenditem(w_item, as_ref)
return w_item
def packitem_maybe_inplace(self, space, w_arg, w_value):
as_int, as_str = convert_to_index(space, w_arg)
if as_str is not None:
try:
try_convert_str_to_int(as_str)
except ValueError:
# really not an int
return self._setitem_str(as_str, w_value, False)
self._appenditem(w_value)
return self
def _unsetitem(self, space, w_arg):
as_int, as_str = convert_to_index(space, w_arg)
if as_str is None:
return self._unsetitem_int(as_int)
else:
return self._unsetitem_str(as_str)
def isset_index(self, space, w_index):
as_int, as_str = convert_to_index(space, w_index)
if as_str is None:
return self._isset_int(as_int)
else:
return self._isset_str(as_str)
def _getitem_int(self, index):
raise NotImplementedError("abstract")
def _getitem_str(self, key):
raise NotImplementedError("abstract")
def _appenditem(self, w_obj, as_ref=False):
raise NotImplementedError("abstract")
def _setitem_int(self, index, w_value, as_ref, unique_item=False):
raise NotImplementedError("abstract")
def _setitem_str(self, key, w_value, as_ref, unique_item=False):
# Note: might be occasionally called with a string like "5" too
raise NotImplementedError("abstract")
def _unsetitem_int(self, index):
raise NotImplementedError("abstract")
def _unsetitem_str(self, key):
raise NotImplementedError("abstract")
def _isset_int(self, index):
raise NotImplementedError("abstract")
def _isset_str(self, key):
raise NotImplementedError("abstract")
def arraylen(self):
raise NotImplementedError("abstract")
def as_rdict(self):
raise NotImplementedError("abstract")
def _each(self, space):
raise NotImplementedError("abstract")
def _current(self):
raise NotImplementedError("abstract")
def next(self, space):
length = self.arraylen()
current_idx = self.current_idx + 1
if current_idx >= length:
self.current_idx = length
return w_False
self.current_idx = current_idx
return self._current()
def _key(self, space):
raise NotImplementedError("abstract")
def _inplace_pop(self, space):
raise NotImplementedError("abstract")
def get_rdict_from_array(self):
raise NotImplementedError("abstract")
def as_dict(self):
"NOT_RPYTHON: for tests only"
return self.as_rdict()
def var_dump(self, space, indent, recursion):
return array_var_dump(self.as_rdict(), space, indent, recursion, self,
'array')
def var_export(self, space, indent, recursion, suffix):
return array_var_export(self.as_rdict(), space, indent, recursion,
self, suffix)
def str(self, space, quiet=False):
if not quiet:
space.ec.notice("Array to string conversion")
return "Array"
def repr(self):
return "Array"
def dump(self):
items = []
next = 0
for key, w_value in self.as_rdict().items():
dumpvalue = w_value.dump()
try:
numkey = int(key)
except ValueError:
items.append('%s=>%s' % (key, dumpvalue))
continue
if numkey == next:
items.append(dumpvalue)
else:
items.append('%d=>%s' % (numkey, dumpvalue))
next = numkey + 1
return "array(%s)" % ', '.join(items)
def as_pair_list(self, space):
pairs = []
with space.iter(self) as w_iter:
while not w_iter.done():
w_key, w_val = w_iter.next_item(space)
pairs.append((w_key, w_val))
return pairs
def eval_static(self, space):
return self
def serialize(self, space, builder, memo):
builder.append("a:")
builder.append(str(self.arraylen()))
builder.append(":{")
memo.add_counter()
with space.iter(self) as itr:
while not itr.done():
w_key, w_value = itr.next_item(space)
w_key.serialize(space, builder, memo)
if w_value.serialize(space, builder, memo):
memo.add_counter()
builder.append("}")
return False # counted above already
def add(self, space, other_array):
assert isinstance(other_array, W_ArrayObject)
d = self.as_rdict()
for k, w_v in other_array.as_rdict().iteritems():
if k not in d:
d[k] = w_v
return space.new_array_from_rdict(d)
def listeq_unroll_case(l1, l2, eqfn):
if jit.isvirtual(l1) and l1.ll_length() < 10:
return True
if jit.isvirtual(l2) and l2.ll_length() < 10:
return True
return False
def ll_dict_getitem(d, key):
i = ll_dict_lookup(d, key, d.keyhash(key))
if not i & HIGHEST_BIT:
return ll_get_value(d, i)
else:
raise KeyError
def ll_dict_setitem(d, key, value):
hash = d.keyhash(key)
i = ll_dict_lookup(d, key, hash)
return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
# It may be safe to look inside always, it has a few branches though, and their
# frequencies needs to be investigated.
@jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
valid = (i & HIGHEST_BIT) == 0
i = i & MASK
ENTRY = lltype.typeOf(d.entries).TO.OF
entry = d.entries[i]
if not d.entries.everused(i):
# a new entry that was never used before
ll_assert(not valid, "valid but not everused")
rc = d.resize_counter - 3
if rc <= 0: # if needed, resize the dict -- before the insertion
ll_dict_resize(d)
i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key'
entry = d.entries[i]
rc = d.resize_counter - 3
ll_assert(rc > 0, "ll_dict_resize failed?")
def from_list_unroll_pred(lst, idx, unroll_to=0):
if not jit.we_are_jitted():
return False
if unroll_to == -1:
return False
return not jit.isvirtual(lst) and idx > unroll_to
class W_CTypeStructOrUnion(W_CType):
_immutable_fields_ = [
'alignment?', 'fields_list?[*]', 'fields_dict?', 'custom_field_pos?',
'with_var_array?'
]
# fields added by complete_struct_or_union():
alignment = -1
fields_list = None
fields_dict = None
custom_field_pos = False
with_var_array = False
def __init__(self, space, name):
W_CType.__init__(self, space, -1, name, len(name))
def check_complete(self, w_errorcls=None):
if self.fields_dict is None:
space = self.space
raise oefmt(w_errorcls or space.w_TypeError,
"'%s' is opaque or not completed yet", self.name)
def _alignof(self):
self.check_complete(w_errorcls=self.space.w_ValueError)
return self.alignment
def _fget(self, attrchar):
if attrchar == 'f': # fields
space = self.space
if self.size < 0:
return space.w_None
result = [None] * len(self.fields_list)
for fname, field in self.fields_dict.iteritems():
i = self.fields_list.index(field)
result[i] = space.newtuple(
[space.wrap(fname), space.wrap(field)])
return space.newlist(result)
return W_CType._fget(self, attrchar)
def convert_to_object(self, cdata):
space = self.space
self.check_complete()
return cdataobj.W_CData(space, cdata, self)
def copy_and_convert_to_object(self, cdata):
space = self.space
self.check_complete()
ob = cdataobj.W_CDataNewOwning(space, self.size, self)
misc._raw_memcopy(cdata, ob._cdata, self.size)
keepalive_until_here(ob)
return ob
def typeoffsetof_field(self, fieldname, following):
self.check_complete()
space = self.space
try:
cfield = self.fields_dict[fieldname]
except KeyError:
raise OperationError(space.w_KeyError, space.wrap(fieldname))
if cfield.bitshift >= 0:
raise OperationError(space.w_TypeError,
space.wrap("not supported for bitfields"))
return (cfield.ctype, cfield.offset)
def _copy_from_same(self, cdata, w_ob):
if isinstance(w_ob, cdataobj.W_CData):
if w_ob.ctype is self and self.size >= 0:
misc._raw_memcopy(w_ob._cdata, cdata, self.size)
keepalive_until_here(w_ob)
return True
return False
def _check_only_one_argument_for_union(self, w_ob):
pass
def convert_from_object(self, cdata, w_ob):
if not self._copy_from_same(cdata, w_ob):
self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@jit.look_inside_iff(
lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob))
def convert_struct_from_object(self, cdata, w_ob, optvarsize):
self._check_only_one_argument_for_union(w_ob)
space = self.space
if (space.isinstance_w(w_ob, space.w_list)
or space.isinstance_w(w_ob, space.w_tuple)):
lst_w = space.listview(w_ob)
if len(lst_w) > len(self.fields_list):
raise oefmt(space.w_ValueError,
"too many initializers for '%s' (got %d)",
self.name, len(lst_w))
for i in range(len(lst_w)):
optvarsize = self.fields_list[i].write_v(
cdata, lst_w[i], optvarsize)
return optvarsize
elif space.isinstance_w(w_ob, space.w_dict):
lst_w = space.fixedview(w_ob)
for i in range(len(lst_w)):
w_key = lst_w[i]
key = space.str_w(w_key)
try:
cf = self.fields_dict[key]
except KeyError:
space.raise_key_error(w_key)
assert 0
optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
optvarsize)
return optvarsize
else:
if optvarsize == -1:
msg = "list or tuple or dict or struct-cdata"
else:
msg = "list or tuple or dict"
raise self._convert_error(msg, w_ob)
@jit.elidable
def _getcfield_const(self, attr):
return self.fields_dict[attr]
def getcfield(self, attr):
if self.fields_dict is not None:
self = jit.promote(self)
attr = jit.promote_string(attr)
try:
return self._getcfield_const(attr)
except KeyError:
pass
return W_CType.getcfield(self, attr)
# int num_live_items;
# int num_ever_used_items;
# int resize_counter;
# {byte, short, int, long} *indexes;
# dictentry *entries;
# lookup_function_no; # one of the four possible functions for different
# # size dicts; the rest of the word is a counter for how
# # many 'entries' at the start are known to be deleted
# (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
# (Function DICTKEY -> int) *fnkeyhash;
# }
#
#
@jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
@jit.oopspec("ordereddict.lookup(d, key, hash, flag)")
def ll_call_lookup_function(d, key, hash, flag):
fun = d.lookup_function_no & FUNC_MASK
# This likely() here forces gcc to compile the check for fun == FUNC_BYTE
# first. Otherwise, this is a regular switch and gcc (at least 4.7)
# compiles this as a series of checks, with the FUNC_BYTE case last.
# It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
if likely(fun == FUNC_BYTE):
return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
elif fun == FUNC_SHORT:
return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
elif IS_64BIT and fun == FUNC_INT:
return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
elif fun == FUNC_LONG:
return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
class W_CTypeStructOrUnion(W_CType):
_immutable_fields_ = [
'alignment?', '_fields_list?[*]', '_fields_dict?',
'_custom_field_pos?', '_with_var_array?'
]
# three possible states:
# - "opaque": for opaque C structs; self.size < 0.
# - "lazy": for non-opaque C structs whose _fields_list, _fields_dict,
# _custom_field_pos and _with_var_array are not filled yet; can be
# filled by calling force_lazy_struct().
# (But self.size and .alignment are already set and won't change.)
# - "forced": for non-opaque C structs which are fully ready.
# fields added by complete_struct_or_union():
alignment = -1
_fields_list = None
_fields_dict = None
_custom_field_pos = False
_with_var_array = False
def __init__(self, space, name):
W_CType.__init__(self, space, -1, name, len(name))
def check_complete(self, w_errorcls=None):
# Check ONLY that are are not opaque. Complain if we are.
if self.size < 0:
space = self.space
raise oefmt(w_errorcls or space.w_TypeError,
"'%s' is opaque or not completed yet", self.name)
def force_lazy_struct(self):
# Force a "lazy" struct to become "forced"; complain if we are "opaque".
if self._fields_list is None:
self.check_complete()
#
from pypy.module._cffi_backend import realize_c_type
realize_c_type.do_realize_lazy_struct(self)
def _alignof(self):
self.check_complete(w_errorcls=self.space.w_ValueError)
if self.alignment == -1:
self.force_lazy_struct()
assert self.alignment > 0
return self.alignment
def _fget(self, attrchar):
if attrchar == 'f': # fields
space = self.space
if self.size < 0:
return space.w_None
self.force_lazy_struct()
result = [None] * len(self._fields_list)
for fname, field in self._fields_dict.iteritems():
i = self._fields_list.index(field)
result[i] = space.newtuple(
[space.wrap(fname), space.wrap(field)])
return space.newlist(result)
return W_CType._fget(self, attrchar)
def convert_to_object(self, cdata):
space = self.space
self.check_complete()
return cdataobj.W_CData(space, cdata, self)
def copy_and_convert_to_object(self, source):
space = self.space
self.check_complete()
ptr = lltype.malloc(rffi.CCHARP.TO,
self.size,
flavor='raw',
zero=False)
misc._raw_memcopy(source, ptr, self.size)
return cdataobj.W_CDataNewStd(space, ptr, self)
def typeoffsetof_field(self, fieldname, following):
self.force_lazy_struct()
space = self.space
try:
cfield = self._fields_dict[fieldname]
except KeyError:
raise OperationError(space.w_KeyError, space.wrap(fieldname))
if cfield.bitshift >= 0:
raise OperationError(space.w_TypeError,
space.wrap("not supported for bitfields"))
return (cfield.ctype, cfield.offset)
def _copy_from_same(self, cdata, w_ob):
if isinstance(w_ob, cdataobj.W_CData):
if w_ob.ctype is self and self.size >= 0:
with w_ob as ptr:
misc._raw_memcopy(ptr, cdata, self.size)
return True
return False
def _check_only_one_argument_for_union(self, w_ob):
pass
def convert_from_object(self, cdata, w_ob):
if not self._copy_from_same(cdata, w_ob):
self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@jit.look_inside_iff(
lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob))
def convert_struct_from_object(self, cdata, w_ob, optvarsize):
self.force_lazy_struct()
self._check_only_one_argument_for_union(w_ob)
space = self.space
if (space.isinstance_w(w_ob, space.w_list)
or space.isinstance_w(w_ob, space.w_tuple)):
lst_w = space.listview(w_ob)
if len(lst_w) > len(self._fields_list):
raise oefmt(space.w_ValueError,
"too many initializers for '%s' (got %d)",
self.name, len(lst_w))
for i in range(len(lst_w)):
optvarsize = self._fields_list[i].write_v(
cdata, lst_w[i], optvarsize)
return optvarsize
elif space.isinstance_w(w_ob, space.w_dict):
lst_w = space.fixedview(w_ob)
for i in range(len(lst_w)):
w_key = lst_w[i]
key = space.str_w(w_key)
try:
cf = self._fields_dict[key]
except KeyError:
space.raise_key_error(w_key)
assert 0
optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
optvarsize)
return optvarsize
else:
if optvarsize == -1:
msg = "list or tuple or dict or struct-cdata"
else:
msg = "list or tuple or dict"
raise self._convert_error(msg, w_ob)
@jit.elidable
def _getcfield_const(self, attr):
return self._fields_dict[attr]
def getcfield(self, attr):
ready = self._fields_dict is not None
if not ready and self.size >= 0:
self.force_lazy_struct()
ready = True
if ready:
self = jit.promote(self)
attr = jit.promote_string(attr)
try:
return self._getcfield_const(attr)
except KeyError:
pass
return W_CType.getcfield(self, attr)
class PyFrame(W_Root):
"""Represents a frame for a regular Python function
that needs to be interpreted.
Public fields:
* 'space' is the object space this frame is running in
* 'code' is the PyCode object this frame runs
* 'w_locals' is the locals dictionary to use, if needed, stored on a
debug object
* 'w_globals' is the attached globals dictionary
* 'builtin' is the attached built-in module
* 'valuestack_w', 'blockstack', control the interpretation
Cell Vars:
my local variables that are exposed to my inner functions
Free Vars:
variables coming from a parent function in which i'm nested
'closure' is a list of Cell instances: the received free vars.
"""
__metaclass__ = extendabletype
frame_finished_execution = False
f_generator_wref = rweakref.dead_ref # for generators/coroutines
f_generator_nowref = None # (only one of the two attrs)
last_instr = -1
f_backref = jit.vref_None
escaped = False # see mark_as_escaped()
debugdata = None
pycode = None # code object executed by that frame
locals_cells_stack_w = None # the list of all locals, cells and the valuestack
valuestackdepth = 0 # number of items on valuestack
lastblock = None
# other fields:
# builtin - builtin cache, only if honor__builtins__ is True
# defaults to False
# there is also self.space which is removed by the annotator
# additionally JIT uses vable_token field that is representing
# frame current virtualizable state as seen by the JIT
def __init__(self, space, code, w_globals, outer_func):
if not we_are_translated():
assert type(self) == space.FrameClass, (
"use space.FrameClass(), not directly PyFrame()")
self = hint(self, access_directly=True, fresh_virtualizable=True)
assert isinstance(code, pycode.PyCode)
self.space = space
self.pycode = code
if code.frame_stores_global(w_globals):
self.getorcreatedebug().w_globals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
size = code.co_nlocals + ncellvars + nfreevars + code.co_stacksize
# the layout of this list is as follows:
# | local vars | cells | stack |
self.locals_cells_stack_w = [None] * size
self.valuestackdepth = code.co_nlocals + ncellvars + nfreevars
make_sure_not_resized(self.locals_cells_stack_w)
check_nonneg(self.valuestackdepth)
#
if space.config.objspace.honor__builtins__:
self.builtin = space.builtin.pick_builtin(w_globals)
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
self.initialize_frame_scopes(outer_func, code)
def getdebug(self):
return self.debugdata
def getorcreatedebug(self):
if self.debugdata is None:
self.debugdata = FrameDebugData(self.pycode)
return self.debugdata
def get_w_globals(self):
debugdata = self.getdebug()
if debugdata is not None:
return debugdata.w_globals
return jit.promote(self.pycode).w_globals
def get_w_f_trace(self):
d = self.getdebug()
if d is None:
return None
return d.w_f_trace
def get_is_being_profiled(self):
d = self.getdebug()
if d is None:
return False
return d.is_being_profiled
def get_w_locals(self):
d = self.getdebug()
if d is None:
return None
return d.w_locals
@not_rpython
def __repr__(self):
# useful in tracebacks
return "<%s.%s executing %s at line %s" % (
self.__class__.__module__, self.__class__.__name__,
self.pycode, self.get_last_lineno())
def _getcell(self, varindex):
cell = self.locals_cells_stack_w[varindex + self.pycode.co_nlocals]
assert isinstance(cell, Cell)
return cell
def mark_as_escaped(self):
"""
Must be called on frames that are exposed to applevel, e.g. by
sys._getframe(). This ensures that the virtualref holding the frame
is properly forced by ec.leave(), and thus the frame will be still
accessible even after the corresponding C stack died.
"""
self.escaped = True
def append_block(self, block):
assert block.previous is self.lastblock
self.lastblock = block
def pop_block(self):
block = self.lastblock
self.lastblock = block.previous
return block
def blockstack_non_empty(self):
return self.lastblock is not None
def get_blocklist(self):
"""Returns a list containing all the blocks in the frame"""
lst = []
block = self.lastblock
while block is not None:
lst.append(block)
block = block.previous
return lst
def set_blocklist(self, lst):
self.lastblock = None
i = len(lst) - 1
while i >= 0:
block = lst[i]
i -= 1
block.previous = self.lastblock
self.lastblock = block
def get_builtin(self):
if self.space.config.objspace.honor__builtins__:
return self.builtin
else:
return self.space.builtin
@jit.unroll_safe
def initialize_frame_scopes(self, outer_func, code):
# regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
# class bodies only have CO_NEWLOCALS.
# CO_NEWLOCALS: make a locals dict unless optimized is also set
# CO_OPTIMIZED: no locals dict needed at all
flags = code.co_flags
if not (flags & pycode.CO_OPTIMIZED):
if flags & pycode.CO_NEWLOCALS:
self.getorcreatedebug().w_locals = self.space.newdict(module=True)
else:
w_globals = self.get_w_globals()
assert w_globals is not None
self.getorcreatedebug().w_locals = w_globals
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
if not nfreevars:
if not ncellvars:
return # no cells needed - fast path
elif outer_func is None:
space = self.space
raise oefmt(space.w_TypeError,
"directly executed code object may not contain free "
"variables")
if outer_func and outer_func.closure:
closure_size = len(outer_func.closure)
else:
closure_size = 0
if closure_size != nfreevars:
raise ValueError("code object received a closure with "
"an unexpected number of free variables")
index = code.co_nlocals
for i in range(ncellvars):
self.locals_cells_stack_w[index] = Cell()
index += 1
for i in range(nfreevars):
self.locals_cells_stack_w[index] = outer_func.closure[i]
index += 1
def _is_generator_or_coroutine(self):
return (self.getcode().co_flags & (pycode.CO_COROUTINE |
pycode.CO_GENERATOR)) != 0
def run(self, name=None, qualname=None):
"""Start this frame's execution."""
if self._is_generator_or_coroutine():
return self.initialize_as_generator(name, qualname)
else:
# untranslated: check that sys_exc_info is exactly
# restored after running any Python function.
# Translated: actually save and restore it, as an attempt to
# work around rare cases that can occur if RecursionError or
# MemoryError is raised at just the wrong place
executioncontext = self.space.getexecutioncontext()
exc_on_enter = executioncontext.sys_exc_info()
if we_are_translated():
try:
return self.execute_frame()
finally:
executioncontext.set_sys_exc_info(exc_on_enter)
else:
# untranslated, we check consistency, but not in case of
# interp-level exceptions different than OperationError
# (e.g. a random failing test, or a pytest Skipped exc.)
try:
w_res = self.execute_frame()
assert exc_on_enter is executioncontext.sys_exc_info()
except OperationError:
assert exc_on_enter is executioncontext.sys_exc_info()
raise
return w_res
run._always_inline_ = True
def initialize_as_generator(self, name, qualname):
space = self.space
if self.getcode().co_flags & pycode.CO_COROUTINE:
from pypy.interpreter.generator import Coroutine
gen = Coroutine(self, name, qualname)
ec = space.getexecutioncontext()
w_wrapper = ec.w_coroutine_wrapper_fn
else:
from pypy.interpreter.generator import GeneratorIterator
gen = GeneratorIterator(self, name, qualname)
ec = None
w_wrapper = None
if space.config.translation.rweakref:
self.f_generator_wref = rweakref.ref(gen)
else:
self.f_generator_nowref = gen
w_gen = gen
if w_wrapper is not None:
if ec.in_coroutine_wrapper:
raise oefmt(space.w_RuntimeError,
"coroutine wrapper %R attempted "
"to recursively wrap %R",
w_wrapper, self.getcode())
ec.in_coroutine_wrapper = True
try:
w_gen = space.call_function(w_wrapper, w_gen)
finally:
ec.in_coroutine_wrapper = False
return w_gen
def execute_frame(self, in_generator=None, w_arg_or_err=None):
"""Execute this frame. Main entry point to the interpreter.
'in_generator' is non-None iff we are starting or resuming
a generator or coroutine frame; in that case, w_arg_or_err
is the input argument -or- an SApplicationException instance.
"""
from pypy.interpreter import pyopcode
# the following 'assert' is an annotation hint: it hides from
# the annotator all methods that are defined in PyFrame but
# overridden in the {,Host}FrameClass subclasses of PyFrame.
assert (isinstance(self, self.space.FrameClass) or
not self.space.config.translating)
executioncontext = self.space.getexecutioncontext()
executioncontext.enter(self)
got_exception = True
w_exitvalue = self.space.w_None
try:
executioncontext.call_trace(self)
#
# Execution starts just after the last_instr. Initially,
# last_instr is -1. After a generator suspends it points to
# the YIELD_VALUE/YIELD_FROM instruction.
try:
try:
if in_generator is None:
assert self.last_instr == -1
next_instr = r_uint(0)
else:
next_instr = in_generator.resume_execute_frame(
self, w_arg_or_err)
except pyopcode.Yield:
w_exitvalue = self.popvalue()
else:
w_exitvalue = self.dispatch(self.pycode, next_instr,
executioncontext)
except OperationError:
raise
except Exception as e: # general fall-back
raise self._convert_unexpected_exception(e)
finally:
executioncontext.return_trace(self, w_exitvalue)
got_exception = False
finally:
executioncontext.leave(self, w_exitvalue, got_exception)
return w_exitvalue
execute_frame.insert_stack_check_here = True
# stack manipulation helpers
def pushvalue(self, w_object):
depth = self.valuestackdepth
self.locals_cells_stack_w[depth] = ll_assert_not_none(w_object)
self.valuestackdepth = depth + 1
def pushvalue_none(self):
depth = self.valuestackdepth
# the entry is already None, and remains None
assert self.locals_cells_stack_w[depth] is None
self.valuestackdepth = depth + 1
def _check_stack_index(self, index):
# will be completely removed by the optimizer if only used in an assert
# and if asserts are disabled
code = self.pycode
ncellvars = len(code.co_cellvars)
nfreevars = len(code.co_freevars)
stackstart = code.co_nlocals + ncellvars + nfreevars
return index >= stackstart
def popvalue(self):
return ll_assert_not_none(self.popvalue_maybe_none())
def popvalue_maybe_none(self):
depth = self.valuestackdepth - 1
assert self._check_stack_index(depth)
assert depth >= 0
w_object = self.locals_cells_stack_w[depth]
self.locals_cells_stack_w[depth] = None
self.valuestackdepth = depth
return w_object
# we need two popvalues that return different data types:
# one in case we want list another in case of tuple
def _new_popvalues():
@jit.unroll_safe
def popvalues(self, n):
values_w = [None] * n
while True:
n -= 1
if n < 0:
break
values_w[n] = self.popvalue()
return values_w
return popvalues
popvalues = _new_popvalues()
popvalues_mutable = _new_popvalues()
del _new_popvalues
@jit.unroll_safe
def peekvalues(self, n):
values_w = [None] * n
base = self.valuestackdepth - n
assert self._check_stack_index(base)
assert base >= 0
while True:
n -= 1
if n < 0:
break
values_w[n] = self.locals_cells_stack_w[base+n]
return values_w
@jit.unroll_safe
def dropvalues(self, n):
n = hint(n, promote=True)
finaldepth = self.valuestackdepth - n
assert self._check_stack_index(finaldepth)
assert finaldepth >= 0
while True:
n -= 1
if n < 0:
break
self.locals_cells_stack_w[finaldepth+n] = None
self.valuestackdepth = finaldepth
@jit.unroll_safe
def pushrevvalues(self, n, values_w): # n should be len(values_w)
make_sure_not_resized(values_w)
while True:
n -= 1
if n < 0:
break
self.pushvalue(values_w[n])
@jit.unroll_safe
def dupvalues(self, n):
delta = n-1
while True:
n -= 1
if n < 0:
break
w_value = self.peekvalue(delta)
self.pushvalue(w_value)
def peekvalue(self, index_from_top=0):
# NOTE: top of the stack is peekvalue(0).
# Contrast this with CPython where it's PEEK(-1).
return ll_assert_not_none(self.peekvalue_maybe_none(index_from_top))
def peekvalue_maybe_none(self, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
assert self._check_stack_index(index)
assert index >= 0
return self.locals_cells_stack_w[index]
def settopvalue(self, w_object, index_from_top=0):
index_from_top = hint(index_from_top, promote=True)
index = self.valuestackdepth + ~index_from_top
assert self._check_stack_index(index)
assert index >= 0
self.locals_cells_stack_w[index] = ll_assert_not_none(w_object)
@jit.unroll_safe
def dropvaluesuntil(self, finaldepth):
depth = self.valuestackdepth - 1
finaldepth = hint(finaldepth, promote=True)
assert finaldepth >= 0
while depth >= finaldepth:
self.locals_cells_stack_w[depth] = None
depth -= 1
self.valuestackdepth = finaldepth
def make_arguments(self, nargs, methodcall=False):
return Arguments(
self.space, self.peekvalues(nargs), methodcall=methodcall)
def argument_factory(self, arguments, keywords, keywords_w, w_star, w_starstar, methodcall=False):
return Arguments(
self.space, arguments, keywords, keywords_w, w_star,
w_starstar, methodcall=methodcall)
def hide(self):
return self.pycode.hidden_applevel
def getcode(self):
return hint(self.pycode, promote=True)
@jit.look_inside_iff(lambda self, scope_w: jit.isvirtual(scope_w))
def setfastscope(self, scope_w):
"""Initialize the fast locals from a list of values,
where the order is according to self.pycode.signature()."""
scope_len = len(scope_w)
if scope_len > self.pycode.co_nlocals:
raise ValueError("new fastscope is longer than the allocated area")
# don't assign directly to 'locals_cells_stack_w[:scope_len]' to be
# virtualizable-friendly
for i in range(scope_len):
self.locals_cells_stack_w[i] = scope_w[i]
self.init_cells()
def getdictscope(self):
"""
Get the locals as a dictionary
"""
self.fast2locals()
return self.debugdata.w_locals
def setdictscope(self, w_locals):
"""
Initialize the locals from a dictionary.
"""
self.getorcreatedebug().w_locals = w_locals
self.locals2fast()
@jit.unroll_safe
def fast2locals(self):
# Copy values from the fastlocals to self.w_locals
d = self.getorcreatedebug()
if d.w_locals is None:
d.w_locals = self.space.newdict()
varnames = self.getcode().getvarnames()
for i in range(min(len(varnames), self.getcode().co_nlocals)):
name = varnames[i]
w_value = self.locals_cells_stack_w[i]
if w_value is not None:
self.space.setitem_str(d.w_locals, name, w_value)
else:
w_name = self.space.newtext(name)
try:
self.space.delitem(d.w_locals, w_name)
except OperationError as e:
if not e.match(self.space, self.space.w_KeyError):
raise
# cellvars are values exported to inner scopes
# freevars are values coming from outer scopes
# (see locals2fast for why CO_OPTIMIZED)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
try:
w_value = cell.get()
except ValueError:
pass
else:
self.space.setitem_str(d.w_locals, name, w_value)
@jit.unroll_safe
def locals2fast(self):
# Copy values from self.w_locals to the fastlocals
w_locals = self.getorcreatedebug().w_locals
assert w_locals is not None
varnames = self.getcode().getvarnames()
numlocals = self.getcode().co_nlocals
new_fastlocals_w = [None] * numlocals
for i in range(min(len(varnames), numlocals)):
name = varnames[i]
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
new_fastlocals_w[i] = w_value
self.setfastscope(new_fastlocals_w)
freevarnames = self.pycode.co_cellvars
if self.pycode.co_flags & consts.CO_OPTIMIZED:
freevarnames = freevarnames + self.pycode.co_freevars
# If the namespace is unoptimized, then one of the
# following cases applies:
# 1. It does not contain free variables, because it
# uses import * or is a top-level namespace.
# 2. It is a class namespace.
# We don't want to accidentally copy free variables
# into the locals dict used by the class.
for i in range(len(freevarnames)):
name = freevarnames[i]
cell = self._getcell(i)
w_value = self.space.finditem_str(w_locals, name)
if w_value is not None:
cell.set(w_value)
@jit.unroll_safe
def init_cells(self):
"""
Initialize cellvars from self.locals_cells_stack_w.
"""
args_to_copy = self.pycode._args_as_cellvars
index = self.pycode.co_nlocals
for i in range(len(args_to_copy)):
argnum = args_to_copy[i]
if argnum >= 0:
cell = self.locals_cells_stack_w[index]
assert isinstance(cell, Cell)
cell.set(self.locals_cells_stack_w[argnum])
index += 1
def getclosure(self):
return None
def fget_code(self, space):
return self.getcode()
def fget_getdictscope(self, space):
return self.getdictscope()
def fget_w_globals(self, space):
# bit silly, but GetSetProperty passes a space
return self.get_w_globals()
### line numbers ###
def fget_f_lineno(self, space):
"Returns the line number of the instruction currently being executed."
if self.get_w_f_trace() is None:
return space.newint(self.get_last_lineno())
else:
return space.newint(self.getorcreatedebug().f_lineno)
def fset_f_lineno(self, space, w_new_lineno):
"Change the line number of the instruction currently being executed."
try:
new_lineno = space.int_w(w_new_lineno)
except OperationError:
raise oefmt(space.w_ValueError, "lineno must be an integer")
if self.get_w_f_trace() is None:
raise oefmt(space.w_ValueError,
"f_lineno can only be set by a trace function.")
line = self.pycode.co_firstlineno
if new_lineno < line:
raise oefmt(space.w_ValueError,
"line %d comes before the current code.", new_lineno)
elif new_lineno == line:
new_lasti = 0
else:
new_lasti = -1
addr = 0
lnotab = self.pycode.co_lnotab
for offset in xrange(0, len(lnotab), 2):
addr += ord(lnotab[offset])
line += ord(lnotab[offset + 1])
if line >= new_lineno:
new_lasti = addr
new_lineno = line
break
if new_lasti == -1:
raise oefmt(space.w_ValueError,
"line %d comes after the current code.", new_lineno)
# Don't jump to a line with an except in it.
code = self.pycode.co_code
if ord(code[new_lasti]) in (DUP_TOP, POP_TOP):
raise oefmt(space.w_ValueError,
"can't jump to 'except' line as there's no exception")
# Don't jump inside or out of an except or a finally block.
# Note that CPython doesn't check except blocks,
# but that results in crashes (tested on 3.5.2+).
f_lasti_handler_addr = -1
new_lasti_handler_addr = -1
blockstack = [] # current blockstack (addresses of SETUP_*)
endblock = [-1] # current finally/except block stack
addr = 0
while addr < len(code):
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
SETUP_ASYNC_WITH):
blockstack.append(addr)
elif op == POP_BLOCK:
if len(blockstack) == 0:
raise oefmt(space.w_SystemError,
"POP_BLOCK not properly nested in this bytecode")
setup_op = ord(code[blockstack.pop()])
if setup_op != SETUP_LOOP:
endblock.append(addr)
elif op == END_FINALLY:
if len(endblock) <= 1:
raise oefmt(space.w_SystemError,
"END_FINALLY not properly nested in this bytecode")
endblock.pop()
if addr == new_lasti:
new_lasti_handler_addr = endblock[-1]
if addr == self.last_instr:
f_lasti_handler_addr = endblock[-1]
if op >= HAVE_ARGUMENT:
addr += 3
else:
addr += 1
if len(blockstack) != 0 or len(endblock) != 1:
raise oefmt(space.w_SystemError,
"blocks not properly nested in this bytecode")
if new_lasti_handler_addr != f_lasti_handler_addr:
raise oefmt(space.w_ValueError,
"can't jump into or out of an 'expect' or "
"'finally' block (%d -> %d)",
f_lasti_handler_addr, new_lasti_handler_addr)
# now we know we're not jumping into or out of a place which
# needs a SysExcInfoRestorer. Check that we're not jumping
# *into* a block, but only (potentially) out of some blocks.
if new_lasti < self.last_instr:
min_addr = new_lasti
max_addr = self.last_instr
else:
min_addr = self.last_instr
max_addr = new_lasti
delta_iblock = min_delta_iblock = 0 # see below for comment
addr = min_addr
while addr < max_addr:
op = ord(code[addr])
if op in (SETUP_LOOP, SETUP_EXCEPT, SETUP_FINALLY, SETUP_WITH,
SETUP_ASYNC_WITH):
delta_iblock += 1
elif op == POP_BLOCK:
delta_iblock -= 1
if delta_iblock < min_delta_iblock:
min_delta_iblock = delta_iblock
if op >= HAVE_ARGUMENT:
addr += 3
else:
addr += 1
# 'min_delta_iblock' is <= 0; its absolute value is the number of
# blocks we exit. 'go_iblock' is the delta number of blocks
# between the last_instr and the new_lasti, in this order.
if new_lasti > self.last_instr:
go_iblock = delta_iblock
else:
go_iblock = -delta_iblock
if go_iblock > min_delta_iblock:
raise oefmt(space.w_ValueError,
"can't jump into the middle of a block")
assert go_iblock <= 0
for ii in range(-go_iblock):
block = self.pop_block()
block.cleanupstack(self)
self.getorcreatedebug().f_lineno = new_lineno
self.last_instr = new_lasti
def get_last_lineno(self):
"Returns the line number of the instruction currently being executed."
return pytraceback.offset2lineno(self.pycode, self.last_instr)
def fget_f_builtins(self, space):
return self.get_builtin().getdict(space)
def fget_f_back(self, space):
f_back = ExecutionContext.getnextframe_nohidden(self)
return f_back
def fget_f_lasti(self, space):
return self.space.newint(self.last_instr)
def fget_f_trace(self, space):
return self.get_w_f_trace()
def fset_f_trace(self, space, w_trace):
if space.is_w(w_trace, space.w_None):
self.getorcreatedebug().w_f_trace = None
else:
d = self.getorcreatedebug()
d.w_f_trace = w_trace
d.f_lineno = self.get_last_lineno()
def fdel_f_trace(self, space):
self.getorcreatedebug().w_f_trace = None
def fget_f_restricted(self, space):
if space.config.objspace.honor__builtins__:
return space.newbool(self.builtin is not space.builtin)
return space.w_False
def get_generator(self):
if self.space.config.translation.rweakref:
return self.f_generator_wref()
else:
return self.f_generator_nowref
def descr_clear(self, space):
"""F.clear(): clear most references held by the frame"""
# Clears a random subset of the attributes: the local variables
# and the w_locals. Note that CPython doesn't clear f_locals
# (which can create leaks) but it's hard to notice because
# the next Python-level read of 'frame.f_locals' will clear it.
if not self.frame_finished_execution:
if not self._is_generator_or_coroutine():
raise oefmt(space.w_RuntimeError,
"cannot clear an executing frame")
gen = self.get_generator()
if gen is not None:
if gen.running:
raise oefmt(space.w_RuntimeError,
"cannot clear an executing frame")
# xxx CPython raises the RuntimeWarning "coroutine was never
# awaited" in this case too. Does it make any sense?
gen.descr_close()
debug = self.getdebug()
if debug is not None:
debug.w_f_trace = None
if debug.w_locals is not None:
debug.w_locals = space.newdict()
# clear the locals, including the cell/free vars, and the stack
for i in range(len(self.locals_cells_stack_w)):
w_oldvalue = self.locals_cells_stack_w[i]
if isinstance(w_oldvalue, Cell):
w_newvalue = Cell()
else:
w_newvalue = None
self.locals_cells_stack_w[i] = w_newvalue
self.valuestackdepth = 0
self.lastblock = None # the FrameBlock chained list
def _convert_unexpected_exception(self, e):
from pypy.interpreter import error
operr = error.get_converted_unexpected_exception(self.space, e)
pytraceback.record_application_traceback(
self.space, operr, self, self.last_instr)
raise operr
class LLHelpers(AbstractLLHelpers):
from rpython.rtyper.annlowlevel import llstr, llunicode
@staticmethod
@jit.elidable
def ll_str_mul(s, times):
if times < 0:
times = 0
try:
size = ovfcheck(len(s.chars) * times)
except OverflowError:
raise MemoryError
newstr = s.malloc(size)
i = 0
if i < size:
s.copy_contents(s, newstr, 0, 0, len(s.chars))
i += len(s.chars)
while i < size:
if i <= size - i:
j = i
else:
j = size - i
s.copy_contents(newstr, newstr, 0, i, j)
i += j
return newstr
@staticmethod
@jit.elidable
def ll_char_mul(ch, times):
if typeOf(ch) is Char:
malloc = mallocstr
else:
malloc = mallocunicode
if times < 0:
times = 0
newstr = malloc(times)
j = 0
# XXX we can use memset here, not sure how useful this is
while j < times:
newstr.chars[j] = ch
j += 1
return newstr
@staticmethod
def ll_strlen(s):
return len(s.chars)
@staticmethod
@signature(types.any(), types.int(), returns=types.any())
def ll_stritem_nonneg(s, i):
chars = s.chars
ll_assert(i >= 0, "negative str getitem index")
ll_assert(i < len(chars), "str getitem index out of bound")
return chars[i]
@staticmethod
def ll_chr2str(ch):
if typeOf(ch) is Char:
malloc = mallocstr
else:
malloc = mallocunicode
s = malloc(1)
s.chars[0] = ch
return s
# @jit.look_inside_iff(lambda str: jit.isconstant(len(str.chars)) and len(str.chars) == 1)
@staticmethod
@jit.oopspec("str.str2unicode(str)")
def ll_str2unicode(str):
lgt = len(str.chars)
s = mallocunicode(lgt)
for i in range(lgt):
if ord(str.chars[i]) > 127:
raise UnicodeDecodeError
s.chars[i] = cast_primitive(UniChar, str.chars[i])
return s
@staticmethod
def ll_str2bytearray(str):
from rpython.rtyper.lltypesystem.rbytearray import BYTEARRAY
lgt = len(str.chars)
b = malloc(BYTEARRAY, lgt)
for i in range(lgt):
b.chars[i] = str.chars[i]
return b
@staticmethod
def ll_strhash(s):
if s:
return jit.conditional_call_elidable(s.hash, LLHelpers._ll_strhash,
s)
else:
return 0
@staticmethod
@dont_inline
@jit.dont_look_inside
def _ll_strhash(s):
# unlike CPython, there is no reason to avoid to return -1
# but our malloc initializes the memory to zero, so we use zero as the
# special non-computed-yet value. Also, jit.conditional_call_elidable
# always checks for zero, for now.
x = ll_hash_string(s)
if x == 0:
x = 29872897
s.hash = x
return x
@staticmethod
def ll_length(s):
return len(s.chars)
@staticmethod
def ll_strfasthash(s):
ll_assert(s.hash != 0, "ll_strfasthash: hash==0")
return s.hash # assumes that the hash is already computed
@staticmethod
@jit.elidable
@jit.oopspec('stroruni.concat(s1, s2)')
def ll_strconcat(s1, s2):
len1 = s1.length()
len2 = s2.length()
# a single '+' like this is allowed to overflow: it gets
# a negative result, and the gc will complain
# the typechecks below are if TP == BYTEARRAY
if typeOf(s1) == Ptr(STR):
newstr = s2.malloc(len1 + len2)
newstr.copy_contents_from_str(s1, newstr, 0, 0, len1)
else:
newstr = s1.malloc(len1 + len2)
newstr.copy_contents(s1, newstr, 0, 0, len1)
if typeOf(s2) == Ptr(STR):
newstr.copy_contents_from_str(s2, newstr, 0, len1, len2)
else:
newstr.copy_contents(s2, newstr, 0, len1, len2)
return newstr
@staticmethod
@jit.elidable
def ll_strip(s, ch, left, right):
s_len = len(s.chars)
if s_len == 0:
return s.empty()
lpos = 0
rpos = s_len - 1
if left:
while lpos < rpos and s.chars[lpos] == ch:
lpos += 1
if right:
while lpos < rpos + 1 and s.chars[rpos] == ch:
rpos -= 1
if rpos < lpos:
return s.empty()
r_len = rpos - lpos + 1
result = s.malloc(r_len)
s.copy_contents(s, result, lpos, 0, r_len)
return result
@staticmethod
@jit.elidable
def ll_strip_default(s, left, right):
s_len = len(s.chars)
if s_len == 0:
return s.empty()
lpos = 0
rpos = s_len - 1
if left:
while lpos < rpos and s.chars[lpos].isspace():
lpos += 1
if right:
while lpos < rpos + 1 and s.chars[rpos].isspace():
rpos -= 1
if rpos < lpos:
return s.empty()
r_len = rpos - lpos + 1
result = s.malloc(r_len)
s.copy_contents(s, result, lpos, 0, r_len)
return result
@staticmethod
@jit.elidable
def ll_strip_multiple(s, s2, left, right):
s_len = len(s.chars)
if s_len == 0:
return s.empty()
lpos = 0
rpos = s_len - 1
if left:
while lpos < rpos and LLHelpers.ll_contains(s2, s.chars[lpos]):
lpos += 1
if right:
while lpos < rpos + 1 and LLHelpers.ll_contains(s2, s.chars[rpos]):
rpos -= 1
if rpos < lpos:
return s.empty()
r_len = rpos - lpos + 1
result = s.malloc(r_len)
s.copy_contents(s, result, lpos, 0, r_len)
return result
@staticmethod
@jit.elidable
def ll_upper(s):
s_chars = s.chars
s_len = len(s_chars)
if s_len == 0:
return s.empty()
i = 0
result = mallocstr(s_len)
# ^^^^^^^^^ specifically to explode on unicode
while i < s_len:
result.chars[i] = LLHelpers.ll_upper_char(s_chars[i])
i += 1
return result
@staticmethod
@jit.elidable
def ll_lower(s):
s_chars = s.chars
s_len = len(s_chars)
if s_len == 0:
return s.empty()
i = 0
result = mallocstr(s_len)
# ^^^^^^^^^ specifically to explode on unicode
while i < s_len:
result.chars[i] = LLHelpers.ll_lower_char(s_chars[i])
i += 1
return result
@staticmethod
def ll_join(s, length, items):
s_chars = s.chars
s_len = len(s_chars)
num_items = length
if num_items == 0:
return s.empty()
itemslen = 0
i = 0
while i < num_items:
try:
itemslen = ovfcheck(itemslen + len(items[i].chars))
except OverflowError:
raise MemoryError
i += 1
try:
seplen = ovfcheck(s_len * (num_items - 1))
except OverflowError:
raise MemoryError
# a single '+' at the end is allowed to overflow: it gets
# a negative result, and the gc will complain
result = s.malloc(itemslen + seplen)
res_index = len(items[0].chars)
s.copy_contents(items[0], result, 0, 0, res_index)
i = 1
while i < num_items:
s.copy_contents(s, result, 0, res_index, s_len)
res_index += s_len
lgt = len(items[i].chars)
s.copy_contents(items[i], result, 0, res_index, lgt)
res_index += lgt
i += 1
return result
@staticmethod
@jit.elidable
@jit.oopspec('stroruni.cmp(s1, s2)')
def ll_strcmp(s1, s2):
if not s1 and not s2:
return True
if not s1 or not s2:
return False
chars1 = s1.chars
chars2 = s2.chars
len1 = len(chars1)
len2 = len(chars2)
if len1 < len2:
cmplen = len1
else:
cmplen = len2
i = 0
while i < cmplen:
diff = ord(chars1[i]) - ord(chars2[i])
if diff != 0:
return diff
i += 1
return len1 - len2
@staticmethod
@jit.elidable
@jit.oopspec('stroruni.equal(s1, s2)')
def ll_streq(s1, s2):
if s1 == s2: # also if both are NULLs
return True
if not s1 or not s2:
return False
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 != len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
while j < len1:
if chars1[j] != chars2[j]:
return False
j += 1
return True
@staticmethod
@jit.elidable
def ll_startswith(s1, s2):
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 < len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
while j < len2:
if chars1[j] != chars2[j]:
return False
j += 1
return True
@staticmethod
def ll_startswith_char(s, ch):
if not len(s.chars):
return False
return s.chars[0] == ch
@staticmethod
@jit.elidable
def ll_endswith(s1, s2):
len1 = len(s1.chars)
len2 = len(s2.chars)
if len1 < len2:
return False
j = 0
chars1 = s1.chars
chars2 = s2.chars
offset = len1 - len2
while j < len2:
if chars1[offset + j] != chars2[j]:
return False
j += 1
return True
@staticmethod
def ll_endswith_char(s, ch):
if not len(s.chars):
return False
return s.chars[len(s.chars) - 1] == ch
@staticmethod
@jit.elidable
@signature(types.any(),
types.any(),
types.int(),
types.int(),
returns=types.int())
def ll_find_char(s, ch, start, end):
i = start
if end > len(s.chars):
end = len(s.chars)
while i < end:
if s.chars[i] == ch:
return i
i += 1
return -1
@staticmethod
@jit.elidable
@signature(types.any(),
types.any(),
types.int(),
types.int(),
returns=types.int())
def ll_rfind_char(s, ch, start, end):
if end > len(s.chars):
end = len(s.chars)
i = end
while i > start:
i -= 1
if s.chars[i] == ch:
return i
return -1
@staticmethod
@jit.elidable
def ll_count_char(s, ch, start, end):
count = 0
i = start
if end > len(s.chars):
end = len(s.chars)
while i < end:
if s.chars[i] == ch:
count += 1
i += 1
return count
@staticmethod
@signature(types.any(),
types.any(),
types.int(),
types.int(),
returns=types.int())
def ll_find(s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return -1
m = len(s2.chars)
if m == 1:
return LLHelpers.ll_find_char(s1, s2.chars[0], start, end)
return LLHelpers.ll_search(s1, s2, start, end, FAST_FIND)
@staticmethod
@signature(types.any(),
types.any(),
types.int(),
types.int(),
returns=types.int())
def ll_rfind(s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return -1
m = len(s2.chars)
if m == 1:
return LLHelpers.ll_rfind_char(s1, s2.chars[0], start, end)
return LLHelpers.ll_search(s1, s2, start, end, FAST_RFIND)
@classmethod
def ll_count(cls, s1, s2, start, end):
if start < 0:
start = 0
if end > len(s1.chars):
end = len(s1.chars)
if end - start < 0:
return 0
m = len(s2.chars)
if m == 1:
return cls.ll_count_char(s1, s2.chars[0], start, end)
res = cls.ll_search(s1, s2, start, end, FAST_COUNT)
assert res >= 0
return res
@staticmethod
@jit.elidable
def ll_search(s1, s2, start, end, mode):
count = 0
n = end - start
m = len(s2.chars)
if m == 0:
if mode == FAST_COUNT:
return end - start + 1
elif mode == FAST_RFIND:
return end
else:
return start
w = n - m
if w < 0:
if mode == FAST_COUNT:
return 0
return -1
mlast = m - 1
skip = mlast - 1
mask = 0
if mode != FAST_RFIND:
for i in range(mlast):
mask = bloom_add(mask, s2.chars[i])
if s2.chars[i] == s2.chars[mlast]:
skip = mlast - i - 1
mask = bloom_add(mask, s2.chars[mlast])
i = start - 1
while i + 1 <= start + w:
i += 1
if s1.chars[i + m - 1] == s2.chars[m - 1]:
for j in range(mlast):
if s1.chars[i + j] != s2.chars[j]:
break
else:
if mode != FAST_COUNT:
return i
count += 1
i += mlast
continue
if i + m < len(s1.chars):
c = s1.chars[i + m]
else:
c = '\0'
if not bloom(mask, c):
i += m
else:
i += skip
else:
if i + m < len(s1.chars):
c = s1.chars[i + m]
else:
c = '\0'
if not bloom(mask, c):
i += m
else:
mask = bloom_add(mask, s2.chars[0])
for i in range(mlast, 0, -1):
mask = bloom_add(mask, s2.chars[i])
if s2.chars[i] == s2.chars[0]:
skip = i - 1
i = start + w + 1
while i - 1 >= start:
i -= 1
if s1.chars[i] == s2.chars[0]:
for j in xrange(mlast, 0, -1):
if s1.chars[i + j] != s2.chars[j]:
break
else:
return i
if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]):
i -= m
else:
i -= skip
else:
if i - 1 >= 0 and not bloom(mask, s1.chars[i - 1]):
i -= m
if mode != FAST_COUNT:
return -1
return count
@staticmethod
@signature(types.int(), types.any(), returns=types.any())
@jit.look_inside_iff(
lambda length, items: jit.loop_unrolling_heuristic(items, length))
def ll_join_strs(length, items):
# Special case for length 1 items, helps both the JIT and other code
if length == 1:
return items[0]
num_items = length
itemslen = 0
i = 0
while i < num_items:
try:
itemslen = ovfcheck(itemslen + len(items[i].chars))
except OverflowError:
raise MemoryError
i += 1
if typeOf(items).TO.OF.TO == STR:
malloc = mallocstr
copy_contents = copy_string_contents
else:
malloc = mallocunicode
copy_contents = copy_unicode_contents
result = malloc(itemslen)
res_index = 0
i = 0
while i < num_items:
item_chars = items[i].chars
item_len = len(item_chars)
copy_contents(items[i], result, 0, res_index, item_len)
res_index += item_len
i += 1
return result
@staticmethod
@jit.look_inside_iff(lambda length, chars, RES: jit.isconstant(length) and
jit.isvirtual(chars))
def ll_join_chars(length, chars, RES):
# no need to optimize this, will be replaced by string builder
# at some point soon
num_chars = length
if RES is StringRepr.lowleveltype:
target = Char
malloc = mallocstr
else:
target = UniChar
malloc = mallocunicode
result = malloc(num_chars)
res_chars = result.chars
i = 0
while i < num_chars:
res_chars[i] = cast_primitive(target, chars[i])
i += 1
return result
@staticmethod
@jit.oopspec('stroruni.slice(s1, start, stop)')
@signature(types.any(), types.int(), types.int(), returns=types.any())
@jit.elidable
def _ll_stringslice(s1, start, stop):
lgt = stop - start
assert start >= 0
# If start > stop, return a empty string. This can happen if the start
# is greater than the length of the string. Use < instead of <= to avoid
# creating another path for the JIT when start == stop.
if lgt < 0:
return s1.empty()
newstr = s1.malloc(lgt)
s1.copy_contents(s1, newstr, start, 0, lgt)
return newstr
@staticmethod
def ll_stringslice_startonly(s1, start):
return LLHelpers._ll_stringslice(s1, start, len(s1.chars))
@staticmethod
@signature(types.any(), types.int(), types.int(), returns=types.any())
def ll_stringslice_startstop(s1, start, stop):
if jit.we_are_jitted():
if stop > len(s1.chars):
stop = len(s1.chars)
else:
if stop >= len(s1.chars):
if start == 0:
return s1
stop = len(s1.chars)
return LLHelpers._ll_stringslice(s1, start, stop)
@staticmethod
def ll_stringslice_minusone(s1):
newlen = len(s1.chars) - 1
return LLHelpers._ll_stringslice(s1, 0, newlen)
@staticmethod
def ll_split_chr(LIST, s, c, max):
chars = s.chars
strlen = len(chars)
count = 1
i = 0
if max == 0:
i = strlen
while i < strlen:
if chars[i] == c:
count += 1
if max >= 0 and count > max:
break
i += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
i = 0
j = 0
resindex = 0
if max == 0:
j = strlen
while j < strlen:
if chars[j] == c:
item = items[resindex] = s.malloc(j - i)
item.copy_contents(s, item, i, 0, j - i)
resindex += 1
i = j + 1
if max >= 0 and resindex >= max:
j = strlen
break
j += 1
item = items[resindex] = s.malloc(j - i)
item.copy_contents(s, item, i, 0, j - i)
return res
@staticmethod
def ll_split(LIST, s, c, max):
count = 1
if max == -1:
max = len(s.chars)
pos = 0
last = len(s.chars)
markerlen = len(c.chars)
pos = s.find(c, 0, last)
while pos >= 0 and count <= max:
pos = s.find(c, pos + markerlen, last)
count += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
pos = 0
count = 0
pos = s.find(c, 0, last)
prev_pos = 0
if pos < 0:
items[0] = s
return res
while pos >= 0 and count < max:
item = items[count] = s.malloc(pos - prev_pos)
item.copy_contents(s, item, prev_pos, 0, pos - prev_pos)
count += 1
prev_pos = pos + markerlen
pos = s.find(c, pos + markerlen, last)
item = items[count] = s.malloc(last - prev_pos)
item.copy_contents(s, item, prev_pos, 0, last - prev_pos)
return res
@staticmethod
def ll_rsplit_chr(LIST, s, c, max):
chars = s.chars
strlen = len(chars)
count = 1
i = 0
if max == 0:
i = strlen
while i < strlen:
if chars[i] == c:
count += 1
if max >= 0 and count > max:
break
i += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
i = strlen
j = strlen
resindex = count - 1
assert resindex >= 0
if max == 0:
j = 0
while j > 0:
j -= 1
if chars[j] == c:
item = items[resindex] = s.malloc(i - j - 1)
item.copy_contents(s, item, j + 1, 0, i - j - 1)
resindex -= 1
i = j
if resindex == 0:
j = 0
break
item = items[resindex] = s.malloc(i - j)
item.copy_contents(s, item, j, 0, i - j)
return res
@staticmethod
def ll_rsplit(LIST, s, c, max):
count = 1
if max == -1:
max = len(s.chars)
pos = len(s.chars)
markerlen = len(c.chars)
pos = s.rfind(c, 0, pos)
while pos >= 0 and count <= max:
pos = s.rfind(c, 0, pos - markerlen)
count += 1
res = LIST.ll_newlist(count)
items = res.ll_items()
pos = 0
pos = len(s.chars)
prev_pos = pos
pos = s.rfind(c, 0, pos)
if pos < 0:
items[0] = s
return res
count -= 1
while pos >= 0 and count > 0:
item = items[count] = s.malloc(prev_pos - pos - markerlen)
item.copy_contents(s, item, pos + markerlen, 0,
prev_pos - pos - markerlen)
count -= 1
prev_pos = pos
pos = s.rfind(c, 0, pos)
item = items[count] = s.malloc(prev_pos)
item.copy_contents(s, item, 0, 0, prev_pos)
return res
@staticmethod
@jit.elidable
def ll_replace_chr_chr(s, c1, c2):
length = len(s.chars)
newstr = s.malloc(length)
src = s.chars
dst = newstr.chars
j = 0
while j < length:
c = src[j]
if c == c1:
c = c2
dst[j] = c
j += 1
return newstr
@staticmethod
@jit.elidable
def ll_contains(s, c):
chars = s.chars
strlen = len(chars)
i = 0
while i < strlen:
if chars[i] == c:
return True
i += 1
return False
@staticmethod
@jit.elidable
def ll_int(s, base):
if not 2 <= base <= 36:
raise ValueError
chars = s.chars
strlen = len(chars)
i = 0
#XXX: only space is allowed as white space for now
while i < strlen and chars[i] == ' ':
i += 1
if not i < strlen:
raise ValueError
#check sign
sign = 1
if chars[i] == '-':
sign = -1
i += 1
elif chars[i] == '+':
i += 1
# skip whitespaces between sign and digits
while i < strlen and chars[i] == ' ':
i += 1
#now get digits
val = 0
oldpos = i
while i < strlen:
c = ord(chars[i])
if ord('a') <= c <= ord('z'):
digit = c - ord('a') + 10
elif ord('A') <= c <= ord('Z'):
digit = c - ord('A') + 10
elif ord('0') <= c <= ord('9'):
digit = c - ord('0')
else:
break
if digit >= base:
break
val = val * base + digit
i += 1
if i == oldpos:
raise ValueError # catch strings like '+' and '+ '
#skip trailing whitespace
while i < strlen and chars[i] == ' ':
i += 1
if not i == strlen:
raise ValueError
return sign * val
# interface to build strings:
# x = ll_build_start(n)
# ll_build_push(x, next_string, 0)
# ll_build_push(x, next_string, 1)
# ...
# ll_build_push(x, next_string, n-1)
# s = ll_build_finish(x)
@staticmethod
def ll_build_start(parts_count):
return malloc(TEMP, parts_count)
@staticmethod
def ll_build_push(builder, next_string, index):
builder[index] = next_string
@staticmethod
def ll_build_finish(builder):
return LLHelpers.ll_join_strs(len(builder), builder)
@staticmethod
@specialize.memo()
def ll_constant(s):
return string_repr.convert_const(s)
@staticmethod
@specialize.memo()
def ll_constant_unicode(s):
return unicode_repr.convert_const(s)
@classmethod
def do_stringformat(cls, hop, sourcevarsrepr):
s_str = hop.args_s[0]
assert s_str.is_constant()
is_unicode = isinstance(s_str, annmodel.SomeUnicodeString)
if is_unicode:
TEMPBUF = TEMP_UNICODE
else:
TEMPBUF = TEMP
s = s_str.const
things = cls.parse_fmt_string(s)
size = inputconst(Signed, len(things)) # could be unsigned?
cTEMP = inputconst(Void, TEMPBUF)
cflags = inputconst(Void, {'flavor': 'gc'})
vtemp = hop.genop("malloc_varsize", [cTEMP, cflags, size],
resulttype=Ptr(TEMPBUF))
argsiter = iter(sourcevarsrepr)
from rpython.rtyper.rclass import InstanceRepr
for i, thing in enumerate(things):
if isinstance(thing, tuple):
code = thing[0]
vitem, r_arg = argsiter.next()
if not hasattr(r_arg, 'll_str'):
raise TyperError("ll_str unsupported for: %r" % r_arg)
if code == 's':
if is_unicode:
# only UniCharRepr and UnicodeRepr has it so far
vchunk = hop.gendirectcall(r_arg.ll_unicode, vitem)
else:
vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
elif code == 'r' and isinstance(r_arg, InstanceRepr):
vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
elif code == 'd':
assert isinstance(r_arg, IntegerRepr)
#vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
vchunk = hop.gendirectcall(ll_str.ll_int2dec, vitem)
elif code == 'f':
#assert isinstance(r_arg, FloatRepr)
vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
elif code == 'x':
assert isinstance(r_arg, IntegerRepr)
vchunk = hop.gendirectcall(ll_str.ll_int2hex, vitem,
inputconst(Bool, False))
elif code == 'o':
assert isinstance(r_arg, IntegerRepr)
vchunk = hop.gendirectcall(ll_str.ll_int2oct, vitem,
inputconst(Bool, False))
else:
raise TyperError("%%%s is not RPython" % (code, ))
else:
if is_unicode:
vchunk = inputconst(unicode_repr, thing)
else:
vchunk = inputconst(string_repr, thing)
i = inputconst(Signed, i)
if is_unicode and vchunk.concretetype != Ptr(UNICODE):
# if we are here, one of the ll_str.* functions returned some
# STR, so we convert it to unicode. It's a bit suboptimal
# because we do one extra copy.
vchunk = hop.gendirectcall(cls.ll_str2unicode, vchunk)
hop.genop('setarrayitem', [vtemp, i, vchunk])
hop.exception_cannot_occur() # to ignore the ZeroDivisionError of '%'
return hop.gendirectcall(cls.ll_join_strs, size, vtemp)
@staticmethod
@jit.dont_look_inside
def ll_string2list(RESLIST, src):
length = len(src.chars)
lst = RESLIST.ll_newlist(length)
dst = lst.ll_items()
SRC = typeOf(src).TO # STR or UNICODE
DST = typeOf(dst).TO # GcArray
assert DST.OF is SRC.chars.OF
# from here, no GC operations can happen
asrc = llmemory.cast_ptr_to_adr(src) + (llmemory.offsetof(
SRC, 'chars') + llmemory.itemoffsetof(SRC.chars, 0))
adst = llmemory.cast_ptr_to_adr(dst) + llmemory.itemoffsetof(DST, 0)
llmemory.raw_memcopy(asrc, adst, llmemory.sizeof(DST.OF) * length)
# end of "no GC" section
keepalive_until_here(src)
keepalive_until_here(dst)
return lst
class W_CTypeStructOrUnion(W_CType):
_immutable_fields_ = [
'alignment?', '_fields_list?[*]', '_fields_dict?',
'_custom_field_pos?', '_with_var_array?'
]
is_indirect_arg_for_call_python = True
# three possible states:
# - "opaque": for opaque C structs; self.size < 0.
# - "lazy": for non-opaque C structs whose _fields_list, _fields_dict,
# _custom_field_pos and _with_var_array are not filled yet; can be
# filled by calling force_lazy_struct().
# (But self.size and .alignment are already set and won't change.)
# - "forced": for non-opaque C structs which are fully ready.
# fields added by complete_struct_or_union():
alignment = -1
_fields_list = None
_fields_dict = None
_custom_field_pos = False
_with_var_array = False
_with_packed_changed = False
def __init__(self, space, name):
W_CType.__init__(self, space, -1, name, len(name))
def check_complete(self, w_errorcls=None):
# Check ONLY that are are not opaque. Complain if we are.
if self.size < 0:
space = self.space
raise oefmt(w_errorcls or space.w_TypeError,
"'%s' is opaque or not completed yet", self.name)
def force_lazy_struct(self):
# Force a "lazy" struct to become "forced"; complain if we are "opaque".
if self._fields_list is None:
self.check_complete()
#
from pypy.module._cffi_backend import realize_c_type
realize_c_type.do_realize_lazy_struct(self)
def _alignof(self):
self.check_complete(w_errorcls=self.space.w_ValueError)
if self.alignment == -1:
self.force_lazy_struct()
assert self.alignment > 0
return self.alignment
def _fget(self, attrchar):
if attrchar == 'f': # fields
space = self.space
if self.size < 0:
return space.w_None
self.force_lazy_struct()
result = [None] * len(self._fields_list)
for fname, field in self._fields_dict.iteritems():
i = self._fields_list.index(field)
result[i] = space.newtuple([space.newtext(fname), field])
return space.newlist(result)
return W_CType._fget(self, attrchar)
def convert_to_object(self, cdata):
space = self.space
self.check_complete()
return cdataobj.W_CData(space, cdata, self)
def copy_and_convert_to_object(self, source):
space = self.space
self.check_complete()
ptr = lltype.malloc(rffi.CCHARP.TO,
self.size,
flavor='raw',
zero=False)
misc._raw_memcopy(source, ptr, self.size)
return cdataobj.W_CDataNewStd(space, ptr, self)
def typeoffsetof_field(self, fieldname, following):
self.force_lazy_struct()
space = self.space
try:
cfield = self._getcfield_const(fieldname)
except KeyError:
raise OperationError(space.w_KeyError, space.newtext(fieldname))
if cfield.bitshift >= 0:
raise oefmt(space.w_TypeError, "not supported for bitfields")
return (cfield.ctype, cfield.offset)
def _copy_from_same(self, cdata, w_ob):
if isinstance(w_ob, cdataobj.W_CData):
if w_ob.ctype is self and self.size >= 0:
with w_ob as ptr:
misc._raw_memcopy(ptr, cdata, self.size)
return True
return False
def convert_from_object(self, cdata, w_ob):
if not self._copy_from_same(cdata, w_ob):
self.convert_struct_from_object(cdata, w_ob, optvarsize=-1)
@jit.look_inside_iff(
lambda self, cdata, w_ob, optvarsize: jit.isvirtual(w_ob))
def convert_struct_from_object(self, cdata, w_ob, optvarsize):
self.force_lazy_struct()
space = self.space
if (space.isinstance_w(w_ob, space.w_list)
or space.isinstance_w(w_ob, space.w_tuple)):
lst_w = space.listview(w_ob)
j = 0
for w_obj in lst_w:
try:
while (self._fields_list[j].flags
& W_CField.BF_IGNORE_IN_CTOR):
j += 1
except IndexError:
raise oefmt(space.w_ValueError,
"too many initializers for '%s' (got %d)",
self.name, len(lst_w))
optvarsize = self._fields_list[j].write_v(
cdata, w_obj, optvarsize)
j += 1
return optvarsize
elif space.isinstance_w(w_ob, space.w_dict):
lst_w = space.fixedview(w_ob)
for i in range(len(lst_w)):
w_key = lst_w[i]
key = space.text_w(w_key)
try:
cf = self._fields_dict[key]
except KeyError:
space.raise_key_error(w_key)
assert 0
optvarsize = cf.write_v(cdata, space.getitem(w_ob, w_key),
optvarsize)
return optvarsize
else:
if optvarsize == -1:
msg = "list or tuple or dict or struct-cdata"
else:
msg = "list or tuple or dict"
raise self._convert_error(msg, w_ob)
@jit.elidable
def _getcfield_const(self, attr):
return self._fields_dict[attr]
def getcfield(self, attr):
# Returns a W_CField. Error cases: returns None if we are an
# opaque struct; or raises KeyError if the particular field
# 'attr' does not exist. The point of not directly building the
# error here is to get the exact ctype in the error message: it
# might be of the kind 'struct foo' or 'struct foo *'.
if self._fields_dict is None:
if self.size < 0:
return None
self.force_lazy_struct()
self = jit.promote(self)
attr = jit.promote_string(attr)
return self._getcfield_const(attr) # <= KeyError here
def cdata_dir(self):
if self.size < 0:
return []
self.force_lazy_struct()
return self._fields_dict.keys()
ll_builder.current_pos = pos + size
ll_builder.copy_raw_to_string(charp, ll_builder.current_buf, pos, size)
# ------------------------------------------------------------
# builder.getlength()
@always_inline
def ll_getlength(ll_builder):
num_chars_missing_from_last_piece = (
ll_builder.current_end - ll_builder.current_pos)
return ll_builder.total_size - num_chars_missing_from_last_piece
# ------------------------------------------------------------
# builder.build()
@jit.look_inside_iff(lambda ll_builder: jit.isvirtual(ll_builder))
def ll_build(ll_builder):
# NB. usually the JIT doesn't look inside this function; it does
# so only in the simplest example where it could virtualize everything
if ll_builder.extra_pieces:
ll_fold_pieces(ll_builder)
elif ll_builder.current_pos != ll_builder.total_size:
ll_shrink_final(ll_builder)
return ll_builder.current_buf
def ll_shrink_final(ll_builder):
final_size = ll_builder.current_pos
ll_assert(final_size <= ll_builder.total_size,
"final_size > ll_builder.total_size?")
buf = rgc.ll_shrink_array(ll_builder.current_buf, final_size)
ll_builder.current_buf = buf
# int num_live_items;
# int num_ever_used_items;
# int resize_counter;
# {byte, short, int, long} *indexes;
# dictentry *entries;
# lookup_function_no; # one of the four possible functions for different
# # size dicts; the rest of the word is a counter for how
# # many 'entries' at the start are known to be deleted
# (Function DICTKEY, DICTKEY -> bool) *fnkeyeq;
# (Function DICTKEY -> int) *fnkeyhash;
# }
#
#
@jit.look_inside_iff(lambda d, key, hash, flag: jit.isvirtual(d))
@jit.oopspec('ordereddict.lookup(d, key, hash, flag)')
def ll_call_lookup_function(d, key, hash, flag):
fun = d.lookup_function_no & FUNC_MASK
# This likely() here forces gcc to compile the check for fun == FUNC_BYTE
# first. Otherwise, this is a regular switch and gcc (at least 4.7)
# compiles this as a series of checks, with the FUNC_BYTE case last.
# It sounds minor, but it is worth 6-7% on a PyPy microbenchmark.
if likely(fun == FUNC_BYTE):
return ll_dict_lookup(d, key, hash, flag, TYPE_BYTE)
elif fun == FUNC_SHORT:
return ll_dict_lookup(d, key, hash, flag, TYPE_SHORT)
elif IS_64BIT and fun == FUNC_INT:
return ll_dict_lookup(d, key, hash, flag, TYPE_INT)
elif fun == FUNC_LONG:
return ll_dict_lookup(d, key, hash, flag, TYPE_LONG)
