def create_iter(self,
shape=None,
backward_broadcast=False,
require_index=False):
if shape is not None and \
support.product(shape) > support.product(self.get_shape()):
r = calculate_broadcast_strides(self.get_strides(),
self.get_backstrides(),
self.get_shape(), shape,
backward_broadcast)
return iter.MultiDimViewIterator(self, self.dtype, self.start,
r[0], r[1], shape)
if not require_index:
return iter.ConcreteArrayIterator(self)
else:
if len(self.get_shape()) == 1:
return iter.OneDimViewIterator(self, self.dtype, self.start,
self.get_strides(),
self.get_shape())
else:
return iter.MultiDimViewIterator(self, self.dtype, self.start,
self.get_strides(),
self.get_backstrides(),
self.get_shape())
def create_iter(self, shape=None, backward_broadcast=False):
if shape is not None and \
support.product(shape) > support.product(self.get_shape()):
r = calculate_broadcast_strides(self.get_strides(),
self.get_backstrides(),
self.get_shape(), shape,
backward_broadcast)
i = ArrayIter(self, support.product(shape), shape, r[0], r[1])
else:
i = ArrayIter(self, self.get_size(), self.shape,
self.strides, self.backstrides)
return i, i.reset()
def set_shape(self, space, orig_array, new_shape):
if len(new_shape) > NPY.MAXDIMS:
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d",
NPY.MAXDIMS)
try:
support.product(new_shape) * self.dtype.elsize
except OverflowError as e:
raise oefmt(space.w_ValueError, "array is too big")
strides, backstrides = calc_strides(new_shape, self.dtype, self.order)
return SliceArray(self.start, strides, backstrides, new_shape, self,
orig_array)
def create_iter(self, shape=None, backward_broadcast=False):
if shape is not None and \
support.product(shape) > support.product(self.get_shape()):
r = calculate_broadcast_strides(self.get_strides(),
self.get_backstrides(),
self.get_shape(), shape,
backward_broadcast)
i = ArrayIter(self, support.product(shape), shape, r[0], r[1])
else:
i = ArrayIter(self, self.get_size(), self.shape, self.strides,
self.backstrides)
return i, i.reset()
def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
order='C', owning=False, w_subtype=None,
w_base=None, writable=True, strides=None, start=0):
from pypy.module.micronumpy import concrete
from pypy.module.micronumpy.strides import (calc_strides,
calc_backstrides)
isize = dtype.elsize
if storage_bytes > 0 :
totalsize = support.product(shape) * isize
if totalsize > storage_bytes:
raise OperationError(space.w_TypeError, space.wrap(
"buffer is too small for requested array"))
else:
storage_bytes = support.product(shape) * isize
if strides is None:
strides, backstrides = calc_strides(shape, dtype, order)
else:
if len(strides) != len(shape):
raise oefmt(space.w_ValueError,
'strides, if given, must be the same length as shape')
for i in range(len(strides)):
if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
raise oefmt(space.w_ValueError,
'strides is incompatible with shape of requested '
'array and size of buffer')
backstrides = calc_backstrides(strides, shape)
if w_base is not None:
if owning:
raise OperationError(space.w_ValueError,
space.wrap("Cannot have owning=True when specifying a buffer"))
if writable:
impl = concrete.ConcreteArrayWithBase(shape, dtype, order,
strides, backstrides, storage, w_base,
start=start)
else:
impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order,
strides, backstrides,
storage, w_base)
elif owning:
# Will free storage when GCd
impl = concrete.ConcreteArray(shape, dtype, order, strides,
backstrides, storage=storage)
else:
impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides,
backstrides, storage)
if w_subtype:
w_ret = space.allocate_instance(W_NDimArray, w_subtype)
W_NDimArray.__init__(w_ret, impl)
space.call_method(w_ret, '__array_finalize__', w_subtype)
return w_ret
return W_NDimArray(impl)
def from_shape_and_storage(space, shape, storage, dtype, storage_bytes=-1,
order='C', owning=False, w_subtype=None,
w_base=None, writable=True, strides=None):
from pypy.module.micronumpy import concrete
from pypy.module.micronumpy.strides import (calc_strides,
calc_backstrides)
isize = dtype.elsize
if storage_bytes > 0 :
totalsize = support.product(shape) * isize
if totalsize > storage_bytes:
raise OperationError(space.w_TypeError, space.wrap(
"buffer is too small for requested array"))
else:
storage_bytes = support.product(shape) * isize
if strides is None:
strides, backstrides = calc_strides(shape, dtype, order)
else:
if len(strides) != len(shape):
raise oefmt(space.w_ValueError,
'strides, if given, must be the same length as shape')
for i in range(len(strides)):
if strides[i] < 0 or strides[i]*shape[i] > storage_bytes:
raise oefmt(space.w_ValueError,
'strides is incompatible with shape of requested '
'array and size of buffer')
backstrides = calc_backstrides(strides, shape)
if w_base is not None:
if owning:
raise OperationError(space.w_ValueError,
space.wrap("Cannot have owning=True when specifying a buffer"))
if writable:
impl = concrete.ConcreteArrayWithBase(shape, dtype, order, strides,
backstrides, storage, w_base)
else:
impl = concrete.ConcreteNonWritableArrayWithBase(shape, dtype, order,
strides, backstrides,
storage, w_base)
elif owning:
# Will free storage when GCd
impl = concrete.ConcreteArray(shape, dtype, order, strides,
backstrides, storage=storage)
else:
impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides,
backstrides, storage)
if w_subtype:
w_ret = space.allocate_instance(W_NDimArray, w_subtype)
W_NDimArray.__init__(w_ret, impl)
space.call_method(w_ret, '__array_finalize__', w_subtype)
return w_ret
return W_NDimArray(impl)
def test_iterator_step(self):
#iteration in C order with #contiguous layout => strides[-1] is 1
#skip less than the shape
shape = [3, 5]
strides = [5, 1]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [10, 4]
i = ArrayIter(MockArray, support.product(shape), shape,
strides, backstrides)
s = i.reset()
s = i.next_skip_x(s, 2)
s = i.next_skip_x(s, 2)
s = i.next_skip_x(s, 2)
assert s.offset == 6
assert not i.done(s)
assert s.indices == [1,1]
#And for some big skips
s = i.next_skip_x(s, 5)
assert s.offset == 11
assert s.indices == [2,1]
s = i.next_skip_x(s, 5)
# Note: the offset does not overflow but recycles,
# this is good for broadcast
assert s.offset == 1
assert s.indices == [0,1]
assert i.done(s)
#Now what happens if the array is transposed? strides[-1] != 1
# therefore layout is non-contiguous
strides = [1, 3]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [2, 12]
i = ArrayIter(MockArray, support.product(shape), shape,
strides, backstrides)
s = i.reset()
s = i.next_skip_x(s, 2)
s = i.next_skip_x(s, 2)
s = i.next_skip_x(s, 2)
assert s.offset == 4
assert s.indices == [1,1]
assert not i.done(s)
s = i.next_skip_x(s, 5)
assert s.offset == 5
assert s.indices == [2,1]
assert not i.done(s)
s = i.next_skip_x(s, 5)
assert s.indices == [0,1]
assert s.offset == 3
assert i.done(s)
def create_iter(self, shape=None, backward_broadcast=False, require_index=False):
if shape is not None and \
support.product(shape) > support.product(self.get_shape()):
r = calculate_broadcast_strides(self.get_strides(),
self.get_backstrides(),
self.get_shape(), shape,
backward_broadcast)
return iter.MultiDimViewIterator(self.parent, self.dtype,
self.start, r[0], r[1], shape)
if len(self.get_shape()) == 1:
return iter.OneDimViewIterator(self.parent, self.dtype, self.start,
self.get_strides(), self.get_shape())
return iter.MultiDimViewIterator(self.parent, self.dtype, self.start,
self.get_strides(),
self.get_backstrides(), self.get_shape())
def _zeros_or_empty(space, w_shape, w_dtype, w_order, zero):
dtype = space.interp_w(descriptor.W_Dtype,
space.call_function(space.gettypefor(descriptor.W_Dtype), w_dtype))
if dtype.is_str_or_unicode() and dtype.elsize < 1:
dtype = descriptor.variable_dtype(space, dtype.char + '1')
shape = shape_converter(space, w_shape, dtype)
for dim in shape:
if dim < 0:
raise OperationError(space.w_ValueError, space.wrap(
"negative dimensions are not allowed"))
try:
support.product(shape)
except OverflowError:
raise OperationError(space.w_ValueError, space.wrap(
"array is too big."))
return W_NDimArray.from_shape(space, shape, dtype=dtype, zero=zero)
def __init__(self, shape, dtype, order, strides, backstrides,
storage=lltype.nullptr(RAW_STORAGE), zero=True):
if storage == lltype.nullptr(RAW_STORAGE):
storage = dtype.itemtype.malloc(support.product(shape) *
dtype.elsize, zero=zero)
ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
storage)
def get_iter(self, space, i):
arr = self.seq[i]
imp = arr.implementation
if arr.is_scalar():
return ConcreteIter(imp, 1, [], [], [], self.op_flags[i], self)
shape = self.shape
if self.external_loop and len(self.seq) < 2 and self.buffered:
# Special case, always return a memory-ordered iterator
stride = imp.dtype.elsize
backstride = imp.size * stride - stride
return ConcreteIter(
imp, imp.get_size(), [support.product(shape)], [stride], [backstride], self.op_flags[i], self
)
backward = imp.order != self.order
# XXX cleanup needed
strides = imp.strides
backstrides = imp.backstrides
if self.allow_backward:
if (abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or (
abs(imp.strides[0]) > abs(imp.strides[-1]) and backward
):
# flip the strides. Is this always true for multidimension?
strides = imp.strides[:]
backstrides = imp.backstrides[:]
shape = imp.shape[:]
strides.reverse()
backstrides.reverse()
shape.reverse()
r = calculate_broadcast_strides(strides, backstrides, imp.shape, shape, backward)
iter_shape = shape
if len(shape) != len(r[0]):
# shape can be shorter when using an external loop, just return a view
iter_shape = imp.shape
return ConcreteIter(imp, imp.get_size(), iter_shape, r[0], r[1], self.op_flags[i], self)
def get_iter(self, space, i):
arr = self.seq[i]
imp = arr.implementation
if arr.is_scalar():
return ConcreteIter(imp, 1, [], [], [], self.op_flags[i], self)
shape = self.shape
if (self.external_loop and len(self.seq) < 2 and self.buffered):
# Special case, always return a memory-ordered iterator
stride = imp.dtype.elsize
backstride = imp.size * stride - stride
return ConcreteIter(imp, imp.get_size(), [support.product(shape)],
[stride], [backstride], self.op_flags[i], self)
backward = imp.order != self.order
# XXX cleanup needed
strides = imp.strides
backstrides = imp.backstrides
if self.allow_backward:
if ((abs(imp.strides[0]) < abs(imp.strides[-1]) and not backward) or \
(abs(imp.strides[0]) > abs(imp.strides[-1]) and backward)):
# flip the strides. Is this always true for multidimension?
strides = imp.strides[:]
backstrides = imp.backstrides[:]
shape = imp.shape[:]
strides.reverse()
backstrides.reverse()
shape.reverse()
r = calculate_broadcast_strides(strides, backstrides, imp.shape, shape,
backward)
iter_shape = shape
if len(shape) != len(r[0]):
# shape can be shorter when using an external loop, just return a view
iter_shape = imp.shape
return ConcreteIter(imp, imp.get_size(), iter_shape, r[0], r[1],
self.op_flags[i], self)
def _zeros_or_empty(space, w_shape, w_dtype, w_order, zero):
dtype = space.interp_w(descriptor.W_Dtype,
space.call_function(space.gettypefor(descriptor.W_Dtype), w_dtype))
if dtype.is_str_or_unicode() and dtype.elsize < 1:
dtype = descriptor.variable_dtype(space, dtype.char + '1')
shape = shape_converter(space, w_shape, dtype)
for dim in shape:
if dim < 0:
raise OperationError(space.w_ValueError, space.wrap(
"negative dimensions are not allowed"))
try:
support.product(shape)
except OverflowError:
raise OperationError(space.w_ValueError, space.wrap(
"array is too big."))
return W_NDimArray.from_shape(space, shape, dtype=dtype, zero=zero)
def __init__(self,
start,
strides,
backstrides,
shape,
parent,
orig_arr,
dtype=None):
self.strides = strides
self.backstrides = backstrides
self.shape = shape
if dtype is None:
dtype = parent.dtype
if isinstance(parent, SliceArray):
parent = parent.parent # one level only
self.parent = parent
self.storage = parent.storage
self.gcstruct = parent.gcstruct
self.order = parent.order
self.dtype = dtype
self.size = support.product(shape) * self.dtype.elsize
self.start = start
self.orig_arr = orig_arr
flags = parent.flags & NPY.ARRAY_ALIGNED
flags |= parent.flags & NPY.ARRAY_WRITEABLE
if is_c_contiguous(self):
flags |= NPY.ARRAY_C_CONTIGUOUS
if is_f_contiguous(self):
flags |= NPY.ARRAY_F_CONTIGUOUS
self.flags = flags
def __init__(self,
shape,
dtype,
order,
strides,
backstrides,
storage,
start=0):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
# already tested for overflow in from_shape_and_storage
self.size = support.product(shape) * dtype.elsize
if order not in (NPY.CORDER, NPY.FORTRANORDER):
raise oefmt(
dtype.itemtype.space.w_ValueError,
"ConcreteArrayNotOwning but order is not 0,1 rather %d", order)
self.order = order
self.dtype = dtype
self.strides = strides
self.backstrides = backstrides
self.storage = storage
self.start = start
self.gcstruct = V_OBJECTSTORE
def __init__(self,
shape,
dtype,
order,
strides,
backstrides,
storage=lltype.nullptr(RAW_STORAGE),
zero=True):
gcstruct = V_OBJECTSTORE
if storage == lltype.nullptr(RAW_STORAGE):
length = support.product(shape)
if dtype.num == NPY.OBJECT:
storage = dtype.itemtype.malloc(length * dtype.elsize,
zero=True)
gcstruct = _create_objectstore(storage, length, dtype.elsize)
else:
storage = dtype.itemtype.malloc(length * dtype.elsize,
zero=zero)
start = calc_start(shape, strides)
ConcreteArrayNotOwning.__init__(self,
shape,
dtype,
order,
strides,
backstrides,
storage,
start=start)
self.gcstruct = gcstruct
self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
if is_c_contiguous(self):
self.flags |= NPY.ARRAY_C_CONTIGUOUS
if is_f_contiguous(self):
self.flags |= NPY.ARRAY_F_CONTIGUOUS
def from_shape(space, shape, dtype, order="C", w_instance=None, zero=True):
from pypy.module.micronumpy import concrete, descriptor, boxes
from pypy.module.micronumpy.strides import calc_strides
if len(shape) > NPY.MAXDIMS:
raise oefmt(space.w_ValueError, "sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
support.product(shape) * dtype.elsize
except OverflowError as e:
raise oefmt(space.w_ValueError, "array is too big")
strides, backstrides = calc_strides(shape, dtype.base, order)
impl = concrete.ConcreteArray(shape, dtype.base, order, strides, backstrides, zero=zero)
if dtype == descriptor.get_dtype_cache(space).w_objectdtype:
impl.fill(space, boxes.W_ObjectBox(space.w_None))
if w_instance:
return wrap_impl(space, space.type(w_instance), w_instance, impl)
return W_NDimArray(impl)
def test_iterator_basic(self):
#Let's get started, simple iteration in C order with
#contiguous layout => strides[-1] is 1
shape = [3, 5]
strides = [5, 1]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [10, 4]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert i.contiguous
s = i.reset()
s = i.next(s)
s = i.next(s)
s = i.next(s)
assert s.offset == 3
assert not i.done(s)
assert s._indices == [0,0]
assert i.indices(s) == [0,3]
#cause a dimension overflow
s = i.next(s)
s = i.next(s)
assert s.offset == 5
assert s._indices == [0,3]
assert i.indices(s) == [1,0]
#Now what happens if the array is transposed? strides[-1] != 1
# therefore layout is non-contiguous
strides = [1, 3]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [2, 12]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert not i.contiguous
s = i.reset()
s = i.next(s)
s = i.next(s)
s = i.next(s)
assert s.offset == 9
assert not i.done(s)
assert s._indices == [0,3]
#cause a dimension overflow
s = i.next(s)
s = i.next(s)
assert s.offset == 1
assert s._indices == [1,0]
def test_iterator_basic(self):
#Let's get started, simple iteration in C order with
#contiguous layout => strides[-1] is 1
shape = [3, 5]
strides = [5, 1]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [10, 4]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert i.contiguous
s = i.reset()
s = i.next(s)
s = i.next(s)
s = i.next(s)
assert s.offset == 3
assert not i.done(s)
assert s._indices == [0,0]
assert i.indices(s) == [0,3]
#cause a dimension overflow
s = i.next(s)
s = i.next(s)
assert s.offset == 5
assert s._indices == [0,3]
assert i.indices(s) == [1,0]
#Now what happens if the array is transposed? strides[-1] != 1
# therefore layout is non-contiguous
strides = [1, 3]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [2, 12]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert not i.contiguous
s = i.reset()
s = i.next(s)
s = i.next(s)
s = i.next(s)
assert s.offset == 9
assert not i.done(s)
assert s._indices == [0,3]
#cause a dimension overflow
s = i.next(s)
s = i.next(s)
assert s.offset == 1
assert s._indices == [1,0]
def set_shape(self, space, orig_array, new_shape):
if not new_shape:
return self
if support.product(new_shape) == 1:
arr = W_NDimArray.from_shape(space, new_shape, self.dtype)
arr_iter = arr.create_iter(new_shape)
arr_iter.setitem(self.value)
return arr.implementation
raise OperationError(space.w_ValueError, space.wrap("total size of the array must be unchanged"))
def from_shape(space, shape, dtype, order='C', w_instance=None, zero=True):
from pypy.module.micronumpy import concrete, descriptor, boxes
from pypy.module.micronumpy.strides import calc_strides
if len(shape) > NPY.MAXDIMS:
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d", NPY.MAXDIMS)
try:
support.product(shape) * dtype.elsize
except OverflowError as e:
raise oefmt(space.w_ValueError, "array is too big")
strides, backstrides = calc_strides(shape, dtype.base, order)
impl = concrete.ConcreteArray(shape, dtype.base, order, strides,
backstrides, zero=zero)
if dtype == descriptor.get_dtype_cache(space).w_objectdtype:
impl.fill(space, boxes.W_ObjectBox(space.w_None))
if w_instance:
return wrap_impl(space, space.type(w_instance), w_instance, impl)
return W_NDimArray(impl)
def set_shape(self, space, orig_array, new_shape):
if not new_shape:
return self
if support.product(new_shape) == 1:
arr = W_NDimArray.from_shape(space, new_shape, self.dtype)
arr_iter = arr.create_iter(new_shape)
arr_iter.setitem(self.value)
return arr.implementation
raise OperationError(space.w_ValueError, space.wrap(
"total size of the array must be unchanged"))
def __init__(self, array, dtype, start, strides, backstrides, shape):
self.indexes = [0] * len(shape)
self.array = array
self.dtype = dtype
self.shape = shape
self.offset = start
self.shapelen = len(shape)
self._done = self.shapelen == 0 or product(shape) == 0
self.strides = strides
self.backstrides = backstrides
self.size = array.size
def __init__(self, shape, dtype, order, strides, backstrides, storage):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
self.size = support.product(shape) * dtype.elsize
self.order = order
self.dtype = dtype
self.strides = strides
self.backstrides = backstrides
self.storage = storage
def __init__(self, array, dtype, start, strides, backstrides, shape):
self.indexes = [0] * len(shape)
self.array = array
self.dtype = dtype
self.shape = shape
self.offset = start
self.shapelen = len(shape)
self._done = self.shapelen == 0 or product(shape) == 0
self.strides = strides
self.backstrides = backstrides
self.size = array.size
def __init__(self, shape, dtype, order, strides, backstrides, storage):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
self.size = support.product(shape) * dtype.elsize
self.order = order
self.dtype = dtype
self.strides = strides
self.backstrides = backstrides
self.storage = storage
def AxisIter(array, shape, axis):
strides = array.get_strides()
backstrides = array.get_backstrides()
if len(shape) == len(strides):
# keepdims = True
strides = strides[:axis] + [0] + strides[axis + 1:]
backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
else:
strides = strides[:axis] + [0] + strides[axis:]
backstrides = backstrides[:axis] + [0] + backstrides[axis:]
return ArrayIter(array, support.product(shape), shape, strides, backstrides)
def set_shape(self, space, orig_array, new_shape):
if len(new_shape) > NPY.MAXDIMS:
raise oefmt(space.w_ValueError,
"sequence too large; must be smaller than %d",
NPY.MAXDIMS)
try:
support.product(new_shape) * self.dtype.elsize
except OverflowError as e:
raise oefmt(space.w_ValueError, "array is too big")
if len(self.get_shape()) < 2 or self.size == 0:
# TODO: this code could be refactored into calc_strides
# but then calc_strides would have to accept a stepping factor
strides = []
backstrides = []
dtype = self.dtype
try:
s = self.get_strides()[0] // dtype.elsize
except IndexError:
s = 1
if self.order == 'C':
new_shape.reverse()
for sh in new_shape:
strides.append(s * dtype.elsize)
backstrides.append(s * (sh - 1) * dtype.elsize)
s *= max(1, sh)
if self.order == 'C':
strides.reverse()
backstrides.reverse()
new_shape.reverse()
return self.__class__(self.start, strides, backstrides, new_shape,
self, orig_array)
new_strides = calc_new_strides(new_shape, self.get_shape(),
self.get_strides(), self.order)
if new_strides is None or len(new_strides) != len(new_shape):
raise oefmt(space.w_AttributeError,
"incompatible shape for a non-contiguous array")
new_backstrides = [0] * len(new_shape)
for nd in range(len(new_shape)):
new_backstrides[nd] = (new_shape[nd] - 1) * new_strides[nd]
return self.__class__(self.start, new_strides, new_backstrides,
new_shape, self, orig_array)
def do_axis_reduce(self, obj, dtype, dim, keepdims):
from pypy.module.micronumpy.interp_numarray import AxisReduce,\
W_NDimArray
if keepdims:
shape = obj.shape[:dim] + [1] + obj.shape[dim + 1:]
else:
shape = obj.shape[:dim] + obj.shape[dim + 1:]
result = W_NDimArray(support.product(shape), shape, dtype)
arr = AxisReduce(self.func, self.name, self.identity, obj.shape, dtype,
result, obj, dim)
loop.compute(arr)
return arr.left
def AxisIter(array, shape, axis):
strides = array.get_strides()
backstrides = array.get_backstrides()
if len(shape) == len(strides):
# keepdims = True
strides = strides[:axis] + [0] + strides[axis + 1:]
backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
else:
strides = strides[:axis] + [0] + strides[axis:]
backstrides = backstrides[:axis] + [0] + backstrides[axis:]
return ArrayIter(array, support.product(shape), shape, strides,
backstrides)
def coalesce_iter(old_iter, op_flags, it, order, flat=True):
"""
We usually iterate through an array one value at a time.
But after coalesce(), getoperand() will return a slice by removing
the fastest varying dimension(s) from the beginning or end of the shape.
If flat is true, then the slice will be 1d, otherwise stack up the shape of
the fastest varying dimension in the slice, so an iterator of a 'C' array
of shape (2,4,3) after two calls to coalesce will iterate 2 times over a slice
of shape (4,3) by setting the offset to the beginning of the data at each iteration
"""
shape = [s + 1 for s in old_iter.shape_m1]
if len(shape) < 1:
return old_iter
strides = old_iter.strides
backstrides = old_iter.backstrides
if order == NPY.FORTRANORDER:
new_shape = shape[1:]
new_strides = strides[1:]
new_backstrides = backstrides[1:]
_stride = old_iter.slice_stride + [strides[0]]
_shape = old_iter.slice_shape + [shape[0]]
_backstride = old_iter.slice_backstride + [strides[0] * (shape[0] - 1)]
fastest = shape[0]
else:
new_shape = shape[:-1]
new_strides = strides[:-1]
new_backstrides = backstrides[:-1]
# use the operand's iterator's rightmost stride,
# even if it is not the fastest (for 'F' or swapped axis)
_stride = [strides[-1]] + old_iter.slice_stride
_shape = [shape[-1]] + old_iter.slice_shape
_backstride = [(shape[-1] - 1) * strides[-1]] + old_iter.slice_backstride
fastest = shape[-1]
if fastest == 0:
return old_iter
if flat:
_shape = [support.product(_shape)]
if len(_stride) > 1:
_stride = [min(_stride[0], _stride[1])]
_backstride = [(shape[0] - 1) * _stride[0]]
return SliceIter(
old_iter.array,
old_iter.size / fastest,
new_shape,
new_strides,
new_backstrides,
_shape,
_stride,
_backstride,
op_flags,
it,
)
def split_iter(arr, axis_flags):
"""Prepare 2 iterators for nested iteration over `arr`.
Arguments:
arr: instance of BaseConcreteArray
axis_flags: list of bools, one for each dimension of `arr`.The inner
iterator operates over the dimensions for which the flag is True
"""
shape = arr.get_shape()
strides = arr.get_strides()
backstrides = arr.get_backstrides()
shapelen = len(shape)
assert len(axis_flags) == shapelen
inner_shape = [-1] * shapelen
inner_strides = [-1] * shapelen
inner_backstrides = [-1] * shapelen
outer_shape = [-1] * shapelen
outer_strides = [-1] * shapelen
outer_backstrides = [-1] * shapelen
for i in range(len(shape)):
if axis_flags[i]:
inner_shape[i] = shape[i]
inner_strides[i] = strides[i]
inner_backstrides[i] = backstrides[i]
outer_shape[i] = 1
outer_strides[i] = 0
outer_backstrides[i] = 0
else:
outer_shape[i] = shape[i]
outer_strides[i] = strides[i]
outer_backstrides[i] = backstrides[i]
inner_shape[i] = 1
inner_strides[i] = 0
inner_backstrides[i] = 0
inner_iter = ArrayIter(arr, support.product(inner_shape),
inner_shape, inner_strides, inner_backstrides)
outer_iter = ArrayIter(arr, support.product(outer_shape),
outer_shape, outer_strides, outer_backstrides)
return inner_iter, outer_iter
def split_iter(arr, axis_flags):
"""Prepare 2 iterators for nested iteration over `arr`.
Arguments:
arr: instance of BaseConcreteArray
axis_flags: list of bools, one for each dimension of `arr`.The inner
iterator operates over the dimensions for which the flag is True
"""
shape = arr.get_shape()
strides = arr.get_strides()
backstrides = arr.get_backstrides()
shapelen = len(shape)
assert len(axis_flags) == shapelen
inner_shape = [-1] * shapelen
inner_strides = [-1] * shapelen
inner_backstrides = [-1] * shapelen
outer_shape = [-1] * shapelen
outer_strides = [-1] * shapelen
outer_backstrides = [-1] * shapelen
for i in range(len(shape)):
if axis_flags[i]:
inner_shape[i] = shape[i]
inner_strides[i] = strides[i]
inner_backstrides[i] = backstrides[i]
outer_shape[i] = 1
outer_strides[i] = 0
outer_backstrides[i] = 0
else:
outer_shape[i] = shape[i]
outer_strides[i] = strides[i]
outer_backstrides[i] = backstrides[i]
inner_shape[i] = 1
inner_strides[i] = 0
inner_backstrides[i] = 0
inner_iter = ArrayIter(arr, support.product(inner_shape), inner_shape,
inner_strides, inner_backstrides)
outer_iter = ArrayIter(arr, support.product(outer_shape), outer_shape,
outer_strides, outer_backstrides)
return inner_iter, outer_iter
def __init__(self,
shape,
dtype,
order,
strides,
backstrides,
storage=lltype.nullptr(RAW_STORAGE),
zero=True):
if storage == lltype.nullptr(RAW_STORAGE):
storage = dtype.itemtype.malloc(support.product(shape) *
dtype.elsize,
zero=zero)
ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides,
backstrides, storage)
def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
# already tested for overflow in from_shape_and_storage
self.size = support.product(shape) * dtype.elsize
self.order = order
self.dtype = dtype
self.strides = strides
self.backstrides = backstrides
self.storage = storage
self.start = start
self.gcstruct = V_OBJECTSTORE
def __init__(self, start, strides, backstrides, shape, parent, orig_arr,
dtype=None):
self.strides = strides
self.backstrides = backstrides
self.shape = shape
if dtype is None:
dtype = parent.dtype
if isinstance(parent, SliceArray):
parent = parent.parent # one level only
self.parent = parent
self.storage = parent.storage
self.order = parent.order
self.dtype = dtype
self.size = support.product(shape) * self.dtype.elsize
self.start = start
self.orig_arr = orig_arr
def test_iterator_goto(self):
shape = [3, 5]
strides = [1, 3]
backstrides = [x * (y - 1) for x, y in zip(strides, shape)]
assert backstrides == [2, 12]
a = MockArray(shape, strides, 42)
i = ArrayIter(a, support.product(shape), shape, strides, backstrides)
assert not i.contiguous
s = i.reset()
assert s.index == 0
assert s._indices == [0, 0]
assert s.offset == a.start
s = i.goto(11)
assert s.index == 11
assert s._indices is None
assert s.offset == a.start + 5
def __init__(self, shape, dtype, order, strides, backstrides, storage, start=0):
make_sure_not_resized(shape)
make_sure_not_resized(strides)
make_sure_not_resized(backstrides)
self.shape = shape
# already tested for overflow in from_shape_and_storage
self.size = support.product(shape) * dtype.elsize
if order not in (NPY.CORDER, NPY.FORTRANORDER):
raise oefmt(dtype.itemtype.space.w_ValueError, "ConcreteArrayNotOwning but order is not 0,1 rather %d", order)
self.order = order
self.dtype = dtype
self.strides = strides
self.backstrides = backstrides
self.storage = storage
self.start = start
self.gcstruct = V_OBJECTSTORE
def test_iterator_goto(self):
shape = [3, 5]
strides = [1, 3]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [2, 12]
a = MockArray(shape, strides, 42)
i = ArrayIter(a, support.product(shape), shape,
strides, backstrides)
assert not i.contiguous
s = i.reset()
assert s.index == 0
assert s._indices == [0, 0]
assert s.offset == a.start
s = i.goto(11)
assert s.index == 11
assert s._indices is None
assert s.offset == a.start + 5
def coalesce_iter(old_iter, op_flags, it, order, flat=True):
'''
We usually iterate through an array one value at a time.
But after coalesce(), getoperand() will return a slice by removing
the fastest varying dimension(s) from the beginning or end of the shape.
If flat is true, then the slice will be 1d, otherwise stack up the shape of
the fastest varying dimension in the slice, so an iterator of a 'C' array
of shape (2,4,3) after two calls to coalesce will iterate 2 times over a slice
of shape (4,3) by setting the offset to the beginning of the data at each iteration
'''
shape = [s + 1 for s in old_iter.shape_m1]
if len(shape) < 1:
return old_iter
strides = old_iter.strides
backstrides = old_iter.backstrides
if order == NPY.FORTRANORDER:
new_shape = shape[1:]
new_strides = strides[1:]
new_backstrides = backstrides[1:]
_stride = old_iter.slice_stride + [strides[0]]
_shape = old_iter.slice_shape + [shape[0]]
_backstride = old_iter.slice_backstride + [strides[0] * (shape[0] - 1)]
fastest = shape[0]
else:
new_shape = shape[:-1]
new_strides = strides[:-1]
new_backstrides = backstrides[:-1]
# use the operand's iterator's rightmost stride,
# even if it is not the fastest (for 'F' or swapped axis)
_stride = [strides[-1]] + old_iter.slice_stride
_shape = [shape[-1]] + old_iter.slice_shape
_backstride = [(shape[-1] - 1) * strides[-1]
] + old_iter.slice_backstride
fastest = shape[-1]
if fastest == 0:
return old_iter
if flat:
_shape = [support.product(_shape)]
if len(_stride) > 1:
_stride = [min(_stride[0], _stride[1])]
_backstride = [(shape[0] - 1) * _stride[0]]
return SliceIter(old_iter.array, old_iter.size / fastest, new_shape,
new_strides, new_backstrides, _shape, _stride,
_backstride, op_flags, it)
def __init__(self, shape, dtype, order, strides, backstrides,
storage=lltype.nullptr(RAW_STORAGE), zero=True):
gcstruct = V_OBJECTSTORE
if storage == lltype.nullptr(RAW_STORAGE):
length = support.product(shape)
if dtype.num == NPY.OBJECT:
storage = dtype.itemtype.malloc(length * dtype.elsize, zero=True)
gcstruct = _create_objectstore(storage, length, dtype.elsize)
else:
storage = dtype.itemtype.malloc(length * dtype.elsize, zero=zero)
start = calc_start(shape, strides)
ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides,
storage, start=start)
self.gcstruct = gcstruct
self.flags = NPY.ARRAY_ALIGNED | NPY.ARRAY_WRITEABLE
if is_c_contiguous(self):
self.flags |= NPY.ARRAY_C_CONTIGUOUS
if is_f_contiguous(self):
self.flags |= NPY.ARRAY_F_CONTIGUOUS
def __init__(self,
shape,
dtype,
order,
strides,
backstrides,
storage=lltype.nullptr(RAW_STORAGE),
zero=True):
gcstruct = V_OBJECTSTORE
if storage == lltype.nullptr(RAW_STORAGE):
length = support.product(shape)
if dtype.num == NPY.OBJECT:
storage = dtype.itemtype.malloc(length * dtype.elsize,
zero=True)
gcstruct = _create_objectstore(storage, length, dtype.elsize)
else:
storage = dtype.itemtype.malloc(length * dtype.elsize,
zero=zero)
ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides,
backstrides, storage)
self.gcstruct = gcstruct
def test_one_in_shape(self):
strides = [16, 4, 8]
shape = [3, 4, 1]
backstrides = [x * (y - 1) for x, y in zip(strides, shape)]
assert backstrides == [32, 12, 0]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert not i.contiguous
s = i.reset()
for j in range(3):
s = i.next(s)
assert s.offset == 12
assert not i.done(s)
assert s._indices == [0, 3, 0]
while not i.done(s):
old_indices = s._indices[:]
old_offset = s.offset
s = i.next(s)
assert s.offset == 0
assert s._indices == [0, 0, 0]
assert old_indices == [2, 3, 0]
assert old_offset == 44
def test_one_in_shape(self):
strides = [16, 4, 8]
shape = [3, 4, 1]
backstrides = [x * (y - 1) for x,y in zip(strides, shape)]
assert backstrides == [32, 12, 0]
i = ArrayIter(MockArray(shape, strides), support.product(shape), shape,
strides, backstrides)
assert not i.contiguous
s = i.reset()
for j in range(3):
s = i.next(s)
assert s.offset == 12
assert not i.done(s)
assert s._indices == [0, 3, 0]
while not i.done(s):
old_indices = s._indices[:]
old_offset = s.offset
s = i.next(s)
assert s.offset == 0
assert s._indices == [0, 0, 0]
assert old_indices == [2, 3, 0]
assert old_offset == 44
def __init__(self,
start,
strides,
backstrides,
shape,
parent,
orig_arr,
dtype=None):
self.strides = strides
self.backstrides = backstrides
self.shape = shape
if dtype is None:
dtype = parent.dtype
if isinstance(parent, SliceArray):
parent = parent.parent # one level only
self.parent = parent
self.storage = parent.storage
self.order = parent.order
self.dtype = dtype
self.size = support.product(shape) * self.dtype.elsize
self.start = start
self.orig_arr = orig_arr
def __init__(self, start, strides, backstrides, shape, parent, orig_arr,
dtype=None):
self.strides = strides
self.backstrides = backstrides
self.shape = shape
if dtype is None:
dtype = parent.dtype
if isinstance(parent, SliceArray):
parent = parent.parent # one level only
self.parent = parent
self.storage = parent.storage
self.gcstruct = parent.gcstruct
self.order = parent.order
self.dtype = dtype
self.size = support.product(shape) * self.dtype.elsize
self.start = start
self.orig_arr = orig_arr
self.flags = parent.flags & NPY.ARRAY_ALIGNED
self.flags |= parent.flags & NPY.ARRAY_WRITEABLE
if is_c_contiguous(self):
self.flags |= NPY.ARRAY_C_CONTIGUOUS
if is_f_contiguous(self):
self.flags |= NPY.ARRAY_F_CONTIGUOUS
def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
from pypy.module.micronumpy import strides
# for anything that isn't already an array, try __array__ method first
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is not None:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
if not isinstance(w_object, W_NDimArray):
w_array = try_interface_method(space, w_object)
if w_array is not None:
w_object = w_array
copy = False
dtype = descriptor.decode_w_dtype(space, w_dtype)
if space.is_none(w_order):
order = 'C'
else:
order = space.str_w(w_order)
if order == 'K':
order = 'C'
if order != 'C': # or order != 'F':
raise oefmt(space.w_ValueError, "Unknown order: %s", order)
if isinstance(w_object, W_NDimArray):
if (dtype is None or w_object.get_dtype() is dtype):
if copy and (subok or type(w_object) is W_NDimArray):
return w_object.descr_copy(space, w_order)
elif not copy and (subok or type(w_object) is W_NDimArray):
return w_object
if subok and not type(w_object) is W_NDimArray:
raise oefmt(space.w_NotImplementedError,
"array(..., subok=True) only partially implemented")
# we have a ndarray, but need to copy or change dtype
if dtype is None:
dtype = w_object.get_dtype()
if dtype != w_object.get_dtype():
# silently reject the copy value
copy = True
if copy:
shape = w_object.get_shape()
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if support.product(shape) == 1:
w_arr.set_scalar_value(dtype.coerce(space,
w_object.implementation.getitem(0)))
else:
loop.setslice(space, shape, w_arr.implementation, w_object.implementation)
return w_arr
else:
imp = w_object.implementation
w_base = w_object
if imp.base() is not None:
w_base = imp.base()
with imp as storage:
sz = support.product(w_object.get_shape()) * dtype.elsize
return W_NDimArray.from_shape_and_storage(space,
w_object.get_shape(), storage, dtype, storage_bytes=sz,
w_base=w_base, start=imp.start)
else:
# not an array
shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
if dtype is None and space.isinstance_w(w_object, space.w_buffer):
dtype = descriptor.get_dtype_cache(space).w_uint8dtype
if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
dtype = find_dtype_for_seq(space, elems_w, dtype)
if dtype is None:
dtype = descriptor.get_dtype_cache(space).w_float64dtype
elif dtype.is_str_or_unicode() and dtype.elsize < 1:
# promote S0 -> S1, U0 -> U1
dtype = descriptor.variable_dtype(space, dtype.char + '1')
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if support.product(shape) == 1: # safe from overflow since from_shape checks
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
return w_arr
def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
from pypy.module.micronumpy import strides
# for anything that isn't already an array, try __array__ method first
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is not None:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
dtype = descriptor.decode_w_dtype(space, w_dtype)
if space.is_none(w_order):
order = 'C'
else:
order = space.str_w(w_order)
if order == 'K':
order = 'C'
if order != 'C': # or order != 'F':
raise oefmt(space.w_ValueError, "Unknown order: %s", order)
if isinstance(w_object, W_NDimArray):
if (dtype is None or w_object.get_dtype() is dtype):
if copy and (subok or type(w_object) is W_NDimArray):
return w_object.descr_copy(space, w_order)
elif not copy and (subok or type(w_object) is W_NDimArray):
return w_object
# we have a ndarray, but need to copy or change dtype or create W_NDimArray
if dtype is None:
dtype = w_object.get_dtype()
if dtype != w_object.get_dtype():
# silently reject the copy value
copy = True
if copy:
shape = w_object.get_shape()
_elems_w = w_object.reshape(space, space.wrap(-1))
elems_w = [None] * w_object.get_size()
for i in range(len(elems_w)):
elems_w[i] = _elems_w.descr_getitem(space, space.wrap(i))
elif subok:
raise oefmt(space.w_NotImplementedError,
"array(...copy=False, subok=True) not implemented yet")
else:
sz = support.product(w_object.get_shape()) * dtype.elsize
return W_NDimArray.from_shape_and_storage(space,
w_object.get_shape(),w_object.implementation.storage,
dtype, storage_bytes=sz, w_base=w_object)
else:
# not an array
shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
dtype = strides.find_dtype_for_seq(space, elems_w, dtype)
if dtype is None:
dtype = descriptor.get_dtype_cache(space).w_float64dtype
elif dtype.is_str_or_unicode() and dtype.elsize < 1:
# promote S0 -> S1, U0 -> U1
dtype = descriptor.variable_dtype(space, dtype.char + '1')
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if len(elems_w) == 1:
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
return w_arr
def _array(space, w_object, w_dtype=None, copy=True, w_order=None, subok=False):
from pypy.module.micronumpy.boxes import W_GenericBox
# numpy testing calls array(type(array([]))) and expects a ValueError
if space.isinstance_w(w_object, space.w_type):
raise oefmt(space.w_ValueError, "cannot create ndarray from type instance")
# for anything that isn't already an array, try __array__ method first
dtype = descriptor.decode_w_dtype(space, w_dtype)
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is None:
if ( not space.isinstance_w(w_object, space.w_str) and
not space.isinstance_w(w_object, space.w_unicode) and
not isinstance(w_object, W_GenericBox)):
# use buffer interface
w_object = _array_from_buffer_3118(space, w_object, dtype)
else:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
dtype = w_object.get_dtype()
if not isinstance(w_object, W_NDimArray):
w_array, _copy = try_interface_method(space, w_object, copy)
if w_array is not None:
w_object = w_array
copy = _copy
dtype = w_object.get_dtype()
if isinstance(w_object, W_NDimArray):
npy_order = order_converter(space, w_order, NPY.ANYORDER)
if (dtype is None or w_object.get_dtype() is dtype) and (subok or
type(w_object) is W_NDimArray):
flags = w_object.get_flags()
must_copy = copy
must_copy |= (npy_order == NPY.CORDER and not flags & NPY.ARRAY_C_CONTIGUOUS)
must_copy |= (npy_order == NPY.FORTRANORDER and not flags & NPY.ARRAY_F_CONTIGUOUS)
if must_copy:
return w_object.descr_copy(space, space.wrap(npy_order))
else:
return w_object
if subok and not type(w_object) is W_NDimArray:
raise oefmt(space.w_NotImplementedError,
"array(..., subok=True) only partially implemented")
# we have a ndarray, but need to copy or change dtype
if dtype is None:
dtype = w_object.get_dtype()
if dtype != w_object.get_dtype():
# silently reject the copy value
copy = True
if copy:
shape = w_object.get_shape()
order = support.get_order_as_CF(w_object.get_order(), npy_order)
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if support.product(shape) == 1:
w_arr.set_scalar_value(dtype.coerce(space,
w_object.implementation.getitem(0)))
else:
loop.setslice(space, shape, w_arr.implementation, w_object.implementation)
return w_arr
else:
imp = w_object.implementation
w_base = w_object
sz = w_base.get_size() * dtype.elsize
if imp.base() is not None:
w_base = imp.base()
if type(w_base) is W_NDimArray:
sz = w_base.get_size() * dtype.elsize
else:
# this must succeed (mmap, buffer, ...)
sz = space.int_w(space.call_method(w_base, 'size'))
with imp as storage:
return W_NDimArray.from_shape_and_storage(space,
w_object.get_shape(), storage, dtype, storage_bytes=sz,
w_base=w_base, strides=imp.strides, start=imp.start)
else:
# not an array
npy_order = order_converter(space, w_order, NPY.CORDER)
shape, elems_w = find_shape_and_elems(space, w_object, dtype)
if dtype is None and space.isinstance_w(w_object, space.w_buffer):
dtype = descriptor.get_dtype_cache(space).w_uint8dtype
if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
dtype = find_dtype_for_seq(space, elems_w, dtype)
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=npy_order)
if support.product(shape) == 1: # safe from overflow since from_shape checks
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
return w_arr
def try_interface_method(space, w_object, copy):
try:
w_interface = space.getattr(w_object, space.wrap("__array_interface__"))
if w_interface is None:
return None, False
version_w = space.finditem(w_interface, space.wrap("version"))
if version_w is None:
raise oefmt(space.w_ValueError, "__array_interface__ found without"
" 'version' key")
if not space.isinstance_w(version_w, space.w_int):
raise oefmt(space.w_ValueError, "__array_interface__ found with"
" non-int 'version' key")
version = space.int_w(version_w)
if version < 3:
raise oefmt(space.w_ValueError,
"__array_interface__ version %d not supported", version)
# make a view into the data
w_shape = space.finditem(w_interface, space.wrap('shape'))
w_dtype = space.finditem(w_interface, space.wrap('typestr'))
w_descr = space.finditem(w_interface, space.wrap('descr'))
w_data = space.finditem(w_interface, space.wrap('data'))
w_strides = space.finditem(w_interface, space.wrap('strides'))
if w_shape is None or w_dtype is None:
raise oefmt(space.w_ValueError,
"__array_interface__ missing one or more required keys: shape, typestr"
)
if w_descr is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ descr not supported yet")
if w_strides is None or space.is_w(w_strides, space.w_None):
strides = None
else:
strides = [space.int_w(i) for i in space.listview(w_strides)]
shape = [space.int_w(i) for i in space.listview(w_shape)]
dtype = descriptor.decode_w_dtype(space, w_dtype)
if dtype is None:
raise oefmt(space.w_ValueError,
"__array_interface__ could not decode dtype %R", w_dtype
)
if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
space.isinstance_w(w_data, space.w_list)):
data_w = space.listview(w_data)
w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
read_only = space.is_true(data_w[1]) or copy
offset = 0
w_base = w_object
if read_only:
w_base = None
return W_NDimArray.from_shape_and_storage(space, shape, w_data,
dtype, w_base=w_base, strides=strides,
start=offset), read_only
if w_data is None:
w_data = w_object
w_offset = space.finditem(w_interface, space.wrap('offset'))
if w_offset is None:
offset = 0
else:
offset = space.int_w(w_offset)
#print 'create view from shape',shape,'dtype',dtype,'data',data
if strides is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ strides not fully supported yet")
arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
new_impl = arr.implementation.reshape(arr, shape)
return W_NDimArray(new_impl), False
except OperationError as e:
if e.match(space, space.w_AttributeError):
return None, False
raise
def _array(space,
w_object,
w_dtype=None,
copy=True,
w_order=None,
subok=False):
from pypy.module.micronumpy import strides
# for anything that isn't already an array, try __array__ method first
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is not None:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
if not isinstance(w_object, W_NDimArray):
w_array = try_interface_method(space, w_object)
if w_array is not None:
w_object = w_array
copy = False
dtype = descriptor.decode_w_dtype(space, w_dtype)
if space.is_none(w_order):
order = 'C'
else:
order = space.str_w(w_order)
if order == 'K':
order = 'C'
if order != 'C': # or order != 'F':
raise oefmt(space.w_ValueError, "Unknown order: %s", order)
if isinstance(w_object, W_NDimArray):
if (dtype is None or w_object.get_dtype() is dtype):
if copy and (subok or type(w_object) is W_NDimArray):
return w_object.descr_copy(space, w_order)
elif not copy and (subok or type(w_object) is W_NDimArray):
return w_object
if subok and not type(w_object) is W_NDimArray:
raise oefmt(space.w_NotImplementedError,
"array(..., subok=True) only partially implemented")
# we have a ndarray, but need to copy or change dtype
if dtype is None:
dtype = w_object.get_dtype()
if dtype != w_object.get_dtype():
# silently reject the copy value
copy = True
if copy:
shape = w_object.get_shape()
elems_w = [None] * w_object.get_size()
elsize = w_object.get_dtype().elsize
# TODO - use w_object.implementation without copying to a list
# unfortunately that causes a union error in translation
for i in range(w_object.get_size()):
elems_w[i] = w_object.implementation.getitem(i * elsize)
else:
imp = w_object.implementation
with imp as storage:
sz = support.product(w_object.get_shape()) * dtype.elsize
return W_NDimArray.from_shape_and_storage(space,
w_object.get_shape(),
storage,
dtype,
storage_bytes=sz,
w_base=w_object,
start=imp.start)
else:
# not an array
shape, elems_w = strides.find_shape_and_elems(space, w_object, dtype)
if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
dtype = strides.find_dtype_for_seq(space, elems_w, dtype)
if dtype is None:
dtype = descriptor.get_dtype_cache(space).w_float64dtype
elif dtype.is_str_or_unicode() and dtype.elsize < 1:
# promote S0 -> S1, U0 -> U1
dtype = descriptor.variable_dtype(space, dtype.char + '1')
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if support.product(shape) == 1:
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
return w_arr
def descr_get_itersize(self, space):
return space.wrap(support.product(self.shape))
def try_interface_method(space, w_object, copy):
try:
w_interface = space.getattr(w_object,
space.newtext("__array_interface__"))
if w_interface is None:
return None, False
version_w = space.finditem(w_interface, space.newtext("version"))
if version_w is None:
raise oefmt(space.w_ValueError, "__array_interface__ found without"
" 'version' key")
if not space.isinstance_w(version_w, space.w_int):
raise oefmt(
space.w_ValueError, "__array_interface__ found with"
" non-int 'version' key")
version = space.int_w(version_w)
if version < 3:
raise oefmt(space.w_ValueError,
"__array_interface__ version %d not supported",
version)
# make a view into the data
w_shape = space.finditem(w_interface, space.newtext('shape'))
w_dtype = space.finditem(w_interface, space.newtext('typestr'))
w_descr = space.finditem(w_interface, space.newtext('descr'))
w_data = space.finditem(w_interface, space.newtext('data'))
w_strides = space.finditem(w_interface, space.newtext('strides'))
if w_shape is None or w_dtype is None:
raise oefmt(
space.w_ValueError,
"__array_interface__ missing one or more required keys: shape, typestr"
)
if w_descr is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ descr not supported yet")
if w_strides is None or space.is_w(w_strides, space.w_None):
strides = None
else:
strides = [space.int_w(i) for i in space.listview(w_strides)]
shape = [space.int_w(i) for i in space.listview(w_shape)]
dtype = descriptor.decode_w_dtype(space, w_dtype)
if dtype is None:
raise oefmt(space.w_ValueError,
"__array_interface__ could not decode dtype %R",
w_dtype)
if w_data is not None and (space.isinstance_w(w_data, space.w_tuple) or
space.isinstance_w(w_data, space.w_list)):
data_w = space.listview(w_data)
w_data = rffi.cast(RAW_STORAGE_PTR, space.int_w(data_w[0]))
read_only = space.is_true(data_w[1]) or copy
offset = 0
w_base = w_object
if read_only:
w_base = None
return W_NDimArray.from_shape_and_storage(space,
shape,
w_data,
dtype,
w_base=w_base,
strides=strides,
start=offset), read_only
if w_data is None:
w_data = w_object
w_offset = space.finditem(w_interface, space.newtext('offset'))
if w_offset is None:
offset = 0
else:
offset = space.int_w(w_offset)
#print 'create view from shape',shape,'dtype',dtype,'data',data
if strides is not None:
raise oefmt(space.w_NotImplementedError,
"__array_interface__ strides not fully supported yet")
arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
new_impl = arr.implementation.reshape(arr, shape)
return W_NDimArray(new_impl), False
except OperationError as e:
if e.match(space, space.w_AttributeError):
return None, False
raise
def descr_get_itersize(self, space):
return space.wrap(support.product(self.shape))
def _array(space,
w_object,
w_dtype=None,
copy=True,
w_order=None,
subok=False):
from pypy.module.micronumpy.boxes import W_GenericBox
# numpy testing calls array(type(array([]))) and expects a ValueError
if space.isinstance_w(w_object, space.w_type):
raise oefmt(space.w_ValueError,
"cannot create ndarray from type instance")
# for anything that isn't already an array, try __array__ method first
dtype = descriptor.decode_w_dtype(space, w_dtype)
if not isinstance(w_object, W_NDimArray):
w_array = try_array_method(space, w_object, w_dtype)
if w_array is None:
if (not space.isinstance_w(w_object, space.w_bytes)
and not space.isinstance_w(w_object, space.w_unicode)
and not isinstance(w_object, W_GenericBox)):
# use buffer interface
w_object = _array_from_buffer_3118(space, w_object, dtype)
else:
# continue with w_array, but do further operations in place
w_object = w_array
copy = False
dtype = w_object.get_dtype()
if not isinstance(w_object, W_NDimArray):
w_array, _copy = try_interface_method(space, w_object, copy)
if w_array is not None:
w_object = w_array
copy = _copy
dtype = w_object.get_dtype()
if isinstance(w_object, W_NDimArray):
npy_order = order_converter(space, w_order, NPY.ANYORDER)
if (dtype is None or w_object.get_dtype() is dtype) and (
subok or type(w_object) is W_NDimArray):
flags = w_object.get_flags()
must_copy = copy
must_copy |= (npy_order == NPY.CORDER
and not flags & NPY.ARRAY_C_CONTIGUOUS)
must_copy |= (npy_order == NPY.FORTRANORDER
and not flags & NPY.ARRAY_F_CONTIGUOUS)
if must_copy:
return w_object.descr_copy(space, space.newint(npy_order))
else:
return w_object
if subok and not type(w_object) is W_NDimArray:
raise oefmt(space.w_NotImplementedError,
"array(..., subok=True) only partially implemented")
# we have a ndarray, but need to copy or change dtype
if dtype is None:
dtype = w_object.get_dtype()
if dtype != w_object.get_dtype():
# silently reject the copy value
copy = True
if copy:
shape = w_object.get_shape()
order = support.get_order_as_CF(w_object.get_order(), npy_order)
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=order)
if support.product(shape) == 1:
w_arr.set_scalar_value(
dtype.coerce(space, w_object.implementation.getitem(0)))
else:
loop.setslice(space, shape, w_arr.implementation,
w_object.implementation)
return w_arr
else:
imp = w_object.implementation
w_base = w_object
sz = w_base.get_size() * dtype.elsize
if imp.base() is not None:
w_base = imp.base()
if type(w_base) is W_NDimArray:
sz = w_base.get_size() * dtype.elsize
else:
# this must succeed (mmap, buffer, ...)
sz = space.int_w(space.call_method(w_base, 'size'))
with imp as storage:
return W_NDimArray.from_shape_and_storage(space,
w_object.get_shape(),
storage,
dtype,
storage_bytes=sz,
w_base=w_base,
strides=imp.strides,
start=imp.start)
else:
# not an array
npy_order = order_converter(space, w_order, NPY.CORDER)
shape, elems_w = find_shape_and_elems(space, w_object, dtype)
if dtype is None and space.isinstance_w(w_object, space.w_buffer):
dtype = descriptor.get_dtype_cache(space).w_uint8dtype
if dtype is None or (dtype.is_str_or_unicode() and dtype.elsize < 1):
dtype = find_dtype_for_seq(space, elems_w, dtype)
w_arr = W_NDimArray.from_shape(space, shape, dtype, order=npy_order)
if support.product(
shape) == 1: # safe from overflow since from_shape checks
w_arr.set_scalar_value(dtype.coerce(space, elems_w[0]))
else:
loop.assign(space, w_arr, elems_w)
return w_arr
