def test_bigarray_resize():
data = zeros(8*PS, dtype=uint32)
f = BigFile_Data(data, PS)
fh = f.fileh_open()
# set first part & ensure it is set correctly
A = BigArray((10,3), uint32, fh)
A[:,:] = arange(10*3, dtype=uint32).reshape((10,3))
a = A[:]
assert array_equal(a.ravel(), arange(10*3, dtype=uint32))
# grow array
A.resize((11,3))
# a as already mapped, should stay the same
assert array_equal(a.ravel(), arange(10*3, dtype=uint32))
# mapping it once again maps it whole with new size
b = A[:]
assert isinstance(b, ndarray)
assert b.shape == (11,3)
assert b.dtype == dtype(uint32)
# head data is the same as a
assert array_equal(a, b[:10,:])
# tail is zeros
assert array_equal(b[10,:], zeros(3, dtype=uint32))
# old mapping stays valid and changes propageate to/from it
assert a[0,0] == 0
assert b[0,0] == 0
a[0,0] = 1
assert b[0,0] == 1
b[0,0] = 2
assert a[0,0] == 2
a[0,0] = 0
assert b[0,0] == 0
assert a[ -1,-1] == 10*3-1
assert b[10-1,-1] == 10*3-1
a[ -1,-1] = 1
assert b[10-1,-1] == 1
b[10-1,-1] = 2
assert a[ -1,-1] == 2
a[ -1,-1] = 10*3-1
assert b[10-1,-1] == 10*3-1
# we cannot access old mapping beyond it's end
assert raises(IndexError, 'a[10,:]')
# we can change tail
b[10,:] = arange(10*3, (10+1)*3)
# map it whole again and ensure we have correct data
c = A[:]
assert array_equal(c.ravel(), arange(11*3, dtype=uint32))
def __init__(self, shape, dtype, order='C', blksize=2*1024*1024):
LivePersistent.__init__(self)
# NOTE BigArray is cooperative to us - it names all helping (= not needing
# to be saved) data members starting with _v_. Were it otherwise, we
# could not inherit from BigArray and would need to keep its instance
# in e.g. ._v_array helper and redirect all BigArray method from us to it.
BigArray._init0(self, shape, dtype, order)
self.zfile = ZBigFile(blksize)
self._v_fileh = None
def test_bigarray_append():
for order in ('C', 'F'):
data = zeros(8*PS, dtype=uint32)
f = BigFile_Data(data, PS)
fh = f.fileh_open()
arange32 = {'C': arange32_c, 'F': arange32_f} [order]
# first make sure arange32 works correctly
x = numpy.append( arange32(0,4), arange32(4,7), axis={'C': 0, 'F': 2} [order] )
#x = numpy.append( arange32(0,4), arange32(4,7), axis=0)
#x = numpy.append( arange32(0,4), arange32(4,7))
assert array_equal(x, arange32(0,7))
assert array_equal(x.ravel(order), arange(7*3*2))
# init empty BigArray of shape (x,3,2)
A = BigArray({'C': (0,3,2), 'F': (2,3,0)} [order], int64, fh, order=order)
assert array_equal(A[:], arange32(0,0))
# append initial data
A.append(arange32(0,2))
assert array_equal(A[:], arange32(0,2))
A.append(arange32(2,3))
assert array_equal(A[:], arange32(0,3))
# append plain list (test for arg conversion)
A.append({'C': [[[18,19], [20,21], [22,23]]], \
'F': [[[18],[20],[22]], [[19],[21],[23]]]} [order])
assert array_equal(A[:], arange32(0,4))
# append with incorrect shape - rejected, original stays the same
assert raises(ValueError, 'A.append(arange(3))')
assert array_equal(A[:], arange32(0,4))
assert raises(ValueError, 'A.append(arange(3*2).reshape(3,2))')
assert array_equal(A[:], arange32(0,4))
# append with correct shape, but incompatible dtype - rejected, original stays the same
assert raises(ValueError,
{'C': 'A.append(asarray([[[0,1], [2,3], [4,"abc"]]], dtype=object))',
'F': 'A.append(asarray([[[0],[1],[2]], [[3],[4],["abc"]]], dtype=object))'} [order] )
assert array_equal(A[:], arange32(0,4))