def test_bigarray_resize(): data = zeros(8*PS, dtype=uint32) f = BigFile_Data(data, PS) fh = f.fileh_open() # set first part & ensure it is set correctly A = BigArray((10,3), uint32, fh) A[:,:] = arange(10*3, dtype=uint32).reshape((10,3)) a = A[:] assert array_equal(a.ravel(), arange(10*3, dtype=uint32)) # grow array A.resize((11,3)) # a as already mapped, should stay the same assert array_equal(a.ravel(), arange(10*3, dtype=uint32)) # mapping it once again maps it whole with new size b = A[:] assert isinstance(b, ndarray) assert b.shape == (11,3) assert b.dtype == dtype(uint32) # head data is the same as a assert array_equal(a, b[:10,:]) # tail is zeros assert array_equal(b[10,:], zeros(3, dtype=uint32)) # old mapping stays valid and changes propageate to/from it assert a[0,0] == 0 assert b[0,0] == 0 a[0,0] = 1 assert b[0,0] == 1 b[0,0] = 2 assert a[0,0] == 2 a[0,0] = 0 assert b[0,0] == 0 assert a[ -1,-1] == 10*3-1 assert b[10-1,-1] == 10*3-1 a[ -1,-1] = 1 assert b[10-1,-1] == 1 b[10-1,-1] = 2 assert a[ -1,-1] == 2 a[ -1,-1] = 10*3-1 assert b[10-1,-1] == 10*3-1 # we cannot access old mapping beyond it's end assert raises(IndexError, 'a[10,:]') # we can change tail b[10,:] = arange(10*3, (10+1)*3) # map it whole again and ensure we have correct data c = A[:] assert array_equal(c.ravel(), arange(11*3, dtype=uint32))
def __init__(self, shape, dtype, order='C', blksize=2*1024*1024): LivePersistent.__init__(self) # NOTE BigArray is cooperative to us - it names all helping (= not needing # to be saved) data members starting with _v_. Were it otherwise, we # could not inherit from BigArray and would need to keep its instance # in e.g. ._v_array helper and redirect all BigArray method from us to it. BigArray._init0(self, shape, dtype, order) self.zfile = ZBigFile(blksize) self._v_fileh = None
def test_bigarray_append(): for order in ('C', 'F'): data = zeros(8*PS, dtype=uint32) f = BigFile_Data(data, PS) fh = f.fileh_open() arange32 = {'C': arange32_c, 'F': arange32_f} [order] # first make sure arange32 works correctly x = numpy.append( arange32(0,4), arange32(4,7), axis={'C': 0, 'F': 2} [order] ) #x = numpy.append( arange32(0,4), arange32(4,7), axis=0) #x = numpy.append( arange32(0,4), arange32(4,7)) assert array_equal(x, arange32(0,7)) assert array_equal(x.ravel(order), arange(7*3*2)) # init empty BigArray of shape (x,3,2) A = BigArray({'C': (0,3,2), 'F': (2,3,0)} [order], int64, fh, order=order) assert array_equal(A[:], arange32(0,0)) # append initial data A.append(arange32(0,2)) assert array_equal(A[:], arange32(0,2)) A.append(arange32(2,3)) assert array_equal(A[:], arange32(0,3)) # append plain list (test for arg conversion) A.append({'C': [[[18,19], [20,21], [22,23]]], \ 'F': [[[18],[20],[22]], [[19],[21],[23]]]} [order]) assert array_equal(A[:], arange32(0,4)) # append with incorrect shape - rejected, original stays the same assert raises(ValueError, 'A.append(arange(3))') assert array_equal(A[:], arange32(0,4)) assert raises(ValueError, 'A.append(arange(3*2).reshape(3,2))') assert array_equal(A[:], arange32(0,4)) # append with correct shape, but incompatible dtype - rejected, original stays the same assert raises(ValueError, {'C': 'A.append(asarray([[[0,1], [2,3], [4,"abc"]]], dtype=object))', 'F': 'A.append(asarray([[[0],[1],[2]], [[3],[4],["abc"]]], dtype=object))'} [order] ) assert array_equal(A[:], arange32(0,4))