def test01a(self): """Testing ctable creation in "r" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) self.assertRaises(RuntimeError, ca.ctable, (a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='r')
def __init__(self, data=None, dshape=None, params=None): # need at least one of the three assert (data is not None) or (dshape is not None) or \ (params.get('storage')) # Extract the relevant carray parameters from the more # general Blaze params object. if params: cparams, rootdir, format_flavor = to_cparams(params) else: rootdir, cparams = None, None if isinstance(data, CArraySource): data = data.ca dshape = dshape if dshape else data.dshape if dshape: shape, dtype = to_numpy(dshape) self.ca = carray.carray(data, dtype=dtype, rootdir=rootdir, cparams=cparams) self.dshape = dshape else: self.ca = carray.carray(data, rootdir=rootdir, cparams=cparams) self.dshape = from_numpy(self.ca.shape, self.ca.dtype)
def test01b(self): """Testing ctable creation in "w" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last ctable t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='w') #print "t->", `t` ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[11] = (11, 12.0) # Check values N = 12 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test00c(self): """Testing ctable opening in "a" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Open t t = ca.open(rootdir=self.rootdir, mode='a') #print "t->", `t` # Check values ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[-1] = (11, 12.0) # Check values N = 12 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test01b(self): """Testing ctable creation in "w" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last ctable t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='w') #print "t->", `t` ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[11] = (11, 12.0) # Check values N = 12 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test00c(self): """Testing ctable opening in "a" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Open t t = ca.open(rootdir=self.rootdir, mode='a') #print "t->", `t` # Check values ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[-1] = (11, 12.0) # Check values N = 12 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test05(self): """Testing `where()` iterator using `skip`""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v <= 5][2:] cwt = [v for v in b.where(ca.carray(a <= 5), skip=2)] #print "numpy ->", [v for v in a if v<=5][2:] #print "where ->", [v for v in b.where(ca.carray(a<=5), skip=2)] self.assert_(wt == cwt, "where() does not work correctly")
def test06(self): """Testing `where()` iterator (using carray bool in fancy indexing)""" a = np.arange(1, 110) b = ca.carray(a, chunklen=10) wt = a[(a < 5) | (a > 9)] cwt = b[ca.carray((a < 5) | (a > 9))] #print "numpy ->", a[(a<5)|(a>9)] #print "where ->", b[ca.carray((a<5)|(a>9))] assert_array_equal(wt, cwt, "where() does not work correctly")
def test03(self): """Testing `where()` iterator (using a boolean carray)""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v<=5] cwt = [v for v in b.where(ca.carray(a<=5))] #print "numpy ->", [v for v in a if v<=5] #print "where ->", [v for v in b.where(ca.carray(a<=5))] self.assert_(wt == cwt, "where() does not work correctly")
def test05(self): """Testing `where()` iterator using `skip`""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v<=5][2:] cwt = [v for v in b.where(ca.carray(a<=5), skip=2)] #print "numpy ->", [v for v in a if v<=5][2:] #print "where ->", [v for v in b.where(ca.carray(a<=5), skip=2)] self.assert_(wt == cwt, "where() does not work correctly")
def test06(self): """Testing `where()` iterator (using carray bool in fancy indexing)""" a = np.arange(1, 110) b = ca.carray(a, chunklen=10) wt = a[(a<5)|(a>9)] cwt = b[ca.carray((a<5)|(a>9))] #print "numpy ->", a[(a<5)|(a>9)] #print "where ->", b[ca.carray((a<5)|(a>9))] assert_array_equal(wt, cwt, "where() does not work correctly")
def test01a(self): """Testing ctable creation in "r" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) self.assertRaises(RuntimeError, ca.ctable, (a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='r')
def test01c(self): """Testing ctable creation in "a" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last ctable self.assertRaises(RuntimeError, ca.ctable, (a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='a')
def test03(self): """Testing `where()` iterator (using a boolean carray)""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v <= 5] cwt = [v for v in b.where(ca.carray(a <= 5))] #print "numpy ->", [v for v in a if v<=5] #print "where ->", [v for v in b.where(ca.carray(a<=5))] self.assert_(wt == cwt, "where() does not work correctly")
def test02(self): """Testing append() with carrays""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = ca.ctable(ra, rootdir=self.rootdir) a = np.arange(N, N+10, dtype='i4') b = np.arange(N, N+10, dtype='f8')*2. t.append((ca.carray(a), ca.carray(b))) ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8') assert_array_equal(t[:], ra, "ctable values are not correct")
def test00a(self): """Testing ctable creation from a tuple of carrays""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) #print "t->", `t` ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test02(self): """Testing append() with carrays""" N = 10 ra = np.fromiter(((i, i * 2.) for i in xrange(N)), dtype='i4,f8') t = ca.ctable(ra, rootdir=self.rootdir) a = np.arange(N, N + 10, dtype='i4') b = np.arange(N, N + 10, dtype='f8') * 2. t.append((ca.carray(a), ca.carray(b))) ra = np.fromiter(((i, i * 2.) for i in xrange(N + 10)), dtype='i4,f8') assert_array_equal(t[:], ra, "ctable values are not correct")
def test06(self): """Testing `where()` iterator using `limit` and `skip`""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v <= 5][1:4] cwt = [v for v in b.where(ca.carray(a <= 5), limit=3, skip=1)] #print "numpy ->", [v for v in a if v<=5][1:4] #print "where ->", [v for v in b.where(ca.carray(a<=5), # limit=3, skip=1)] self.assert_(wt == cwt, "where() does not work correctly")
def test00a(self): """Testing ctable creation from a tuple of carrays""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) #print "t->", `t` ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct")
def test04(self): """Testing fancy indexing with __setitem__ (bool carray)""" a = np.arange(1, 1e2) b = ca.carray(a, chunklen=10) bc = (a > 5) & (a < 40) sl = ca.carray(bc) b[sl] = 3. a[bc] = 3. #print "b[%s] -> %r" % (sl, b) assert_array_equal(b[:], a, "fancy indexing does not work correctly")
def test06(self): """Testing `where()` iterator using `limit` and `skip`""" a = np.arange(1, 11) b = ca.carray(a) wt = [v for v in a if v<=5][1:4] cwt = [v for v in b.where(ca.carray(a<=5), limit=3, skip=1)] #print "numpy ->", [v for v in a if v<=5][1:4] #print "where ->", [v for v in b.where(ca.carray(a<=5), # limit=3, skip=1)] self.assert_(wt == cwt, "where() does not work correctly")
def test04(self): """Testing fancy indexing with __setitem__ (bool carray)""" a = np.arange(1,1e2) b = ca.carray(a, chunklen=10) bc = (a > 5) & (a < 40) sl = ca.carray(bc) b[sl] = 3. a[bc] = 3. #print "b[%s] -> %r" % (sl, b) assert_array_equal(b[:], a, "fancy indexing does not work correctly")
def test01c(self): """Testing ctable creation in "a" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last ctable self.assertRaises(RuntimeError, ca.ctable, (a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='a')
def test07(self): """Testing `where()` iterator using `limit` and `skip` (zeros)""" a = np.arange(10000) b = ca.carray(a,) wt = [v for v in a if v<=5000][1010:2020] cwt = [v for v in b.where(ca.carray(a<=5000, chunklen=100), limit=1010, skip=1010)] # print "numpy ->", [v for v in a if v>=5000][1010:2020] # print "where ->", [v for v in b.where(ca.carray(a>=5000,chunklen=100), # limit=1010, skip=1010)] self.assert_(wt == cwt, "where() does not work correctly")
def test07(self): """Testing `where()` iterator using `limit` and `skip` (zeros)""" a = np.arange(10000) b = ca.carray(a, ) wt = [v for v in a if v <= 5000][1010:2020] cwt = [ v for v in b.where( ca.carray(a <= 5000, chunklen=100), limit=1010, skip=1010) ] # print "numpy ->", [v for v in a if v>=5000][1010:2020] # print "where ->", [v for v in b.where(ca.carray(a>=5000,chunklen=100), # limit=1010, skip=1010)] self.assert_(wt == cwt, "where() does not work correctly")
def test01d(self): """Testing `__getitem()__` method with only a (large) start""" a = np.arange(1e4) b = ca.carray(a, rootdir=self.rootdir) sl = -2 # second last element #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test03b(self): """Testing `iter()` method with start, stop, step""" a = np.arange(101) b = ca.carray(a, chunklen=2, rootdir=self.rootdir) #print "sum iter->", sum(b.iter(3, 24, 4)) self.assert_( sum(a[3:24:4]) == sum(b.iter(3, 24, 4)), "Sums are not equal")
def test04a(self): """Testing `__getitem()__` method with long ranges""" a = np.arange(1e3) b = ca.carray(a, chunklen=100, rootdir=self.rootdir) sl = slice(1, 8000) #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02(self): """Testing fancy indexing (empty list)""" a = np.arange(101) b = ca.carray(a) c = b[[]] r = a[[]] assert_array_equal(c, r, "fancy indexing does not work correctly")
def test00(self): """Testing fancy indexing (short list)""" a = np.arange(1,111) b = ca.carray(a) c = b[[3,1]] r = a[[3,1]] assert_array_equal(c, r, "fancy indexing does not work correctly")
def test04(self): """Testing `iter()` method with large zero arrays""" a = np.zeros(1e4, dtype='f8') b = ca.carray(a, chunklen=100, rootdir=self.rootdir) c = ca.fromiter((v for v in b), dtype='f8', count=len(a)) #print "c ->", repr(c) assert_array_equal(a, c[:], "iterator fails on zeros")
def test03(self): """Testing copy() with no shuffle""" a = np.linspace(-1., 1., 1e4) b = ca.carray(a, rootdir=self.rootdir) c = b.copy(cparams=ca.cparams(shuffle=False)) #print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes self.assert_(b.cbytes < c.cbytes, "shuffle not changed")
def test04d(self): """Testing `__getitem()__` method with no start and no stop""" a = np.arange(1e3) b = ca.carray(a, chunklen=100, rootdir=self.rootdir) sl = slice(None, None, 2) #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test01b(self): """Testing `__getitem()__` method with only a (negative) start""" a = np.arange(1e2) b = ca.carray(a, chunklen=10, rootdir=self.rootdir) sl = slice(-1) #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test03d(self): """Testing `__getitem()__` method with ranges and steps (IV)""" a = np.arange(1e3) b = ca.carray(a, chunklen=10, rootdir=self.rootdir) sl = slice(4, 80, 3000) #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02c(self): """Testing `__getitem()__` method with ranges (negative stop)""" a = np.arange(1e3) b = ca.carray(a, chunklen=10, rootdir=self.rootdir) sl = slice(1, -3) #print "b[sl]->", `b[sl]` assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02c(self): """Testing `iter()` method with positive start, negative stop""" a = np.arange(101) b = ca.carray(a, chunklen=2, rootdir=self.rootdir) #print "sum iter->", sum(b.iter(24, -3)) self.assert_( sum(a[24:-3]) == sum(b.iter(24, -3)), "Sums are not equal")
def test03(self): """Testing fancy indexing (list of floats)""" a = np.arange(1, 101) b = ca.carray(a) c = b[[1.1, 3.3]] r = a[[1.1, 3.3]] assert_array_equal(c, r, "fancy indexing does not work correctly")
def test02(self): """Testing copy() with lesser compression""" a = np.linspace(-1., 1., 1e4) b = ca.carray(a, rootdir=self.rootdir) c = b.copy(cparams=ca.cparams(clevel=1)) #print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes self.assert_(b.cbytes < c.cbytes, "clevel not changed")
def test02(self): """Testing __sizeof__() (small carrays)""" a = np.arange(111) b = ca.carray(a) #print "size b uncompressed-->", b.nbytes #print "size b compressed -->", b.cbytes self.assert_(sys.getsizeof(b) > b.nbytes, "carray compress too much??")
def test00a(self): """Testing ctable opening in "r" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8') + 1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Open t t = ca.open(rootdir=self.rootdir, mode='r') #print "t->", `t` ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses self.assertRaises(RuntimeError, t.__setitem__, 1, (0, 0.0)) self.assertRaises(RuntimeError, t.append, (0, 0.0))
def test05(self): """Testing `__getitem()__` method with negative steps""" a = np.arange(1e3) b = ca.carray(a, chunklen=10, rootdir=self.rootdir) sl = slice(None, None, -3) #print "b[sl]->", `b[sl]` self.assertRaises(NotImplementedError, b.__getitem__, sl)
def test00(self): """Testing fancy indexing (short list)""" a = np.arange(1, 111) b = ca.carray(a) c = b[[3, 1]] r = a[[3, 1]] assert_array_equal(c, r, "fancy indexing does not work correctly")
def test03(self): """Testing fancy indexing (list of floats)""" a = np.arange(1,101) b = ca.carray(a) c = b[[1.1, 3.3]] r = a[[1.1, 3.3]] assert_array_equal(c, r, "fancy indexing does not work correctly")
def test02c(self): """Testing `iter()` method with positive start, negative stop""" a = np.arange(101) b = ca.carray(a, chunklen=2, rootdir=self.rootdir) #print "sum iter->", sum(b.iter(24, -3)) self.assert_(sum(a[24:-3]) == sum(b.iter(24, -3)), "Sums are not equal")
def test03a(self): """Testing `iter()` method with only step""" a = np.arange(101) b = ca.carray(a, chunklen=2, rootdir=self.rootdir) #print "sum iter->", sum(b.iter(step=4)) self.assert_(sum(a[::4]) == sum(b.iter(step=4)), "Sums are not equal")
def test00a(self): """Testing ctable opening in "r" mode""" N = 1e1 a = ca.carray(np.arange(N, dtype='i4')) b = ca.carray(np.arange(N, dtype='f8')+1) t = ca.ctable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Open t t = ca.open(rootdir=self.rootdir, mode='r') #print "t->", `t` ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "ctable values are not correct") # Now check some accesses self.assertRaises(RuntimeError, t.__setitem__, 1, (0, 0.0)) self.assertRaises(RuntimeError, t.append, (0, 0.0))
def test03b(self): """Testing `iter()` method with start, stop, step""" a = np.arange(101) b = ca.carray(a, chunklen=2, rootdir=self.rootdir) #print "sum iter->", sum(b.iter(3, 24, 4)) self.assert_(sum(a[3:24:4]) == sum(b.iter(3, 24, 4)), "Sums are not equal")
def test01c(self): """Testing `__getitem()__` method with only a (start,)""" a = np.arange(1e2) b = ca.carray(a, chunklen=10, rootdir=self.rootdir) #print "b[(1,)]->", `b[(1,)]` self.assert_(a[(1, )] == b[(1, )], "Values with key (1,) are not equal")
def test01(self): """Testing fancy indexing (large list, numpy)""" a = np.arange(1,1e4) b = ca.carray(a) idx = np.random.randint(1000, size=1000) c = b[idx] r = a[idx] assert_array_equal(c, r, "fancy indexing does not work correctly")