def test01b(self): """Testing btable creation in "w" mode""" N = 1e1 a = blz.barray(np.arange(N, dtype='i4')) b = blz.barray(np.arange(N, dtype='f8')+1) t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last btable t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='w') #print "t->", `t` ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[11] = (11, 12.0) # Check values N = 12 a = blz.barray(np.arange(N, dtype='i4')) b = blz.barray(np.arange(N, dtype='f8')+1) ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct")
def test04(self): """Testing append() with another btable""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) ra2 = np.fromiter(((i, i*2.) for i in xrange(N, N+10)), dtype='i4,f8') t2 = blz.btable(ra2) t.append(t2) ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8') assert_array_equal(t[:], ra, "btable values are not correct")
def test00b(self): """Testing btable creation from a tuple of lists""" t = blz.btable(([1,2,3],[4,5,6]), ('f0', 'f1'), rootdir=self.rootdir) #print "t->", `t` ra = np.rec.fromarrays([[1,2,3],[4,5,6]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct")
def test00c(self): """Testing btable opening in "a" mode""" N = 1e1 a = blz.barray(np.arange(N, dtype='i4')) b = blz.barray(np.arange(N, dtype='f8')+1) t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Open t t = blz.open(rootdir=self.rootdir, mode='a') #print "t->", `t` # Check values ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct") # Now check some accesses t.append((10, 11.0)) t.append((10, 11.0)) t[-1] = (11, 12.0) # Check values N = 12 a = blz.barray(np.arange(N, dtype='i4')) b = blz.barray(np.arange(N, dtype='f8')+1) ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self): """Testing append() with scalar values""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) t.append((N, N*2)) ra = np.fromiter(((i, i*2.) for i in xrange(N+1)), dtype='i4,f8') assert_array_equal(t[:], ra, "btable values are not correct")
def test02(self): """Testing btable creation from an structured array""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) #print "t->", `t` #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct")
def test01b(self): """Testing bparams when adding a new column (numpy flavor)""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, bparams=blz.bparams(1), rootdir=self.rootdir) c = np.arange(N, dtype='i8')*3 t.addcol(c, 'f2') self.assert_(t['f2'].bparams.clevel == 1, "Incorrect clevel")
def test00(self): """Testing __getitem__ with only a start""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) start = 9 #print "t->", `t` #print "ra[:]", ra[:] assert_array_equal(t[start], ra[start], "btable values are not correct")
def test01c(self): """Testing btable creation in "a" mode""" N = 1e1 a = blz.barray(np.arange(N, dtype='i4')) b = blz.barray(np.arange(N, dtype='f8')+1) t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir) # Overwrite the last btable self.assertRaises(RuntimeError, blz.btable, (a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='a')
def test01(self): """Testing __sizeof__() (big btables)""" N = int(1e4) ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) #print "size t uncompressed ->", t.nbytes #print "size t compressed ->", t.cbytes self.assert_(sys.getsizeof(t) < t.nbytes, "btable does not seem to compress at all")
def test03(self): """Testing copy() with no shuffle""" N = 10*1000 ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra) # print "t:", t, t.rootdir t2 = t.copy(bparams=blz.bparams(shuffle=False), rootdir=self.rootdir) #print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test02(self): """Testing __sizeof__() (small btables)""" N = int(111) ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) #print "size t uncompressed ->", t.nbytes #print "size t compressed ->", t.cbytes self.assert_(sys.getsizeof(t) > t.nbytes, "btable compress too much??")
def test02(self): """Testing fancy indexing with an empty list""" N = 10*1000 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) rt = t[[]] rar = ra[[]] #print "rt->", rt #print "rar->", rar assert_array_equal(rt, rar, "btable values are not correct")
def test02(self): """Testing __getitem__ with start, stop, step""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) start, stop, step = 3, 9, 2 #print "t->", `t[start:stop:step]` #print "ra->", ra[start:stop:step] assert_array_equal(t[start:stop:step], ra[start:stop:step], "btable values are not correct")
def test02(self): """Testing copy() with lower clevel""" N = 10*1000 ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) t2 = t.copy(bparams=blz.bparams(clevel=1)) self.assert_(t.bparams.clevel == blz.bparams().clevel) self.assert_(t2.bparams.clevel == 1) #print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test03(self): """Testing __getitem__ with a column name""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) colname = "f1" #print "t->", `t[colname]` #print "ra->", ra[colname] assert_array_equal(t[colname][:], ra[colname], "btable values are not correct")
def test03(self): """Testing fancy indexing (list of floats)""" N = 101 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) rt = t[[2.3, 5.6]] rar = ra[[2.3, 5.6]] #print "rt->", rt #print "rar->", rar assert_array_equal(rt, rar, "btable values are not correct")
def test01(self): """Testing btable.iter() without params""" N = 10 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra, chunklen=4, rootdir=self.rootdir) cl = [r.f1 for r in t.iter()] nl = [r['f1'] for r in ra] #print "cl ->", cl #print "nl ->", nl self.assert_(cl == nl, "iter not working correctily")
def test02(self): """Testing append() with barrays""" N = 10 ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8') t = blz.btable(ra, rootdir=self.rootdir) a = np.arange(N, N+10, dtype='i4') b = np.arange(N, N+10, dtype='f8')*2. t.append((blz.barray(a), blz.barray(b))) ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8') assert_array_equal(t[:], ra, "btable values are not correct")
def test07(self): """Testing btable.iter() with start, stop, step and limit, skip""" N = 10 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra, chunklen=4, rootdir=self.rootdir) cl = [r.f1 for r in t.iter(1,9,2, limit=2, skip=1)] nl = [r['f1'] for r in ra[1:9:2][1:3]] #print "cl ->", cl #print "nl ->", nl self.assert_(cl == nl, "iter not working correctily")
def test04(self): """Testing btable.iter() with start,stop,step and outcols""" N = 10 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra, chunklen=4, rootdir=self.rootdir) cl = [r for r in t.iter(1,9,3, 'f2, nrow__ f0')] nl = [(r['f2'], r['f0'], r['f0']) for r in ra[1:9:3]] #print "cl ->", cl #print "nl ->", nl self.assert_(cl == nl, "iter not working correctily")
def test03(self): """Testing btable.iter() with outcols""" N = 10 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra, chunklen=4, rootdir=self.rootdir) cl = [tuple(r) for r in t.iter(outcols='f2, nrow__, f0')] nl = [(r['f2'], i, r['f0']) for i, r in enumerate(ra)] #print "cl ->", cl #print "nl ->", nl self.assert_(cl == nl, "iter not working correctily")
def test01(self): """Testing btable creation from a tuple of numpy arrays""" N = 1e1 a = np.arange(N, dtype='i4') b = np.arange(N, dtype='f8')+1 t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir) #print "t->", `t` ra = np.rec.fromarrays([a,b]).view(np.ndarray) #print "ra[:]", ra[:] assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self): """Testing `wherechunks` method with only an expression""" N = int(1e4) ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0 for block in blz.whereblocks(t, 'f1 < f2'): l += len(block) s += block['f0'].sum() self.assert_(l == N - 1) self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test07(self): """Testing `wherechunks` method with a `limit`, `skip` parameter""" N, M = int(1e4), 101 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0 for block in blz.whereblocks(t, 'f1 < f2', limit=N-M-2, skip=M): l += len(block) s += block['f0'].sum() self.assert_(l == N - M - 2) self.assert_(s == np.arange(M+1, N-1).sum())
def test05(self): """Testing `wherechunks` method with a `limit` parameter""" N, M = int(1e4), 101 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0 for block in blz.whereblocks(t, 'f1 < f2', limit=M): l += len(block) s += block['f0'].sum() self.assert_(l == M) self.assert_(s == M * ((M + 1) / 2)) # Gauss summation formula
def test03(self): """Testing `iterchunks` method with all parameters set""" N, blen = int(1e4), 100 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0 for block in blz.iterblocks(t, blen, blen-1, 3*blen+2): l += len(block) s += block['f2'].sum() self.assert_(l == 2*blen + 3) self.assert_(s == (np.arange(blen-1, 3*blen+2)*3).sum())
def test02(self): """Testing `iterchunks` method with no stop""" N, blen = int(1e4), 100 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0. for block in blz.iterblocks(t, blen, blen-1): l += len(block) s += block['f1'].sum() self.assert_(l == (N - (blen - 1))) self.assert_(s == (np.arange(blen-1, N, dtype='f8')*2).sum())
def test00(self): """Testing `iterchunks` method with no blen, no start, no stop""" N = int(1e4) ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra) l, s = 0, 0 for block in blz.iterblocks(t): l += len(block) s += block['f0'].sum() self.assert_(l == N) self.assert_(s == (N - 1) * (N / 2)) # as per Gauss summation formula
def test03b(self): """Testing fancy indexing (setitem) with a boolean array (all true)""" N = 1000 ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8') t = blz.btable(ra, chunklen=10) sl = np.ones(N, dtype="bool") t[sl] = [(-1, -2, -3)] ra[sl] = [(-1, -2, -3)] #print "t[%s] -> %r" % (sl, t) #print "ra[%s] -> %r" % (sl, ra) assert_array_equal(t[:], ra, "btable values are not correct")