def test01b(self):
"""Testing btable creation in "w" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype='i4'))
b = blz.barray(np.arange(N, dtype='f8')+1)
t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir)
# Overwrite the last btable
t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir, mode='w')
#print "t->", `t`
ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
# Now check some accesses
t.append((10, 11.0))
t.append((10, 11.0))
t[11] = (11, 12.0)
# Check values
N = 12
a = blz.barray(np.arange(N, dtype='i4'))
b = blz.barray(np.arange(N, dtype='f8')+1)
ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test04(self):
"""Testing append() with another btable"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
ra2 = np.fromiter(((i, i*2.) for i in xrange(N, N+10)), dtype='i4,f8')
t2 = blz.btable(ra2)
t.append(t2)
ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8')
assert_array_equal(t[:], ra, "btable values are not correct")
def test00b(self):
"""Testing btable creation from a tuple of lists"""
t = blz.btable(([1,2,3],[4,5,6]), ('f0', 'f1'), rootdir=self.rootdir)
#print "t->", `t`
ra = np.rec.fromarrays([[1,2,3],[4,5,6]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test00c(self):
"""Testing btable opening in "a" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype='i4'))
b = blz.barray(np.arange(N, dtype='f8')+1)
t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir)
# Open t
t = blz.open(rootdir=self.rootdir, mode='a')
#print "t->", `t`
# Check values
ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
# Now check some accesses
t.append((10, 11.0))
t.append((10, 11.0))
t[-1] = (11, 12.0)
# Check values
N = 12
a = blz.barray(np.arange(N, dtype='i4'))
b = blz.barray(np.arange(N, dtype='f8')+1)
ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self):
"""Testing append() with scalar values"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
t.append((N, N*2))
ra = np.fromiter(((i, i*2.) for i in xrange(N+1)), dtype='i4,f8')
assert_array_equal(t[:], ra, "btable values are not correct")
def test02(self):
"""Testing btable creation from an structured array"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
#print "t->", `t`
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test01b(self):
"""Testing bparams when adding a new column (numpy flavor)"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, bparams=blz.bparams(1), rootdir=self.rootdir)
c = np.arange(N, dtype='i8')*3
t.addcol(c, 'f2')
self.assert_(t['f2'].bparams.clevel == 1, "Incorrect clevel")
def test00(self):
"""Testing __getitem__ with only a start"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
start = 9
#print "t->", `t`
#print "ra[:]", ra[:]
assert_array_equal(t[start], ra[start], "btable values are not correct")
def test01c(self):
"""Testing btable creation in "a" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype='i4'))
b = blz.barray(np.arange(N, dtype='f8')+1)
t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir)
# Overwrite the last btable
self.assertRaises(RuntimeError, blz.btable, (a, b), ('f0', 'f1'),
rootdir=self.rootdir, mode='a')
def test01(self):
"""Testing __sizeof__() (big btables)"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
#print "size t uncompressed ->", t.nbytes
#print "size t compressed ->", t.cbytes
self.assert_(sys.getsizeof(t) < t.nbytes,
"btable does not seem to compress at all")
def test03(self):
"""Testing copy() with no shuffle"""
N = 10*1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra)
# print "t:", t, t.rootdir
t2 = t.copy(bparams=blz.bparams(shuffle=False), rootdir=self.rootdir)
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test02(self):
"""Testing __sizeof__() (small btables)"""
N = int(111)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
#print "size t uncompressed ->", t.nbytes
#print "size t compressed ->", t.cbytes
self.assert_(sys.getsizeof(t) > t.nbytes,
"btable compress too much??")
def test02(self):
"""Testing fancy indexing with an empty list"""
N = 10*1000
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
rt = t[[]]
rar = ra[[]]
#print "rt->", rt
#print "rar->", rar
assert_array_equal(rt, rar, "btable values are not correct")
def test02(self):
"""Testing __getitem__ with start, stop, step"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
start, stop, step = 3, 9, 2
#print "t->", `t[start:stop:step]`
#print "ra->", ra[start:stop:step]
assert_array_equal(t[start:stop:step], ra[start:stop:step],
"btable values are not correct")
def test02(self):
"""Testing copy() with lower clevel"""
N = 10*1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
t2 = t.copy(bparams=blz.bparams(clevel=1))
self.assert_(t.bparams.clevel == blz.bparams().clevel)
self.assert_(t2.bparams.clevel == 1)
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test03(self):
"""Testing __getitem__ with a column name"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
colname = "f1"
#print "t->", `t[colname]`
#print "ra->", ra[colname]
assert_array_equal(t[colname][:], ra[colname],
"btable values are not correct")
def test03(self):
"""Testing fancy indexing (list of floats)"""
N = 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
rt = t[[2.3, 5.6]]
rar = ra[[2.3, 5.6]]
#print "rt->", rt
#print "rar->", rar
assert_array_equal(rt, rar, "btable values are not correct")
def test01(self):
"""Testing btable.iter() without params"""
N = 10
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r.f1 for r in t.iter()]
nl = [r['f1'] for r in ra]
#print "cl ->", cl
#print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test02(self):
"""Testing append() with barrays"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = blz.btable(ra, rootdir=self.rootdir)
a = np.arange(N, N+10, dtype='i4')
b = np.arange(N, N+10, dtype='f8')*2.
t.append((blz.barray(a), blz.barray(b)))
ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8')
assert_array_equal(t[:], ra, "btable values are not correct")
def test07(self):
"""Testing btable.iter() with start, stop, step and limit, skip"""
N = 10
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r.f1 for r in t.iter(1,9,2, limit=2, skip=1)]
nl = [r['f1'] for r in ra[1:9:2][1:3]]
#print "cl ->", cl
#print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test04(self):
"""Testing btable.iter() with start,stop,step and outcols"""
N = 10
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r for r in t.iter(1,9,3, 'f2, nrow__ f0')]
nl = [(r['f2'], r['f0'], r['f0']) for r in ra[1:9:3]]
#print "cl ->", cl
#print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test03(self):
"""Testing btable.iter() with outcols"""
N = 10
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [tuple(r) for r in t.iter(outcols='f2, nrow__, f0')]
nl = [(r['f2'], i, r['f0']) for i, r in enumerate(ra)]
#print "cl ->", cl
#print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test01(self):
"""Testing btable creation from a tuple of numpy arrays"""
N = 1e1
a = np.arange(N, dtype='i4')
b = np.arange(N, dtype='f8')+1
t = blz.btable((a, b), ('f0', 'f1'), rootdir=self.rootdir)
#print "t->", `t`
ra = np.rec.fromarrays([a,b]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self):
"""Testing `wherechunks` method with only an expression"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2'):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test07(self):
"""Testing `wherechunks` method with a `limit`, `skip` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=N-M-2, skip=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - M - 2)
self.assert_(s == np.arange(M+1, N-1).sum())
def test05(self):
"""Testing `wherechunks` method with a `limit` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == M)
self.assert_(s == M * ((M + 1) / 2)) # Gauss summation formula
def test03(self):
"""Testing `iterchunks` method with all parameters set"""
N, blen = int(1e4), 100
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.iterblocks(t, blen, blen-1, 3*blen+2):
l += len(block)
s += block['f2'].sum()
self.assert_(l == 2*blen + 3)
self.assert_(s == (np.arange(blen-1, 3*blen+2)*3).sum())
def test02(self):
"""Testing `iterchunks` method with no stop"""
N, blen = int(1e4), 100
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0.
for block in blz.iterblocks(t, blen, blen-1):
l += len(block)
s += block['f1'].sum()
self.assert_(l == (N - (blen - 1)))
self.assert_(s == (np.arange(blen-1, N, dtype='f8')*2).sum())
def test00(self):
"""Testing `iterchunks` method with no blen, no start, no stop"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.iterblocks(t):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N)
self.assert_(s == (N - 1) * (N / 2)) # as per Gauss summation formula
def test03b(self):
"""Testing fancy indexing (setitem) with a boolean array (all true)"""
N = 1000
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra, chunklen=10)
sl = np.ones(N, dtype="bool")
t[sl] = [(-1, -2, -3)]
ra[sl] = [(-1, -2, -3)]
#print "t[%s] -> %r" % (sl, t)
#print "ra[%s] -> %r" % (sl, ra)
assert_array_equal(t[:], ra, "btable values are not correct")
