def test01b(self):
"""Testing btable creation in "w" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype="i4"))
b = blz.barray(np.arange(N, dtype="f8") + 1)
t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
# Overwrite the last btable
t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir, mode="w")
# print "t->", `t`
ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
# Now check some accesses
t.append((10, 11.0))
t.append((10, 11.0))
t[11] = (11, 12.0)
# Check values
N = 12
a = blz.barray(np.arange(N, dtype="i4"))
b = blz.barray(np.arange(N, dtype="f8") + 1)
ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test04(self):
"""Testing append() with another btable"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
ra2 = np.fromiter(((i, i * 2.0) for i in xrange(N, N + 10)), dtype="i4,f8")
t2 = blz.btable(ra2)
t.append(t2)
ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 10)), dtype="i4,f8")
assert_array_equal(t[:], ra, "btable values are not correct")
def test00b(self):
"""Testing btable creation from a tuple of lists"""
t = blz.btable(([1, 2, 3], [4, 5, 6]), ("f0", "f1"), rootdir=self.rootdir)
# print "t->", `t`
ra = np.rec.fromarrays([[1, 2, 3], [4, 5, 6]]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test00c(self):
"""Testing btable opening in "a" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype="i4"))
b = blz.barray(np.arange(N, dtype="f8") + 1)
t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
# Open t
t = blz.open(rootdir=self.rootdir, mode="a")
# print "t->", `t`
# Check values
ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
# Now check some accesses
t.append((10, 11.0))
t.append((10, 11.0))
t[-1] = (11, 12.0)
# Check values
N = 12
a = blz.barray(np.arange(N, dtype="i4"))
b = blz.barray(np.arange(N, dtype="f8") + 1)
ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test01b(self):
"""Testing bparams when adding a new column (numpy flavor)"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, bparams=blz.bparams(1), rootdir=self.rootdir)
c = np.arange(N, dtype="i8") * 3
t.addcol(c, "f2")
self.assert_(t["f2"].bparams.clevel == 1, "Incorrect clevel")
def test01c(self):
"""Testing btable creation in "a" mode"""
N = 1e1
a = blz.barray(np.arange(N, dtype="i4"))
b = blz.barray(np.arange(N, dtype="f8") + 1)
t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
# Overwrite the last btable
self.assertRaises(RuntimeError, blz.btable, (a, b), ("f0", "f1"), rootdir=self.rootdir, mode="a")
def test01(self):
"""Testing __sizeof__() (big btables)"""
N = int(1e4)
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
# print "size t uncompressed ->", t.nbytes
# print "size t compressed ->", t.cbytes
self.assert_(sys.getsizeof(t) < t.nbytes, "btable does not seem to compress at all")
def test02(self):
"""Testing __sizeof__() (small btables)"""
N = int(111)
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
# print "size t uncompressed ->", t.nbytes
# print "size t compressed ->", t.cbytes
self.assert_(sys.getsizeof(t) > t.nbytes, "btable compress too much??")
def test02(self):
"""Testing btable creation from an structured array"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
# print "t->", `t`
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self):
"""Testing append() with scalar values"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
t.append((N, N * 2))
ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 1)), dtype="i4,f8")
assert_array_equal(t[:], ra, "btable values are not correct")
def test00(self):
"""Testing __getitem__ with only a start"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
start = 9
# print "t->", `t`
# print "ra[:]", ra[:]
assert_array_equal(t[start], ra[start], "btable values are not correct")
def test03(self):
"""Testing copy() with no shuffle"""
N = 10 * 1000
ra = np.fromiter(((i, i ** 2.2) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra)
# print "t:", t, t.rootdir
t2 = t.copy(bparams=blz.bparams(shuffle=False), rootdir=self.rootdir)
# print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t["f1"].cbytes < t2["f1"].cbytes, "clevel not changed")
def test02(self):
"""Testing __getitem__ with start, stop, step"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
start, stop, step = 3, 9, 2
# print "t->", `t[start:stop:step]`
# print "ra->", ra[start:stop:step]
assert_array_equal(t[start:stop:step], ra[start:stop:step], "btable values are not correct")
def test03(self):
"""Testing __getitem__ with a column name"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
colname = "f1"
# print "t->", `t[colname]`
# print "ra->", ra[colname]
assert_array_equal(t[colname][:], ra[colname], "btable values are not correct")
def test02(self):
"""Testing fancy indexing with an empty list"""
N = 10 * 1000
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
rt = t[[]]
rar = ra[[]]
# print "rt->", rt
# print "rar->", rar
assert_array_equal(rt, rar, "btable values are not correct")
def test02(self):
"""Testing copy() with lower clevel"""
N = 10 * 1000
ra = np.fromiter(((i, i ** 2.2) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
t2 = t.copy(bparams=blz.bparams(clevel=1))
self.assert_(t.bparams.clevel == blz.bparams().clevel)
self.assert_(t2.bparams.clevel == 1)
# print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t["f1"].cbytes < t2["f1"].cbytes, "clevel not changed")
def test02(self):
"""Testing append() with barrays"""
N = 10
ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
t = blz.btable(ra, rootdir=self.rootdir)
a = np.arange(N, N + 10, dtype="i4")
b = np.arange(N, N + 10, dtype="f8") * 2.0
t.append((blz.barray(a), blz.barray(b)))
ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 10)), dtype="i4,f8")
assert_array_equal(t[:], ra, "btable values are not correct")
def test01(self):
"""Testing btable creation from a tuple of numpy arrays"""
N = 1e1
a = np.arange(N, dtype="i4")
b = np.arange(N, dtype="f8") + 1
t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
# print "t->", `t`
ra = np.rec.fromarrays([a, b]).view(np.ndarray)
# print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "btable values are not correct")
def test03(self):
"""Testing fancy indexing (list of floats)"""
N = 101
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
rt = t[[2.3, 5.6]]
rar = ra[[2.3, 5.6]]
# print "rt->", rt
# print "rar->", rar
assert_array_equal(rt, rar, "btable values are not correct")
def test07(self):
"""Testing btable.iter() with start, stop, step and limit, skip"""
N = 10
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r.f1 for r in t.iter(1, 9, 2, limit=2, skip=1)]
nl = [r["f1"] for r in ra[1:9:2][1:3]]
# print "cl ->", cl
# print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test04(self):
"""Testing btable.iter() with start,stop,step and outcols"""
N = 10
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r for r in t.iter(1, 9, 3, "f2, nrow__ f0")]
nl = [(r["f2"], r["f0"], r["f0"]) for r in ra[1:9:3]]
# print "cl ->", cl
# print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test03(self):
"""Testing btable.iter() with outcols"""
N = 10
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [tuple(r) for r in t.iter(outcols="f2, nrow__, f0")]
nl = [(r["f2"], i, r["f0"]) for i, r in enumerate(ra)]
# print "cl ->", cl
# print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test01(self):
"""Testing btable.iter() without params"""
N = 10
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
cl = [r.f1 for r in t.iter()]
nl = [r["f1"] for r in ra]
# print "cl ->", cl
# print "nl ->", nl
self.assert_(cl == nl, "iter not working correctily")
def test00(self):
"""Testing `wherechunks` method with only an expression"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2'):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test07(self):
"""Testing `wherechunks` method with a `limit`, `skip` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=N-M-2, skip=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - M - 2)
self.assert_(s == np.arange(M+1, N-1).sum())
def test05(self):
"""Testing `wherechunks` method with a `limit` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == M)
self.assert_(s == M * ((M + 1) / 2)) # Gauss summation formula
def test03(self):
"""Testing `iterchunks` method with all parameters set"""
N, blen = int(1e4), 100
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
l, s = 0, 0
for block in blz.iterblocks(t, blen, blen - 1, 3 * blen + 2):
l += len(block)
s += block["f2"].sum()
self.assert_(l == 2 * blen + 3)
self.assert_(s == (np.arange(blen - 1, 3 * blen + 2) * 3).sum())
def test02(self):
"""Testing `iterchunks` method with no stop"""
N, blen = int(1e4), 100
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
l, s = 0, 0.0
for block in blz.iterblocks(t, blen, blen - 1):
l += len(block)
s += block["f1"].sum()
self.assert_(l == (N - (blen - 1)))
self.assert_(s == (np.arange(blen - 1, N, dtype="f8") * 2).sum())
def test00(self):
"""Testing `iterchunks` method with no blen, no start, no stop"""
N = int(1e4)
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra)
l, s = 0, 0
for block in blz.iterblocks(t):
l += len(block)
s += block["f0"].sum()
self.assert_(l == N)
self.assert_(s == (N - 1) * (N / 2)) # as per Gauss summation formula
def test03b(self):
"""Testing fancy indexing (setitem) with a boolean array (all true)"""
N = 1000
ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
t = blz.btable(ra, chunklen=10)
sl = np.ones(N, dtype="bool")
t[sl] = [(-1, -2, -3)]
ra[sl] = [(-1, -2, -3)]
# print "t[%s] -> %r" % (sl, t)
# print "ra[%s] -> %r" % (sl, ra)
assert_array_equal(t[:], ra, "btable values are not correct")