Exemple #1
0
    def test01b(self):
        """Testing btable creation in "w" mode"""
        N = 1e1
        a = blz.barray(np.arange(N, dtype="i4"))
        b = blz.barray(np.arange(N, dtype="f8") + 1)
        t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
        # Overwrite the last btable
        t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir, mode="w")
        # print "t->", `t`
        ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
        # print "ra[:]", ra[:]
        assert_array_equal(t[:], ra, "btable values are not correct")

        # Now check some accesses
        t.append((10, 11.0))
        t.append((10, 11.0))
        t[11] = (11, 12.0)

        # Check values
        N = 12
        a = blz.barray(np.arange(N, dtype="i4"))
        b = blz.barray(np.arange(N, dtype="f8") + 1)
        ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
        # print "ra[:]", ra[:]
        assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #2
0
 def test04(self):
     """Testing append() with another btable"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     ra2 = np.fromiter(((i, i * 2.0) for i in xrange(N, N + 10)), dtype="i4,f8")
     t2 = blz.btable(ra2)
     t.append(t2)
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 10)), dtype="i4,f8")
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #3
0
 def test00b(self):
     """Testing btable creation from a tuple of lists"""
     t = blz.btable(([1, 2, 3], [4, 5, 6]), ("f0", "f1"), rootdir=self.rootdir)
     # print "t->", `t`
     ra = np.rec.fromarrays([[1, 2, 3], [4, 5, 6]]).view(np.ndarray)
     # print "ra[:]", ra[:]
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #4
0
    def test00c(self):
        """Testing btable opening in "a" mode"""
        N = 1e1
        a = blz.barray(np.arange(N, dtype="i4"))
        b = blz.barray(np.arange(N, dtype="f8") + 1)
        t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
        # Open t
        t = blz.open(rootdir=self.rootdir, mode="a")
        # print "t->", `t`

        # Check values
        ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
        # print "ra[:]", ra[:]
        assert_array_equal(t[:], ra, "btable values are not correct")

        # Now check some accesses
        t.append((10, 11.0))
        t.append((10, 11.0))
        t[-1] = (11, 12.0)

        # Check values
        N = 12
        a = blz.barray(np.arange(N, dtype="i4"))
        b = blz.barray(np.arange(N, dtype="f8") + 1)
        ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
        # print "ra[:]", ra[:]
        assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #5
0
 def test01b(self):
     """Testing bparams when adding a new column (numpy flavor)"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, bparams=blz.bparams(1), rootdir=self.rootdir)
     c = np.arange(N, dtype="i8") * 3
     t.addcol(c, "f2")
     self.assert_(t["f2"].bparams.clevel == 1, "Incorrect clevel")
Exemple #6
0
 def test01c(self):
     """Testing btable creation in "a" mode"""
     N = 1e1
     a = blz.barray(np.arange(N, dtype="i4"))
     b = blz.barray(np.arange(N, dtype="f8") + 1)
     t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
     # Overwrite the last btable
     self.assertRaises(RuntimeError, blz.btable, (a, b), ("f0", "f1"), rootdir=self.rootdir, mode="a")
Exemple #7
0
 def test01(self):
     """Testing __sizeof__() (big btables)"""
     N = int(1e4)
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     # print "size t uncompressed ->", t.nbytes
     # print "size t compressed   ->", t.cbytes
     self.assert_(sys.getsizeof(t) < t.nbytes, "btable does not seem to compress at all")
Exemple #8
0
 def test02(self):
     """Testing __sizeof__() (small btables)"""
     N = int(111)
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     # print "size t uncompressed ->", t.nbytes
     # print "size t compressed   ->", t.cbytes
     self.assert_(sys.getsizeof(t) > t.nbytes, "btable compress too much??")
Exemple #9
0
 def test02(self):
     """Testing btable creation from an structured array"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     # print "t->", `t`
     # print "ra[:]", ra[:]
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #10
0
 def test00(self):
     """Testing append() with scalar values"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     t.append((N, N * 2))
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 1)), dtype="i4,f8")
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #11
0
 def test00(self):
     """Testing __getitem__ with only a start"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     start = 9
     # print "t->", `t`
     # print "ra[:]", ra[:]
     assert_array_equal(t[start], ra[start], "btable values are not correct")
Exemple #12
0
 def test03(self):
     """Testing copy() with no shuffle"""
     N = 10 * 1000
     ra = np.fromiter(((i, i ** 2.2) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra)
     # print "t:", t, t.rootdir
     t2 = t.copy(bparams=blz.bparams(shuffle=False), rootdir=self.rootdir)
     # print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
     self.assert_(t["f1"].cbytes < t2["f1"].cbytes, "clevel not changed")
Exemple #13
0
 def test02(self):
     """Testing __getitem__ with start, stop, step"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     start, stop, step = 3, 9, 2
     # print "t->", `t[start:stop:step]`
     # print "ra->", ra[start:stop:step]
     assert_array_equal(t[start:stop:step], ra[start:stop:step], "btable values are not correct")
Exemple #14
0
 def test03(self):
     """Testing __getitem__ with a column name"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     colname = "f1"
     # print "t->", `t[colname]`
     # print "ra->", ra[colname]
     assert_array_equal(t[colname][:], ra[colname], "btable values are not correct")
Exemple #15
0
 def test02(self):
     """Testing fancy indexing with an empty list"""
     N = 10 * 1000
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     rt = t[[]]
     rar = ra[[]]
     # print "rt->", rt
     # print "rar->", rar
     assert_array_equal(rt, rar, "btable values are not correct")
Exemple #16
0
 def test02(self):
     """Testing copy() with lower clevel"""
     N = 10 * 1000
     ra = np.fromiter(((i, i ** 2.2) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     t2 = t.copy(bparams=blz.bparams(clevel=1))
     self.assert_(t.bparams.clevel == blz.bparams().clevel)
     self.assert_(t2.bparams.clevel == 1)
     # print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
     self.assert_(t["f1"].cbytes < t2["f1"].cbytes, "clevel not changed")
Exemple #17
0
 def test02(self):
     """Testing append() with barrays"""
     N = 10
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N)), dtype="i4,f8")
     t = blz.btable(ra, rootdir=self.rootdir)
     a = np.arange(N, N + 10, dtype="i4")
     b = np.arange(N, N + 10, dtype="f8") * 2.0
     t.append((blz.barray(a), blz.barray(b)))
     ra = np.fromiter(((i, i * 2.0) for i in xrange(N + 10)), dtype="i4,f8")
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #18
0
 def test01(self):
     """Testing btable creation from a tuple of numpy arrays"""
     N = 1e1
     a = np.arange(N, dtype="i4")
     b = np.arange(N, dtype="f8") + 1
     t = blz.btable((a, b), ("f0", "f1"), rootdir=self.rootdir)
     # print "t->", `t`
     ra = np.rec.fromarrays([a, b]).view(np.ndarray)
     # print "ra[:]", ra[:]
     assert_array_equal(t[:], ra, "btable values are not correct")
Exemple #19
0
 def test03(self):
     """Testing fancy indexing (list of floats)"""
     N = 101
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     rt = t[[2.3, 5.6]]
     rar = ra[[2.3, 5.6]]
     # print "rt->", rt
     # print "rar->", rar
     assert_array_equal(rt, rar, "btable values are not correct")
Exemple #20
0
 def test07(self):
     """Testing btable.iter() with start, stop, step and limit, skip"""
     N = 10
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
     cl = [r.f1 for r in t.iter(1, 9, 2, limit=2, skip=1)]
     nl = [r["f1"] for r in ra[1:9:2][1:3]]
     # print "cl ->", cl
     # print "nl ->", nl
     self.assert_(cl == nl, "iter not working correctily")
Exemple #21
0
 def test04(self):
     """Testing btable.iter() with start,stop,step and outcols"""
     N = 10
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
     cl = [r for r in t.iter(1, 9, 3, "f2, nrow__ f0")]
     nl = [(r["f2"], r["f0"], r["f0"]) for r in ra[1:9:3]]
     # print "cl ->", cl
     # print "nl ->", nl
     self.assert_(cl == nl, "iter not working correctily")
Exemple #22
0
 def test03(self):
     """Testing btable.iter() with outcols"""
     N = 10
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
     cl = [tuple(r) for r in t.iter(outcols="f2, nrow__, f0")]
     nl = [(r["f2"], i, r["f0"]) for i, r in enumerate(ra)]
     # print "cl ->", cl
     # print "nl ->", nl
     self.assert_(cl == nl, "iter not working correctily")
Exemple #23
0
 def test01(self):
     """Testing btable.iter() without params"""
     N = 10
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra, chunklen=4, rootdir=self.rootdir)
     cl = [r.f1 for r in t.iter()]
     nl = [r["f1"] for r in ra]
     # print "cl ->", cl
     # print "nl ->", nl
     self.assert_(cl == nl, "iter not working correctily")
Exemple #24
0
 def test00(self):
     """Testing `wherechunks` method with only an expression"""
     N = int(1e4)
     ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
     t = blz.btable(ra)
     l, s = 0, 0
     for block in blz.whereblocks(t, 'f1 < f2'):
         l += len(block)
         s += block['f0'].sum()
     self.assert_(l == N - 1)
     self.assert_(s == (N - 1) * (N / 2))  # Gauss summation formula
Exemple #25
0
 def test07(self):
     """Testing `wherechunks` method with a `limit`, `skip` parameter"""
     N, M = int(1e4), 101
     ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
     t = blz.btable(ra)
     l, s = 0, 0
     for block in blz.whereblocks(t, 'f1 < f2', limit=N-M-2, skip=M):
         l += len(block)
         s += block['f0'].sum()
     self.assert_(l == N - M - 2)
     self.assert_(s == np.arange(M+1, N-1).sum())
Exemple #26
0
 def test05(self):
     """Testing `wherechunks` method with a `limit` parameter"""
     N, M = int(1e4), 101
     ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
     t = blz.btable(ra)
     l, s = 0, 0
     for block in blz.whereblocks(t, 'f1 < f2', limit=M):
         l += len(block)
         s += block['f0'].sum()
     self.assert_(l == M)
     self.assert_(s == M * ((M + 1) / 2))  # Gauss summation formula
Exemple #27
0
 def test03(self):
     """Testing `iterchunks` method with all parameters set"""
     N, blen = int(1e4), 100
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     l, s = 0, 0
     for block in blz.iterblocks(t, blen, blen - 1, 3 * blen + 2):
         l += len(block)
         s += block["f2"].sum()
     self.assert_(l == 2 * blen + 3)
     self.assert_(s == (np.arange(blen - 1, 3 * blen + 2) * 3).sum())
Exemple #28
0
 def test02(self):
     """Testing `iterchunks` method with no stop"""
     N, blen = int(1e4), 100
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     l, s = 0, 0.0
     for block in blz.iterblocks(t, blen, blen - 1):
         l += len(block)
         s += block["f1"].sum()
     self.assert_(l == (N - (blen - 1)))
     self.assert_(s == (np.arange(blen - 1, N, dtype="f8") * 2).sum())
Exemple #29
0
 def test00(self):
     """Testing `iterchunks` method with no blen, no start, no stop"""
     N = int(1e4)
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra)
     l, s = 0, 0
     for block in blz.iterblocks(t):
         l += len(block)
         s += block["f0"].sum()
     self.assert_(l == N)
     self.assert_(s == (N - 1) * (N / 2))  # as per Gauss summation formula
Exemple #30
0
 def test03b(self):
     """Testing fancy indexing (setitem) with a boolean array (all true)"""
     N = 1000
     ra = np.fromiter(((i, i * 2.0, i * 3) for i in xrange(N)), dtype="i4,f8,i8")
     t = blz.btable(ra, chunklen=10)
     sl = np.ones(N, dtype="bool")
     t[sl] = [(-1, -2, -3)]
     ra[sl] = [(-1, -2, -3)]
     # print "t[%s] -> %r" % (sl, t)
     # print "ra[%s] -> %r" % (sl, ra)
     assert_array_equal(t[:], ra, "btable values are not correct")