def test00b(self):
"""Testing evaluation of ndcarrays (bool out, NumPy)"""
a = np.arange(np.prod(self.shape)).reshape(self.shape)
b = bcolz.arange(np.prod(self.shape)).reshape(self.shape)
outa = eval("a>0")
outb = bcolz.eval("b>0", out_flavor='numpy')
assert_array_equal(outa, outb, "Arrays are not equal")
def test02b(self):
"""Testing `reshape()` (ndim -> ndim, II)"""
a = np.arange(24, dtype="i4").reshape((2, 3, 4))
c = bcolz.arange(24, dtype="i4").reshape((4, 3, 2))
b = c.reshape((2, 3, 4))
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def test01(self):
"""Testing evaluation of ndcarrays (int out)"""
a = np.arange(np.prod(self.shape)).reshape(self.shape)
b = bcolz.arange(np.prod(self.shape)).reshape(self.shape)
outa = eval("a*2.+1")
outb = bcolz.eval("b*2.+1")
assert_array_equal(outa, outb, "Arrays are not equal")
def test00b(self):
"""Testing `carray` reshape (large shape)"""
a = np.arange(16000).reshape((20, 20, 40))
b = bcolz.arange(16000, rootdir=self.rootdir).reshape((20, 20, 40))
if self.open:
b = bcolz.open(rootdir=self.rootdir)
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def test02(self):
"""Testing iter() in combination with a list constructor"""
b = bcolz.arange(self.N, dtype="f4", rootdir=self.rootdir)
blist = list(b)
blist1 = [r for r in b.iter(3, 10)]
self.assertTrue(blist1 == list(range(3, 10)))
blist2 = list(b)
self.assertTrue(blist == blist2, "iter() not working correctly")
def test00a(self):
"""Testing `carray` reshape"""
a = np.arange(16).reshape((2, 2, 4))
b = bcolz.arange(16, rootdir=self.rootdir).reshape((2, 2, 4))
if self.open:
b = bcolz.open(rootdir=self.rootdir)
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def test02(self):
"""Testing iter() in combination with a list constructor"""
b = bcolz.arange(self.N, dtype="f4", rootdir=self.rootdir)
blist = list(b)
blist1 = [r for r in b.iter(3, 10)]
self.assertTrue(blist1 == list(range(3, 10)))
blist2 = list(b)
self.assertTrue(blist == blist2, "iter() not working correctly")
def test02(self):
"""Testing evaluation of ndcarrays (reduction, no axis)"""
a = np.arange(np.prod(self.shape)).reshape(self.shape)
b = bcolz.arange(np.prod(self.shape)).reshape(self.shape)
if bcolz.defaults.eval_vm == "python":
assert_array_equal(sum(a), bcolz.eval("sum(b)"),
"Arrays are not equal")
else:
self.assertEqual(a.sum(), bcolz.eval("sum(b)"))
def test01a(self):
"""Testing where() in combination with a list constructor"""
a = bcolz.zeros(self.N, dtype="bool", rootdir=self.rootdir)
a[30:40] = bcolz.ones(10, dtype="bool")
b = bcolz.arange(self.N, dtype="f4")
blist = list(b)
blist1 = [r for r in b.where(a)]
self.assertTrue(blist1 == list(range(30, 40)))
blist2 = list(b)
self.assertTrue(blist == blist2, "where() not working correctly")
def test01a(self):
"""Testing where() in combination with a list constructor"""
a = bcolz.zeros(self.N, dtype="bool", rootdir=self.rootdir)
a[30:40] = bcolz.ones(10, dtype="bool")
b = bcolz.arange(self.N, dtype="f4")
blist = list(b)
blist1 = [r for r in b.where(a)]
self.assertTrue(blist1 == list(range(30, 40)))
blist2 = list(b)
self.assertTrue(blist == blist2, "where() not working correctly")
def test02b(self):
"""Testing evaluation of ndcarrays (reduction, with axis)"""
a = np.arange(np.prod(self.shape)).reshape(self.shape)
b = bcolz.arange(np.prod(self.shape)).reshape(self.shape)
if bcolz.defaults.eval_vm == "python":
# The Python VM does not have support for `axis` param
assert_array_equal(sum(a), bcolz.eval("sum(b)"),
"Arrays are not equal")
else:
assert_array_equal(a.sum(axis=1), bcolz.eval("sum(b, axis=1)"),
"Arrays are not equal")
def arange(start=None, stop=None, step=None, dtype=None):
# Check start, stop, step values
if (start, stop) == (None, None):
raise ValueError("You must pass a `stop` value at least.")
elif stop is None:
start, stop = 0, start
elif start is None:
start, stop = 0, stop
if step is None:
step = 1
carr = bcolz.arange(start, stop, step, dtype, expectedlen=len(range(start, stop, step)))
return ResultColumn(carr)
def test01b(self):
"""Testing where() with a multidimensional array"""
a = bcolz.zeros((self.N, 10), dtype="bool", rootdir=self.rootdir)
a[30:40] = bcolz.ones(10, dtype="bool")
b = bcolz.arange(self.N * 10, dtype="f4").reshape((self.N, 10))
self.assertRaises(NotImplementedError, b.where, a)
PY2 = sys.version_info[0] == 2
if not PY2:
xrange = range
def range(*args):
return list(xrange(*args))
filepath = 'fromhdf5.h5'
nodepath = '/ctable'
NR = int(1e6)
NC = 10
dsize = (NR * NC * 4) / 2. ** 30
bcolz.cparams.setdefaults(clevel=5)
a = bcolz.arange(NR, dtype='i4')
#ra = np.rec.fromarrays([a]*NC, names=['f%d'%i for i in range(NC)])
ra = bcolz.ctable((a,)*NC)[:]
t0 = time()
f = tb.open_file(filepath, "w")
f.create_table(f.root, nodepath[1:], ra)
f.close()
tt = time() - t0
print("time for storing the HDF5 table: %.2f (%.2f GB/s)" % (tt, dsize / tt))
# Using an iterator
t0 = time()
f = tb.open_file(filepath)
t = f.get_node(nodepath)
t = bcolz.fromiter((r[:] for r in t), dtype=t.dtype, count=len(t))
from time import time
import numpy as np
import bcolz
N = 1e8
dtype = 'i4'
start, stop, step = 5, N, 4
t0 = time()
a = np.arange(start, stop, step, dtype=dtype)
print("Time numpy.arange() --> %.3f" % (time() - t0))
t0 = time()
ac = bcolz.arange(start, stop, step, dtype=dtype)
print("Time bcolsz.arange() --> %.3f" % (time() - t0))
print("ac-->", repr(ac))
# assert(np.all(a == ac))
from time import time
import numpy as np
import bcolz
N = 1e8
dtype = 'i4'
start, stop, step = 5, N, 4
t0 = time()
a = np.arange(start, stop, step, dtype=dtype)
print("Time numpy.arange() --> %.3f" % (time() - t0))
t0 = time()
ac = bcolz.arange(start, stop, step, dtype=dtype)
print("Time bcolsz.arange() --> %.3f" % (time() - t0))
print("ac-->", repr(ac))
# assert(np.all(a == ac))
def time_arange(self):
ac = bcolz.arange(self.start, self.stop, self.step, dtype=self.dtype)
t = time.time()
yield
if message:
print(message + ":\t", end="")
print(round(time.time() - t, 4), "sec")
carootdir = "carraypickle.bcolz"
if os.path.exists(carootdir):
shutil.rmtree(carootdir)
ctrootdir = "ctablepickle.bcolz"
if os.path.exists(ctrootdir):
shutil.rmtree(ctrootdir)
N = int(1e7)
a = bcolz.arange(N, dtype="int32")
b = bcolz.arange(N, dtype="float32")
ca = bcolz.carray(a, rootdir=carootdir)
ct = bcolz.ctable([ca, b], names=['a', 'b'], rootdir=ctrootdir)
with ctime("Time spent pickling carray with N=%d" % N):
s = pickle.dumps(ca)
with ctime("Time spent unpickling carray with N=%d" % N):
ca2 = pickle.loads(s)
np.testing.assert_allclose(ca2[:], a)
with ctime("Time spent pickling ctable with N=%d" % N):
s = pickle.dumps(ct)
def test01b(self):
"""Testing where() with a multidimensional array"""
a = bcolz.zeros((self.N, 10), dtype="bool", rootdir=self.rootdir)
a[30:40] = bcolz.ones(10, dtype="bool")
b = bcolz.arange(self.N * 10, dtype="f4").reshape((self.N, 10))
self.assertRaises(NotImplementedError, b.where, a)