def test02(self):
"""Testing copy() with lower clevel"""
N = 10*1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra, rootdir=self.rootdir)
t2 = t.copy(cparams=ca.cparams(clevel=1))
self.assert_(t.cparams.clevel == ca.cparams().clevel)
self.assert_(t2.cparams.clevel == 1)
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test02(self):
"""Testing copy() with lesser compression"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a, rootdir=self.rootdir)
c = b.copy(cparams=ca.cparams(clevel=1))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "clevel not changed")
def __init__(self, columns=None, names=None, **kwargs):
# Important optional params
self._cparams = kwargs.get('cparams', ca.cparams())
self.rootdir = kwargs.get('rootdir', None)
"The directory where this object is saved."
self.mode = kwargs.get('mode', 'a')
"The mode in which the object is created/opened."
# Setup the columns accessor
self.cols = cols(self.rootdir, self.mode)
"The ctable columns accessor."
# The length counter of this array
self.len = 0
# Create a new ctable or open it from disk
if columns is not None:
self.create_ctable(columns, names, **kwargs)
_new = True
else:
self.open_ctable()
_new = False
# Attach the attrs to this object
self.attrs = attrs.attrs(self.rootdir, self.mode, _new=_new)
# Cache a structured array of len 1 for ctable[int] acceleration
self._arr1 = np.empty(shape=(1,), dtype=self.dtype)
def test03(self):
"""Testing copy() with no shuffle"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a, rootdir=self.rootdir)
c = b.copy(cparams=ca.cparams(shuffle=False))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "shuffle not changed")
def test03(self):
"""Testing copy() with no shuffle"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a, rootdir=self.rootdir)
c = b.copy(cparams=ca.cparams(shuffle=False))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "shuffle not changed")
def test02(self):
"""Testing copy() with lesser compression"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a, rootdir=self.rootdir)
c = b.copy(cparams=ca.cparams(clevel=1))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "clevel not changed")
def test01b(self):
"""Testing cparams when adding a new column (numpy flavor)"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra, cparams=ca.cparams(1), rootdir=self.rootdir)
c = np.arange(N, dtype='i8')*3
t.addcol(c, 'f2')
self.assert_(t['f2'].cparams.clevel == 1, "Incorrect clevel")
def test01b(self):
"""Testing cparams when adding a new column (numpy flavor)"""
N = 10
ra = np.fromiter(((i, i * 2.) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra, cparams=ca.cparams(1), rootdir=self.rootdir)
c = np.arange(N, dtype='i8') * 3
t.addcol(c, 'f2')
self.assert_(t['f2'].cparams.clevel == 1, "Incorrect clevel")
def test03(self):
"""Testing copy() with no shuffle"""
N = 10 * 1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra)
# print "t:", t, t.rootdir
t2 = t.copy(cparams=ca.cparams(shuffle=False), rootdir=self.rootdir)
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test03(self):
"""Testing copy() with no shuffle"""
N = 10*1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra)
# print "t:", t, t.rootdir
t2 = t.copy(cparams=ca.cparams(shuffle=False), rootdir=self.rootdir)
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t['f1'].cbytes < t2['f1'].cbytes, "clevel not changed")
def test01(self):
"""Testing copy() with higher clevel"""
N = 10*1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra, rootdir=self.rootdir)
if self.disk:
# Copy over the same location should give an error
self.assertRaises(RuntimeError,
t.copy,cparams=ca.cparams(clevel=9),
rootdir=self.rootdir, mode='w')
return
else:
t2 = t.copy(cparams=ca.cparams(clevel=9),
rootdir=self.rootdir, mode='w')
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t.cparams.clevel == ca.cparams().clevel)
self.assert_(t2.cparams.clevel == 9)
self.assert_(t['f1'].cbytes > t2['f1'].cbytes, "clevel not changed")
def test01(self):
"""Testing copy() with higher clevel"""
N = 10 * 1000
ra = np.fromiter(((i, i**2.2) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra, rootdir=self.rootdir)
if self.disk:
# Copy over the same location should give an error
self.assertRaises(RuntimeError,
t.copy,
cparams=ca.cparams(clevel=9),
rootdir=self.rootdir,
mode='w')
return
else:
t2 = t.copy(cparams=ca.cparams(clevel=9),
rootdir=self.rootdir,
mode='w')
#print "cbytes in f1, f2:", t['f1'].cbytes, t2['f1'].cbytes
self.assert_(t.cparams.clevel == ca.cparams().clevel)
self.assert_(t2.cparams.clevel == 9)
self.assert_(t['f1'].cbytes > t2['f1'].cbytes, "clevel not changed")
def to_cparams(params):
"""Convert params to cparams. roodir and format_flavor also extracted."""
cparams = {}; rootdir = format_flavor = None
for key, val in params.iteritems():
if key == 'storage':
rootdir = val
continue
if key == 'format_flavor':
format_flavor = val
continue
cparams[key] = val
return carray.cparams(**cparams), rootdir, format_flavor
def test_compressed():
filename = 'output'
# hackish, just experimenting!
arr = carray(xrange(10000), cparams(clevel=5, shuffle=True)).chunks
ca = [bytes(chunk.viewof) for chunk in arr]
pack_list(ca, {}, filename, {'typesize': 8, 'clevel': 5, 'shuffle': True})
out_list, meta_info = unpack_file('output')
assert out_list[0] == ca[0]
assert out_list[1] == ca[1]
def test01a(self):
"""Testing `__setitem()__` method with start,stop (scalar)"""
a = np.ones((500,200), dtype="i4")*3
b = ca.fill((500,200), 3, dtype="i4", rootdir=self.rootdir,
cparams=ca.cparams())
sl = slice(100,400)
a[sl,:] = 0
b[sl] = 0
if self.open:
b.flush()
b = ca.open(rootdir=self.rootdir)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test01a(self):
"""Testing `__setitem()__` method with start,stop (scalar)"""
a = np.ones((500, 200), dtype="i4") * 3
b = ca.fill((500, 200),
3,
dtype="i4",
rootdir=self.rootdir,
cparams=ca.cparams())
sl = slice(100, 400)
a[sl, :] = 0
b[sl] = 0
if self.open:
b.flush()
b = ca.open(rootdir=self.rootdir)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def __init__(self, data=None, dshape=None, params=None):
""" CArray object passed directly into the constructor,
ostensibly this is just a thin wrapper that consumes a
reference.
"""
# need at least one of the three
assert (data is not None) or (dshape is not None) or (params.get("storage"))
# TODO: clean up ugly conditionals
if params:
rootdir = params.get("storage")
# compatabaility
cparams = carray.cparams(params.get("clevel"), params.get("shuffle"))
else:
rootdir = None
cparams = None
if dshape:
dtype = to_numpy(dshape)
self.ca = carray.carray(data, dtype, rootdir=rootdir)
else:
self.ca = carray.carray(data, rootdir=rootdir, cparams=cparams)
def test04(self):
"""Testing `__getitem()__` method with long ranges"""
a = np.arange(1e4)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1:8000]->", `b[1:8000]`
assert_array_equal(a[1:8000], b[1:8000], "Arrays are not equal")
def test03(self):
"""Testing `__getitem()__` method with ranges and steps"""
a = np.arange(1e3)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1:8:3]->", `b[1:8:3]`
assert_array_equal(a[1:8:3], b[1:8:3], "Arrays are not equal")
def test01(self):
"""Testing `__getitem()__` method with scalars"""
a = np.arange(1e3)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1]->", `b[1]`
self.assert_(a[1] == b[1], "Values in key 1 are not equal")
def test04(self):
"""Testing `__getitem()__` method with long ranges"""
a = np.arange(1e4)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1:8000]->", `b[1:8000]`
assert_array_equal(a[1:8000], b[1:8000], "Arrays are not equal")
def test03(self):
"""Testing `__getitem()__` method with ranges and steps"""
a = np.arange(1e3)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1:8:3]->", `b[1:8:3]`
assert_array_equal(a[1:8:3], b[1:8:3], "Arrays are not equal")
def test01(self):
"""Testing `__getitem()__` method with scalars"""
a = np.arange(1e3)
b = chunk(a, atom=a.dtype, cparams=ca.cparams())
#print "b[1]->", `b[1]`
self.assert_(a[1] == b[1], "Values in key 1 are not equal")