def test06(self):
"""Testing `where()` iterator (using carray bool in fancy indexing)"""
a = np.arange(1, 110)
b = ca.carray(a, chunklen=10)
wt = a[(a<5)|(a>9)]
cwt = b[ca.carray((a<5)|(a>9))]
#print "numpy ->", a[(a<5)|(a>9)]
#print "where ->", b[ca.carray((a<5)|(a>9))]
assert_array_equal(wt, cwt, "where() does not work correctly")
def test05(self):
"""Testing `where()` iterator using `skip`"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v<=5][2:]
cwt = [v for v in b.where(ca.carray(a<=5), skip=2)]
#print "numpy ->", [v for v in a if v<=5][2:]
#print "where ->", [v for v in b.where(ca.carray(a<=5), skip=2)]
self.assert_(wt == cwt, "where() does not work correctly")
def test04(self):
"""Testing eval() with a mix of carray, ndarray and scalars"""
a, b = np.arange(self.N), np.arange(1, self.N + 1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("a + 2 * d - 3")
nr = a + 2 * b - 3
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test01(self):
"""Testing eval() with only carrays"""
a, b = np.arange(self.N), np.arange(1, self.N+1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("c * d")
nr = a * b
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test05(self):
"""Testing `where()` iterator using `skip`"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v <= 5][2:]
cwt = [v for v in b.where(ca.carray(a <= 5), skip=2)]
#print "numpy ->", [v for v in a if v<=5][2:]
#print "where ->", [v for v in b.where(ca.carray(a<=5), skip=2)]
self.assert_(wt == cwt, "where() does not work correctly")
def test03(self):
"""Testing `where()` iterator (using a boolean carray)"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v <= 5]
cwt = [v for v in b.where(ca.carray(a <= 5))]
#print "numpy ->", [v for v in a if v<=5]
#print "where ->", [v for v in b.where(ca.carray(a<=5))]
self.assert_(wt == cwt, "where() does not work correctly")
def test06(self):
"""Testing `where()` iterator (using carray bool in fancy indexing)"""
a = np.arange(1, 110)
b = ca.carray(a, chunklen=10)
wt = a[(a < 5) | (a > 9)]
cwt = b[ca.carray((a < 5) | (a > 9))]
#print "numpy ->", a[(a<5)|(a>9)]
#print "where ->", b[ca.carray((a<5)|(a>9))]
assert_array_equal(wt, cwt, "where() does not work correctly")
def test01(self):
"""Testing eval() with only carrays"""
a, b = np.arange(self.N), np.arange(1, self.N + 1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("c * d")
nr = a * b
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test03(self):
"""Testing `where()` iterator (using a boolean carray)"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v<=5]
cwt = [v for v in b.where(ca.carray(a<=5))]
#print "numpy ->", [v for v in a if v<=5]
#print "where ->", [v for v in b.where(ca.carray(a<=5))]
self.assert_(wt == cwt, "where() does not work correctly")
def test04(self):
"""Testing eval() with a mix of carray, ndarray and scalars"""
a, b = np.arange(self.N), np.arange(1, self.N+1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("a + 2 * d - 3")
nr = a + 2 * b - 3
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test12(self):
"""Testing eval() with `out_flavor` == 'numpy'"""
a, b = np.arange(self.N), np.arange(1, self.N+1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("c + 2 * d - 3", out_flavor='numpy')
nr = a + 2 * b - 3
#print "ca.eval ->", cr, type(cr)
#print "numpy ->", nr
self.assert_(type(cr) == np.ndarray)
assert_array_equal(cr, nr, "eval does not work correctly")
def test12(self):
"""Testing eval() with `out_flavor` == 'numpy'"""
a, b = np.arange(self.N), np.arange(1, self.N + 1)
c, d = ca.carray(a), ca.carray(b)
cr = ca.eval("c + 2 * d - 3", out_flavor='numpy')
nr = a + 2 * b - 3
#print "ca.eval ->", cr, type(cr)
#print "numpy ->", nr
self.assert_(type(cr) == np.ndarray)
assert_array_equal(cr, nr, "eval does not work correctly")
def test02(self):
"""Testing append() with carrays"""
N = 10
ra = np.fromiter(((i, i * 2.) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra)
a = np.arange(N, N + 10, dtype='i4')
b = np.arange(N, N + 10, dtype='f8') * 2.
t.append((ca.carray(a), ca.carray(b)))
ra = np.fromiter(((i, i * 2.) for i in xrange(N + 10)), dtype='i4,f8')
assert_array_equal(t[:], ra, "ctable values are not correct")
def test00(self):
"""Testing ctable creation from a tuple of carrays"""
N = 1e1
a = ca.carray(np.arange(N, dtype='i4'))
b = ca.carray(np.arange(N, dtype='f8') + 1)
t = ca.ctable((a, b), ('f0', 'f1'))
#print "t->", `t`
ra = np.rec.fromarrays([a[:], b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "ctable values are not correct")
def test02(self):
"""Testing append() with carrays"""
N = 10
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = ca.ctable(ra)
a = np.arange(N, N+10, dtype='i4')
b = np.arange(N, N+10, dtype='f8')*2.
t.append((ca.carray(a), ca.carray(b)))
ra = np.fromiter(((i, i*2.) for i in xrange(N+10)), dtype='i4,f8')
assert_array_equal(t[:], ra, "ctable values are not correct")
def test04(self):
"""Testing fancy indexing with __setitem__ (bool carray)"""
a = np.arange(1, 1e2)
b = ca.carray(a, chunklen=10)
bc = (a > 5) & (a < 40)
sl = ca.carray(bc)
b[sl] = 3.
a[bc] = 3.
#print "b[%s] -> %r" % (sl, b)
assert_array_equal(b[:], a, "fancy indexing does not work correctly")
def test06(self):
"""Testing `where()` iterator using `limit` and `skip`"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v<=5][1:4]
cwt = [v for v in b.where(ca.carray(a<=5), limit=3, skip=1)]
#print "numpy ->", [v for v in a if v<=5][1:4]
#print "where ->", [v for v in b.where(ca.carray(a<=5),
# limit=3, skip=1)]
self.assert_(wt == cwt, "where() does not work correctly")
def test06(self):
"""Testing `where()` iterator using `limit` and `skip`"""
a = np.arange(1, 11)
b = ca.carray(a)
wt = [v for v in a if v <= 5][1:4]
cwt = [v for v in b.where(ca.carray(a <= 5), limit=3, skip=1)]
#print "numpy ->", [v for v in a if v<=5][1:4]
#print "where ->", [v for v in b.where(ca.carray(a<=5),
# limit=3, skip=1)]
self.assert_(wt == cwt, "where() does not work correctly")
def test04(self):
"""Testing fancy indexing with __setitem__ (bool carray)"""
a = np.arange(1,1e2)
b = ca.carray(a, chunklen=10)
bc = (a > 5) & (a < 40)
sl = ca.carray(bc)
b[sl] = 3.
a[bc] = 3.
#print "b[%s] -> %r" % (sl, b)
assert_array_equal(b[:], a, "fancy indexing does not work correctly")
def test00(self):
"""Testing ctable creation from a tuple of carrays"""
N = 1e1
a = ca.carray(np.arange(N, dtype='i4'))
b = ca.carray(np.arange(N, dtype='f8')+1)
t = ca.ctable((a, b), ('f0', 'f1'))
#print "t->", `t`
ra = np.rec.fromarrays([a[:],b[:]]).view(np.ndarray)
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "ctable values are not correct")
def test07(self):
"""Testing `where()` iterator using `limit` and `skip` (zeros)"""
a = np.arange(10000)
b = ca.carray(a,)
wt = [v for v in a if v<=5000][1010:2020]
cwt = [v for v in b.where(ca.carray(a<=5000, chunklen=100),
limit=1010, skip=1010)]
# print "numpy ->", [v for v in a if v>=5000][1010:2020]
# print "where ->", [v for v in b.where(ca.carray(a>=5000,chunklen=100),
# limit=1010, skip=1010)]
self.assert_(wt == cwt, "where() does not work correctly")
def test11(self):
"""Testing eval() with functions like `np.sin()`"""
a, b = np.arange(self.N), np.arange(1, self.N+1)
c, d = ca.carray(a), ca.carray(b)
if self.vm == "python":
cr = ca.eval("np.sin(c) + 2 * np.log(d) - 3")
else:
cr = ca.eval("sin(c) + 2 * log(d) - 3")
nr = np.sin(a) + 2 * np.log(b) - 3
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test11(self):
"""Testing eval() with functions like `np.sin()`"""
a, b = np.arange(self.N), np.arange(1, self.N + 1)
c, d = ca.carray(a), ca.carray(b)
if self.vm == "python":
cr = ca.eval("np.sin(c) + 2 * np.log(d) - 3")
else:
cr = ca.eval("sin(c) + 2 * log(d) - 3")
nr = np.sin(a) + 2 * np.log(b) - 3
#print "ca.eval ->", cr
#print "numpy ->", nr
assert_array_equal(cr[:], nr, "eval does not work correctly")
def test07(self):
"""Testing `where()` iterator using `limit` and `skip` (zeros)"""
a = np.arange(10000)
b = ca.carray(a, )
wt = [v for v in a if v <= 5000][1010:2020]
cwt = [
v for v in b.where(
ca.carray(a <= 5000, chunklen=100), limit=1010, skip=1010)
]
# print "numpy ->", [v for v in a if v>=5000][1010:2020]
# print "where ->", [v for v in b.where(ca.carray(a>=5000,chunklen=100),
# limit=1010, skip=1010)]
self.assert_(wt == cwt, "where() does not work correctly")
def test02(self):
"""Testing copy() with lesser compression"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a)
c = b.copy(cparams=ca.cparams(clevel=1))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "clevel not changed")
def test03b(self):
"""Testing `iter()` method with start, stop, step"""
a = np.arange(101)
b = ca.carray(a, chunklen=2)
#print "sum iter->", sum(b.iter(3, 24, 4))
self.assert_(sum(a[3:24:4]) == sum(b.iter(3, 24, 4)),
"Sums are not equal")
def test02c(self):
"""Testing `iter()` method with positive start, negative stop"""
a = np.arange(101)
b = ca.carray(a, chunklen=2)
#print "sum iter->", sum(b.iter(24, -3))
self.assert_(sum(a[24:-3]) == sum(b.iter(24, -3)),
"Sums are not equal")
def test01d(self):
"""Testing `__getitem()__` method with only a (large) start"""
a = np.arange(1e4)
b = ca.carray(a)
sl = -2 # second last element
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test05(self):
"""Testing `__getitem()__` method with negative steps"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=10)
sl = slice(None, None, -3)
#print "b[sl]->", `b[sl]`
self.assertRaises(NotImplementedError, b.__getitem__, sl)
def test01c(self):
"""Testing `__getitem()__` method with only a (start,)"""
a = np.arange(1e2)
b = ca.carray(a, chunklen=10)
#print "b[(1,)]->", `b[(1,)]`
self.assert_(a[(1, )] == b[(1, )],
"Values with key (1,) are not equal")
def test03b(self):
"""Testing `iter()` method with start, stop, step"""
a = np.arange(101)
b = ca.carray(a, chunklen=2)
#print "sum iter->", sum(b.iter(3, 24, 4))
self.assert_(
sum(a[3:24:4]) == sum(b.iter(3, 24, 4)), "Sums are not equal")
def test00(self):
"""Testing fancy indexing (short list)"""
a = np.arange(1, 111)
b = ca.carray(a)
c = b[[3, 1]]
r = a[[3, 1]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def test03(self):
"""Testing copy() with no shuffle"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a)
c = b.copy(cparams=ca.cparams(shuffle=False))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "shuffle not changed")
def test03(self):
"""Testing fancy indexing (list of floats)"""
a = np.arange(1,101)
b = ca.carray(a)
c = b[[1.1, 3.3]]
r = a[[1.1, 3.3]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def test03a(self):
"""Testing `iter()` method with only step"""
a = np.arange(101)
b = ca.carray(a, chunklen=2)
#print "sum iter->", sum(b.iter(step=4))
self.assert_(sum(a[::4]) == sum(b.iter(step=4)),
"Sums are not equal")
def test03a(self):
"""Testing `__getitem()__` method with several slices (I)"""
a = np.arange(12).reshape((4, 3))
b = ca.carray(a)
sl = (slice(1, 3, 1), slice(1, 4, 2))
# print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test04c(self):
"""Testing `__getitem()__` method with shape reduction (III)"""
a = np.arange(60).reshape((5, 4, 3))
b = ca.carray(a)
sl = (1, slice(1, 4, 2), 2)
# print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test03(self):
"""Testing copy() with no shuffle"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a)
c = b.copy(cparams=ca.cparams(shuffle=False))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "shuffle not changed")
def test02c(self):
"""Testing `iter()` method with positive start, negative stop"""
a = np.arange(101)
b = ca.carray(a, chunklen=2)
#print "sum iter->", sum(b.iter(24, -3))
self.assert_(
sum(a[24:-3]) == sum(b.iter(24, -3)), "Sums are not equal")
def test03(self):
"""Testing fancy indexing (list of floats)"""
a = np.arange(1, 101)
b = ca.carray(a)
c = b[[1.1, 3.3]]
r = a[[1.1, 3.3]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def test04(self):
"""Testing `iter()` method with large zero arrays"""
a = np.zeros(1e4, dtype='f8')
b = ca.carray(a, chunklen=100)
c = ca.fromiter((v for v in b), dtype='f8', count=len(a))
#print "c ->", repr(c)
assert_array_equal(a, c[:], "iterator fails on zeros")
def test02(self):
"""Testing fancy indexing (empty list)"""
a = np.arange(101)
b = ca.carray(a)
c = b[[]]
r = a[[]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def test02(self):
"""Testing copy() with lesser compression"""
a = np.linspace(-1., 1., 1e4)
b = ca.carray(a)
c = b.copy(cparams=ca.cparams(clevel=1))
#print "b.cbytes, c.cbytes:", b.cbytes, c.cbytes
self.assert_(b.cbytes < c.cbytes, "clevel not changed")
def test04a(self):
"""Testing `__getitem()__` method with long ranges"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=100)
sl = slice(1, 8000)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test01d(self):
"""Testing `__getitem()__` method with only a (large) start"""
a = np.arange(1e4)
b = ca.carray(a)
sl = -2 # second last element
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test00(self):
"""Testing fancy indexing (short list)"""
a = np.arange(1,111)
b = ca.carray(a)
c = b[[3,1]]
r = a[[3,1]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def addcol(self, newcol, name=None, pos=None, **kwargs):
"""
addcol(newcol, name=None, pos=None, **kwargs)
Add a new `newcol` carray or ndarray as column.
Parameters
----------
newcol : carray or ndarray
If a carray is passed, no conversion will be carried out.
If conversion to a carray has to be done, `kwargs` will
apply.
name : string, optional
The name for the new column. If not passed, it will
receive an automatic name.
pos : int, optional
The column position. If not passed, it will be appended
at the end.
kwargs : list of parameters or dictionary
Any parameter supported by the carray constructor.
Notes
-----
You should not specificy both `name` and `pos` arguments,
unless they are compatible.
See Also
--------
delcol
"""
# Check params
if pos is None:
pos = len(self.names)
else:
if pos and type(pos) != int:
raise ValueError, "`pos` must be an int"
if pos < 0 or pos > len(self.names):
raise ValueError, "`pos` must be >= 0 and <= len(self.cols)"
if name is None:
name = "f%d" % pos
else:
if type(name) != str:
raise ValueError, "`name` must be a string"
if name in self.names:
raise ValueError, "'%s' column already exists" % name
if len(newcol) != self.len:
raise ValueError, "`newcol` must have the same length than ctable"
if isinstance(newcol, np.ndarray):
if 'cparams' not in kwargs:
kwargs['cparams'] = self.cparams
newcol = ca.carray(newcol, **kwargs)
# Insert the column
self.names.insert(pos, name)
self.cols[name] = newcol
# Update _arr1
self._arr1 = np.empty(shape=(1, ), dtype=self.dtype)
def test02b(self):
"""Testing `__getitem()__` method with ranges (negative start)"""
a = np.arange(1e2)
b = ca.carray(a, chunklen=10)
sl = slice(-3)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02(self):
"""Testing fancy indexing (empty list)"""
a = np.arange(101)
b = ca.carray(a)
c = b[[]]
r = a[[]]
assert_array_equal(c, r, "fancy indexing does not work correctly")
def test02e(self):
"""Testing `__getitem()__` method with start > stop"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=10)
sl = slice(4, 3, 30)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test04d(self):
"""Testing `__getitem()__` method with no start and no stop"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=100)
sl = slice(None, None, 2)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test03d(self):
"""Testing `__getitem()__` method with ranges and steps (IV)"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=10)
sl = slice(4, 80, 3000)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02b(self):
"""Testing `__getitem()__` method with ranges (negative start)"""
a = np.arange(1e2)
b = ca.carray(a, chunklen=10)
sl = slice(-3)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test04a(self):
"""Testing `__getitem()__` method with long ranges"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=100)
sl = slice(1, 8000)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test04d(self):
"""Testing `__getitem()__` method with no start and no stop"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=100)
sl = slice(None, None, 2)
#print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test05(self):
"""Testing `__getitem()__` method with negative steps"""
a = np.arange(1e3)
b = ca.carray(a, chunklen=10)
sl = slice(None, None, -3)
#print "b[sl]->", `b[sl]`
self.assertRaises(NotImplementedError, b.__getitem__, sl)
def addcol(self, newcol, name=None, pos=None, **kwargs):
"""
addcol(newcol, name=None, pos=None, **kwargs)
Add a new `newcol` carray or ndarray as column.
Parameters
----------
newcol : carray or ndarray
If a carray is passed, no conversion will be carried out.
If conversion to a carray has to be done, `kwargs` will
apply.
name : string, optional
The name for the new column. If not passed, it will
receive an automatic name.
pos : int, optional
The column position. If not passed, it will be appended
at the end.
kwargs : list of parameters or dictionary
Any parameter supported by the carray constructor.
Notes
-----
You should not specificy both `name` and `pos` arguments,
unless they are compatible.
See Also
--------
delcol
"""
# Check params
if pos is None:
pos = len(self.names)
else:
if pos and type(pos) != int:
raise ValueError, "`pos` must be an int"
if pos < 0 or pos > len(self.names):
raise ValueError, "`pos` must be >= 0 and <= len(self.cols)"
if name is None:
name = "f%d" % pos
else:
if type(name) != str:
raise ValueError, "`name` must be a string"
if name in self.names:
raise ValueError, "'%s' column already exists" % name
if len(newcol) != self.len:
raise ValueError, "`newcol` must have the same length than ctable"
if isinstance(newcol, np.ndarray):
if 'cparams' not in kwargs:
kwargs['cparams'] = self.cparams
newcol = ca.carray(newcol, **kwargs)
# Insert the column
self.names.insert(pos, name)
self.cols[name] = newcol
# Update _arr1
self._arr1 = np.empty(shape=(1,), dtype=self.dtype)
def test02(self):
"""Testing __sizeof__() (small carrays)"""
a = np.arange(111)
b = ca.carray(a)
#print "size b uncompressed-->", b.nbytes
#print "size b compressed -->", b.cbytes
self.assert_(sys.getsizeof(b) > b.nbytes, "carray compress too much??")
def open(rootdir, mode='a'):
"""
open(rootdir, mode='a')
Open a disk-based carray/ctable.
Parameters
----------
rootdir : pathname (string)
The directory hosting the carray/ctable object.
mode : the open mode (string)
Specifies the mode in which the object is opened. The supported
values are:
* 'r' for read-only
* 'w' for emptying the previous underlying data
* 'a' for allowing read/write on top of existing data
Returns
-------
out : a carray/ctable object or None (if not objects are found)
"""
# First try with a carray
obj = None
try:
obj = ca.carray(rootdir=rootdir, mode=mode)
except IOError:
# Not a carray. Now with a ctable
try:
obj = ca.ctable(rootdir=rootdir, mode=mode)
except IOError:
# Not a ctable
pass
return obj
def test04(self):
"""Testing `iter()` method with large zero arrays"""
a = np.zeros(1e4, dtype='f8')
b = ca.carray(a, chunklen=100)
c = ca.fromiter((v for v in b), dtype='f8', count=len(a))
#print "c ->", repr(c)
assert_array_equal(a, c[:], "iterator fails on zeros")