def test01a(self):
"""Testing fromiter (long iter)"""
N = 1e4
a = (i for i in xrange(int(N)))
b = ca.fromiter(a, dtype='f8', count=int(N))
c = np.arange(N)
assert_array_equal(b[:], c, "fromiter does not work correctly")
def test01b(self):
"""Testing fromiter (long iter, chunk is multiple of iter length)"""
N = 1e4
a = (i for i in xrange(int(N)))
b = ca.fromiter(a, dtype='f8', chunklen=1000, count=int(N))
c = np.arange(N)
assert_array_equal(b[:], c, "fromiter does not work correctly")
def test01b(self):
"""Testing fromiter (long iter, chunk is multiple of iter length)"""
N = 1e4
a = (i for i in xrange(int(N)))
b = ca.fromiter(a, dtype='f8', chunklen=1000, count=int(N))
c = np.arange(N)
assert_array_equal(b[:], c, "fromiter does not work correctly")
def test01a(self):
"""Testing fromiter (long iter)"""
N = 1e4
a = (i for i in xrange(int(N)))
b = ca.fromiter(a, dtype='f8', count=int(N))
c = np.arange(N)
assert_array_equal(b[:], c, "fromiter does not work correctly")
def test04(self):
"""Testing `iter()` method with large zero arrays"""
a = np.zeros(1e4, dtype='f8')
b = ca.carray(a, chunklen=100, rootdir=self.rootdir)
c = ca.fromiter((v for v in b), dtype='f8', count=len(a))
#print "c ->", repr(c)
assert_array_equal(a, c[:], "iterator fails on zeros")
def test04(self):
"""Testing `iter()` method with large zero arrays"""
a = np.zeros(1e4, dtype='f8')
b = ca.carray(a, chunklen=100, rootdir=self.rootdir)
c = ca.fromiter((v for v in b), dtype='f8', count=len(a))
#print "c ->", repr(c)
assert_array_equal(a, c[:], "iterator fails on zeros")
def test00(self):
"""Testing resize() (decreasing)"""
N = 100
ra = np.fromiter(((i, i*2.) for i in xrange(N-2)), dtype='i4,f8')
t = ca.fromiter(((i, i*2.) for i in xrange(N)), 'i4,f8', N,
rootdir=self.rootdir)
t.resize(N-2)
assert_array_equal(t[:], ra, "ctable values are not correct")
def test02(self):
"""Testing trim() with a complete trim"""
N = 100
ra = np.fromiter(((i, i*2.) for i in xrange(0)), dtype='i4,f8')
t = ca.fromiter(((i, i*2.) for i in xrange(N)), 'i4,f8', N,
rootdir=self.rootdir)
t.trim(N)
self.assert_(len(ra) == len(t), "Lengths are not equal")
def test07(self):
"""Testing `iter()` method with `limit` and `skip`"""
a = np.arange(1e4, dtype='f8')
b = ca.carray(a, chunklen=100, rootdir=self.rootdir)
c = ca.fromiter((v for v in b.iter(limit=1010, skip=1010)), dtype='f8',
count=1010)
#print "c ->", repr(c)
assert_array_equal(a[1010:2020], c, "iterator fails on zeros")
def fromiter(iterable, dshape, params=None):
""" Create an Array and fill it with values from `iterable`.
Parameters
----------
iterable : iterable object
An iterable object providing data for the carray.
dshape : str, blaze.dshape instance
Specifies the datashape of the outcome object. Only 1d shapes
are supported right now. When the `iterator` should return an
unknown number of items, a ``TypeVar`` can be used.
params : blaze.params object
Any parameter supported by the backend library.
Returns
-------
out : an Array object.
"""
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
shape, dtype = dshape.parameters[:-1], dshape.parameters[-1]
# Check the shape part
if len(shape) > 1:
raise ValueError("shape can be only 1-dimensional")
length = shape[0]
count = -1
if type(length) == TypeVar:
count = -1
elif type(length) == Fixed:
count = length.val
dtype = dtype.to_dtype()
# Now, create the Array itself (using the carray backend)
cparams, rootdir, format_flavor = to_cparams(params or _params())
if rootdir is not None:
carray.fromiter(iterable,
dtype,
count=count,
rootdir=rootdir,
cparams=cparams)
return open(rootdir)
else:
ica = carray.fromiter(iterable, dtype, count=count, cparams=cparams)
source = CArraySource(ica, params=params)
return Array(source)
def test01(self):
"""Testing trim() with NumPy scalar values"""
N = 10000
ra = np.fromiter(((i, i*2.) for i in xrange(N-200)), dtype='i4,f8')
t = ca.fromiter(((i, i*2.) for i in xrange(N)), 'i4,f8', N,
rootdir=self.rootdir)
t.trim(np.int(200))
assert_array_equal(t[:], ra, "ctable values are not correct")
def getobject(self):
if self.flavor == 'carray':
obj = ca.zeros(10, dtype="i1", rootdir=self.rootdir)
assert type(obj) == ca.carray
elif self.flavor == 'ctable':
obj = ca.fromiter(((i,i*2) for i in range(10)), dtype='i2,f4',
count=10, rootdir=self.rootdir)
assert type(obj) == ca.ctable
return obj
def test03a(self):
"""Testing ctable creation from large iterator"""
N = 10*1000
ra = np.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8')
t = ca.fromiter(((i, i*2.) for i in xrange(N)), dtype='i4,f8',
count=N, rootdir=self.rootdir)
#print "t->", `t`
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "ctable values are not correct")
def test07(self):
"""Testing `iter()` method with `limit` and `skip`"""
a = np.arange(1e4, dtype='f8')
b = ca.carray(a, chunklen=100, rootdir=self.rootdir)
c = ca.fromiter((v for v in b.iter(limit=1010, skip=1010)),
dtype='f8',
count=1010)
#print "c ->", repr(c)
assert_array_equal(a[1010:2020], c, "iterator fails on zeros")
def test01(self):
"""Testing resize() (increasing)"""
N = 100
ra = np.fromiter(((i, i*2.) for i in xrange(N+4)), dtype='i4,f8')
t = ca.fromiter(((i, i*2.) for i in xrange(N)), 'i4,f8', N,
rootdir=self.rootdir)
t.resize(N+4)
ra['f0'][N:] = np.zeros(4)
ra['f1'][N:] = np.zeros(4)
assert_array_equal(t[:], ra, "ctable values are not correct")
def test00(self):
"""Testing resize() (decreasing)"""
N = 100
ra = np.fromiter(((i, i * 2.) for i in xrange(N - 2)), dtype='i4,f8')
t = ca.fromiter(((i, i * 2.) for i in xrange(N)),
'i4,f8',
N,
rootdir=self.rootdir)
t.resize(N - 2)
assert_array_equal(t[:], ra, "ctable values are not correct")
def test02(self):
"""Testing trim() with a complete trim"""
N = 100
ra = np.fromiter(((i, i * 2.) for i in xrange(0)), dtype='i4,f8')
t = ca.fromiter(((i, i * 2.) for i in xrange(N)),
'i4,f8',
N,
rootdir=self.rootdir)
t.trim(N)
self.assert_(len(ra) == len(t), "Lengths are not equal")
def fromiter(iterable, dshape, params=None):
""" Create an Array and fill it with values from `iterable`.
Parameters
----------
iterable : iterable object
An iterable object providing data for the carray.
dshape : str, blaze.dshape instance
Specifies the datashape of the outcome object. Only 1d shapes
are supported right now. When the `iterator` should return an
unknown number of items, a ``TypeVar`` can be used.
params : blaze.params object
Any parameter supported by the backend library.
Returns
-------
out : an Array object.
"""
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
shape, dtype = dshape.parameters[:-1], dshape.parameters[-1]
# Check the shape part
if len(shape) > 1:
raise ValueError("shape can be only 1-dimensional")
length = shape[0]
count = -1
if type(length) == TypeVar:
count = -1
elif type(length) == Fixed:
count = length.val
dtype = dtype.to_dtype()
# Now, create the Array itself (using the carray backend)
cparams, rootdir, format_flavor = to_cparams(params or _params())
if rootdir is not None:
carray.fromiter(iterable, dtype, count=count,
rootdir=rootdir, cparams=cparams)
return open(rootdir)
else:
ica = carray.fromiter(iterable, dtype, count=count, cparams=cparams)
source = CArraySource(ica, params=params)
return Array(source)
def test01(self):
"""Testing trim() with NumPy scalar values"""
N = 10000
ra = np.fromiter(((i, i * 2.) for i in xrange(N - 200)), dtype='i4,f8')
t = ca.fromiter(((i, i * 2.) for i in xrange(N)),
'i4,f8',
N,
rootdir=self.rootdir)
t.trim(np.int(200))
assert_array_equal(t[:], ra, "ctable values are not correct")
def test03a(self):
"""Testing ctable creation from large iterator"""
N = 10 * 1000
ra = np.fromiter(((i, i * 2.) for i in xrange(N)), dtype='i4,f8')
t = ca.fromiter(((i, i * 2.) for i in xrange(N)),
dtype='i4,f8',
count=N,
rootdir=self.rootdir)
#print "t->", `t`
#print "ra[:]", ra[:]
assert_array_equal(t[:], ra, "ctable values are not correct")
def test01(self):
"""Testing resize() (increasing)"""
N = 100
ra = np.fromiter(((i, i * 2.) for i in xrange(N + 4)), dtype='i4,f8')
t = ca.fromiter(((i, i * 2.) for i in xrange(N)),
'i4,f8',
N,
rootdir=self.rootdir)
t.resize(N + 4)
ra['f0'][N:] = np.zeros(4)
ra['f1'][N:] = np.zeros(4)
assert_array_equal(t[:], ra, "ctable values are not correct")
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
def test03(self):
"""Testing fromiter (dtype conversion)"""
a = np.arange(101, dtype="f8")
b = ca.fromiter(iter(a), dtype='f4', count=len(a))
assert_array_equal(b[:], a, "fromiter does not work correctly")
def test02(self):
"""Testing fromiter (empty iter)"""
a = np.array([], dtype="f8")
b = ca.fromiter(iter(a), dtype='f8', count=-1)
assert_array_equal(b[:], a, "fromiter does not work correctly")
def from_dict(self, data):
dtype = dtype_from_dict(data)
return fromiter(izip(*data.itervalues()), dtype, -1)
def test04b(self):
"""Testing fromiter method with large iterator with a hint"""
N = 10 * 1000
a = np.fromiter((i * 2 for i in xrange(N)), dtype='f8', count=N)
b = ca.fromiter((i * 2 for i in xrange(N)), dtype='f8', count=N)
assert_array_equal(b[:], a, "iterator with a hint fails")
def test00(self):
"""Testing fromiter (short iter)"""
a = np.arange(1, 111)
b = ca.fromiter(iter(a), dtype='i4', count=len(a))
assert_array_equal(b[:], a, "fromiter does not work correctly")
def test02(self):
"""Testing fromiter (empty iter)"""
a = np.array([], dtype="f8")
b = ca.fromiter(iter(a), dtype='f8', count=-1)
assert_array_equal(b[:], a, "fromiter does not work correctly")
def test03(self):
"""Testing fromiter (dtype conversion)"""
a = np.arange(101, dtype="f8")
b = ca.fromiter(iter(a), dtype='f4', count=len(a))
assert_array_equal(b[:], a, "fromiter does not work correctly")
def test00(self):
"""Testing fromiter (short iter)"""
a = np.arange(1,111)
b = ca.fromiter(iter(a), dtype='i4', count=len(a))
assert_array_equal(b[:], a, "fromiter does not work correctly")
def test04b(self):
"""Testing fromiter method with large iterator with a hint"""
N = 10*1000
a = np.fromiter((i*2 for i in xrange(N)), dtype='f8', count=N)
b = ca.fromiter((i*2 for i in xrange(N)), dtype='f8', count=N)
assert_array_equal(b[:], a, "iterator with a hint fails")
