def test_bitwise_xor_bool(self):
t = blaze.array(True)
f = blaze.array(False)
self.assertEqual(dd_as_py((t ^ t)._data), False)
self.assertEqual(dd_as_py((t ^ f)._data), True)
self.assertEqual(dd_as_py((f ^ t)._data), True)
self.assertEqual(dd_as_py((f ^ f)._data), False)
def test_bitwise_and_bool(self):
t = blaze.array(True)
f = blaze.array(False)
self.assertEqual(dd_as_py((t & t)._data), True)
self.assertEqual(dd_as_py((t & f)._data), False)
self.assertEqual(dd_as_py((f & t)._data), False)
self.assertEqual(dd_as_py((f & f)._data), False)
def test_product(self):
# Sanity check of reduction op
self.assertEqual(dd_as_py(blaze.eval(blaze.product(5))._data), 5)
self.assertEqual(dd_as_py(blaze.eval(blaze.product([]))._data), 1)
self.assertEqual(dd_as_py(blaze.eval(blaze.product([3, -2]))._data),
-6)
self.assertEqual(dd_as_py(blaze.eval(blaze.product([1.5, 2.0]))._data),
3.0)
def test_add(self):
types = ['int8', 'int16', 'int32', 'int64']
for type_ in types:
a = blaze.array(range(3), dshape=type_)
c = blaze.eval(a+a)
self.assertEqual(dd_as_py(c._data), [0, 2, 4])
c = blaze.eval(((a+a)*a))
self.assertEqual(dd_as_py(c._data), [0, 2, 8])
def test_descriptor_setitem(self):
dd = HDF5DataDescriptor(self.hdf5_file, '/g/a2')
self.assertEqual(dd.dshape, datashape.dshape('2, 3, int64'))
dd[1,2] = 10
self.assertEqual(dd_as_py(dd[1,2]), 10)
dd[1] = [10, 11, 12]
self.assertEqual(dd_as_py(dd[1]), [10, 11, 12])
def test_max(self):
# Sanity check of reduction op
self.assertEqual(dd_as_py(blaze.eval(blaze.max(5))._data), 5)
self.assertRaises(ValueError, blaze.eval, blaze.max([]))
self.assertEqual(dd_as_py(blaze.eval(blaze.max([3, -2]))._data),
3)
self.assertEqual(dd_as_py(blaze.eval(blaze.max([1.5, 2.0]))._data),
2.0)
def test_any(self):
# Sanity check of reduction op
self.assertEqual(dd_as_py(blaze.eval(blaze.any(True))._data), True)
self.assertEqual(dd_as_py(blaze.eval(blaze.any(False))._data), False)
self.assertEqual(dd_as_py(blaze.eval(blaze.any(blaze.array([], dshape='0 * bool')))._data), False)
self.assertEqual(dd_as_py(blaze.eval(blaze.any([False, True]))._data),
True)
self.assertEqual(dd_as_py(blaze.eval(blaze.any([False, False]))._data),
False)
def test_descriptor_getitem_types(self):
dd = HDF5DataDescriptor(self.hdf5_file, '/g/a2')
self.assertEqual(dd.dshape, datashape.dshape('2, 3, int64'))
# Indexing should produce DyNDDataDescriptor instances
self.assertTrue(isinstance(dd[0], DyNDDataDescriptor))
self.assertEqual(dd_as_py(dd[0]), [1,2,3])
self.assertTrue(isinstance(dd[1,2], DyNDDataDescriptor))
self.assertEqual(dd_as_py(dd[1,2]), 1)
def test_create_record(self):
# A simple record array
a = blaze.array([(10, 3.5), (15, 2.25)], dshape="var, {val: int32; flt: float32}")
self.assertEqual(dd_as_py(a._data), [{"val": 10, "flt": 3.5}, {"val": 15, "flt": 2.25}])
# Test field access via attributes
aval = a.val
self.assertEqual(dd_as_py(aval._data), [10, 15])
aflt = a.flt
self.assertEqual(dd_as_py(aflt._data), [3.5, 2.25])
def test_var_dim(self):
a = nd.array([[1,2,3], [4,5], [6]])
dd = DyNDDataDescriptor(a)
self.assertEqual(dd.dshape, datashape.dshape('3, var, int32'))
self.assertEqual(dd_as_py(dd), [[1,2,3], [4,5], [6]])
self.assertEqual(dd_as_py(dd[0]), [1,2,3])
self.assertEqual(dd_as_py(dd[1]), [4,5])
self.assertEqual(dd_as_py(dd[2]), [6])
def test_ragged(self):
a = blaze.array([[1], [2, 3], [4, 5, 6]])
b = blaze.array([[1, 2, 3], [4, 5], [6]])
c = blaze.eval(a + b)
self.assertEqual(dd_as_py(c._data),
[[2, 3, 4], [6, 8], [10, 11, 12]])
c = blaze.eval(2 * a - b)
self.assertEqual(dd_as_py(c._data),
[[1, 0, -1], [0, 1], [2, 4, 6]])
def test_add_with_pyobj(self):
a = blaze.array(3) + 3
self.assertEqual(dd_as_py(a._data), 6)
a = 3 + blaze.array(4)
self.assertEqual(dd_as_py(a._data), 7)
a = blaze.array([1, 2]) + 4
self.assertEqual(dd_as_py(a._data), [5, 6])
a = [1, 2] + blaze.array(5)
self.assertEqual(dd_as_py(a._data), [6, 7])
def test_descriptor_getitem_types(self):
a = np.arange(6).reshape(2,3)
dd = NumPyDataDescriptor(a)
# Indexing should produce NumPyDataDescriptor instances
self.assertTrue(isinstance(dd[0], NumPyDataDescriptor))
self.assertEqual(dd_as_py(dd[0]), [0,1,2])
self.assertTrue(isinstance(dd[1,2], NumPyDataDescriptor))
self.assertEqual(dd_as_py(dd[1,2]), 5)
def test_descriptor_getitem_types(self):
a = nd.array([[1, 2, 3], [4, 5, 6]])
dd = DyNDDataDescriptor(a)
self.assertEqual(dd.dshape, datashape.dshape('2, 3, int32'))
# Indexing should produce DyNDDataDescriptor instances
self.assertTrue(isinstance(dd[0], DyNDDataDescriptor))
self.assertEqual(dd_as_py(dd[0]), [1,2,3])
self.assertTrue(isinstance(dd[1,2], DyNDDataDescriptor))
self.assertEqual(dd_as_py(dd[1,2]), 6)
def test_scalar(self):
a = ffi.new('int *', 3)
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('int32'))
self.assertEqual(dd_as_py(dd), 3)
self.assertTrue(isinstance(dd_as_py(dd), int))
a = ffi.new('float *', 3.25)
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('float32'))
self.assertEqual(dd_as_py(dd), 3.25)
self.assertTrue(isinstance(dd_as_py(dd), float))
def test_1d_array(self):
# An array where the size is in the type
a = ffi.new('short[32]', [2*i for i in range(32)])
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * int16'))
self.assertEqual(dd_as_py(dd), [2*i for i in range(32)])
# An array where the size is not in the type
a = ffi.new('double[]', [1.5*i for i in range(32)])
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * float64'))
self.assertEqual(dd_as_py(dd), [1.5*i for i in range(32)])
def test_scalar(self):
a = ctypes.c_int(3)
dd = data_descriptor_from_ctypes(a, writable=True)
self.assertEqual(dd.dshape, dshape('int32'))
self.assertEqual(dd_as_py(dd), 3)
self.assertTrue(isinstance(dd_as_py(dd), int))
a = ctypes.c_float(3.25)
dd = data_descriptor_from_ctypes(a, writable=True)
self.assertEqual(dd.dshape, dshape('float32'))
self.assertEqual(dd_as_py(dd), 3.25)
self.assertTrue(isinstance(dd_as_py(dd), float))
def test_2d_array(self):
# An array where the leading array size is in the type
vals = [[2**i + j for i in range(35)] for j in range(32)]
a = ffi.new('long long[32][35]', vals)
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * 35 * int64'))
self.assertEqual(dd_as_py(dd), vals)
# An array where the leading array size is not in the type
vals = [[a + b*2 for a in range(35)] for b in range(32)]
a = ffi.new('unsigned char[][35]', vals)
dd = data_descriptor_from_cffi(ffi, a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * 35 * uint8'))
self.assertEqual(dd_as_py(dd), vals)
def test_1d_array(self):
a = (ctypes.c_short * 32)()
for i in range(32):
a[i] = 2*i
dd = data_descriptor_from_ctypes(a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * int16'))
self.assertEqual(dd_as_py(dd), [2*i for i in range(32)])
a = (ctypes.c_double * 32)()
for i in range(32):
a[i] = 1.5*i
dd = data_descriptor_from_ctypes(a, writable=True)
self.assertEqual(dd.dshape, dshape('32 * float64'))
self.assertEqual(dd_as_py(dd), [1.5*i for i in range(32)])
def test_sum_zerosize(self):
# Empty sum operations should produce 0, the reduction identity
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([]))._data), 0)
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([],
keepdims=True))._data),
[0])
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []]))._data), 0)
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []],
keepdims=True))._data),
[[0]])
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []],
axis=-1))._data),
[0, 0])
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []],
axis=-1,
keepdims=True))._data),
[[0], [0]])
# If we're only reducing on a non-empty dimension, we might still
# end up with zero-sized outputs
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []],
axis=0))._data),
[])
self.assertEqual(dd_as_py(blaze.eval(blaze.sum([[], []],
axis=0,
keepdims=True))._data),
[[]])
def test_assign_scalar_to_two_d_array(self):
# Set up our data buffers
src = data_descriptor_from_ctypes(ctypes.c_float(-50), writable=False)
dst = data_descriptor_from_ctypes((ctypes.c_double * 3 * 4)(), writable=True)
# Do the assignment
execute_unary_single(dst, src, datashape.float64, datashape.float32, self.sqr)
self.assertEqual(dd_as_py(dst), [[-50.0 * -50] * 3] * 4)
def test_uints(self):
types = ['uint8', 'uint16', 'uint32', 'uint64']
for type_ in types:
a = blaze.array(np.arange(3), dshape=type_)
dtype = to_numpy_dtype(a.dshape)
self.assertEqual(dtype, np.dtype(type_))
self.assertEqual(dd_as_py(a._data), [0, 1, 2])
def test_basic_object_type(self):
self.assertTrue(issubclass(DyNDDataDescriptor, IDataDescriptor))
a = nd.array([[1, 2, 3], [4, 5, 6]])
dd = DyNDDataDescriptor(a)
# Make sure the right type is returned
self.assertTrue(isinstance(dd, IDataDescriptor))
self.assertEqual(dd_as_py(dd), [[1, 2, 3], [4, 5, 6]])
def test_getitem_start_step(self):
dd = CSVDataDescriptor(self.csv_file, schema=csv_schema)
el = dd[1::2]
self.assertTrue(isinstance(el, DyNDDataDescriptor))
vals = dd_as_py(el)
self.assertEqual(vals, [
{u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True}])
def test_element_write(self):
a = nd.array([1, 2, 3, 4, 5], access='rw')
dd = DyNDDataDescriptor(a)
self.assertEqual(dd.dshape, datashape.dshape('5, int32'))
ge = dd.element_writer(1)
self.assertTrue(isinstance(ge, IElementWriter))
x = ctypes.c_int32(123)
ge.write_single((1,), ctypes.addressof(x))
self.assertEqual(dd_as_py(dd), [1,123,3,4,5])
with ge.buffered_ptr((3,)) as dst_ptr:
x = ctypes.c_int32(456)
ctypes.memmove(dst_ptr, ctypes.addressof(x), 4)
self.assertEqual(dd_as_py(dd), [1,123,3,456,5])
def test_assign_scalar(self):
# Set up our data buffers
src = data_descriptor_from_ctypes(ctypes.c_float(1028), writable=False)
dst = data_descriptor_from_ctypes(ctypes.c_double(-100), writable=True)
# Do the assignment
execute_unary_single(dst, src, datashape.float64, datashape.float32, self.sqr)
self.assertEqual(dd_as_py(dst), 1028.0*1028.0)
def function(self):
A = getattr(self, storage1 + 'A')
B = getattr(self, storage2 + 'B')
Rd = kernel(A, B)
self.assert_(isinstance(Rd, blaze.Array))
self.assert_(Rd._data.capabilities.deferred)
p = _store(storage3 + 'Rd') if storage3 == 'dsk' else None
try:
Rc = blaze.eval(Rd, storage=p)
self.assert_(isinstance(Rc, blaze.Array))
npy_data = getattr(self, 'npy' + R)
assert_allclose(np.array(dd_as_py(Rc._data)), npy_data)
if storage3 == 'dsk':
self.assert_(Rc._data.capabilities.persistent)
else:
self.assert_(not Rc._data.capabilities.persistent)
finally:
try:
if p is not None:
blaze.drop(p)
except:
pass # show the real error...
def test_append2(self):
persist = blaze.Storage(self.rooturi, format="blz")
a = blaze.empty('0 * 2 * float64', storage=persist)
self.assertTrue(isinstance(a, blaze.Array))
lvals = [[i,i*2] for i in range(10)]
append(a,lvals)
self.assertEqual(dd_as_py(a._data), lvals)
def test_getitem_stop(self):
dd = CSVDataDescriptor(self.csv_file, schema=csv_schema)
el = dd[:1]
self.assertTrue(isinstance(el, DyNDDataDescriptor))
vals = dd_as_py(el)
self.assertEqual(vals, [
{u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False}])
def test_create(self):
persist = blaze.Storage(self.rooturi, format="blz")
a = blaze.array([], 'float64', storage=persist)
self.assertTrue(isinstance(a, blaze.Array))
print("->", a.dshape.shape)
self.assertTrue(a.dshape.shape == (0,))
self.assertEqual(dd_as_py(a._data), [])
def test_append(self):
store = blaze.Storage(self.url, mode='r+')
a = blaze.open(store, schema=csv_schema)
blaze.append(a, ["k4", "v4", 4, True])
self.assertEqual(dd_as_py(a._data), csv_ldict + \
[{u'f0': u'k4', u'f1': u'v4', u'f2': 4, u'f3': True}])
def test_deprecated_open(self):
url = "csv://" + self.fname
store = blaze.Storage(url, mode='r')
a = blaze.open(store, schema=csv_schema)
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), csv_ldict)
def test_open(self):
store = blaze.Storage(self.url, mode='r')
a = blaze.open(store, schema=json_schema)
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), [1, 2, 3, 4, 5])
def test_2d(self):
a = blaze.array(np.arange(3 * 3).reshape(3, 3), caps=self.caps)
a[1] = 2
self.assertEqual(dd_as_py(a[1]._data), [2, 2, 2])
def test_scalar(self):
a = blaze.array(np.arange(3), caps=self.caps)
a[0] = 1
self.assertEqual(dd_as_py(a[0]._data), 1)
def test_compute(self):
ra = blaze.array(RemoteDataDescriptor('%s/py_arr' % self.baseurl))
result = ra + 1
result = blaze.eval(result)
self.assertEqual(result.dshape, datashape.dshape('5, int32'))
self.assertEqual(dd_as_py(result._data), [2, 3, 4, 5, 6])
def test_1d(self):
a = blaze.array(np.arange(3), caps=self.caps)
# print("a:", a, self.caps)
self.assertEqual(dd_as_py(a[0:2]._data), [0, 1])
def test_open_has_header(self):
store = blaze.Storage(self.url, mode='r')
a = blaze.open(store, schema=csv_schema, has_header=False)
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), csv_ldict)
def test_bitwise_not_uint64(self):
x = 0x3192573469a2b3a1
a = blaze.array(x, 'uint64')
self.assertEqual(dd_as_py((~a)._data), x ^ 0xffffffffffffffff)
self.assertEqual(dd_as_py(blaze.bitwise_not(a)._data),
x ^ 0xffffffffffffffff)
def test_open_dialect(self):
store = blaze.Storage(self.url, mode='r')
a = blaze.open(store, schema=csv_schema, dialect='excel')
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), csv_ldict)
def test_bitwise_xor_uint64(self):
x, y = 0x3192573469a2b3a1, 0x9274a2e219c27638
a = blaze.array(x, 'uint64')
b = blaze.array(y, 'uint64')
self.assertEqual(dd_as_py((a ^ b)._data), x ^ y)
self.assertEqual(dd_as_py(blaze.bitwise_xor(a, b)._data), x ^ y)
'''Sample module showing how to read JSON files into blaze arrays'''
from __future__ import absolute_import, division, print_function
import blaze
from blaze.datadescriptor import dd_as_py
json_buf = u"[1, 2, 3, 4, 5]"
json_schema = "var, int8"
# Create a temporary JSON file in URI and open the dataset
dname = '/tmp/test.json'
store = blaze.Storage(dname)
print "store:", store
with file(store.path, "wb") as f:
f.write(json_buf)
arr = blaze.open(store, json_schema)
#print('Blaze array:', arr) # XXX This does not work yet
#print('Blaze array:', nd.array(arr)) # XXX idem
# Convert the data to a native Python object
print('Blaze array:', dd_as_py(arr._data))
# Remove the temporary JSON file
blaze.drop(store)
def test_get_arr(self):
ra = blaze.array(RemoteDataDescriptor('%s/csv_arr' % self.baseurl))
la = blaze.catalog.get('/csv_arr')
self.assertEqual(la.dshape, ra.dshape)
self.assertEqual(dd_as_py(la._data), dd_as_py(blaze.eval(ra)._data))
def test_bitwise_not_bool(self):
t = blaze.array(True)
f = blaze.array(False)
self.assertEqual(dd_as_py((~t)._data), False)
self.assertEqual(dd_as_py((~f)._data), True)
def test_deprecated_open(self):
url = "json://" + self.fname
store = blaze.Storage(url, mode='r')
a = blaze.open(store, schema=json_schema)
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), [1, 2, 3, 4, 5])
def test_basic_object_type(self):
self.assertTrue(issubclass(HDF5DataDescriptor, IDataDescriptor))
dd = HDF5DataDescriptor(self.hdf5_file, '/a1')
# Make sure the right type is returned
self.assertTrue(isinstance(dd, IDataDescriptor))
self.assertEqual(dd_as_py(dd), [[1, 2, 3], [4, 5, 6]])
def test_1d(self):
a = blaze.array(np.arange(3), caps=self.caps)
a[0:2] = 2
self.assertEqual(dd_as_py(a[0:2]._data), [2, 2])