def test_ckernel_deferred_from_pyfunc(self):
# Test wrapping make_assignment_ckernel as a deferred ckernel
def instantiate_assignment(out_ckb, ckb_offset, types, meta,
kerntype, ectx):
out_ckb = _lowlevel.CKernelBuilderStruct.from_address(out_ckb)
return _lowlevel.make_assignment_ckernel(out_ckb, ckb_offset,
types[0], meta[0],
types[1], meta[1],
'expr', kerntype, ectx)
ckd = _lowlevel.ckernel_deferred_from_pyfunc(instantiate_assignment,
[ndt.string, ndt.date])
self.assertEqual(nd.as_py(ckd.types), [ndt.string, ndt.date])
out = nd.empty(ndt.string)
in0 = nd.array('2012-11-05', ndt.date)
ckd.__call__(out, in0)
self.assertEqual(nd.as_py(out), '2012-11-05')
# Also test it as a lifted kernel
ckd_lifted = _lowlevel.lift_ckernel_deferred(ckd,
['3 * var * string', '3 * var * date'])
self.assertEqual(nd.as_py(ckd_lifted.types),
[ndt.type('3 * var * string'), ndt.type('3 * var * date')])
out = nd.empty('3 * var * string')
from datetime import date
in0 = nd.array([['2013-03-11', date(2010, 10, 10)],
[date(1999, 12, 31)],
[]], type='3 * var * date')
ckd_lifted.__call__(out, in0)
self.assertEqual(nd.as_py(out),
[['2013-03-11', '2010-10-10'],
['1999-12-31'], []])
def empty(dshape, ddesc=None):
"""Create an array with uninitialized data.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
ddesc : data descriptor instance
This comes with the necessary info for storing the data. If
None, a DyND_DDesc will be used.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
if ddesc is None:
ddesc = DyND_DDesc(nd.empty(str(dshape)))
return Array(ddesc)
if isinstance(ddesc, BLZ_DDesc):
shape, dt = to_numpy(dshape)
ddesc.blzarr = blz.zeros(shape, dt, rootdir=ddesc.path,
mode=ddesc.mode, **ddesc.kwargs)
elif isinstance(ddesc, HDF5_DDesc):
obj = nd.as_numpy(nd.empty(str(dshape)))
with tb.open_file(ddesc.path, mode=ddesc.mode) as f:
where, name = split_path(ddesc.datapath)
f.create_earray(where, name, filters=ddesc.filters, obj=obj)
ddesc.mode = 'a' # change into 'a'ppend mode for further operations
return Array(ddesc)
def test_single_struct_array(self):
a = nd.empty('3 * {x:int32, y:int32}')
a[...] = [(0, 0), (3, 5), (12, 10)]
self.assertEqual(nd.as_py(a.x), [0, 3, 12])
self.assertEqual(nd.as_py(a.y), [0, 5, 10])
a[...] = [{'x': 1, 'y': 2}, {'x': 4, 'y': 7}, {'x': 14, 'y': 190}]
self.assertEqual(nd.as_py(a.x), [1, 4, 14])
self.assertEqual(nd.as_py(a.y), [2, 7, 190])
a = nd.empty('2 * var * {count:int32, size:fixed_string[1,"A"]}')
a[...] = [[(3, 'X')], [(10, 'L'), (12, 'M')]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']])
a[...] = [[{
'count': 6,
'size': 'M'
}], [{
'count': 3,
'size': 'F'
}, {
'count': 16,
'size': 'D'
}]]
self.assertEqual(nd.as_py(a.count), [[6], [3, 16]])
self.assertEqual(nd.as_py(a.size), [['M'], ['F', 'D']])
a[...] = {'count': 1, 'size': 'Z'}
self.assertEqual(nd.as_py(a.count), [[1], [1, 1]])
self.assertEqual(nd.as_py(a.size), [['Z'], ['Z', 'Z']])
a[...] = [[(10, 'A')], [(5, 'B')]]
self.assertEqual(nd.as_py(a.count), [[10], [5, 5]])
self.assertEqual(nd.as_py(a.size), [['A'], ['B', 'B']])
def test_empty(self):
# Constructor from scalar type
a = nd.empty(ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.int32)
# Constructor from type with fixed dimension
a = nd.empty('3 * int32')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3,))
# Constructor from shape as single integer
a = nd.empty(3, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_strided_dim(ndt.int32))
self.assertEqual(a.shape, (3,))
# Constructor from shape as tuple
a = nd.empty((3,4), ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('strided * strided * int32'))
self.assertEqual(a.shape, (3,4))
# Constructor from shape as variadic arguments
a = nd.empty(3, 4, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('strided * strided * int32'))
self.assertEqual(a.shape, (3,4))
def test_object_in_struct_arr(self):
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', np.int64), ('y', object)])
self.assertEqual(nd.as_py(a), [{
'x': 1,
'y': 'test'
}, {
'x': 2,
'y': 'one'
}, {
'x': 3,
'y': 'two'
}])
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([('opposite', 4)],
dtype=[('y', object), ('x', np.int64)])
self.assertEqual(nd.as_py(a), [{'x': 4, 'y': 'opposite'}] * 3)
a = nd.empty('var * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', object), ('y', object)])
self.assertEqual(nd.as_py(a), [{
'x': 1,
'y': 'test'
}, {
'x': 2,
'y': 'one'
}, {
'x': 3,
'y': 'two'
}])
def test_single_struct_array(self):
a = nd.empty('3 * {x:int32, y:int32}')
a[...] = [(0,0), (3,5), (12,10)]
self.assertEqual(nd.as_py(a.x), [0, 3, 12])
self.assertEqual(nd.as_py(a.y), [0, 5, 10])
a[...] = [{'x':1,'y':2}, {'x':4,'y':7}, {'x':14,'y':190}]
self.assertEqual(nd.as_py(a.x), [1, 4, 14])
self.assertEqual(nd.as_py(a.y), [2, 7, 190])
a = nd.empty('2 * var * {count:int32, size:string[1,"A"]}')
a[...] = [[(3, 'X')], [(10, 'L'), (12, 'M')]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']])
a[...] = [[{'count':6, 'size':'M'}],
[{'count':3, 'size':'F'}, {'count':16, 'size':'D'}]]
self.assertEqual(nd.as_py(a.count), [[6], [3, 16]])
self.assertEqual(nd.as_py(a.size), [['M'], ['F', 'D']])
a[...] = {'count':1, 'size':'Z'}
self.assertEqual(nd.as_py(a.count), [[1], [1, 1]])
self.assertEqual(nd.as_py(a.size), [['Z'], ['Z', 'Z']])
a[...] = [[(10, 'A')], [(5, 'B')]]
self.assertEqual(nd.as_py(a.count), [[10], [5, 5]])
self.assertEqual(nd.as_py(a.size), [['A'], ['B', 'B']])
def op_alloc(self, op):
dshape = op.type
# Allocate a chunk instead of the whole thing
if len(dshape.shape) == self.result_ndim:
chunk = nd.empty(self.chunk_size, str(dshape.subarray(1)))
else:
chunk = nd.empty(str(dshape))
self.values[op] = blaze.array(chunk)
def test_assign_to_struct(self):
value = [(8, u'world', 4.5), (16, u'!', 8.75)]
# Assign list of tuples
a = nd.empty('2 * { i : int32, msg : string, price : float64 }')
a[:] = value
self.assertEqual(nd.as_py(a, tuple=True), value)
# Assign iterator of tuples
a = nd.empty('2 * { i : int32, msg : string, price : float64 }')
a[:] = iter(value)
self.assertEqual(nd.as_py(a, tuple=True), value)
def test_single_struct(self):
a = nd.empty('{x:int32, y:string, z:bool}')
a[...] = [3, 'test', False]
self.assertEqual(nd.as_py(a.x), 3)
self.assertEqual(nd.as_py(a.y), 'test')
self.assertEqual(nd.as_py(a.z), False)
a = nd.empty('{x:int32, y:string, z:bool}')
a[...] = {'x':10, 'y':'testing', 'z':True}
self.assertEqual(nd.as_py(a.x), 10)
self.assertEqual(nd.as_py(a.y), 'testing')
self.assertEqual(nd.as_py(a.z), True)
def test_single_struct(self):
a = nd.empty('{x:int32, y:string, z:bool}')
a[...] = [3, 'test', False]
self.assertEqual(nd.as_py(a.x), 3)
self.assertEqual(nd.as_py(a.y), 'test')
self.assertEqual(nd.as_py(a.z), False)
a = nd.empty('{x:int32, y:string, z:bool}')
a[...] = {'x': 10, 'y': 'testing', 'z': True}
self.assertEqual(nd.as_py(a.x), 10)
self.assertEqual(nd.as_py(a.y), 'testing')
self.assertEqual(nd.as_py(a.z), True)
def test_nested_struct(self):
a = nd.empty('{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float32]}')
a[...] = [[1,2], ['test', 3.5], [3j]]
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
a = nd.empty('{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float32]}')
a[...] = {'x':[1,2], 'y':{'a':'test', 'b':3.5}, 'z':[3j]}
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
def test_nested_struct(self):
a = nd.empty('{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float32]}')
a[...] = [[1,2], ['test', 3.5], [3j]]
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
a = nd.empty('{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float32]}')
a[...] = {'x':[1,2], 'y':{'a':'test', 'b':3.5}, 'z':[3j]}
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
def test_pyconvert_overflow(self):
a = nd.empty(ndt.uint128)
def assign_val(x, val):
x[...] = val
self.assertRaises(OverflowError, assign_val, a, -1)
self.assertRaises(OverflowError, assign_val, a, -2**127 - 1)
self.assertRaises(OverflowError, assign_val, a, 2**128)
def test_object_arr(self):
a = nd.empty('3 * int')
a[...] = np.array([1, 2, 3.0], dtype=object)
self.assertEqual(nd.as_py(a), [1, 2, 3])
a = nd.empty('3 * string')
a[...] = np.array(['testing', 'one', u'two'], dtype=object)
self.assertEqual(nd.as_py(a), ['testing', 'one', 'two'])
a = nd.empty('3 * string')
a[...] = np.array(['broadcast_string'], dtype=object)
self.assertEqual(nd.as_py(a), ['broadcast_string'] * 3)
a = nd.empty('3 * string')
a[...] = np.array('testing', dtype=object)
self.assertEqual(nd.as_py(a), ['testing'] * 3)
def test_lift_ckernel(self):
# First get a ckernel from numpy
requiregil = False
ckd = _lowlevel.ckernel_deferred_from_ufunc(np.ldexp,
(np.float64, np.float64, np.int32),
requiregil)
self.assertEqual(nd.as_py(ckd.types),
[ndt.float64, ndt.float64, ndt.int32])
# Now lift it
ckd_lifted = _lowlevel.lift_ckernel_deferred(ckd,
['var * var * float64', 'strided * var * float64',
'strided * 1 * int32'])
self.assertEqual(nd.as_py(ckd_lifted.types),
[ndt.type(x) for x in ['var * var * float64',
'strided * var * float64', 'strided * 1 * int32']])
# Create some compatible arguments
out = nd.empty('var * var * float64')
in0 = nd.array([[1, 2, 3], [4, 5], [6], [7,9,10]], type='strided * var * float64')
in1 = nd.array([[-1], [10], [100], [-12]], type='strided * 1 * int32')
# Instantiate and call the kernel on these arguments
ckd_lifted.__call__(out, in0, in1)
# Verify that we got the expected result
self.assertEqual(nd.as_py(out),
[[0.5, 1.0, 1.5],
[4096.0, 5120.0],
[float(6*2**100)],
[0.001708984375, 0.002197265625, 0.00244140625]])
def test_object_arr(self):
a = nd.empty('3 * int')
a[...] = np.array([1, 2, 3.0], dtype=object)
self.assertEqual(nd.as_py(a), [1, 2, 3])
a = nd.empty('3 * string')
a[...] = np.array(['testing', 'one', u'two'], dtype=object)
self.assertEqual(nd.as_py(a), ['testing', 'one', 'two'])
a = nd.empty('3 * string')
a[...] = np.array(['broadcast_string'], dtype=object)
self.assertEqual(nd.as_py(a), ['broadcast_string'] * 3)
a = nd.empty('3 * string')
a[...] = np.array('testing', dtype=object)
self.assertEqual(nd.as_py(a), ['testing'] * 3)
def ones(dshape, caps={'efficient-write': True}, storage=None):
"""Create an array and fill it with ones.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for data storage.
Returns
-------
out: a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if storage is not None:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.ones(shape, dt,
rootdir=storage.path))
elif 'efficient-write' in caps:
# TODO: Handle var dimension properly (raise exception?)
dyndarr = nd.empty(str(dshape))
dyndarr[...] = True
dd = DyNDDataDescriptor(dyndarr)
elif 'compress' in caps:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.ones(shape, dt))
return Array(dd)
def empty(dshape, caps={'efficient-write': True}, storage=None):
"""Create an array with uninitialized data.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for data storage.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if storage is not None:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.zeros(shape, dt, rootdir=storage.path))
elif 'efficient-write' in caps:
dd = DyNDDataDescriptor(nd.empty(str(dshape)))
elif 'compress' in caps:
dd = BLZDataDescriptor(blz.zeros(shape, dt))
return Array(dd)
def ones(dshape, caps={'efficient-write': True}, storage=None):
"""Create an array and fill it with ones.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for data storage.
Returns
-------
out: a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if storage is not None:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.ones(shape, dt, rootdir=storage.path))
elif 'efficient-write' in caps:
# TODO: Handle var dimension properly (raise exception?)
dyndarr = nd.empty(str(dshape))
dyndarr[...] = True
dd = DyNDDataDescriptor(dyndarr)
elif 'compress' in caps:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.ones(shape, dt))
return Array(dd)
def empty(dshape, caps={'efficient-write': True}, storage=None):
"""Create an array with uninitialized data.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for data storage.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if storage is not None:
shape, dt = to_numpy(dshape)
dd = BLZDataDescriptor(blz.zeros(shape, dt,
rootdir=storage.path))
elif 'efficient-write' in caps:
dd = DyNDDataDescriptor(nd.empty(str(dshape)))
elif 'compress' in caps:
dd = BLZDataDescriptor(blz.zeros(shape, dt))
return Array(dd)
def finalize(bases):
shape = bases[0].shape[:2]
out = nd.empty(shape, dshape)
for path, finalizer, inds in zip(paths, finalizers, indices):
arr = reduce(getattr, path, out)
np_arr = nd.as_numpy(arr.view_scalars(arr.dtype.value_type))
np_arr[:] = finalizer(*get(inds, bases))
return out
def finalize(bases):
shape = bases[0].shape[:2]
out = nd.empty(shape, dshape)
for path, finalizer, inds in zip(paths, finalizers, indices):
arr = reduce(getattr, path, out)
np_arr = nd.as_numpy(arr.view_scalars(arr.dtype.value_type))
np_arr[:] = finalizer(*get(inds, bases))
return out
def test_sum_1d(self):
# Use the numpy add ufunc for this lifting test
af = _lowlevel.arrfunc_from_ufunc(np.add,
(np.int32, np.int32, np.int32),
False)
in0 = nd.array([3, 12, -5, 10, 2])
# Simple lift
sum = _lowlevel.lift_reduction_arrfunc(af, 'fixed * int32')
out = nd.empty(ndt.int32)
sum.execute(out, in0)
self.assertEqual(nd.as_py(out), 22)
# Lift with keepdims
sum = _lowlevel.lift_reduction_arrfunc(af, 'fixed * int32',
keepdims=True)
out = nd.empty(1, ndt.int32)
sum.execute(out, in0)
self.assertEqual(nd.as_py(out), [22])
def test_sum_1d(self):
# Use the numpy add ufunc for this lifting test
ckd = _lowlevel.ckernel_deferred_from_ufunc(np.add,
(np.int32, np.int32, np.int32),
False)
in0 = nd.array([3, 12, -5, 10, 2])
# Simple lift
sum = _lowlevel.lift_reduction_ckernel_deferred(ckd, 'strided * int32')
out = nd.empty(ndt.int32)
sum.__call__(out, in0)
self.assertEqual(nd.as_py(out), 22)
# Lift with keepdims
sum = _lowlevel.lift_reduction_ckernel_deferred(ckd, 'strided * int32',
keepdims=True)
out = nd.empty(1, ndt.int32)
sum.__call__(out, in0)
self.assertEqual(nd.as_py(out), [22])
def test_missing_field(self):
a = nd.empty('{x:int32, y:int32, z:int32}')
def assign(x, y):
x[...] = y
self.assertRaises(nd.BroadcastError, assign, a, [0, 1])
self.assertRaises(nd.BroadcastError, assign, a, {'x': 0, 'z': 1})
def test_empty(self):
# Constructor from scalar type
a = nd.empty(ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.int32)
# Constructor from type with fixed dimension
a = nd.empty('3 * int32')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3, ))
# Constructor from type with fixed dimension, accesskwarg
a = nd.empty('3 * int32', access='rw')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3, ))
# Constructor from shape as single integer
a = nd.empty(3, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * int32'))
self.assertEqual(a.shape, (3, ))
# Constructor from shape as tuple
a = nd.empty((3, 4), ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3, 4))
# Constructor from shape as variadic arguments
a = nd.empty(3, 4, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3, 4))
# Constructor from shape as variadic arguments, access kwarg
a = nd.empty(3, 4, ndt.int32, access='rw')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3, 4))
def test_simple_var(self):
a = nd.empty('var * int32')
a[...] = np.int64(1)
self.assertEqual(nd.as_py(a), [1])
a = nd.empty('var * int32')
a[...] = np.array(2.0)
self.assertEqual(nd.as_py(a), [2])
a = nd.empty('var * int32')
a[...] = np.array([3], dtype=np.int8)
self.assertEqual(nd.as_py(a), [3])
a = nd.empty('var * int32')
a[...] = np.array([1, 2, 3])
self.assertEqual(nd.as_py(a), [1, 2, 3])
a[...] = np.array([4])
self.assertEqual(nd.as_py(a), [4] * 3)
def test_object_in_struct_arr(self):
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', np.int64), ('y', object)])
self.assertEqual(nd.as_py(a),
[{'x': 1, 'y': 'test'}, {'x': 2, 'y': 'one'}, {'x': 3, 'y': 'two'}])
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([('opposite', 4)],
dtype=[('y', object), ('x', np.int64)])
self.assertEqual(nd.as_py(a),
[{'x': 4, 'y': 'opposite'}] * 3)
a = nd.empty('var * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', object), ('y', object)])
self.assertEqual(nd.as_py(a),
[{'x': 1, 'y': 'test'}, {'x': 2, 'y': 'one'}, {'x': 3, 'y': 'two'}])
def test_simple_var(self):
a = nd.empty('var * int32')
a[...] = np.int64(1)
self.assertEqual(nd.as_py(a), [1])
a = nd.empty('var * int32')
a[...] = np.array(2.0)
self.assertEqual(nd.as_py(a), [2])
a = nd.empty('var * int32')
a[...] = np.array([3], dtype=np.int8)
self.assertEqual(nd.as_py(a), [3])
a = nd.empty('var * int32')
a[...] = np.array([1, 2, 3])
self.assertEqual(nd.as_py(a), [1, 2, 3])
a[...] = np.array([4])
self.assertEqual(nd.as_py(a), [4] * 3)
def test_simple_strided(self):
a = nd.empty('3 * int32')
a[...] = np.int64(1)
self.assertEqual(nd.as_py(a), [1] * 3)
a[...] = np.array(2.0)
self.assertEqual(nd.as_py(a), [2] * 3)
a[...] = np.array([3], dtype=np.int8)
self.assertEqual(nd.as_py(a), [3] * 3)
a[...] = np.array([1, 2, 3])
self.assertEqual(nd.as_py(a), [1, 2, 3])
def test_nested_struct_array(self):
a = nd.empty('3 * {x:{a:int16, b:int16}, y:int32}')
a[...] = [((0,1),0), ((2,2),5), ((100,10),10)]
self.assertEqual(nd.as_py(a.x.a), [0, 2, 100])
self.assertEqual(nd.as_py(a.x.b), [1, 2, 10])
self.assertEqual(nd.as_py(a.y), [0, 5, 10])
a[...] = [{'x':{'a':1,'b':2},'y':5},
{'x':{'a':3,'b':6},'y':7},
{'x':{'a':1001,'b':110},'y':110}]
self.assertEqual(nd.as_py(a.x.a), [1, 3, 1001])
self.assertEqual(nd.as_py(a.x.b), [2, 6, 110])
self.assertEqual(nd.as_py(a.y), [5, 7, 110])
a = nd.empty('2 * var * {count:int32, size:{name:fixed_string[1,"A"], id: int8}}')
a[...] = [[(3, ('X', 10))], [(10, ('L', 7)), (12, ('M', 5))]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size.name), [['X'], ['L', 'M']])
self.assertEqual(nd.as_py(a.size.id), [[10], [7, 5]])
def test_simple_strided(self):
a = nd.empty('3 * int32')
a[...] = np.int64(1)
self.assertEqual(nd.as_py(a), [1] * 3)
a[...] = np.array(2.0)
self.assertEqual(nd.as_py(a), [2] * 3)
a[...] = np.array([3], dtype=np.int8)
self.assertEqual(nd.as_py(a), [3] * 3)
a[...] = np.array([1, 2, 3])
self.assertEqual(nd.as_py(a), [1, 2, 3])
def test_nested_struct_array(self):
a = nd.empty('3 * {x:{a:int16, b:int16}, y:int32}')
a[...] = [((0,1),0), ((2,2),5), ((100,10),10)]
self.assertEqual(nd.as_py(a.x.a), [0, 2, 100])
self.assertEqual(nd.as_py(a.x.b), [1, 2, 10])
self.assertEqual(nd.as_py(a.y), [0, 5, 10])
a[...] = [{'x':{'a':1,'b':2},'y':5},
{'x':{'a':3,'b':6},'y':7},
{'x':{'a':1001,'b':110},'y':110}]
self.assertEqual(nd.as_py(a.x.a), [1, 3, 1001])
self.assertEqual(nd.as_py(a.x.b), [2, 6, 110])
self.assertEqual(nd.as_py(a.y), [5, 7, 110])
a = nd.empty('2 * var * {count:int32, size:{name:string[1,"A"], id: int8}}')
a[...] = [[(3, ('X', 10))], [(10, ('L', 7)), (12, ('M', 5))]]
self.assertEqual(nd.as_py(a.count), [[3], [10, 12]])
self.assertEqual(nd.as_py(a.size.name), [['X'], ['L', 'M']])
self.assertEqual(nd.as_py(a.size.id), [[10], [7, 5]])
def test_ctypes_callback_deferred(self):
# Create a deferred ckernel via a closure
def instantiate_ckernel(out_ckb, ckb_offset, types, meta, kerntype,
ectx):
out_ckb = _lowlevel.CKernelBuilder(out_ckb)
def my_kernel_func_single(dst_ptr, src_ptr, kdp):
dst = ctypes.c_int32.from_address(dst_ptr)
src = ctypes.c_double.from_address(src_ptr[0])
dst.value = int(src.value * 3.5)
def my_kernel_func_strided(dst_ptr, dst_stride, src_ptr,
src_stride, count, kdp):
src_ptr0 = src_ptr[0]
src_stride0 = src_stride[0]
for i in range(count):
my_kernel_func_single(dst_ptr, [src_ptr0], kdp)
dst_ptr += dst_stride
src_ptr0 += src_stride0
if kerntype == 'single':
kfunc = _lowlevel.ExprSingleOperation(my_kernel_func_single)
else:
kfunc = _lowlevel.ExprStridedOperation(my_kernel_func_strided)
return ckernel.wrap_ckernel_func(out_ckb, ckb_offset, kfunc, kfunc)
ckd = _lowlevel.ckernel_deferred_from_pyfunc(instantiate_ckernel,
[ndt.int32, ndt.float64])
# Test calling the ckd
out = nd.empty(ndt.int32)
in0 = nd.array(4.0, type=ndt.float64)
ckd.__call__(out, in0)
self.assertEqual(nd.as_py(out), 14)
# Also call it lifted
ckd_lifted = _lowlevel.lift_ckernel_deferred(
ckd, ['2 * var * int32', '2 * var * float64'])
out = nd.empty('2 * var * int32')
in0 = nd.array([[1.0, 3.0, 2.5], [1.25, -1.5]],
type='2 * var * float64')
ckd_lifted.__call__(out, in0)
self.assertEqual(nd.as_py(out), [[3, 10, 8], [4, -5]])
def test_pyconvert(self):
# Conversions to/from python longs
a = nd.empty(ndt.uint128)
a[...] = 1
self.assertEqual(nd.as_py(a), 1)
a[...] = 12345
self.assertEqual(nd.as_py(a), 12345)
a[...] = 0
self.assertEqual(nd.as_py(a), 0)
a[...] = 2**128 - 1
self.assertEqual(nd.as_py(a), 2**128 - 1)
def test_creation(self):
af = nd.empty('arrfunc')
self.assertEqual(nd.type_of(af).type_id, 'arrfunc')
# Test there is a string version of a NULL arrfunc
self.assertTrue(str(af) != '')
self.assertEqual(nd.as_py(af.proto), ndt.type())
# Test there is a string version of an initialized arrfunc
af = _lowlevel.make_arrfunc_from_assignment(
ndt.float32, ndt.int64, "nocheck")
self.assertTrue(str(af) != '')
self.assertEqual(nd.as_py(af.proto), ndt.type("(int64) -> float32"))
def run(self, size):
if self.cuda:
dst_tp = ndt.type('cuda_device[{} * float64]'.format(size))
else:
dst_tp = ndt.type('{} * float64'.format(size))
dst = nd.empty(dst_tp)
with CUDATimer() if self.cuda else Timer() as timer:
nd.uniform(dst_tp=dst_tp)
return timer.elapsed_time()
def run(self, size):
if self.cuda:
dst_tp = ndt.type('cuda_device[{} * float64]'.format(size))
else:
dst_tp = ndt.type('{} * float64'.format(size))
dst = nd.empty(dst_tp)
with CUDATimer() if self.cuda else Timer() as timer:
nd.uniform(dst_tp = dst_tp)
return timer.elapsed_time()
def test_creation(self):
ckd = nd.empty('ckernel_deferred')
self.assertEqual(nd.type_of(ckd).type_id, 'ckernel_deferred')
# Test there is a string version of a NULL ckernel_deferred
self.assertTrue(str(ckd) != '')
self.assertEqual(nd.as_py(ckd.types), [])
# Test there is a string version of an initialized ckernel_deferred
ckd = _lowlevel.make_ckernel_deferred_from_assignment(
ndt.float32, ndt.int64,
"unary", "none")
self.assertTrue(str(ckd) != '')
self.assertEqual(nd.as_py(ckd.types), [ndt.float32, ndt.int64])
def test_extra_field(self):
a = nd.empty('{x:int32, y:int32, z:int32}')
def assign(x, y):
x[...] = y
self.assertRaises(nd.BroadcastError, assign, a, [0, 1, 2, 3])
self.assertRaises(nd.BroadcastError, assign, a, {
'x': 0,
'y': 1,
'z': 2,
'w': 3
})
def test_sum_2d_axisall(self):
# Use the numpy add ufunc for this lifting test
af = _lowlevel.arrfunc_from_ufunc(np.add,
(np.int32, np.int32, np.int32),
False)
in0 = nd.array([[3, 12, -5], [10, 2, 3]])
# Simple lift
sum = _lowlevel.lift_reduction_arrfunc(af,
'fixed * fixed * int32',
commutative=True,
associative=True)
out = nd.empty(ndt.int32)
sum.execute(out, in0)
self.assertEqual(nd.as_py(out), 25)
def test_sum_2d_axisall(self):
# Use the numpy add ufunc for this lifting test
ckd = _lowlevel.ckernel_deferred_from_ufunc(np.add,
(np.int32, np.int32, np.int32),
False)
in0 = nd.array([[3, 12, -5], [10, 2, 3]])
# Simple lift
sum = _lowlevel.lift_reduction_ckernel_deferred(ckd,
'strided * strided * int32',
commutative=True,
associative=True)
out = nd.empty(ndt.int32)
sum.__call__(out, in0)
self.assertEqual(nd.as_py(out), 25)
def test_ctypes_callback_deferred(self):
# Create a deferred ckernel via a closure
def instantiate_ckernel(out_ckb, ckb_offset, types, meta,
kerntype, ectx):
out_ckb = _lowlevel.CKernelBuilder(out_ckb)
def my_kernel_func_single(dst_ptr, src_ptr, kdp):
dst = ctypes.c_int32.from_address(dst_ptr)
src = ctypes.c_double.from_address(src_ptr[0])
dst.value = int(src.value * 3.5)
def my_kernel_func_strided(dst_ptr, dst_stride, src_ptr, src_stride, count, kdp):
src_ptr0 = src_ptr[0]
src_stride0 = src_stride[0]
for i in range(count):
my_kernel_func_single(dst_ptr, [src_ptr0], kdp)
dst_ptr += dst_stride
src_ptr0 += src_stride0
if kerntype == 'single':
kfunc = _lowlevel.ExprSingleOperation(my_kernel_func_single)
else:
kfunc = _lowlevel.ExprStridedOperation(my_kernel_func_strided)
return ckernel.wrap_ckernel_func(out_ckb, ckb_offset,
kfunc, kfunc)
ckd = _lowlevel.ckernel_deferred_from_pyfunc(instantiate_ckernel,
[ndt.int32, ndt.float64])
# Test calling the ckd
out = nd.empty(ndt.int32)
in0 = nd.array(4.0, type=ndt.float64)
ckd.__call__(out, in0)
self.assertEqual(nd.as_py(out), 14)
# Also call it lifted
ckd_lifted = _lowlevel.lift_ckernel_deferred(ckd,
['2 * var * int32', '2 * var * float64'])
out = nd.empty('2 * var * int32')
in0 = nd.array([[1.0, 3.0, 2.5], [1.25, -1.5]], type='2 * var * float64')
ckd_lifted.__call__(out, in0)
self.assertEqual(nd.as_py(out), [[3, 10, 8], [4, -5]])
def test_sum_2d_axis1(self):
# Use the numpy add ufunc for this lifting test
af = _lowlevel.arrfunc_from_ufunc(np.add,
(np.int32, np.int32, np.int32),
False)
# Reduce along axis 1
sum = _lowlevel.lift_reduction_arrfunc(af,
'strided * strided * int32',
axis=1,
commutative=True,
associative=True)
in0 = nd.array([[3, 12, -5], [10, 2, 3]])
out = nd.empty(2, ndt.int32)
sum.execute(out, in0)
self.assertEqual(nd.as_py(out), [10, 15])
def test_strided_dim(self):
a = nd.empty(100, ndt.int32)
a[...] = nd.range(100)
self.assertEqual(nd.type_of(a), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('A * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[0:1]), ndt.type('A * int32'))
self.assertEqual(nd.as_py(a[0]), 0)
self.assertEqual(nd.as_py(a[99]), 99)
self.assertEqual(nd.as_py(a[-1]), 99)
self.assertEqual(nd.as_py(a[-100]), 0)
self.assertRaises(IndexError, lambda x : x[-101], a)
self.assertRaises(IndexError, lambda x : x[100], a)
self.assertRaises(IndexError, lambda x : x[-101:], a)
self.assertRaises(IndexError, lambda x : x[-5:101:2], a)
def __next__(self):
# Copy the previous element to the array if it is buffered
self.flush()
if self._index < self._len:
i = self._index
self._index = i + 1
if self._usebuffer:
if self._buffer is None:
self._buffer = nd.empty(self._c_dtype)
self._buffer_index = i
return _lowlevel.data_address_of(self._buffer)
else:
return _lowlevel.data_address_of(self._dyndarr[i])
else:
raise StopIteration
def empty(dshape, ddesc=None):
"""Create an array with uninitialized data.
Parameters
----------
dshape : datashape
The datashape for the resulting array.
ddesc : data descriptor instance
This comes with the necessary info for storing the data. If
None, a DyND_DDesc will be used.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
if ddesc is None:
ddesc = DyND_DDesc(nd.empty(str(dshape)))
return Array(ddesc)
if isinstance(ddesc, BLZ_DDesc):
shape, dt = to_numpy(dshape)
ddesc.blzarr = blz.zeros(shape,
dt,
rootdir=ddesc.path,
mode=ddesc.mode,
**ddesc.kwargs)
elif isinstance(ddesc, HDF5_DDesc):
obj = nd.as_numpy(nd.empty(str(dshape)))
with tb.open_file(ddesc.path, mode=ddesc.mode) as f:
where, name = split_path(ddesc.datapath)
f.create_earray(where, name, filters=ddesc.filters, obj=obj)
ddesc.mode = 'a' # change into 'a'ppend mode for further operations
return Array(ddesc)
def test_fixed_dim(self):
a = nd.empty(100, ndt.int32)
a[...] = nd.range(100)
b = list(range(100))
self.assertEqual(nd.type_of(a), ndt.type('100 * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('100 * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[0:1]), ndt.type('1 * int32'))
self.assertEqual(nd.as_py(a[0]), b[0])
self.assertEqual(nd.as_py(a[99]), b[99])
self.assertEqual(nd.as_py(a[-1]), b[-1])
self.assertEqual(nd.as_py(a[-100]), b[-100])
self.assertEqual(nd.as_py(a[-101:]), b[-101:])
self.assertEqual(nd.as_py(a[-5:101:2]), b[-5:101:2])
self.assertRaises(IndexError, lambda x: x[-101], a)
self.assertRaises(IndexError, lambda x: x[100], a)
def test_strided_dim(self):
a = nd.empty(100, ndt.int32)
a[...] = nd.range(100)
a[0] = 1000
self.assertEqual(nd.as_py(a[0]), 1000)
a[1:8:3] = 120
self.assertEqual(nd.as_py(a[:11]),
[1000, 120, 2, 3, 120, 5, 6, 120, 8, 9, 10])
a[5:2:-1] = [-10, -20, -30]
self.assertEqual(nd.as_py(a[:11]),
[1000, 120, 2, -30, -20, -10, 6, 120, 8, 9, 10])
a[1] = False
self.assertEqual(nd.as_py(a[1]), 0)
a[2] = True
self.assertEqual(nd.as_py(a[2]), 1)
a[3] = -10.0
self.assertEqual(nd.as_py(a[3]), -10)
def test_var_dim(self):
# TODO: Reenable tests below when var dim slicing is implemented properly
a = nd.empty('var * int32')
a[...] = nd.range(100)
b = list(range(100))
self.assertEqual(nd.type_of(a), ndt.type('var * int32'))
self.assertEqual(nd.type_of(a[...]), ndt.type('var * int32'))
self.assertEqual(nd.type_of(a[:]), ndt.type('var * int32'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
# self.assertEqual(nd.type_of(a[0:1]), ndt.type('fixed * int32'))
self.assertEqual(nd.as_py(a[0]), b[0])
self.assertEqual(nd.as_py(a[99]), b[99])
self.assertEqual(nd.as_py(a[-1]), b[-1])
self.assertEqual(nd.as_py(a[-100]), b[-100])
#self.assertEqual(nd.as_py(a[-101:]), b[-101:])
#self.assertEqual(nd.as_py(a[-5:101:2]), b[-5:101:2])
self.assertRaises(IndexError, lambda x: x[-101], a)
self.assertRaises(IndexError, lambda x: x[100], a)
def dynd_arr(self):
# TODO: This should really use blz
if self._dynd_result is not None:
return self._dynd_result
# Allocate empty dynd array
length = sum(len(chunk) for chunk in self.query_result)
ds = DataShape(length, self.dshape.measure)
result = nd.empty(str(ds))
# Fill dynd array with chunks
offset = 0
for chunk in self.query_result:
result[offset:offset + len(chunk)] = chunk
offset += len(chunk)
self._dynd_result = result
return result
def test_simple_fixed_dim(self):
# Assign to a strided dim from a generator
a = nd.empty(10, ndt.int32)
a[...] = (x + 2 for x in range(10))
self.assertEqual(len(a), 10)
self.assertEqual(nd.as_py(a), [x + 2 for x in range(10)])
# If we assign from a generator with one element, it broadcasts
a[...] = (x + 3 for x in range(5,6))
self.assertEqual(len(a), 10)
self.assertEqual(nd.as_py(a), [8]*10)
def assign(x, y):
x[...] = y
# If we assign from a generator with too few elements, it errors
self.assertRaises(nd.BroadcastError, assign, a,
(x + 2 for x in range(9)))
# If we assign from a generator with too many elements, it errors
self.assertRaises(nd.BroadcastError, assign, a,
(x + 2 for x in range(11)))
def dynd_chunk_iterator(result, chunk_size=1024):
"""
Turn a query Result into a bunch of DyND arrays
"""
cursor = result.cursor
chunk_size = max(cursor.arraysize, chunk_size)
while True:
try:
results = cursor.fetchmany(chunk_size)
except db.Error:
break
if not results:
break
dshape = DataShape(len(results), result.dshape.measure)
chunk = nd.empty(str(dshape))
chunk[:] = list(iter_result(results, dshape))
yield chunk
def test_bool(self):
a = nd.empty('var * bool')
a[...] = [True, False, 1, 0, 'true', 'false', 'on', 'off']
self.assertEqual(nd.as_py(a), [True, False] * 4)
def create_dynd_array(x, dshape=None):
return nd.empty(str(dshape))
def into(a, df):
schema = discover(df)
arr = nd.empty(str(schema))
for i in range(len(df.columns)):
arr[:, i] = np.asarray(df[df.columns[i]])
return arr