def execplan(context, plan):
""" Takes a list of of instructions from the Pipeline and
then allocates the necessary memory needed for the
intermediates are temporaries. Then executes the plan
returning the result. """
instructions = context["instructions"] # [ Instruction(...) ]
symbols = context["symbols"] # { %0 -> Array(...){...}
operands = context["operand_dict"] # { Array(...){...} -> Blaze Array }
def getop(symbol):
term = symbols[symbol]
term_id = term.annotation.meta[0].label
op = operands[term_id]
return op
h = Heap()
ret = None
for instruction in instructions:
ops = map(getop, instruction.args)
if not instruction.lhs:
# lhs = h.allocate(instruction.lhs.size())
lhs = blaze.zeros(instruction.datashape)
else:
lhs = getop(instruction.lhs)
ret = instruction.execute(ops, lhs)
# h.finalize()
return ret
def execplan(context, plan):
""" Takes a list of of instructions from the Pipeline and
then allocates the necessary memory needed for the
intermediates are temporaries. Then executes the plan
returning the result. """
instructions = context["instructions"] # [ Instruction(...) ]
symbols = context["symbols"] # { %0 -> Array(...){...}
operands = context["operand_dict"] # { Array(...){...} -> Blaze Array }
def getop(symbol):
term = symbols[symbol]
term_id = term.annotation.meta[0].label
op = operands[term_id]
return op
h = Heap()
ret = None
for instruction in instructions:
ops = map(getop, instruction.args)
if not instruction.lhs:
# lhs = h.allocate(instruction.lhs.size())
lhs = blaze.zeros(instruction.datashape)
else:
lhs = getop(instruction.lhs)
ret = instruction.execute(ops, lhs)
# h.finalize()
return ret
def test_metadata_all_prop():
a = blaze.ones(blaze.dshape('20, 20, float64'))
b = blaze.zeros(blaze.dshape('20, 20, float64'))
c = blaze.NDTable([(1.0, 1.0), (1.0, 1.0)], dshape='2, {x: int32; y: float32}')
assert blaze.metadata.all_prop((a, b), blaze.metadata.arraylike)
assert not blaze.metadata.all_prop((a, b, c), blaze.metadata.arraylike)
def test_binary_kerneltree_lifted(self):
# Create some simple blaze funcs, using Numba
def _add(a,b):
return a + b
def _mul(a,b):
return a * b
add = BlazeFunc('add',[('f8(f8,f8)', _add),
('c16(c16,c16)', _add)])
mul = BlazeFunc('mul', {(double,)*3: _mul,
(c128,)*3: _mul})
# Array data and expression
af = blaze.array([[1,2,3], [4,5,6]],dshape=double)
bf = blaze.array([2,3,4],dshape=double)
cf = add(af,bf)
df = mul(cf,cf)
lifted_kernel = df._data.kerneltree.fuse().kernel.lift(1, 'C')
ubck = lifted_kernel.unbound_single_ckernel
# Allocate the result, and run the kernel across the arguments
result = blaze.zeros(df.dshape)
args = [arg.arr._data for arg in df._data.args]
ck = ubck.bind(result._data, args)
execute_expr_single(result._data, args,
result._data.dshape.subarray(-2),
[a.dshape.subarray(-2) for a in args],
ck)
self.assertEqual(dd_as_py(result._data),
[[(a+b) * (a+b) for a, b in izip(a1, b1)]
for a1, b1 in izip(
[[1,2,3], [4,5,6]], [[2,3,4]]*2)])
def test_dot_out_exception():
'''Output array of wrong size should raise exception.'''
a = blaze.ones(blaze.dshape('20, 20, float64'))
b = blaze.ones(blaze.dshape('20, 30, float64'))
out = blaze.zeros(blaze.dshape('20, 20, float64'))
with assert_raises(ValueError):
dot(a, b, out=out)
def test_metadata_all_prop():
a = blaze.ones(blaze.dshape('20, 20, float64'))
b = blaze.zeros(blaze.dshape('20, 20, float64'))
c = blaze.NDTable([(1.0, 1.0), (1.0, 1.0)],
dshape='2, {x: int32; y: float32}')
assert blaze.metadata.all_prop((a, b), blaze.metadata.arraylike)
assert not blaze.metadata.all_prop((a, b, c), blaze.metadata.arraylike)
def add(a, b):
from blaze import zeros
ds_a, dd_a = a
ds_b, dd_b = b
out = zeros(ds_a)
iters = [desc.as_chunked_iterator() for desc in [dd_a, dd_b]]
for a, b in zip(*iters):
pass
return out
def add(a, b):
from blaze import zeros
ds_a, dd_a = a
ds_b, dd_b = b
out = zeros(ds_a)
iters = [desc.as_chunked_iterator() for desc in [dd_a, dd_b]]
for a,b in zip(*iters):
pass
return out
def _test():
import blaze
arr = blaze.array([2, 3, 4.0])
print(arr.dshape)
print(array2string(arr._data))
arr = blaze.zeros('30, 30, 30, float32')
print(arr.dshape)
print(array2string(arr._data))
def eval_in_mem(arr, iter_dims, chunk=1, dump_result=False):
logging.info("starting in-mem with dims=%s chunk=%d",
str(iter_dims), chunk)
t = time.time()
res = blaze.zeros(arr.dshape)
simple_execute_write(arr._data, res._data,
iter_dims=iter_dims, chunk=chunk)
logging.info("took %f secs.\n", time.time()-t);
if dump_result:
logging.debug(res)
return res
def _test():
import blaze
arr = blaze.array([2, 3, 4.0])
print(arr.dshape)
print(array2string(arr._data))
arr = blaze.zeros('30, 30, 30, float32')
print(arr.dshape)
print(array2string(arr._data))
def test_binary_kerneltree(self):
# Create some simple blaze funcs, using Numba
def _add(a,b):
return a + b
def _mul(a,b):
return a * b
add = BlazeFunc('add',[('f8(f8,f8)', _add),
('c16(c16,c16)', _add)])
mul = BlazeFunc('mul', {(double,)*3: _mul,
(c128,)*3: _mul})
# Array data and expression
af = blaze.array([[1,2,3], [4,5,6]],dshape=double)
bf = blaze.array([2,3,4],dshape=double)
cf = add(af,bf)
df = mul(cf,cf)
ubck = df._data.kerneltree.unbound_single_ckernel
# Allocate the result, and run the kernel across the arguments
result = blaze.zeros(df.dshape)
args = [arg.arr._data for arg in df._data.args]
ck = ubck.bind(result._data, args)
execute_expr_single(result._data, args,
df._data.kerneltree.kernel.dshapes[-1],
df._data.kerneltree.kernel.dshapes[:-1], ck)
self.assertEqual(dd_as_py(result._data),
[[(a+b) * (a+b) for a, b in izip(a1, b1)]
for a1, b1 in izip(
[[1,2,3], [4,5,6]], [[2,3,4]]*2)])
# Use blaze.eval to evaluate cf and df into concrete arrays
cf2 = blaze.eval(cf)
self.assertEqual(dd_as_py(cf2._data),
[[(a+b) for a, b in izip(a1, b1)]
for a1, b1 in izip(
[[1,2,3], [4,5,6]], [[2,3,4]]*2)])
df2 = blaze.eval(df)
self.assertEqual(dd_as_py(df2._data),
[[(a+b) * (a+b) for a, b in izip(a1, b1)]
for a1, b1 in izip(
[[1,2,3], [4,5,6]], [[2,3,4]]*2)])
def test_append(self):
persist = blaze.Storage(self.rooturi, format="blz")
a = blaze.zeros('0 * float64', storage=persist)
self.assertTrue(isinstance(a, blaze.Array))
append(a,list(range(10)))
self.assertEqual(dd_as_py(a._data), list(range(10)))
e = blaze.array([1, 2, 3], dshape='3 * float32')
print(e)
# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:
f = blaze.array([1, 2, 3], dshape='float32')
print(f)
# --------------------------------------------------------------------
print_section('Alternative constructors')
# Arrays can be created to be all zeros:
g = blaze.zeros('10 * 10 * int32')
print(g)
# All ones
h = blaze.ones('10 * 10 * float64')
print(h)
# --------------------------------------------------------------------
print_section('Indexing')
print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10 * 10 * 10 * 10 * float32')
def describe_array(label, array):
e = blaze.array([1, 2, 3], dshape='3 * float32')
print(e)
# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:
f = blaze.array([1, 2, 3], dshape='float32')
print(f)
# --------------------------------------------------------------------
print_section('Alternative constructors')
# Arrays can be created to be all zeros:
g = blaze.zeros('10 * 10 * int32')
print(g)
# All ones
h = blaze.ones('10 * 10 * float64')
print(h)
# --------------------------------------------------------------------
print_section('Indexing')
print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10 * 10 * 10 * 10 * float32')
def describe_array(label, array):
def test_append(self):
ddesc = HDF5_DDesc(path=self.file, datapath='/earray', mode='a')
a = blaze.zeros('0 * float64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
append(a, list(range(10)))
self.assertEqual(list(a), list(range(10)))
def test_append(self):
ddesc = BLZ_DDesc(path=self.rootdir, mode='w')
a = blaze.zeros('0 * float64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
append(a, list(range(10)))
self.assertEqual(list(a), list(range(10)))
def make_test_array(datashape):
"""TODO: Use something more interesting"""
return blaze.zeros(datashape)
def make_test_array(datashape):
"""TODO: Use something more interesting"""
return blaze.zeros(datashape)
def test_create_zeros(self):
# A default array
a = blaze.zeros('10 * int64')
self.assertTrue(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), [0]*10)
def dot(a, b, out=None, outname='out'):
"""
Matrix multiplication of two 2-D arrays.
Parameters
----------
a : array
First argument.
b : array
Second argument.
out : array, optional
Output argument. This must have the exact kind that would be
returned if it was not used. In particular, it must have the
right type, must be C-contiguous, and its dtype must be the
dtype that would be returned for `dot(a,b)`. This is a
performance feature. Therefore, if these conditions are not
met, an exception is raised, instead of attempting to be
flexible.
outname : str, optional
If provided this will be the name for the output matrix storage.
This parameter is only used when `out` is not provided.
Returns
-------
output : array
Returns the dot product of `a` and `b`. If `a` and `b` are
both scalars or both 1-D arrays then a scalar is returned;
otherwise an array is returned.
If `out` is given, then it is returned.
Raises
------
ValueError
If the last dimension of `a` is not the same size as the
second-to-last dimension of `b`.
"""
a_shape = tuple(i.val for i in a.datashape.parameters[:-1])
b_shape = tuple(i.val for i in b.datashape.parameters[:-1])
if len(a_shape) != 2 or len(b_shape) != 2:
raise (ValueError, "only 2-D matrices supported")
if a_shape[1] != b_shape[0]:
raise (ValueError,
"last dimension of `a` does not match first dimension of `b`")
l, m, n = a_shape[0], a_shape[1], b_shape[1]
if out:
out_shape = tuple(i.val for i in out.datashape.parameters[:-1])
if out_shape != (l, n):
raise (ValueError, "`out` array does not have the correct shape")
else:
parms = blaze.params(clevel=5, storage=outname)
a_dtype = a.datashape.parameters[-1].to_dtype()
dshape = blaze.dshape('%d, %d, %s' % (l, n, a_dtype))
out = blaze.zeros(dshape, parms)
# Compute a good block size
out_dtype = out.datashape.parameters[-1].to_dtype()
bl = math.sqrt(OOC_BUFFER_SIZE / out_dtype.itemsize)
bl = 2**int(math.log(bl, 2))
for i in range(0, l, bl):
for j in range(0, n, bl):
for k in range(0, m, bl):
a0 = a[i:min(i+bl, l), k:min(k+bl, m)]
b0 = b[k:min(k+bl, m), j:min(j+bl, n)]
out[i:i+bl, j:j+bl] += np.dot(a0, b0)
return out
def test_zeros(self):
for t in self.full_types:
a = zeros(t)
self.assertEqual(a.datashape, dshape(t))
def test_create_zeros(self):
# A default array (backed by NumPy)
a = blaze.zeros('10, int64')
self.assert_(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), [0]*10)
def test_append(self):
persist = blaze.Persist(self.rooturi)
a = blaze.zeros('0, float64', persist=persist)
self.assert_(isinstance(a, blaze.Array))
a.append(list(range(10)))
self.assertEqual(dd_as_py(a._data), list(range(10)))
def test_create_compress_zeros(self):
# A compressed array (backed by BLZ)
a = blaze.zeros('10 * int64', caps={'compress': True})
self.assertTrue(isinstance(a, blaze.Array))
self.assertEqual(dd_as_py(a._data), [0]*10)
e = blaze.array([1, 2, 3], dshape='3, float32')
print(e)
# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:
f = blaze.array([1, 2, 3], dshape='float32')
print(f)
# --------------------------------------------------------------------
print_section('Alternative constructors')
# Arrays can be created to be all zeros:
g = blaze.zeros('10, 10, int32')
print(g)
# All ones
h = blaze.ones('10, 10, float64')
print(h)
# --------------------------------------------------------------------
print_section('Indexing')
print_section('Indexing for read', level=1)
print('starting with a 4d array')
array4d = blaze.ones('10,10,10,10, float32')
e = blaze.array([ 1, 2, 3], dshape='3, float32')
print(e)
# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:
f = blaze.array([ 1, 2, 3], dshape='float32')
print(f)
# --------------------------------------------------------------------
print_section('Alternative constructors')
# Arrays can be created to be all zeros:
g = blaze.zeros('10, 10, int32')
print(g)
# All ones
h = blaze.ones('10, 10, float64')
print(h)
# --------------------------------------------------------------------
print_section('Indexing')
print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10,10,10,10, float32')
def describe_array(label, array):
def test_append(self):
ddesc = BLZ_DDesc(path=self.rootdir, mode='w')
a = blaze.zeros('0 * float64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
append(a, list(range(10)))
self.assertEqual(list(a), list(range(10)))
def test_zeros(self):
for t in self.full_types:
a = zeros(t)
self.assertEqual(a.datashape, dshape(t))
def test_append(self):
ddesc = HDF5_DDesc(path=self.file, datapath='/earray', mode='a')
a = blaze.zeros('0 * float64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
append(a, list(range(10)))
self.assertEqual(list(a), list(range(10)))
def test_create_compress_zeros(self):
# A compressed array (backed by BLZ)
ddesc = BLZ_DDesc(mode='w', bparams=blz.bparams(clevel=5))
a = blaze.zeros('10 * int64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
self.assertEqual(list(a), [0]*10)
def test_create_compress_zeros(self):
# A compressed array (backed by BLZ)
ddesc = BLZ_DDesc(mode='w', bparams=blz.bparams(clevel=5))
a = blaze.zeros('10 * int64', ddesc=ddesc)
self.assertTrue(isinstance(a, blaze.Array))
self.assertEqual(list(a), [0] * 10)