def execute_datadescriptor_ooc(dd, res_name=None):
# only lift by one
res_ds = dd.dshape
res_shape, res_dt = to_numpy(dd.dshape)
lifted = dd.kerneltree._fused.kernel.lift(1,'C')
cf = lifted.ctypes_func
# element readers for operands
args = [(ct._type_,
arr.arr._data.element_reader(1),
arr.arr.dshape.shape[1:])
for ct, arr in izip(cf.argtypes[:-1], dd.args)]
res_dd = BLZDataDescriptor(blz.zeros((0,) + res_shape[1:],
dtype = res_dt,
rootdir = res_name))
res_ct = ctypes.c_double*3
res_buffer = res_ct()
res_buffer_entry = (cf.argtypes[-1]._type_,
ctypes.pointer(res_buffer),
res_shape[1:])
with res_dd.element_appender() as ea:
for i in xrange(res_shape[0]):
args_i = [(t, er.read_single((i,)), sh)
for t, er, sh in args]
args_i.append(res_buffer_entry)
cf_args = [_convert(*foo) for foo in args_i]
cf(*[ctypes.byref(x) for x in cf_args])
ea.append(ctypes.addressof(res_buffer),1)
return blaze.Array(res_dd)
def execute_datadescriptor_ooc_2(dd, res_name=None):
res_ds = dd.dshape
res_shape, res_dt = to_numpy(dd.dshape)
lifted = dd.kerneltree._fused.kernel.lift(1,'C')
cf = lifted.ctypes_func
res_ctype = cf.argtypes[-1]._type_
args = [(ct._type_,
arr.arr._data.element_reader(1),
arr.arr.dshape.shape[1:])
for ct, arr in izip(cf.argtypes[:-1], dd.args)]
res_dd = BLZDataDescriptor(blz.zeros((0,) + res_shape[1:],
dtype = res_dt,
rootdir = res_name))
with res_dd.element_appender() as dst:
for i in xrange(res_shape[0]):
# advance sources
tpl = (i,)
cf_args = [_mk_array_c_ref(t, er.read_single(tpl), sh)
for t, er, sh in args ]
with dst.buffered_ptr() as dst_ptr:
cf_args.append(_mk_array_c_ref(res_ctype,
dst_ptr,
res_shape[1:]))
cf(*cf_args)
return blaze.Array(res_dd)
def __init__(self, data=None, dshape=None, params=None):
# need at least one of the three
assert (data is not None) or (dshape is not None) or \
(params.get('storage'))
if isinstance(data, ctable):
self.ca = data
return
# Extract the relevant carray parameters from the more
# general Blaze params object.
if params:
cparams, rootdir, format_flavor = to_cparams(params)
else:
rootdir, cparams = None, None
# Extract the relevant carray parameters from the more
# general Blaze params object.
if dshape:
shape, dtype = to_numpy(dshape)
if len(data) == 0:
data = np.empty(0, dtype=dtype)
self.ca = ctable(data, rootdir=rootdir, cparams=cparams)
else:
self.ca = ctable(data, dtype=dtype, rootdir=rootdir)
else:
self.ca = ctable(data, rootdir=rootdir, cparams=cparams)
def __init__(self, data=None, dshape=None, params=None):
# need at least one of the three
assert (data is not None) or (dshape is not None) or \
(params.get('storage'))
# Extract the relevant carray parameters from the more
# general Blaze params object.
if params:
cparams, rootdir, format_flavor = to_cparams(params)
else:
rootdir, cparams = None, None
if isinstance(data, CArraySource):
data = data.ca
dshape = dshape if dshape else data.dshape
if dshape:
shape, dtype = to_numpy(dshape)
self.ca = carray.carray(data,
dtype=dtype,
rootdir=rootdir,
cparams=cparams)
self.dshape = dshape
else:
self.ca = carray.carray(data, rootdir=rootdir, cparams=cparams)
self.dshape = from_numpy(self.ca.shape, self.ca.dtype)
def zeros(dshape, params=None, eclass=_eclass.manifest):
""" Create an Array and fill it with zeros.
Parameters
----------
dshape : str, blaze.dshape instance
Specifies the datashape of the outcome object.
params : blaze.params object
Any parameter supported by the backend library.
Returns
-------
out : an Array object.
"""
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
shape, dtype = to_numpy(dshape)
cparams, rootdir, format_flavor = to_cparams(params or _params())
if rootdir is not None:
carray.zeros(shape, dtype, rootdir=rootdir, cparams=cparams)
return open(rootdir)
else:
source = CArraySource(carray.zeros(shape, dtype, cparams=cparams),
params=params)
if eclass is _eclass.manifest:
return Array(source)
elif eclass is _eclass.delayed:
return NDArray(source)
def __init__(self, data=None, dshape=None, params=None):
# need at least one of the three
assert (data is not None) or (dshape is not None) or \
(params.get('storage'))
if isinstance(data, ctable):
self.ca = data
return
# Extract the relevant carray parameters from the more
# general Blaze params object.
if params:
cparams, rootdir, format_flavor = to_cparams(params)
else:
rootdir,cparams = None, None
# Extract the relevant carray parameters from the more
# general Blaze params object.
if dshape:
shape, dtype = to_numpy(dshape)
if len(data) == 0:
data = np.empty(0, dtype=dtype)
self.ca = ctable(data, rootdir=rootdir, cparams=cparams)
else:
self.ca = ctable(data, dtype=dtype, rootdir=rootdir)
else:
self.ca = ctable(data, rootdir=rootdir, cparams=cparams)
def zeros(dshape, params=None, eclass=_eclass.manifest):
""" Create an Array and fill it with zeros.
Parameters
----------
dshape : str, blaze.dshape instance
Specifies the datashape of the outcome object.
params : blaze.params object
Any parameter supported by the backend library.
Returns
-------
out : an Array object.
"""
if isinstance(dshape, basestring):
dshape = _dshape(dshape)
shape, dtype = to_numpy(dshape)
cparams, rootdir, format_flavor = to_cparams(params or _params())
if rootdir is not None:
carray.zeros(shape, dtype, rootdir=rootdir, cparams=cparams)
return open(rootdir)
else:
source = CArraySource(carray.zeros(shape, dtype, cparams=cparams),
params=params)
if eclass is _eclass.manifest:
return Array(source)
elif eclass is _eclass.delayed:
return NDArray(source)
def test_dims(d,outer):
banner("Executer mem %d iters" % outer)
ex =BlazeExecutor(d._data, outer)
shape, dtype = to_numpy(d._data.dshape)
res_dd = NumPyDataDescriptor(np.empty(shape, dtype=dtype))
ex.run_write(res_dd)
res = blaze.Array(res_dd)
print (res)
def execute_datadescriptor(dd):
# make a lifted fused func...
lifted = dd.kerneltree._fused.kernel.lift(2,'C')
cf = lifted.ctypes_func
# the actual ctypes function to call
args = [(ct._type_,
arr.arr._data.element_reader(0).read_single(()),
arr.arr.dshape.shape)
for ct, arr in izip(cf.argtypes[:-1], dd.args)]
res_dd = NumPyDataDescriptor(np.empty(*to_numpy(dd.dshape)))
with res_dd.element_writer(0).buffered_ptr(()) as dst_ptr:
args.append((cf.argtypes[-1]._type_, dst_ptr, res_dd.shape))
cf_args = [_convert(*foo) for foo in args]
cf(*[ctypes.byref(x) for x in cf_args])
return blaze.Array(res_dd)
def __init__(self, data=None, dshape=None, params=None):
# need at least one of the three
assert (data is not None) or (dshape is not None) or \
(params.get('storage'))
# Extract the relevant carray parameters from the more
# general Blaze params object.
if params:
cparams, rootdir, format_flavor = to_cparams(params)
else:
rootdir,cparams = None, None
if dshape:
shape, dtype = to_numpy(dshape)
self.ca = carray.carray(data, dtype=dtype, rootdir=rootdir, cparams=cparams)
self.dshape = dshape
else:
self.ca = carray.carray(data, rootdir=rootdir, cparams=cparams)
self.dshape = from_numpy(self.ca.shape, self.ca.dtype)
def execute_datadescriptor_outerdim(dd):
# only lift by one
lifted = dd.kerneltree._fused.kernel.lift(1,'C')
cf = lifted.ctypes_func
print(dir(cf))
# element readers for operands
args = [(ct._type_,
arr.arr._data.element_reader(1),
arr.arr.dshape.shape[1:])
for ct, arr in izip(cf.argtypes[:-1], dd.args)]
res_dd = NumPyDataDescriptor(np.empty(*to_numpy(dd.dshape)))
outer_dimension = res_dd.shape[0]
dst = res_dd.element_writer(1)
for i in xrange(outer_dimension):
args_i = [(t, er.read_single((i,)), sh) for t, er, sh in args]
with dst.buffered_ptr((i,)) as dst_ptr:
args_i.append((cf.argtypes[-1]._type_, dst_ptr, res_dd.shape[1:]))
cf_args = [_convert(*foo) for foo in args_i]
cf(*[ctypes.byref(x) for x in cf_args])
return blaze.Array(res_dd)
def promote_scalars(a, b):
"""Promote two CTypes"""
return CType.from_numpy_dtype(np.result_type(to_numpy(a), to_numpy(b)))
res = execute_datadescriptor_ooc_2(d._data, 'foo.blz')
banner("result (ooc_2)")
print(res)
blaze.drop(blaze.Persist('foo.blz'))
del(res)
res = execute_datadescriptor_ooc(d._data, 'bar.blz')
banner("result (ooc)")
print(res)
blaze.drop(blaze.Persist('bar.blz'))
del(res)
banner("Executor iterating")
ex = BlazeExecutor(d._data, 1)
shape, dtype = to_numpy(d._data.dshape)
res_dd = BLZDataDescriptor(blz.zeros((0,)+shape[1:],
dtype=dtype,
rootdir='baz.blz'))
ex.run_append(res_dd)
res = blaze.Array(res_dd)
print(res)
blaze.drop(blaze.Persist('baz.blz'))
del res
del res_dd
del ex
banner("Executor memory no iter")
ex = BlazeExecutor(d._data, 0)
def promote_scalars(a, b):
"""Promote two CTypes"""
try:
return CType.from_numpy_dtype(np.result_type(to_numpy(a), to_numpy(b)))
except TypeError, e:
raise TypeError("Cannot promote %s and %s: %s" % (a, b, e))