def test_object_unicode():
ds = blaze.dshape('x, blob')
c = blaze.Array([u'a' * i for i in range(10)], ds)
for i, v in enumerate(c):
# The outcome are 0-dim arrays (that might change in the future)
assert v[()] == u'a' * i
def apply_function(blaze_func, *args, **kwargs):
"""
Apply blaze kernel `kernel` to the given arguments.
Returns: a Deferred node representation the delayed computation
"""
# -------------------------------------------------
# Merge input contexts
args, kwargs = blaze_args(args, kwargs)
ctxs = collect_contexts(chain(args, kwargs.values()))
ctx = merge_contexts(ctxs)
# -------------------------------------------------
# Find match to overloaded function
overload, args = blaze_func.dispatcher.lookup_dispatcher(args, kwargs)
# -------------------------------------------------
# Construct graph
term = construct(blaze_func, ctx, overload, args)
desc = DeferredDescriptor(term.dshape, (term, ctx))
# TODO: preserve `user` metadata
return blaze.Array(desc)
def test_object_blob():
ds = blaze.dshape('x, blob')
c = blaze.Array([(i, str(i * .2)) for i in range(10)], ds)
for i, v in enumerate(c):
assert v[0] == i
assert v[1] == str(i * .2)
def test_intfloat_blob():
ds = blaze.dshape('x, blob')
c = blaze.Array([(i, i * .2) for i in range(10)], ds)
for i, v in enumerate(c):
print "v:", v, v[0], type(v[0])
assert v[0] == i
assert v[1] == i * .2
def test_simple_persistent_blob():
td = tempfile.mkdtemp()
tmppath = os.path.join(td, 'c')
ds = blaze.dshape('x, blob')
c = blaze.Array(["s1", "sss2"], ds, params=blaze.params(storage=tmppath))
assert c[0] == "s1"
assert c[1] == "sss2"
# Remove everything under the temporary dir
shutil.rmtree(td)
def test_object_persistent_blob():
td = tempfile.mkdtemp()
tmppath = os.path.join(td, 'c')
ds = blaze.dshape('x, blob')
c = blaze.Array([(i, str(i * .2)) for i in range(10)],
ds,
params=blaze.params(storage=tmppath))
for i, v in enumerate(c):
assert v[0] == i
assert v[1] == str(i * .2)
# Remove everything under the temporary dir
shutil.rmtree(td)
def test_perserve():
shape = (3, 4)
arr = np.ones(shape)
dshape = "%s,%s, float64" % (shape[0], shape[1])
path = "p.blz"
if os.path.exists(path):
shutil.rmtree(path)
bparams = blz.params(storage=path)
barray = blz.Array(arr, dshape, params=bparams)
print "barray:", repr(barray)
barray2 = blz.open(path)
print "barray2:", repr(barray2)
assert (str(barray.datashape) == str(barray2.datashape))
def interpret(func, env, storage=None, **kwds):
args = env['runtime.arglist']
if storage is None:
# Evaluate once
values = dict(zip(func.args, args))
interp = CKernelInterp(values)
visit(interp, func)
return interp.result
else:
result_ndim = env['result-ndim']
res_shape, res_dt = datashape.to_numpy(func.type.restype)
dim_size = operator.index(res_shape[0])
row_size = ndt.type(str(func.type.restype.subarray(1))).data_size
chunk_size = min(max(1, (1024 * 1024) // row_size), dim_size)
# Evaluate by streaming the outermost dimension,
# and using the BLZ data descriptor's append
dst_dd = BLZDataDescriptor(
blz.zeros((0, ) + res_shape[1:], res_dt, rootdir=storage.path))
# Loop through all the chunks
for chunk_start in range(0, dim_size, chunk_size):
# Tell the interpreter which chunk size to use (last
# chunk might be smaller)
chunk_size = min(chunk_size, dim_size - chunk_start)
# Evaluate the chunk
args_chunk = [
arg[chunk_start:chunk_start + chunk_size]
if len(arg.dshape.shape) == result_ndim else arg
for arg in args
]
values = dict(zip(func.args, args_chunk))
interp = CKernelChunkInterp(values, chunk_size, result_ndim)
visit(interp, func)
chunk = interp.result._data.dynd_arr()
dst_dd.append(chunk)
return blaze.Array(dst_dd)
def method(self, *args, **kwargs):
result = blaze.eval(blaze.Array(self))
self._result = result
method = getattr(result.ddesc, methname)
return method(*args, **kwargs)
def op_convert(self, op):
input = self.values[op.args[0]]
input = input.ddesc.dynd_arr()
result = nd.array(input, type=ndt.type(str(op.type)))
result = blaze.Array(DyND_DDesc(result))
self.values[op] = result
def convert(filetxt, storage):
import os.path
if not os.path.exists(storage):
blaze.Array(np.loadtxt(filetxt), params=blaze.params(storage=storage))
def _create(dshape, n, conn, chunk_size=1024, overlap=0):
sdshape = scidb_dshape(dshape, chunk_size, overlap)
query = build(sdshape, n)
return blaze.Array(SciDB_DDesc(dshape, query, conn))
import shutil
"""
Benchmark that compares the storing of objects in both Blaze and PyTables
"""
from time import time
import blaze
import tables
N = 500
if os.path.exists('c'):
shutil.rmtree('c')
t0 = time()
c = blaze.Array([], 'x, object', params=blaze.params(storage='c', clevel=5))
for i in xrange(N):
c.append(u"s" * N * i)
c.commit()
print "time taken for writing in Blaze: %.3f" % (time() - t0)
t0 = time()
c2 = blaze.open('c')
#c2 = c
#print c2.datashape
tlen = 0
for i in range(N):
#print "i:", i, repr(c2[i]), type(c2[i])
tlen += len(c2[i][()])
def test_simple_blob():
ds = blaze.dshape('x, blob')
c = blaze.Array(["s1", "sss2"], ds)
assert c[0] == "s1"
assert c[1] == "sss2"
def op_convert(self, op):
input = self.values[op.args[0]]
input = input._data.dynd_arr()
result = nd.array(input, type=ndt.type(str(op.type)))
result = blaze.Array(DyNDDataDescriptor(result))
self.values[op] = result
