def test_object_unicode(): ds = blaze.dshape('x, blob') c = blaze.Array([u'a' * i for i in range(10)], ds) for i, v in enumerate(c): # The outcome are 0-dim arrays (that might change in the future) assert v[()] == u'a' * i
def apply_function(blaze_func, *args, **kwargs): """ Apply blaze kernel `kernel` to the given arguments. Returns: a Deferred node representation the delayed computation """ # ------------------------------------------------- # Merge input contexts args, kwargs = blaze_args(args, kwargs) ctxs = collect_contexts(chain(args, kwargs.values())) ctx = merge_contexts(ctxs) # ------------------------------------------------- # Find match to overloaded function overload, args = blaze_func.dispatcher.lookup_dispatcher(args, kwargs) # ------------------------------------------------- # Construct graph term = construct(blaze_func, ctx, overload, args) desc = DeferredDescriptor(term.dshape, (term, ctx)) # TODO: preserve `user` metadata return blaze.Array(desc)
def test_object_blob(): ds = blaze.dshape('x, blob') c = blaze.Array([(i, str(i * .2)) for i in range(10)], ds) for i, v in enumerate(c): assert v[0] == i assert v[1] == str(i * .2)
def test_intfloat_blob(): ds = blaze.dshape('x, blob') c = blaze.Array([(i, i * .2) for i in range(10)], ds) for i, v in enumerate(c): print "v:", v, v[0], type(v[0]) assert v[0] == i assert v[1] == i * .2
def test_simple_persistent_blob(): td = tempfile.mkdtemp() tmppath = os.path.join(td, 'c') ds = blaze.dshape('x, blob') c = blaze.Array(["s1", "sss2"], ds, params=blaze.params(storage=tmppath)) assert c[0] == "s1" assert c[1] == "sss2" # Remove everything under the temporary dir shutil.rmtree(td)
def test_object_persistent_blob(): td = tempfile.mkdtemp() tmppath = os.path.join(td, 'c') ds = blaze.dshape('x, blob') c = blaze.Array([(i, str(i * .2)) for i in range(10)], ds, params=blaze.params(storage=tmppath)) for i, v in enumerate(c): assert v[0] == i assert v[1] == str(i * .2) # Remove everything under the temporary dir shutil.rmtree(td)
def test_perserve(): shape = (3, 4) arr = np.ones(shape) dshape = "%s,%s, float64" % (shape[0], shape[1]) path = "p.blz" if os.path.exists(path): shutil.rmtree(path) bparams = blz.params(storage=path) barray = blz.Array(arr, dshape, params=bparams) print "barray:", repr(barray) barray2 = blz.open(path) print "barray2:", repr(barray2) assert (str(barray.datashape) == str(barray2.datashape))
def interpret(func, env, storage=None, **kwds): args = env['runtime.arglist'] if storage is None: # Evaluate once values = dict(zip(func.args, args)) interp = CKernelInterp(values) visit(interp, func) return interp.result else: result_ndim = env['result-ndim'] res_shape, res_dt = datashape.to_numpy(func.type.restype) dim_size = operator.index(res_shape[0]) row_size = ndt.type(str(func.type.restype.subarray(1))).data_size chunk_size = min(max(1, (1024 * 1024) // row_size), dim_size) # Evaluate by streaming the outermost dimension, # and using the BLZ data descriptor's append dst_dd = BLZDataDescriptor( blz.zeros((0, ) + res_shape[1:], res_dt, rootdir=storage.path)) # Loop through all the chunks for chunk_start in range(0, dim_size, chunk_size): # Tell the interpreter which chunk size to use (last # chunk might be smaller) chunk_size = min(chunk_size, dim_size - chunk_start) # Evaluate the chunk args_chunk = [ arg[chunk_start:chunk_start + chunk_size] if len(arg.dshape.shape) == result_ndim else arg for arg in args ] values = dict(zip(func.args, args_chunk)) interp = CKernelChunkInterp(values, chunk_size, result_ndim) visit(interp, func) chunk = interp.result._data.dynd_arr() dst_dd.append(chunk) return blaze.Array(dst_dd)
def method(self, *args, **kwargs): result = blaze.eval(blaze.Array(self)) self._result = result method = getattr(result.ddesc, methname) return method(*args, **kwargs)
def op_convert(self, op): input = self.values[op.args[0]] input = input.ddesc.dynd_arr() result = nd.array(input, type=ndt.type(str(op.type))) result = blaze.Array(DyND_DDesc(result)) self.values[op] = result
def convert(filetxt, storage): import os.path if not os.path.exists(storage): blaze.Array(np.loadtxt(filetxt), params=blaze.params(storage=storage))
def _create(dshape, n, conn, chunk_size=1024, overlap=0): sdshape = scidb_dshape(dshape, chunk_size, overlap) query = build(sdshape, n) return blaze.Array(SciDB_DDesc(dshape, query, conn))
import shutil """ Benchmark that compares the storing of objects in both Blaze and PyTables """ from time import time import blaze import tables N = 500 if os.path.exists('c'): shutil.rmtree('c') t0 = time() c = blaze.Array([], 'x, object', params=blaze.params(storage='c', clevel=5)) for i in xrange(N): c.append(u"s" * N * i) c.commit() print "time taken for writing in Blaze: %.3f" % (time() - t0) t0 = time() c2 = blaze.open('c') #c2 = c #print c2.datashape tlen = 0 for i in range(N): #print "i:", i, repr(c2[i]), type(c2[i]) tlen += len(c2[i][()])
def test_simple_blob(): ds = blaze.dshape('x, blob') c = blaze.Array(["s1", "sss2"], ds) assert c[0] == "s1" assert c[1] == "sss2"
def op_convert(self, op): input = self.values[op.args[0]] input = input._data.dynd_arr() result = nd.array(input, type=ndt.type(str(op.type))) result = blaze.Array(DyNDDataDescriptor(result)) self.values[op] = result