Example #1
0
def execplan(context, plan):
    """ Takes a list of of instructions from the Pipeline and
    then allocates the necessary memory needed for the
    intermediates are temporaries. Then executes the plan
    returning the result. """

    instructions = context["instructions"]  # [ Instruction(...) ]
    symbols = context["symbols"]  # { %0 -> Array(...){...}
    operands = context["operand_dict"]  # { Array(...){...} -> Blaze Array }

    def getop(symbol):
        term = symbols[symbol]
        term_id = term.annotation.meta[0].label
        op = operands[term_id]
        return op

    h = Heap()
    ret = None

    for instruction in instructions:
        ops = map(getop, instruction.args)

        if not instruction.lhs:
            # lhs = h.allocate(instruction.lhs.size())
            lhs = blaze.zeros(instruction.datashape)
        else:
            lhs = getop(instruction.lhs)

        ret = instruction.execute(ops, lhs)

    # h.finalize()
    return ret
Example #2
0
def execplan(context, plan):
    """ Takes a list of of instructions from the Pipeline and
    then allocates the necessary memory needed for the
    intermediates are temporaries. Then executes the plan
    returning the result. """

    instructions = context["instructions"]  # [ Instruction(...) ]
    symbols = context["symbols"]            # { %0 -> Array(...){...}
    operands = context["operand_dict"]      # { Array(...){...} -> Blaze Array }

    def getop(symbol):
        term = symbols[symbol]
        term_id = term.annotation.meta[0].label
        op = operands[term_id]
        return op

    h = Heap()
    ret = None

    for instruction in instructions:
        ops = map(getop, instruction.args)

        if not instruction.lhs:
            # lhs = h.allocate(instruction.lhs.size())
            lhs = blaze.zeros(instruction.datashape)
        else:
            lhs = getop(instruction.lhs)

        ret = instruction.execute(ops, lhs)

    # h.finalize()
    return ret
Example #3
0
def test_metadata_all_prop():
    a = blaze.ones(blaze.dshape('20, 20, float64'))
    b = blaze.zeros(blaze.dshape('20, 20, float64'))
    c = blaze.NDTable([(1.0, 1.0), (1.0, 1.0)], dshape='2, {x: int32; y: float32}')

    assert blaze.metadata.all_prop((a, b), blaze.metadata.arraylike)
    assert not blaze.metadata.all_prop((a, b, c), blaze.metadata.arraylike)
    def test_binary_kerneltree_lifted(self):
        # Create some simple blaze funcs, using Numba
        def _add(a,b):
            return a + b

        def _mul(a,b):
            return a * b
        add = BlazeFunc('add',[('f8(f8,f8)', _add),
                               ('c16(c16,c16)', _add)])
        mul = BlazeFunc('mul', {(double,)*3: _mul,
                                (c128,)*3: _mul})
        # Array data and expression
        af = blaze.array([[1,2,3], [4,5,6]],dshape=double)
        bf = blaze.array([2,3,4],dshape=double)
        cf = add(af,bf)
        df = mul(cf,cf)
        lifted_kernel = df._data.kerneltree.fuse().kernel.lift(1, 'C')
        ubck = lifted_kernel.unbound_single_ckernel

        # Allocate the result, and run the kernel across the arguments
        result = blaze.zeros(df.dshape)
        args = [arg.arr._data for arg in df._data.args]
        ck = ubck.bind(result._data, args)
        execute_expr_single(result._data, args,
                        result._data.dshape.subarray(-2),
                        [a.dshape.subarray(-2) for a in args],
                        ck)
        self.assertEqual(dd_as_py(result._data),
                        [[(a+b) * (a+b) for a, b in izip(a1, b1)]
                                for a1, b1 in izip(
                                    [[1,2,3], [4,5,6]], [[2,3,4]]*2)])
Example #5
0
def test_dot_out_exception():
    '''Output array of wrong size should raise exception.'''
    a = blaze.ones(blaze.dshape('20, 20, float64'))
    b = blaze.ones(blaze.dshape('20, 30, float64'))
    out = blaze.zeros(blaze.dshape('20, 20, float64'))

    with assert_raises(ValueError):
        dot(a, b, out=out)
Example #6
0
def test_metadata_all_prop():
    a = blaze.ones(blaze.dshape('20, 20, float64'))
    b = blaze.zeros(blaze.dshape('20, 20, float64'))
    c = blaze.NDTable([(1.0, 1.0), (1.0, 1.0)],
                      dshape='2, {x: int32; y: float32}')

    assert blaze.metadata.all_prop((a, b), blaze.metadata.arraylike)
    assert not blaze.metadata.all_prop((a, b, c), blaze.metadata.arraylike)
Example #7
0
def add(a, b):
    from blaze import zeros

    ds_a, dd_a = a
    ds_b, dd_b = b
    out = zeros(ds_a)

    iters = [desc.as_chunked_iterator() for desc in [dd_a, dd_b]]
    for a, b in zip(*iters):
        pass
    return out
Example #8
0
def add(a, b):
    from blaze import zeros

    ds_a, dd_a = a
    ds_b, dd_b = b
    out = zeros(ds_a)

    iters = [desc.as_chunked_iterator() for desc in [dd_a, dd_b]]
    for a,b in zip(*iters):
        pass
    return out
Example #9
0
def _test():
    import blaze

    arr = blaze.array([2, 3, 4.0])
    print(arr.dshape)

    print(array2string(arr._data))

    arr = blaze.zeros('30, 30, 30, float32')
    print(arr.dshape)

    print(array2string(arr._data))
Example #10
0
def eval_in_mem(arr, iter_dims, chunk=1, dump_result=False):
    logging.info("starting in-mem with dims=%s chunk=%d",
                 str(iter_dims), chunk)
    t = time.time()
    res = blaze.zeros(arr.dshape)
    simple_execute_write(arr._data, res._data,
                         iter_dims=iter_dims, chunk=chunk)
    logging.info("took %f secs.\n", time.time()-t);
    if dump_result:
        logging.debug(res)

    return res
Example #11
0
def _test():
    import blaze

    arr = blaze.array([2, 3, 4.0])
    print(arr.dshape)

    print(array2string(arr._data))

    arr = blaze.zeros('30, 30, 30, float32')
    print(arr.dshape)

    print(array2string(arr._data))
    def test_binary_kerneltree(self):
        # Create some simple blaze funcs, using Numba
        def _add(a,b):
            return a + b

        def _mul(a,b):
            return a * b
        add = BlazeFunc('add',[('f8(f8,f8)', _add),
                               ('c16(c16,c16)', _add)])
        mul = BlazeFunc('mul', {(double,)*3: _mul,
                                (c128,)*3: _mul})
        # Array data and expression
        af = blaze.array([[1,2,3], [4,5,6]],dshape=double)
        bf = blaze.array([2,3,4],dshape=double)
        cf = add(af,bf)
        df = mul(cf,cf)
        ubck = df._data.kerneltree.unbound_single_ckernel

        # Allocate the result, and run the kernel across the arguments
        result = blaze.zeros(df.dshape)
        args = [arg.arr._data for arg in df._data.args]
        ck = ubck.bind(result._data, args)
        execute_expr_single(result._data, args,
                        df._data.kerneltree.kernel.dshapes[-1],
                        df._data.kerneltree.kernel.dshapes[:-1], ck)
        self.assertEqual(dd_as_py(result._data),
                        [[(a+b) * (a+b) for a, b in izip(a1, b1)]
                                for a1, b1 in izip(
                                    [[1,2,3], [4,5,6]], [[2,3,4]]*2)])

        # Use blaze.eval to evaluate cf and df into concrete arrays
        cf2 = blaze.eval(cf)
        self.assertEqual(dd_as_py(cf2._data),
                        [[(a+b) for a, b in izip(a1, b1)]
                                for a1, b1 in izip(
                                    [[1,2,3], [4,5,6]], [[2,3,4]]*2)])
        df2 = blaze.eval(df)
        self.assertEqual(dd_as_py(df2._data),
                        [[(a+b) * (a+b) for a, b in izip(a1, b1)]
                                for a1, b1 in izip(
                                    [[1,2,3], [4,5,6]], [[2,3,4]]*2)])
Example #13
0
 def test_append(self):
     persist = blaze.Storage(self.rooturi, format="blz")
     a = blaze.zeros('0 * float64', storage=persist)
     self.assertTrue(isinstance(a, blaze.Array))
     append(a,list(range(10)))
     self.assertEqual(dd_as_py(a._data), list(range(10)))
Example #14
0
e =  blaze.array([1, 2, 3], dshape='3 * float32')
print(e)

# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:

f = blaze.array([1, 2, 3], dshape='float32')
print(f)

# --------------------------------------------------------------------

print_section('Alternative  constructors')

# Arrays can be created to be all zeros:
g = blaze.zeros('10 * 10 * int32')
print(g)

# All ones
h = blaze.ones('10 * 10 * float64')
print(h)

# --------------------------------------------------------------------

print_section('Indexing')

print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10 * 10 * 10 * 10 * float32')

def describe_array(label, array):
e =  blaze.array([1, 2, 3], dshape='3 * float32')
print(e)

# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:

f = blaze.array([1, 2, 3], dshape='float32')
print(f)

# --------------------------------------------------------------------

print_section('Alternative  constructors')

# Arrays can be created to be all zeros:
g = blaze.zeros('10 * 10 * int32')
print(g)

# All ones
h = blaze.ones('10 * 10 * float64')
print(h)

# --------------------------------------------------------------------

print_section('Indexing')

print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10 * 10 * 10 * 10 * float32')

def describe_array(label, array):
 def test_append(self):
     ddesc = HDF5_DDesc(path=self.file, datapath='/earray', mode='a')
     a = blaze.zeros('0 * float64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     append(a, list(range(10)))
     self.assertEqual(list(a), list(range(10)))
 def test_append(self):
     ddesc = BLZ_DDesc(path=self.rootdir, mode='w')
     a = blaze.zeros('0 * float64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     append(a, list(range(10)))
     self.assertEqual(list(a), list(range(10)))
Example #18
0
def make_test_array(datashape):
    """TODO: Use something more interesting"""
    return blaze.zeros(datashape)
Example #19
0
def make_test_array(datashape):
    """TODO: Use something more interesting"""
    return blaze.zeros(datashape)
Example #20
0
 def test_create_zeros(self):
     # A default array
     a = blaze.zeros('10 * int64')
     self.assertTrue(isinstance(a, blaze.Array))
     self.assertEqual(dd_as_py(a._data), [0]*10)
Example #21
0
def dot(a, b, out=None, outname='out'):
    """
    Matrix multiplication of two 2-D arrays.

    Parameters
    ----------
    a : array
        First argument.
    b : array
        Second argument.
    out : array, optional
        Output argument. This must have the exact kind that would be
        returned if it was not used. In particular, it must have the
        right type, must be C-contiguous, and its dtype must be the
        dtype that would be returned for `dot(a,b)`. This is a
        performance feature. Therefore, if these conditions are not
        met, an exception is raised, instead of attempting to be
        flexible.
    outname : str, optional
       If provided this will be the name for the output matrix storage.
       This parameter is only used when `out` is not provided.

    Returns
    -------
    output : array
        Returns the dot product of `a` and `b`.  If `a` and `b` are
        both scalars or both 1-D arrays then a scalar is returned;
        otherwise an array is returned.
        If `out` is given, then it is returned.

    Raises
    ------
    ValueError
        If the last dimension of `a` is not the same size as the
        second-to-last dimension of `b`.
    """

    a_shape = tuple(i.val for i in a.datashape.parameters[:-1])
    b_shape = tuple(i.val for i in b.datashape.parameters[:-1])

    if len(a_shape) != 2 or len(b_shape) != 2:
        raise (ValueError, "only 2-D matrices supported")

    if a_shape[1] != b_shape[0]:
        raise (ValueError,
               "last dimension of `a` does not match first dimension of `b`")

    l, m, n = a_shape[0], a_shape[1], b_shape[1]

    if out:
        out_shape = tuple(i.val for i in out.datashape.parameters[:-1])
        if out_shape != (l, n):
            raise (ValueError, "`out` array does not have the correct shape")
    else:
        parms = blaze.params(clevel=5, storage=outname)
        a_dtype = a.datashape.parameters[-1].to_dtype()
        dshape = blaze.dshape('%d, %d, %s' % (l, n, a_dtype))
        out = blaze.zeros(dshape, parms)


    # Compute a good block size
    out_dtype = out.datashape.parameters[-1].to_dtype()
    bl = math.sqrt(OOC_BUFFER_SIZE / out_dtype.itemsize)
    bl = 2**int(math.log(bl, 2))
    for i in range(0, l, bl):
        for j in range(0, n, bl):
            for k in range(0, m, bl):
                a0 = a[i:min(i+bl, l), k:min(k+bl, m)]
                b0 = b[k:min(k+bl, m), j:min(j+bl, n)]
                out[i:i+bl, j:j+bl] += np.dot(a0, b0)

    return out
Example #22
0
 def test_zeros(self):
     for t in self.full_types:
         a = zeros(t)
         self.assertEqual(a.datashape, dshape(t))
Example #23
0
 def test_create_zeros(self):
     # A default array (backed by NumPy)
     a = blaze.zeros('10, int64')
     self.assert_(isinstance(a, blaze.Array))
     self.assertEqual(dd_as_py(a._data), [0]*10)
Example #24
0
 def test_append(self):
     persist = blaze.Persist(self.rooturi)
     a = blaze.zeros('0, float64', persist=persist)
     self.assert_(isinstance(a, blaze.Array))
     a.append(list(range(10)))
     self.assertEqual(dd_as_py(a._data), list(range(10)))
Example #25
0
 def test_create_compress_zeros(self):
     # A compressed array (backed by BLZ)
     a = blaze.zeros('10 * int64', caps={'compress': True})
     self.assertTrue(isinstance(a, blaze.Array))
     self.assertEqual(dd_as_py(a._data), [0]*10)
Example #26
0
e = blaze.array([1, 2, 3], dshape='3, float32')
print(e)

# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:

f = blaze.array([1, 2, 3], dshape='float32')
print(f)

# --------------------------------------------------------------------

print_section('Alternative  constructors')

# Arrays can be created to be all zeros:
g = blaze.zeros('10, 10, int32')
print(g)

# All ones
h = blaze.ones('10, 10, float64')
print(h)

# --------------------------------------------------------------------

print_section('Indexing')

print_section('Indexing for read', level=1)
print('starting with a 4d array')
array4d = blaze.ones('10,10,10,10, float32')

Example #27
0
e =  blaze.array([ 1, 2, 3], dshape='3, float32') 
print(e)

# Note that the dimensions in the datashape when creating from a
# collection can be omitted. If that's the case, the dimensions will
# be inferred. The following is thus equivalent:

f = blaze.array([ 1, 2, 3], dshape='float32')
print(f)

# --------------------------------------------------------------------

print_section('Alternative  constructors')

# Arrays can be created to be all zeros:
g = blaze.zeros('10, 10, int32')
print(g)

# All ones
h = blaze.ones('10, 10, float64')
print(h)

# --------------------------------------------------------------------

print_section('Indexing')

print_section('Indexing for read', level=1)
print ('starting with a 4d array')
array4d = blaze.ones('10,10,10,10, float32')

def describe_array(label, array):
 def test_append(self):
     ddesc = BLZ_DDesc(path=self.rootdir, mode='w')
     a = blaze.zeros('0 * float64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     append(a, list(range(10)))
     self.assertEqual(list(a), list(range(10)))
 def test_zeros(self):
     for t in self.full_types:
         a = zeros(t)
         self.assertEqual(a.datashape, dshape(t))
 def test_append(self):
     ddesc = HDF5_DDesc(path=self.file, datapath='/earray', mode='a')
     a = blaze.zeros('0 * float64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     append(a, list(range(10)))
     self.assertEqual(list(a), list(range(10)))
 def test_create_compress_zeros(self):
     # A compressed array (backed by BLZ)
     ddesc = BLZ_DDesc(mode='w', bparams=blz.bparams(clevel=5))
     a = blaze.zeros('10 * int64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     self.assertEqual(list(a), [0]*10)
 def test_create_compress_zeros(self):
     # A compressed array (backed by BLZ)
     ddesc = BLZ_DDesc(mode='w', bparams=blz.bparams(clevel=5))
     a = blaze.zeros('10 * int64', ddesc=ddesc)
     self.assertTrue(isinstance(a, blaze.Array))
     self.assertEqual(list(a), [0] * 10)