Exemplo n.º 1
0
    def test01b(self):
        """Creating a barray in "w" mode."""

        N = 50000
        cn = blz.zeros(N, dtype="i1", rootdir=self.rootdir)
        self.assert_(len(cn) == N)

        cn = blz.zeros(N-2, dtype="i1", rootdir=self.rootdir, mode='w')
        self.assert_(len(cn) == N-2)

        # Now check some accesses (no errors should be raised)
        cn.append([1,1])
        self.assert_(len(cn) == N)
        cn[1] = 2
        self.assert_(cn[1] == 2)
Exemplo n.º 2
0
 def test01c(self):
     """Testing zeros() constructor, with a string type."""
     a = np.zeros(self.N, dtype='S5')
     ac = blz.zeros(self.N, dtype='S5', rootdir=self.rootdir)
     #print "ac-->", `ac`
     self.assert_(a.dtype == ac.dtype)
     self.assert_(np.all(a == ac[:]))
Exemplo n.º 3
0
 def test01b(self):
     """Testing zeros() constructor, with a `dtype`."""
     a = np.zeros(self.N, dtype='i4')
     ac = blz.zeros(self.N, dtype='i4', rootdir=self.rootdir)
     #print "dtypes-->", a.dtype, ac.dtype
     self.assert_(a.dtype == ac.dtype)
     self.assert_(np.all(a == ac[:]))
Exemplo n.º 4
0
def interpret(func, env, args, storage=None, **kwds):
    assert len(args) == len(func.args)

    # Make a copy, since we're going to mutate our IR!
    func = copy_function(func)

    # If it's a BLZ output, we want an interpreter that streams
    # the processing through in chunks
    if storage is not None:
        if len(func.type.restype.shape) == 0:
            raise TypeError('Require an array, not a scalar, for outputting to BLZ')
        env['stream-outer'] = True
        result_ndim = env['result-ndim'] = len(func.type.restype.shape)
    else:
        # Convert any persistent inputs to memory
        # TODO: should stream the computation in this case
        for i, arg in enumerate(args):
            if isinstance(arg._data, BLZDataDescriptor):
                args[i] = arg[:]

    # Update environment with dynd type information
    dynd_types = dict((arg, get_dynd_type(array))
                          for arg, array in zip(func.args, args)
                              if isinstance(array._data, DyNDDataDescriptor))
    env['dynd-types'] = dynd_types

    # Lift ckernels
    func, env = run_pipeline(func, env, run_time_passes)

    if storage is None:
        # Evaluate once
        values = dict(zip(func.args, args))
        interp = CKernelInterp(values)
        visit(interp, func)
        return interp.result
    else:
        res_shape, res_dt = datashape.to_numpy(func.type.restype)
        dim_size = operator.index(res_shape[0])
        row_size = ndt.type(str(func.type.restype.subarray(1))).data_size
        chunk_size = min(max(1, (1024*1024) // row_size), dim_size)
        # Evaluate by streaming the outermost dimension,
        # and using the BLZ data descriptor's append
        dst_dd = BLZDataDescriptor(blz.zeros((0,)+res_shape[1:], res_dt,
                                             rootdir=storage.path))
        # Loop through all the chunks
        for chunk_start in range(0, dim_size, chunk_size):
            # Tell the interpreter which chunk size to use (last
            # chunk might be smaller)
            chunk_size = min(chunk_size, dim_size - chunk_start)
            # Evaluate the chunk
            args_chunk = [arg[chunk_start:chunk_start+chunk_size]
                            if len(arg.dshape.shape) == result_ndim
                            else arg for arg in args]
            values = dict(zip(func.args, args_chunk))
            interp = CKernelChunkInterp(values, chunk_size, result_ndim)
            visit(interp, func)
            chunk = interp.result._data.dynd_arr()
            dst_dd.append(chunk)
        return blaze.Array(dst_dd)
Exemplo n.º 5
0
 def test01c(self):
     """Testing `zeros` constructor (III)"""
     a = np.zeros((2,2), dtype='(4,)i4')
     b = blz.zeros((2,2), dtype='(4,)i4', rootdir=self.rootdir)
     if self.open:
         b = blz.open(rootdir=self.rootdir)
     #print "b->", `b`
     assert_array_equal(a, b, "Arrays are not equal")
Exemplo n.º 6
0
 def getobject(self):
     if self.flavor == 'barray':
         obj = blz.zeros(10, dtype="i1", rootdir=self.rootdir)
         self.assertEqual(type(obj), blz.barray)
     elif self.flavor == 'btable':
         obj = blz.fromiter(((i,i*2) for i in range(10)), dtype='i2,f4',
                           count=10, rootdir=self.rootdir)
         self.assertEqual(type(obj), blz.btable)
     return obj
Exemplo n.º 7
0
 def test00a(self):
     """Testing wheretrue() in combination with a list constructor"""
     a = blz.zeros(self.N, dtype="bool")
     a[30:40] = blz.ones(10, dtype="bool")
     alist = list(a)
     blist1 = [r for r in a.wheretrue()]
     self.assert_(blist1 == list(range(30,40)))
     alist2 = list(a)
     self.assert_(alist == alist2, "wheretrue() not working correctly")
Exemplo n.º 8
0
    def test01c(self):
        """Creating a barray in "a" mode."""

        N = 30003
        cn = blz.zeros(N, dtype="i1", rootdir=self.rootdir)
        self.assert_(len(cn) == N)

        self.assertRaises(RuntimeError, blz.zeros,
                          N-2, dtype="i1", rootdir=self.rootdir, mode='a')
Exemplo n.º 9
0
    def test02a(self):
        """Opening a barray in "r" mode."""

        N = 10001
        cn = blz.zeros(N, dtype="i1", rootdir=self.rootdir)
        self.assert_(len(cn) == N)

        cn = blz.barray(rootdir=self.rootdir, mode='r')
        self.assert_(len(cn) == N)

        # Now check some accesses
        self.assertRaises(RuntimeError, cn.__setitem__, 1, 1)
        self.assertRaises(RuntimeError, cn.append, 1)
Exemplo n.º 10
0
    def test02b(self):
        """Opening a barray in "w" mode."""

        N = 100001
        cn = blz.zeros(N, dtype="i1", rootdir=self.rootdir)
        self.assert_(len(cn) == N)

        cn = blz.barray(rootdir=self.rootdir, mode='w')
        self.assert_(len(cn) == 0)

        # Now check some accesses (no errors should be raised)
        cn.append([1,1])
        self.assert_(len(cn) == 2)
        cn[1] = 2
        self.assert_(cn[1] == 2)
Exemplo n.º 11
0
    def test01(self):
        """Creating an extremely large barray (> 2**32) on disk."""

        cn = blz.zeros(5e9, dtype="i1", rootdir=self.rootdir)
        self.assertEqual(len(cn), int(5e9))

        # Now check some accesses
        cn[1] = 1
        self.assertEqual(cn[1], 1)
        cn[int(2e9)] = 2
        self.assertEqual(cn[int(2e9)], 2)
        cn[long(3e9)] = 3
        self.assertEqual(cn[long(3e9)], 3)
        cn[-1] = 4
        self.assertEqual(cn[-1], 4)

        self.assertEqual(cn.sum(), 10)
Exemplo n.º 12
0
    def test07(self):
        """Checking barray constructor from another barray.

        Test introduced after it was seen failing (blaze issue #30)
        """
        types = [np.int8, np.int16, np.int32, np.int64,
                 np.uint8, np.uint16, np.uint32, np.uint64,
                 np.float16, np.float32, np.float64,
                 np.complex64, np.complex128]
        if hasattr(np, 'float128'):
            types.extend([np.float128, np.complex256])
        shapes = [(10,), (10,10), (10,10,10)]
        for shape in shapes:
            for t in types:
                a = blz.zeros(shape, t)
                b = blz.barray(a)
                self.assertEqual(a.dtype, b.dtype)
                self.assertEqual(a.shape, b.shape)
                self.assertEqual(a.shape, shape)
Exemplo n.º 13
0
 def test01b(self):
     """Testing where() with a multidimensional array"""
     a = blz.zeros((self.N, 10), dtype="bool")
     a[30:40] = blz.ones(10, dtype="bool")
     b = blz.arange(self.N*10, dtype="f4").reshape((self.N, 10))
     self.assertRaises(NotImplementedError, b.where, a)
Exemplo n.º 14
0
 def test00b(self):
     """Testing wheretrue() with a multidimensional array"""
     a = blz.zeros((self.N, 10), dtype="bool")
     a[30:40] = blz.ones(10, dtype="bool")
     self.assertRaises(NotImplementedError, a.wheretrue)
Exemplo n.º 15
0
 def test01a(self):
     """Testing zeros() constructor."""
     a = np.zeros(self.N)
     ac = blz.zeros(self.N, rootdir=self.rootdir)
     self.assert_(a.dtype == ac.dtype)
     self.assert_(np.all(a == ac[:]))
Exemplo n.º 16
0
 def test02(self):
     """Testing sum() with strings (TypeError)."""
     ac = blz.zeros(10, 'S3')
     self.assertRaises(TypeError, ac.sum)