def test_dynd_view_of_numpy_array(self): # Tests viewing a numpy array as a dynd.array nonnative = self.nonnative a = np.arange(10, dtype=np.int32) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.int32) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) a = np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.make_byteswap(ndt.int32)) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) a = np.arange(49, dtype='i1') a = a[1:].view(dtype=np.int32).reshape(4,3) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.make_unaligned(ndt.int32)) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) a = np.arange(49, dtype='i1') a = a[1:].view(dtype=(nonnative + 'i4')).reshape(2,2,3) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.make_unaligned(ndt.make_byteswap(ndt.int32))) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides)
def test_readwrite_access_flags(self): def assign_to(x,y): x[0] = y # Tests that read/write access control is preserved to/from numpy a = np.arange(10.) # Writeable b = nd.view(a) b[0] = 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Readonly view of writeable b = nd.view(a, access='r') self.assertRaises(RuntimeError, assign_to, b, 3.0) # should still be 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Not writeable a.flags.writeable = False b = nd.view(a) self.assertRaises(RuntimeError, assign_to, b, 3.0) # should still be 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Trying to get a readwrite view raises an error self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_readwrite_access_flags(self): def assign_to(x, y): x[0] = y # Tests that read/write access control is preserved to/from numpy a = np.arange(10.) # Writeable b = nd.view(a) b[0] = 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Readonly view of writeable b = nd.view(a, access='r') self.assertRaises(RuntimeError, assign_to, b, 3.0) # should still be 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Not writeable a.flags.writeable = False b = nd.view(a) self.assertRaises(RuntimeError, assign_to, b, 3.0) # should still be 2.0 self.assertEqual(nd.as_py(b[0]), 2.0) self.assertEqual(a[0], 2.0) # Trying to get a readwrite view raises an error self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_dynd_view_of_numpy_array(self): # Tests viewing a numpy array as a dynd.array nonnative = self.nonnative a = np.arange(10, dtype=np.int32) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.int32) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) """ a = np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.make_byteswap(ndt.int32)) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) """ a = np.arange(49, dtype='i1') a = a[1:].view(dtype=np.int32).reshape(4,3) n = nd.view(a) self.assertEqual(nd.dtype_of(n), ndt.make_unaligned(ndt.int32)) self.assertEqual(nd.ndim_of(n), a.ndim) self.assertEqual(n.shape, a.shape) self.assertEqual(n.strides, a.strides) """
def test_numpy_view_of_dynd_array(self): # Tests viewing a dynd.array as a numpy array nonnative = self.nonnative n = nd.range(10, dtype=ndt.int32) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(np.int32)) self.assertTrue(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1] = 100 self.assertEqual(nd.as_py(n[1]), 100) n = nd.view(np.arange(12, dtype=(nonnative + 'i4')).reshape(3, 4)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(nonnative + 'i4')) self.assertTrue(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1, 2] = 100 self.assertEqual(nd.as_py(n[1, 2]), 100) n = nd.view( np.arange(49, dtype='i1')[1:].view(dtype=np.int32).reshape(4, 3)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(np.int32)) self.assertFalse(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1, 2] = 100 self.assertEqual(nd.as_py(n[1, 2]), 100) n = nd.view( np.arange(49, dtype='i1')[1:].view(dtype=(nonnative + 'i4')).reshape( 2, 2, 3)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(nonnative + 'i4')) self.assertFalse(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1, 1, 1] = 100 self.assertEqual(nd.as_py(n[1, 1, 1]), 100)
def test_access_from_immutable_array(self): # `a` is an immutable array a = nd.array([1, 2, 3], access='r') b = nd.view(a) self.assertEqual(b.access_flags, 'immutable') b = nd.view(a, access='immutable') self.assertEqual(b.access_flags, 'immutable') b = nd.view(a, access='readonly') self.assertEqual(b.access_flags, 'immutable') b = nd.view(a, access='r') self.assertEqual(b.access_flags, 'immutable') # Can't create a readwrite view from a readonly array self.assertRaises(RuntimeError, nd.view, b, access='readwrite') self.assertRaises(RuntimeError, nd.view, b, access='rw')
def test_access_from_immutable_array(self): # `a` is an immutable array a = nd.array([1, 2, 3]) b = nd.view(a) self.assertEqual(b.access_flags, "immutable") b = nd.view(a, access="immutable") self.assertEqual(b.access_flags, "immutable") b = nd.view(a, access="readonly") self.assertEqual(b.access_flags, "immutable") b = nd.view(a, access="r") self.assertEqual(b.access_flags, "immutable") # Can't create a readwrite view from a readonly array self.assertRaises(RuntimeError, nd.view, a, access="readwrite") self.assertRaises(RuntimeError, nd.view, a, access="rw")
def test_numpy_view_of_dynd_array(self): # Tests viewing a dynd.array as a numpy array nonnative = self.nonnative n = nd.range(10, dtype=ndt.int32) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(np.int32)) self.assertTrue(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1] = 100 self.assertEqual(nd.as_py(n[1]), 100) n = nd.view(np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(nonnative + 'i4')) self.assertTrue(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1,2] = 100 self.assertEqual(nd.as_py(n[1,2]), 100) n = nd.view(np.arange(49, dtype='i1')[1:].view(dtype=np.int32).reshape(4,3)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(np.int32)) self.assertFalse(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1,2] = 100 self.assertEqual(nd.as_py(n[1,2]), 100) n = nd.view(np.arange(49, dtype='i1')[1:].view( dtype=(nonnative + 'i4')).reshape(2,2,3)) a = np.asarray(n) self.assertEqual(a.dtype, np.dtype(nonnative + 'i4')) self.assertFalse(a.flags.aligned) self.assertEqual(a.ndim, nd.ndim_of(n)) self.assertEqual(a.shape, n.shape) self.assertEqual(a.strides, n.strides) # Make sure it's a view a[1,1,1] = 100 self.assertEqual(nd.as_py(n[1,1,1]), 100)
def test_simple(self): a = nd.array([1, 2, 3], access='rw') # Modifying 'a' should affect 'b', because it's a view b = nd.view(a) a[1] = 10 self.assertEqual(nd.as_py(b), [1, 10, 3]) # Can't construct a view of a python list self.assertRaises(RuntimeError, nd.view, [1, 2, 3])
def test_simple(self): a = nd.array([1, 2, 3]) # Modifying 'a' should affect 'b', because it's a view b = nd.view(a) a[1] = 10 self.assertEqual(nd.as_py(b), [1, 10, 3]) # Can't construct a view of a python list self.assertRaises(TypeError, nd.view, [1, 2, 3])
def test_dynd_scalar_view(self): a = np.array(3, dtype='int64') n = nd.view(a) self.assertEqual(nd.type_of(n), ndt.int64) self.assertEqual(nd.as_py(n), 3) self.assertEqual(n.access_flags, 'readwrite') # Ensure it's a view n[...] = 4 self.assertEqual(a[()], 4)
def test_numpy_dynd_fixedstring_interop(self): # Tests converting fixed-size string arrays to/from numpy # ASCII Numpy -> dynd a = np.array(['abc', 'testing', 'array']) b = nd.view(a) if sys.version_info >= (3, 0): self.assertEqual(ndt.make_fixedstring(7, 'utf_32'), nd.dtype_of(b)) else: self.assertEqual(ndt.make_fixedstring(7, 'ascii'), nd.dtype_of(b)) self.assertEqual(nd.dtype_of(b), ndt.type(a.dtype)) # Make sure it's ascii a = a.astype('S7') b = nd.view(a) # ASCII dynd -> Numpy c = np.asarray(b) self.assertEqual(a.dtype, c.dtype) assert_array_equal(a, c) # verify 'a' and 'c' are looking at the same data a[1] = 'modify' assert_array_equal(a, c) # ASCII dynd -> UTF32 dynd b_u = b.ucast(ndt.make_fixedstring(7, 'utf_32')) self.assertEqual( ndt.make_convert( ndt.make_fixedstring(7, 'utf_32'), ndt.make_fixedstring(7, 'ascii')), nd.dtype_of(b_u)) # Evaluate to its value array b_u = b_u.eval() self.assertEqual( ndt.make_fixedstring(7, 'utf_32'), nd.dtype_of(b_u)) # UTF32 dynd -> Numpy c_u = np.asarray(b_u) self.assertEqual(nd.dtype_of(b_u), ndt.type(c_u.dtype)) assert_array_equal(a.astype('U'), c_u) # 'a' and 'c_u' are not looking at the same data a[1] = 'diff' self.assertFalse(np.all(a == c_u))
def test_numpy_dynd_fixed_string_interop(self): # Tests converting fixed-size string arrays to/from numpy # ASCII Numpy -> dynd a = np.array(['abc', 'testing', 'array']) b = nd.view(a) if sys.version_info >= (3, 0): self.assertEqual(ndt.make_fixed_string(7, 'utf_32'), nd.dtype_of(b)) else: self.assertEqual(ndt.make_fixed_string(7, 'ascii'), nd.dtype_of(b)) self.assertEqual(nd.dtype_of(b), ndt.type(a.dtype)) # Make sure it's ascii a = a.astype('S7') b = nd.view(a) # ASCII dynd -> Numpy c = np.asarray(b) self.assertEqual(a.dtype, c.dtype) assert_array_equal(a, c) # verify 'a' and 'c' are looking at the same data a[1] = 'modify' assert_array_equal(a, c) # ASCII dynd -> UTF32 dynd b_u = b.ucast(ndt.make_fixed_string(7, 'utf_32')) self.assertEqual( ndt.make_convert( ndt.make_fixed_string(7, 'utf_32'), ndt.make_fixed_string(7, 'ascii')), nd.dtype_of(b_u)) # Evaluate to its value array b_u = b_u.eval() self.assertEqual( ndt.make_fixed_string(7, 'utf_32'), nd.dtype_of(b_u)) # UTF32 dynd -> Numpy c_u = np.asarray(b_u) self.assertEqual(nd.dtype_of(b_u), ndt.type(c_u.dtype)) assert_array_equal(a.astype('U'), c_u) # 'a' and 'c_u' are not looking at the same data a[1] = 'diff' self.assertFalse(np.all(a == c_u))
def test_misaligned_datetime_from_numpy(self): a = np.array([(1, "2000-12-25T00:00:01Z"), (2, "2001-12-25T00:00:01Z"), (3, "2002-12-25T00:00:01Z")], dtype=[('id', 'int8'), ('ts', 'M8[us]')]) b = nd.view(a) self.assertEqual(nd.type_of(b), ndt.type("strided * c{id : int8, ts: adapt[(unaligned[int64]) -> datetime[tz='UTC'], 'microseconds since 1970']}")) self.assertEqual(nd.as_py(b, tuple=True), [(1, datetime(2000, 12, 25, 0, 0, 1)), (2, datetime(2001, 12, 25, 0, 0, 1)), (3, datetime(2002, 12, 25, 0, 0, 1))])
def test_misaligned_datetime_from_numpy(self): a = np.array([(1, "2000-12-25T00:00:01Z"), (2, "2001-12-25T00:00:01Z"), (3, "2002-12-25T00:00:01Z")], dtype=[('id', 'int8'), ('ts', 'M8[us]')]) b = nd.view(a) self.assertEqual( nd.type_of(b), ndt.type( "3 * {id : int8, ts: adapt[(unaligned[int64]) -> datetime[tz='UTC'], 'microseconds since 1970']}" )) self.assertEqual(nd.as_py(b), [{ 'id': 1, 'ts': datetime(2000, 12, 25, 0, 0, 1) }, { 'id': 2, 'ts': datetime(2001, 12, 25, 0, 0, 1) }, { 'id': 3, 'ts': datetime(2002, 12, 25, 0, 0, 1) }])
def array(obj, dshape=None, caps={'efficient-write': True}, storage=None): """Create a Blaze array. Parameters ---------- obj : array_like Initial contents for the array. dshape : datashape The datashape for the resulting array. By default the datashape will be inferred from data. If an explicit dshape is provided, the input data will be coerced into the provided dshape. caps : capabilities dictionary A dictionary containing the desired capabilities of the array. storage : Storage instance A Storage object with the necessary info for storing the data. Returns ------- out : a concrete blaze array. Bugs ---- Right now the explicit dshape is ignored. This needs to be corrected. When the data cannot be coerced to an explicit dshape an exception should be raised. """ dshape = _normalize_dshape(dshape) storage = _storage_convert(storage) if isinstance(obj, Array): return obj elif isinstance(obj, IDataDescriptor): # TODO: Validate the 'caps', convert to another kind # of data descriptor if necessary # Note by Francesc: but if it is already an IDataDescriptor I wonder # if `caps` should be ignored. Hmm, probably not... # # Note by Oscar: Maybe we shouldn't accept a datadescriptor at # all at this level. If you've got a DataDescriptor you are # playing with internal datastructures anyways, go to the # Array constructor directly. If you want to transform to # another datadescriptor... convert it yourself (you are # playing with internal datastructures, remember? you should # be able to do it in your own. dd = obj elif storage is not None: dt = None if dshape is None else to_numpy_dtype(dshape) if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor(blz.barray(obj, dtype=dt, count=-1, rootdir=storage.path)) else: dd = BLZDataDescriptor( blz.barray(obj, dtype=dt, rootdir=storage.path)) elif 'efficient-write' in caps and caps['efficient-write'] is True: # In-Memory array if dshape is None: dd = DyNDDataDescriptor(nd.asarray(obj, access='rw')) else: # Use the uniform/full dtype specification in dynd depending # on whether the datashape has a uniform dim dt = ndt.type(str(dshape)) if dt.ndim > 0: dd = DyNDDataDescriptor(nd.array(obj, type=dt, access='rw')) else: dd = DyNDDataDescriptor(nd.array(obj, dtype=dt, access='rw')) elif 'compress' in caps and caps['compress'] is True: dt = None if dshape is None else to_numpy_dtype(dshape) # BLZ provides compression if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1)) else: dd = BLZDataDescriptor(blz.barray(obj, dtype=dt)) elif isinstance(obj, np.ndarray): dd = DyNDDataDescriptor(nd.view(obj)) elif isinstance(obj, nd.array): dd = DyNDDataDescriptor(obj) elif isinstance(obj, blz.barray): dd = BLZDataDescriptor(obj) else: raise TypeError(('Failed to construct blaze array from ' 'object of type %r') % type(obj)) return Array(dd)
def array(obj, dshape=None, caps={'efficient-write': True}, storage=None): """Create a Blaze array. Parameters ---------- obj : array_like Initial contents for the array. dshape : datashape The datashape for the resulting array. By default the datashape will be inferred from data. If an explicit dshape is provided, the input data will be coerced into the provided dshape. caps : capabilities dictionary A dictionary containing the desired capabilities of the array. storage : Storage instance A Storage object with the necessary info for storing the data. Returns ------- out : a concrete blaze array. Bugs ---- Right now the explicit dshape is ignored. This needs to be corrected. When the data cannot be coerced to an explicit dshape an exception should be raised. """ dshape = _normalize_dshape(dshape) storage = _storage_convert(storage) if isinstance(obj, Array): return obj elif isinstance(obj, IDataDescriptor): # TODO: Validate the 'caps', convert to another kind # of data descriptor if necessary # Note by Francesc: but if it is already an IDataDescriptor I wonder # if `caps` should be ignored. Hmm, probably not... # # Note by Oscar: Maybe we shouldn't accept a datadescriptor at # all at this level. If you've got a DataDescriptor you are # playing with internal datastructures anyways, go to the # Array constructor directly. If you want to transform to # another datadescriptor... convert it yourself (you are # playing with internal datastructures, remember? you should # be able to do it in your own. dd = obj elif storage is not None: dt = None if dshape is None else to_numpy_dtype(dshape) if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor( blz.barray(obj, dtype=dt, count=-1, rootdir=storage.path)) else: dd = BLZDataDescriptor( blz.barray(obj, dtype=dt, rootdir=storage.path)) elif 'efficient-write' in caps and caps['efficient-write'] is True: # In-Memory array if dshape is None: dd = DyNDDataDescriptor(nd.asarray(obj, access='rw')) else: # Use the uniform/full dtype specification in dynd depending # on whether the datashape has a uniform dim dt = ndt.type(str(dshape)) if dt.ndim > 0: dd = DyNDDataDescriptor(nd.array(obj, type=dt, access='rw')) else: dd = DyNDDataDescriptor(nd.array(obj, dtype=dt, access='rw')) elif 'compress' in caps and caps['compress'] is True: dt = None if dshape is None else to_numpy_dtype(dshape) # BLZ provides compression if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1)) else: dd = BLZDataDescriptor(blz.barray(obj, dtype=dt)) elif isinstance(obj, np.ndarray): dd = DyNDDataDescriptor(nd.view(obj)) elif isinstance(obj, nd.array): dd = DyNDDataDescriptor(obj) elif isinstance(obj, blz.barray): dd = BLZDataDescriptor(obj) else: raise TypeError(('Failed to construct blaze array from ' 'object of type %r') % type(obj)) return Array(dd)