def test_dynd_view_of_numpy_array(self):
# Tests viewing a numpy array as a dynd.array
nonnative = self.nonnative
a = np.arange(10, dtype=np.int32)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.int32)
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
a = np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.make_byteswap(ndt.int32))
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
a = np.arange(49, dtype='i1')
a = a[1:].view(dtype=np.int32).reshape(4,3)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.make_unaligned(ndt.int32))
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
a = np.arange(49, dtype='i1')
a = a[1:].view(dtype=(nonnative + 'i4')).reshape(2,2,3)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n),
ndt.make_unaligned(ndt.make_byteswap(ndt.int32)))
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
def test_readwrite_access_flags(self):
def assign_to(x,y):
x[0] = y
# Tests that read/write access control is preserved to/from numpy
a = np.arange(10.)
# Writeable
b = nd.view(a)
b[0] = 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Readonly view of writeable
b = nd.view(a, access='r')
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Not writeable
a.flags.writeable = False
b = nd.view(a)
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Trying to get a readwrite view raises an error
self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_readwrite_access_flags(self):
def assign_to(x, y):
x[0] = y
# Tests that read/write access control is preserved to/from numpy
a = np.arange(10.)
# Writeable
b = nd.view(a)
b[0] = 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Readonly view of writeable
b = nd.view(a, access='r')
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Not writeable
a.flags.writeable = False
b = nd.view(a)
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Trying to get a readwrite view raises an error
self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_dynd_view_of_numpy_array(self):
# Tests viewing a numpy array as a dynd.array
nonnative = self.nonnative
a = np.arange(10, dtype=np.int32)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.int32)
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
"""
a = np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.make_byteswap(ndt.int32))
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
"""
a = np.arange(49, dtype='i1')
a = a[1:].view(dtype=np.int32).reshape(4,3)
n = nd.view(a)
self.assertEqual(nd.dtype_of(n), ndt.make_unaligned(ndt.int32))
self.assertEqual(nd.ndim_of(n), a.ndim)
self.assertEqual(n.shape, a.shape)
self.assertEqual(n.strides, a.strides)
"""
def test_numpy_view_of_dynd_array(self):
# Tests viewing a dynd.array as a numpy array
nonnative = self.nonnative
n = nd.range(10, dtype=ndt.int32)
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(np.int32))
self.assertTrue(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1] = 100
self.assertEqual(nd.as_py(n[1]), 100)
n = nd.view(np.arange(12, dtype=(nonnative + 'i4')).reshape(3, 4))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(nonnative + 'i4'))
self.assertTrue(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1, 2] = 100
self.assertEqual(nd.as_py(n[1, 2]), 100)
n = nd.view(
np.arange(49, dtype='i1')[1:].view(dtype=np.int32).reshape(4, 3))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(np.int32))
self.assertFalse(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1, 2] = 100
self.assertEqual(nd.as_py(n[1, 2]), 100)
n = nd.view(
np.arange(49,
dtype='i1')[1:].view(dtype=(nonnative + 'i4')).reshape(
2, 2, 3))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(nonnative + 'i4'))
self.assertFalse(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1, 1, 1] = 100
self.assertEqual(nd.as_py(n[1, 1, 1]), 100)
def test_access_from_immutable_array(self):
# `a` is an immutable array
a = nd.array([1, 2, 3], access='r')
b = nd.view(a)
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='immutable')
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='readonly')
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='r')
self.assertEqual(b.access_flags, 'immutable')
# Can't create a readwrite view from a readonly array
self.assertRaises(RuntimeError, nd.view, b, access='readwrite')
self.assertRaises(RuntimeError, nd.view, b, access='rw')
def test_access_from_immutable_array(self):
# `a` is an immutable array
a = nd.array([1, 2, 3], access='r')
b = nd.view(a)
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='immutable')
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='readonly')
self.assertEqual(b.access_flags, 'immutable')
b = nd.view(a, access='r')
self.assertEqual(b.access_flags, 'immutable')
# Can't create a readwrite view from a readonly array
self.assertRaises(RuntimeError, nd.view, b, access='readwrite')
self.assertRaises(RuntimeError, nd.view, b, access='rw')
def test_access_from_immutable_array(self):
# `a` is an immutable array
a = nd.array([1, 2, 3])
b = nd.view(a)
self.assertEqual(b.access_flags, "immutable")
b = nd.view(a, access="immutable")
self.assertEqual(b.access_flags, "immutable")
b = nd.view(a, access="readonly")
self.assertEqual(b.access_flags, "immutable")
b = nd.view(a, access="r")
self.assertEqual(b.access_flags, "immutable")
# Can't create a readwrite view from a readonly array
self.assertRaises(RuntimeError, nd.view, a, access="readwrite")
self.assertRaises(RuntimeError, nd.view, a, access="rw")
def test_numpy_view_of_dynd_array(self):
# Tests viewing a dynd.array as a numpy array
nonnative = self.nonnative
n = nd.range(10, dtype=ndt.int32)
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(np.int32))
self.assertTrue(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1] = 100
self.assertEqual(nd.as_py(n[1]), 100)
n = nd.view(np.arange(12, dtype=(nonnative + 'i4')).reshape(3,4))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(nonnative + 'i4'))
self.assertTrue(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1,2] = 100
self.assertEqual(nd.as_py(n[1,2]), 100)
n = nd.view(np.arange(49, dtype='i1')[1:].view(dtype=np.int32).reshape(4,3))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(np.int32))
self.assertFalse(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1,2] = 100
self.assertEqual(nd.as_py(n[1,2]), 100)
n = nd.view(np.arange(49, dtype='i1')[1:].view(
dtype=(nonnative + 'i4')).reshape(2,2,3))
a = np.asarray(n)
self.assertEqual(a.dtype, np.dtype(nonnative + 'i4'))
self.assertFalse(a.flags.aligned)
self.assertEqual(a.ndim, nd.ndim_of(n))
self.assertEqual(a.shape, n.shape)
self.assertEqual(a.strides, n.strides)
# Make sure it's a view
a[1,1,1] = 100
self.assertEqual(nd.as_py(n[1,1,1]), 100)
def test_simple(self):
a = nd.array([1, 2, 3], access='rw')
# Modifying 'a' should affect 'b', because it's a view
b = nd.view(a)
a[1] = 10
self.assertEqual(nd.as_py(b), [1, 10, 3])
# Can't construct a view of a python list
self.assertRaises(RuntimeError, nd.view, [1, 2, 3])
def test_simple(self):
a = nd.array([1, 2, 3])
# Modifying 'a' should affect 'b', because it's a view
b = nd.view(a)
a[1] = 10
self.assertEqual(nd.as_py(b), [1, 10, 3])
# Can't construct a view of a python list
self.assertRaises(TypeError, nd.view, [1, 2, 3])
def test_dynd_scalar_view(self):
a = np.array(3, dtype='int64')
n = nd.view(a)
self.assertEqual(nd.type_of(n), ndt.int64)
self.assertEqual(nd.as_py(n), 3)
self.assertEqual(n.access_flags, 'readwrite')
# Ensure it's a view
n[...] = 4
self.assertEqual(a[()], 4)
def test_dynd_scalar_view(self):
a = np.array(3, dtype='int64')
n = nd.view(a)
self.assertEqual(nd.type_of(n), ndt.int64)
self.assertEqual(nd.as_py(n), 3)
self.assertEqual(n.access_flags, 'readwrite')
# Ensure it's a view
n[...] = 4
self.assertEqual(a[()], 4)
def test_numpy_dynd_fixedstring_interop(self):
# Tests converting fixed-size string arrays to/from numpy
# ASCII Numpy -> dynd
a = np.array(['abc', 'testing', 'array'])
b = nd.view(a)
if sys.version_info >= (3, 0):
self.assertEqual(ndt.make_fixedstring(7, 'utf_32'), nd.dtype_of(b))
else:
self.assertEqual(ndt.make_fixedstring(7, 'ascii'), nd.dtype_of(b))
self.assertEqual(nd.dtype_of(b), ndt.type(a.dtype))
# Make sure it's ascii
a = a.astype('S7')
b = nd.view(a)
# ASCII dynd -> Numpy
c = np.asarray(b)
self.assertEqual(a.dtype, c.dtype)
assert_array_equal(a, c)
# verify 'a' and 'c' are looking at the same data
a[1] = 'modify'
assert_array_equal(a, c)
# ASCII dynd -> UTF32 dynd
b_u = b.ucast(ndt.make_fixedstring(7, 'utf_32'))
self.assertEqual(
ndt.make_convert(
ndt.make_fixedstring(7, 'utf_32'),
ndt.make_fixedstring(7, 'ascii')),
nd.dtype_of(b_u))
# Evaluate to its value array
b_u = b_u.eval()
self.assertEqual(
ndt.make_fixedstring(7, 'utf_32'),
nd.dtype_of(b_u))
# UTF32 dynd -> Numpy
c_u = np.asarray(b_u)
self.assertEqual(nd.dtype_of(b_u), ndt.type(c_u.dtype))
assert_array_equal(a.astype('U'), c_u)
# 'a' and 'c_u' are not looking at the same data
a[1] = 'diff'
self.assertFalse(np.all(a == c_u))
def test_numpy_dynd_fixed_string_interop(self):
# Tests converting fixed-size string arrays to/from numpy
# ASCII Numpy -> dynd
a = np.array(['abc', 'testing', 'array'])
b = nd.view(a)
if sys.version_info >= (3, 0):
self.assertEqual(ndt.make_fixed_string(7, 'utf_32'), nd.dtype_of(b))
else:
self.assertEqual(ndt.make_fixed_string(7, 'ascii'), nd.dtype_of(b))
self.assertEqual(nd.dtype_of(b), ndt.type(a.dtype))
# Make sure it's ascii
a = a.astype('S7')
b = nd.view(a)
# ASCII dynd -> Numpy
c = np.asarray(b)
self.assertEqual(a.dtype, c.dtype)
assert_array_equal(a, c)
# verify 'a' and 'c' are looking at the same data
a[1] = 'modify'
assert_array_equal(a, c)
# ASCII dynd -> UTF32 dynd
b_u = b.ucast(ndt.make_fixed_string(7, 'utf_32'))
self.assertEqual(
ndt.make_convert(
ndt.make_fixed_string(7, 'utf_32'),
ndt.make_fixed_string(7, 'ascii')),
nd.dtype_of(b_u))
# Evaluate to its value array
b_u = b_u.eval()
self.assertEqual(
ndt.make_fixed_string(7, 'utf_32'),
nd.dtype_of(b_u))
# UTF32 dynd -> Numpy
c_u = np.asarray(b_u)
self.assertEqual(nd.dtype_of(b_u), ndt.type(c_u.dtype))
assert_array_equal(a.astype('U'), c_u)
# 'a' and 'c_u' are not looking at the same data
a[1] = 'diff'
self.assertFalse(np.all(a == c_u))
def test_misaligned_datetime_from_numpy(self):
a = np.array([(1, "2000-12-25T00:00:01Z"),
(2, "2001-12-25T00:00:01Z"),
(3, "2002-12-25T00:00:01Z")],
dtype=[('id', 'int8'), ('ts', 'M8[us]')])
b = nd.view(a)
self.assertEqual(nd.type_of(b),
ndt.type("strided * c{id : int8, ts: adapt[(unaligned[int64]) -> datetime[tz='UTC'], 'microseconds since 1970']}"))
self.assertEqual(nd.as_py(b, tuple=True),
[(1, datetime(2000, 12, 25, 0, 0, 1)),
(2, datetime(2001, 12, 25, 0, 0, 1)),
(3, datetime(2002, 12, 25, 0, 0, 1))])
def test_misaligned_datetime_from_numpy(self):
a = np.array([(1, "2000-12-25T00:00:01Z"), (2, "2001-12-25T00:00:01Z"),
(3, "2002-12-25T00:00:01Z")],
dtype=[('id', 'int8'), ('ts', 'M8[us]')])
b = nd.view(a)
self.assertEqual(
nd.type_of(b),
ndt.type(
"3 * {id : int8, ts: adapt[(unaligned[int64]) -> datetime[tz='UTC'], 'microseconds since 1970']}"
))
self.assertEqual(nd.as_py(b), [{
'id': 1,
'ts': datetime(2000, 12, 25, 0, 0, 1)
}, {
'id': 2,
'ts': datetime(2001, 12, 25, 0, 0, 1)
}, {
'id': 3,
'ts': datetime(2002, 12, 25, 0, 0, 1)
}])
def array(obj, dshape=None, caps={'efficient-write': True},
storage=None):
"""Create a Blaze array.
Parameters
----------
obj : array_like
Initial contents for the array.
dshape : datashape
The datashape for the resulting array. By default the
datashape will be inferred from data. If an explicit dshape is
provided, the input data will be coerced into the provided
dshape.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for storing the data.
Returns
-------
out : a concrete blaze array.
Bugs
----
Right now the explicit dshape is ignored. This needs to be
corrected. When the data cannot be coerced to an explicit dshape
an exception should be raised.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if isinstance(obj, Array):
return obj
elif isinstance(obj, IDataDescriptor):
# TODO: Validate the 'caps', convert to another kind
# of data descriptor if necessary
# Note by Francesc: but if it is already an IDataDescriptor I wonder
# if `caps` should be ignored. Hmm, probably not...
#
# Note by Oscar: Maybe we shouldn't accept a datadescriptor at
# all at this level. If you've got a DataDescriptor you are
# playing with internal datastructures anyways, go to the
# Array constructor directly. If you want to transform to
# another datadescriptor... convert it yourself (you are
# playing with internal datastructures, remember? you should
# be able to do it in your own.
dd = obj
elif storage is not None:
dt = None if dshape is None else to_numpy_dtype(dshape)
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(blz.barray(obj, dtype=dt, count=-1,
rootdir=storage.path))
else:
dd = BLZDataDescriptor(
blz.barray(obj, dtype=dt, rootdir=storage.path))
elif 'efficient-write' in caps and caps['efficient-write'] is True:
# In-Memory array
if dshape is None:
dd = DyNDDataDescriptor(nd.asarray(obj, access='rw'))
else:
# Use the uniform/full dtype specification in dynd depending
# on whether the datashape has a uniform dim
dt = ndt.type(str(dshape))
if dt.ndim > 0:
dd = DyNDDataDescriptor(nd.array(obj, type=dt, access='rw'))
else:
dd = DyNDDataDescriptor(nd.array(obj, dtype=dt, access='rw'))
elif 'compress' in caps and caps['compress'] is True:
dt = None if dshape is None else to_numpy_dtype(dshape)
# BLZ provides compression
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1))
else:
dd = BLZDataDescriptor(blz.barray(obj, dtype=dt))
elif isinstance(obj, np.ndarray):
dd = DyNDDataDescriptor(nd.view(obj))
elif isinstance(obj, nd.array):
dd = DyNDDataDescriptor(obj)
elif isinstance(obj, blz.barray):
dd = BLZDataDescriptor(obj)
else:
raise TypeError(('Failed to construct blaze array from '
'object of type %r') % type(obj))
return Array(dd)
def array(obj, dshape=None, caps={'efficient-write': True}, storage=None):
"""Create a Blaze array.
Parameters
----------
obj : array_like
Initial contents for the array.
dshape : datashape
The datashape for the resulting array. By default the
datashape will be inferred from data. If an explicit dshape is
provided, the input data will be coerced into the provided
dshape.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for storing the data.
Returns
-------
out : a concrete blaze array.
Bugs
----
Right now the explicit dshape is ignored. This needs to be
corrected. When the data cannot be coerced to an explicit dshape
an exception should be raised.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if isinstance(obj, Array):
return obj
elif isinstance(obj, IDataDescriptor):
# TODO: Validate the 'caps', convert to another kind
# of data descriptor if necessary
# Note by Francesc: but if it is already an IDataDescriptor I wonder
# if `caps` should be ignored. Hmm, probably not...
#
# Note by Oscar: Maybe we shouldn't accept a datadescriptor at
# all at this level. If you've got a DataDescriptor you are
# playing with internal datastructures anyways, go to the
# Array constructor directly. If you want to transform to
# another datadescriptor... convert it yourself (you are
# playing with internal datastructures, remember? you should
# be able to do it in your own.
dd = obj
elif storage is not None:
dt = None if dshape is None else to_numpy_dtype(dshape)
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(
blz.barray(obj, dtype=dt, count=-1, rootdir=storage.path))
else:
dd = BLZDataDescriptor(
blz.barray(obj, dtype=dt, rootdir=storage.path))
elif 'efficient-write' in caps and caps['efficient-write'] is True:
# In-Memory array
if dshape is None:
dd = DyNDDataDescriptor(nd.asarray(obj, access='rw'))
else:
# Use the uniform/full dtype specification in dynd depending
# on whether the datashape has a uniform dim
dt = ndt.type(str(dshape))
if dt.ndim > 0:
dd = DyNDDataDescriptor(nd.array(obj, type=dt, access='rw'))
else:
dd = DyNDDataDescriptor(nd.array(obj, dtype=dt, access='rw'))
elif 'compress' in caps and caps['compress'] is True:
dt = None if dshape is None else to_numpy_dtype(dshape)
# BLZ provides compression
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1))
else:
dd = BLZDataDescriptor(blz.barray(obj, dtype=dt))
elif isinstance(obj, np.ndarray):
dd = DyNDDataDescriptor(nd.view(obj))
elif isinstance(obj, nd.array):
dd = DyNDDataDescriptor(obj)
elif isinstance(obj, blz.barray):
dd = BLZDataDescriptor(obj)
else:
raise TypeError(('Failed to construct blaze array from '
'object of type %r') % type(obj))
return Array(dd)