def test_readwrite_access_flags(self):
def assign_to(x,y):
x[0] = y
# Tests that read/write access control is preserved to/from numpy
a = np.arange(10.)
# Writeable
b = nd.view(a)
b[0] = 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Readonly view of writeable
b = nd.view(a, access='r')
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Not writeable
a.flags.writeable = False
b = nd.view(a)
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Trying to get a readwrite view raises an error
self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_numpy_struct_scalar(self):
# Create a NumPy struct scalar object, by indexing into
# a structured array
a = np.array([(10, 11, 12)], dtype='i4,i8,f8')[0]
aligned_tp = ndt.type('c{f0: int32, f1: int64, f2: float64}')
val = {'f0': 10, 'f1': 11, 'f2': 12}
# Construct using nd.array
b = nd.array(a)
self.assertEqual(nd.type_of(b), aligned_tp)
self.assertEqual(nd.as_py(b), val)
self.assertEqual(b.access_flags, 'immutable')
b = nd.array(a, access='rw')
self.assertEqual(nd.type_of(b), aligned_tp)
self.assertEqual(nd.as_py(b), val)
self.assertEqual(b.access_flags, 'readwrite')
# Construct using nd.asarray
b = nd.asarray(a)
self.assertEqual(nd.type_of(b), aligned_tp)
self.assertEqual(nd.as_py(b), val)
self.assertEqual(b.access_flags, 'immutable')
b = nd.asarray(a, access='rw')
self.assertEqual(nd.type_of(b), aligned_tp)
self.assertEqual(nd.as_py(b), val)
self.assertEqual(b.access_flags, 'readwrite')
# nd.view should fail
self.assertRaises(RuntimeError, nd.view, a)
def test_assignment_arrfunc(self):
af = _lowlevel.make_arrfunc_from_assignment(
ndt.float32, ndt.int64, "nocheck")
self.assertEqual(nd.as_py(af.proto), ndt.type("(int64) -> float32"))
a = nd.array(1234, type=ndt.int64)
b = af(a)
self.assertEqual(nd.type_of(b), ndt.float32)
self.assertEqual(nd.as_py(b), 1234)
# Instantiate as a strided kernel
with _lowlevel.ckernel.CKernelBuilder() as ckb:
meta = (ctypes.c_void_p * 2)()
ectx = nd.eval_context()
_lowlevel.arrfunc_instantiate(af, ckb, 0, ndt.float32, 0,
[ndt.int64], [0], "strided",
ectx._ectx_ptr)
ck = ckb.ckernel(_lowlevel.ExprStridedOperation)
# Do an assignment using ctypes
i64 = (ctypes.c_int64 * 3)()
for i, v in enumerate([3,7,21]):
i64[i] = v
pi64 = ctypes.pointer(i64)
i64_stride = c_ssize_t(8)
f32 = (ctypes.c_float * 3)()
ck(ctypes.addressof(f32), 4,
ctypes.addressof(pi64), ctypes.pointer(i64_stride),
3)
self.assertEqual([f32[i] for i in range(3)], [3,7,21])
def test_array_from_ptr(self):
# cfixed_dim arrmeta is redundant so this is ok
a = (ctypes.c_int32 * 3)()
a[0] = 3
a[1] = 6
a[2] = 9
# Readwrite version using cfixed
b = _lowlevel.array_from_ptr('cfixed[3] * int32',
ctypes.addressof(a), a, 'readwrite')
self.assertEqual(_lowlevel.data_address_of(b), ctypes.addressof(a))
self.assertEqual(nd.dshape_of(b), '3 * int32')
self.assertEqual(nd.as_py(b), [3, 6, 9])
b[1] = 10
self.assertEqual(a[1], 10)
# Readonly version using cfixed
b = _lowlevel.array_from_ptr('cfixed[3] * int32',
ctypes.addressof(a), a, 'readonly')
self.assertEqual(nd.as_py(b), [3, 10, 9])
def assign_to(b):
b[1] = 100
self.assertRaises(RuntimeError, assign_to, b)
# Using a fixed dim default-constructs the arrmeta, so works too
b = _lowlevel.array_from_ptr('3 * int32', ctypes.addressof(a),
a, 'readonly')
self.assertEqual(nd.as_py(b), [3, 10, 9])
# Should get an error if we try strided, because the size is unknown
self.assertRaises(RuntimeError,
lambda: _lowlevel.array_from_ptr('strided * int32',
ctypes.addressof(a),
a, 'readonly'))
def test_iter(self):
# Iteration of a 1D array
a = nd.array([1, 2, 3])
self.assertEqual([nd.as_py(x) for x in a], [1, 2, 3])
# Iteration of a 2D array
a = nd.array([[1, 2, 3], [4,5,6]])
self.assertEqual([nd.as_py(x) for x in a], [[1, 2, 3], [4,5,6]])
def test_lift_arrfunc(self):
# First get a ckernel from numpy
requiregil = False
af = _lowlevel.arrfunc_from_ufunc(np.ldexp,
(np.float64, np.float64, np.int32),
requiregil)
self.assertEqual(nd.as_py(af.proto),
ndt.type("(float64, int32) -> float64"))
# Now lift it
af_lifted = _lowlevel.lift_arrfunc(af)
self.assertEqual(nd.as_py(af_lifted.proto),
ndt.type("(Dims... * float64, Dims... * int32) -> Dims... * float64"))
# Create some compatible arguments
in0 = nd.array([[1, 2, 3], [4, 5], [6], [7,9,10]],
type='fixed * var * float64')
in1 = nd.array([[-1], [10], [100], [-12]], type='fixed * 1 * int32')
# Instantiate and call the kernel on these arguments
out = af_lifted(in0, in1)
# Verify that we got the expected result
self.assertEqual(nd.as_py(out),
[[0.5, 1.0, 1.5],
[4096.0, 5120.0],
[float(6*2**100)],
[0.001708984375, 0.002197265625, 0.00244140625]])
def test_dynamic_fromiter_onetype(self):
# Constructing with an iterator like this uses a dynamic
# array construction method. In this simple case, we
# use generators that have a consistent type
# bool result
a = nd.array(iter([True, False]))
self.assertEqual(nd.type_of(a), ndt.type('2 * bool'))
self.assertEqual(nd.as_py(a), [True, False])
# int32 result
a = nd.array(iter([1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('4 * int32'))
self.assertEqual(nd.as_py(a), [1, 2, 1, 0])
# int64 result
a = nd.array(iter([10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('5 * int64'))
self.assertEqual(nd.as_py(a), [10000000000, 1, 2, 1, 0])
# float64 result
a = nd.array(iter([3.25, 10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('6 * float64'))
self.assertEqual(nd.as_py(a), [3.25, 10000000000, 1, 2, 1, 0])
# complex[float64] result
a = nd.array(iter([3.25j, 3.25, 10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('7 * complex[float64]'))
self.assertEqual(nd.as_py(a), [3.25j, 3.25, 10000000000, 1, 2, 1, 0])
"""
def test_arrfunc_from_instantiate_pyfunc(self):
# Test wrapping make_assignment_ckernel as an arrfunc
def instantiate_assignment(out_ckb, ckb_offset, dst_tp, dst_arrmeta,
src_tp, src_arrmeta, kernreq, ectx):
out_ckb = _lowlevel.CKernelBuilderStruct.from_address(out_ckb)
return _lowlevel.make_assignment_ckernel(out_ckb, ckb_offset,
dst_tp, dst_arrmeta,
src_tp[0], src_arrmeta[0],
kernreq, ectx)
af = _lowlevel.arrfunc_from_instantiate_pyfunc(
instantiate_assignment, "(date) -> string")
self.assertEqual(nd.as_py(af.proto), ndt.type("(date) -> string"))
in0 = nd.array('2012-11-05', ndt.date)
out = af(in0)
self.assertEqual(nd.as_py(out), '2012-11-05')
# Also test it as a lifted kernel
af_lifted = _lowlevel.lift_arrfunc(af)
self.assertEqual(nd.as_py(af_lifted.proto),
ndt.type("(Dims... * date) -> Dims... * string"))
from datetime import date
in0 = nd.array([['2013-03-11', date(2010, 10, 10)],
[date(1999, 12, 31)],
[]], type='3 * var * date')
out = af_lifted(in0)
self.assertEqual(nd.as_py(out),
[['2013-03-11', '2010-10-10'],
['1999-12-31'], []])
def test_date_parse(self):
# By default, don't allow ambiguous year interpretations
self.assertRaises(ValueError, nd.array('01/02/03').cast('date').eval)
self.assertEqual(
nd.as_py(
nd.array('01/02/03').cast('date').eval(ectx=nd.eval_context(
date_parse_order='YMD'))), date(2001, 2, 3))
self.assertEqual(
nd.as_py(
nd.array('01/02/03').cast('date').eval(ectx=nd.eval_context(
date_parse_order='MDY'))), date(2003, 1, 2))
self.assertEqual(
nd.as_py(
nd.array('01/02/03').cast('date').eval(ectx=nd.eval_context(
date_parse_order='DMY'))), date(2003, 2, 1))
# Can also change the two year handling
# century_window of 0 means don't allow
self.assertRaises(ValueError,
nd.array('01/02/03').cast('date').eval,
ectx=nd.eval_context(date_parse_order='YMD',
century_window=0))
# century_window of 10 means a sliding window starting 10 years ago
self.assertEqual(
nd.as_py(
nd.array('01/02/03').cast('date').eval(ectx=nd.eval_context(
date_parse_order='YMD', century_window=10))),
date(2101, 2, 3))
# century_window of 1850 means a fixed window starting at 1850
self.assertEqual(
nd.as_py(
nd.array('01/02/03').cast('date').eval(ectx=nd.eval_context(
date_parse_order='YMD', century_window=1850))),
date(1901, 2, 3))
def test_date_parse(self):
# By default, don't allow ambiguous year interpretations
self.assertRaises(ValueError, nd.array('01/02/03').cast('date').eval)
self.assertEqual(nd.as_py(nd.array('01/02/03').cast('date').eval(
ectx=nd.eval_context(date_parse_order='YMD'))),
date(2001, 2, 3))
self.assertEqual(nd.as_py(nd.array('01/02/03').cast('date').eval(
ectx=nd.eval_context(date_parse_order='MDY'))),
date(2003, 1, 2))
self.assertEqual(nd.as_py(nd.array('01/02/03').cast('date').eval(
ectx=nd.eval_context(date_parse_order='DMY'))),
date(2003, 2, 1))
# Can also change the two year handling
# century_window of 0 means don't allow
self.assertRaises(ValueError, nd.array('01/02/03').cast('date').eval,
ectx=nd.eval_context(date_parse_order='YMD',
century_window=0))
# century_window of 10 means a sliding window starting 10 years ago
self.assertEqual(nd.as_py(nd.array('01/02/03').cast('date').eval(
ectx=nd.eval_context(date_parse_order='YMD',
century_window=10))),
date(2101, 2, 3))
# century_window of 1850 means a fixed window starting at 1850
self.assertEqual(nd.as_py(nd.array('01/02/03').cast('date').eval(
ectx=nd.eval_context(date_parse_order='YMD',
century_window=1850))),
date(1901, 2, 3))
def test_stack(stack_data):
descriptors = [CSV(fn, schema='2 * int32') for fn in sorted(stack_data)]
dd = Stack(descriptors)
assert dd.dshape == 3 * descriptors[0].dshape
expected = (((1, 1), (2, 2)),
((3, 3), (4, 4)),
((5, 5), (6, 6)))
assert tuplify(tuple(dd.as_py())) == expected
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
assert nd.as_py(result) == nd.as_py(expected2)
assert tuplify(tuple(dd)) == expected
assert tuplify(tuple(dd)) == expected # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
assert tuple(dd[[0, 2], 0, 0]) == (1, 5)
assert tuplify(tuple(dd[0])) == ((1, 1), (2, 2))
res = dd[0, :, [1]]
x = tuple(res)
assert tuplify(x) == ((1,), (2,))
assert tuplify(tuple(dd[0])) == expected[0]
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_filesystem(self):
with filetexts(data) as filenames:
dd = Files(sorted(filenames), CSV, subdshape='var * 2 * int32')
self.assertEqual(dd.filenames, ['a.csv', 'b.csv', 'c.csv'])
self.assertEqual(str(dd.schema), '2 * int32')
self.assertEqual(str(dd.dshape), 'var * 2 * int32')
expected = [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]]
self.assertEqual(dd.as_py(), expected)
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
self.assertEqual(nd.as_py(result),
nd.as_py(expected2))
self.assertEqual(list(dd), expected)
self.assertEqual(list(dd), expected) # Not one use only
chunks = list(dd.chunks(blen=3))
expected = [nd.array([[1, 1], [2, 2], [3, 3]], dtype='int32'),
nd.array([[4, 4], [5, 5], [6, 6]], dtype='int32')]
assert all(nd.as_py(a) == nd.as_py(b) for a, b in zip(chunks, expected))
def test_dynamic_fromiter_onetype(self):
# Constructing with an iterator like this uses a dynamic
# array construction method. In this simple case, we
# use generators that have a consistent type
# bool result
a = nd.array(iter([True, False]))
self.assertEqual(nd.type_of(a), ndt.type('2 * bool'))
self.assertEqual(nd.as_py(a), [True, False])
# int32 result
a = nd.array(iter([1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('4 * int32'))
self.assertEqual(nd.as_py(a), [1, 2, 1, 0])
# int64 result
a = nd.array(iter([10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('5 * int64'))
self.assertEqual(nd.as_py(a), [10000000000, 1, 2, 1, 0])
# float64 result
a = nd.array(iter([3.25, 10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('6 * float64'))
self.assertEqual(nd.as_py(a), [3.25, 10000000000, 1, 2, 1, 0])
# complex[float64] result
a = nd.array(iter([3.25j, 3.25, 10000000000, 1, 2, True, False]))
self.assertEqual(nd.type_of(a), ndt.type('7 * complex[float64]'))
self.assertEqual(nd.as_py(a), [3.25j, 3.25, 10000000000, 1, 2, 1, 0])
"""
def check_constructor(self, cons, value):
# Constructor from scalar type
a = cons(ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.int32)
self.assertEqual(nd.as_py(a), value)
# Constructor from type with fixed dimension
a = cons('3 * int32')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3,))
self.assertEqual(nd.as_py(a), [value]*3)
# Constructor from shape as single integer
a = cons(3, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * int32'))
self.assertEqual(a.shape, (3,))
self.assertEqual(nd.as_py(a), [value]*3)
# Constructor from shape as tuple
a = cons((3,4), ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3,4))
self.assertEqual(nd.as_py(a), [[value]*4]*3)
# Constructor from shape as variadic arguments
a = cons(3, 4, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3,4))
self.assertEqual(nd.as_py(a), [[value]*4]*3)
def test_nested_struct(self):
"""
# FIX ME
a = nd.array([[1,2], ['test', 3.5], [3j]],
type='{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float32]}')
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
"""
a = nd.array(
{
'x': [1, 2],
'y': {
'a': 'test',
'b': 3.5
},
'z': [3j]
},
type=
'{x: 2 * int16, y: {a: string, b: float64}, z: 1 * complex[float64]}'
)
self.assertEqual(nd.as_py(a.x), [1, 2])
self.assertEqual(nd.as_py(a.y.a), 'test')
self.assertEqual(nd.as_py(a.y.b), 3.5)
self.assertEqual(nd.as_py(a.z), [3j])
def test_Stack(self):
with filetexts(self.data) as filenames:
descriptors = [CSV(fn, schema='2 * int32')
for fn in sorted(filenames)]
dd = Stack(descriptors)
self.assertEqual(dd.dshape, 3 * descriptors[0].dshape)
expected = (((1, 1), (2, 2)),
((3, 3), (4, 4)),
((5, 5), (6, 6)))
self.assertEqual(tuplify(tuple(dd.as_py())), expected)
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
self.assertEqual(nd.as_py(result),
nd.as_py(expected2))
self.assertEqual(tuplify(tuple(dd)), expected)
self.assertEqual(tuplify(tuple(dd)), expected) # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
self.assertEqual(tuple(dd[[0, 2], 0, 0]), (1, 5))
self.assertEqual(tuplify(tuple(dd[0])), ((1, 1), (2, 2)))
self.assertEqual(tuplify(tuple(dd[0, :, [1]])), ((1,), (2,)))
self.assertEqual(tuplify(tuple(dd[0])), expected[0])
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_Concat(self):
with filetexts(self.data) as filenames:
descriptors = [CSV(fn, schema='2 * int32')
for fn in sorted(filenames)]
dd = Concat(descriptors)
self.assertEqual(str(dd.schema), '2 * int32')
self.assertEqual(str(dd.dshape), 'var * 2 * int32')
expected = ((1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7))
self.assertEqual(tuplify(tuple(dd)), expected)
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
self.assertEqual(nd.as_py(result),
nd.as_py(expected2))
self.assertEqual(tuplify(tuple(dd)), expected)
self.assertEqual(tuplify(tuple(dd)), expected) # Not one use only
chunks = list(dd.chunks())
assert all(isinstance(chunk, nd.array) for chunk in chunks)
self.assertEqual(tuple(dd[[0, 2], 0]), (1, 3))
self.assertEqual(tuple(dd[2, [1, 0]]), (3, 3))
assert isinstance(dd[:, 0], Iterator)
def test_stack(stack_data):
descriptors = [CSV(fn, schema='2 * int32') for fn in sorted(stack_data)]
dd = Stack(descriptors)
assert dd.dshape == 3 * descriptors[0].dshape
expected = (((1, 1), (2, 2)), ((3, 3), (4, 4)), ((5, 5), (6, 6)))
assert tuplify(tuple(dd.as_py())) == expected
result = dd.as_dynd()
expected2 = nd.array(expected, dtype='int32')
assert nd.as_py(result) == nd.as_py(expected2)
assert tuplify(tuple(dd)) == expected
assert tuplify(tuple(dd)) == expected # Not one use only
chunks = dd.chunks()
assert all(isinstance(chunk, nd.array) for chunk in chunks)
assert tuple(dd[[0, 2], 0, 0]) == (1, 5)
assert tuplify(tuple(dd[0])) == ((1, 1), (2, 2))
res = dd[0, :, [1]]
x = tuple(res)
assert tuplify(x) == ((1, ), (2, ))
assert tuplify(tuple(dd[0])) == expected[0]
assert isinstance(dd[:, 0], Iterator)
assert isinstance(dd[:], Iterator)
def test_ndt_type_from_h5py_special(self):
# h5py 2.3 style "special dtype"
dt = np.dtype(object, metadata={'vlen' : str})
self.assertEqual(ndt.type(dt), ndt.string)
if sys.version_info < (3, 0):
dt = np.dtype(object, metadata={'vlen' : unicode})
self.assertEqual(ndt.type(dt), ndt.string)
# h5py 2.2 style "special dtype"
dt = np.dtype(('O', [( ({'type': str},'vlen'), 'O' )] ))
self.assertEqual(ndt.type(dt), ndt.string)
if sys.version_info < (3, 0):
dt = np.dtype(('O', [( ({'type': unicode},'vlen'), 'O' )] ))
self.assertEqual(ndt.type(dt), ndt.string)
# Should be able to roundtrip dynd -> numpy -> dynd
x = nd.array(['testing', 'one', 'two'])
self.assertEqual(nd.type_of(x), ndt.type('3 * string'))
y = nd.as_numpy(x, allow_copy=True)
self.assertEqual(y.dtype.kind, 'O')
if sys.version_info < (3, 0):
self.assertEqual(y.dtype.metadata, {'vlen' : unicode})
else:
self.assertEqual(y.dtype.metadata, {'vlen' : str})
z = nd.array(y)
self.assertEqual(nd.type_of(z), nd.type_of(x))
self.assertEqual(nd.as_py(z), nd.as_py(x))
def test_readwrite_access_flags(self):
def assign_to(x, y):
x[0] = y
# Tests that read/write access control is preserved to/from numpy
a = np.arange(10.)
# Writeable
b = nd.view(a)
b[0] = 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Readonly view of writeable
b = nd.view(a, access='r')
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Not writeable
a.flags.writeable = False
b = nd.view(a)
self.assertRaises(RuntimeError, assign_to, b, 3.0)
# should still be 2.0
self.assertEqual(nd.as_py(b[0]), 2.0)
self.assertEqual(a[0], 2.0)
# Trying to get a readwrite view raises an error
self.assertRaises(RuntimeError, nd.view, a, access='rw')
def test_simple(self):
a = nd.asarray([1, 2, 3], access='rw')
self.assertEqual(nd.type_of(a), ndt.type('3 * int32'))
# Modifying 'a' should affect 'b', because it's a view
b = nd.asarray(a)
self.assertEqual(nd.as_py(b), [1, 2, 3])
a[1] = 10
self.assertEqual(nd.as_py(b), [1, 10, 3])
# Can take a readonly view, but still modify the original
b = nd.asarray(a, access='r')
self.assertEqual(nd.as_py(b), [1, 10, 3])
a[1] = 20
self.assertEqual(nd.as_py(b), [1, 20, 3])
# The readonly view we took can't be written to
def assign_at(x, i, y):
x[i] = y
self.assertRaises(RuntimeError, assign_at, b, 1, 30)
# Asking for immutable makes a copy instead of a view
b = nd.asarray(a, access='immutable')
self.assertEqual(nd.as_py(b), [1, 20, 3])
a[1] = 40
self.assertEqual(nd.as_py(b), [1, 20, 3])
# Asking for immutable from a non-immutable
# readonly array makes a copy
aprime = nd.asarray(a, access='r')
b = nd.asarray(aprime, access='immutable')
self.assertEqual(nd.as_py(aprime), [1, 40, 3])
self.assertEqual(nd.as_py(b), [1, 40, 3])
a[1] = 50
self.assertEqual(nd.as_py(aprime), [1, 50, 3])
self.assertEqual(nd.as_py(b), [1, 40, 3])
def test_struct(self):
a = nd.parse_json('{x:int32, y:string, z:float32}',
'{"x":20, "y":"testing one two three", "z":-3.25}')
self.assertEqual(nd.type_of(a),
ndt.type('{x:int32, y:string, z:float32}'))
self.assertEqual(nd.type_of(a[...]),
ndt.type('{x:int32, y:string, z:float32}'))
self.assertEqual(nd.type_of(a[0]), ndt.int32)
self.assertEqual(nd.type_of(a[1]), ndt.string)
self.assertEqual(nd.type_of(a[2]), ndt.float32)
self.assertEqual(nd.type_of(a[-3]), ndt.int32)
self.assertEqual(nd.type_of(a[-2]), ndt.string)
self.assertEqual(nd.type_of(a[-1]), ndt.float32)
self.assertEqual(
nd.type_of(a[1:]),
ndt.make_struct([ndt.string, ndt.float32], ['y', 'z']))
self.assertEqual(nd.type_of(a[::-2]),
ndt.make_struct([ndt.float32, ndt.int32], ['z', 'x']))
self.assertEqual(nd.as_py(a[0]), 20)
self.assertEqual(nd.as_py(a[1]), "testing one two three")
self.assertEqual(nd.as_py(a[2]), -3.25)
self.assertEqual(nd.as_py(a[1:]), {
'y': 'testing one two three',
'z': -3.25
})
self.assertEqual(nd.as_py(a[::-2]), {'x': 20, 'z': -3.25})
def test__type_from_h5py_special(self):
# h5py 2.3 style "special dtype"
dt = np.dtype(object, metadata={'vlen' : str})
self.assertEqual(ndt.type(dt), ndt.string)
if sys.version_info < (3, 0):
dt = np.dtype(object, metadata={'vlen' : unicode})
self.assertEqual(ndt.type(dt), ndt.string)
# h5py 2.2 style "special dtype"
dt = np.dtype(('O', [( ({'type': str},'vlen'), 'O' )] ))
self.assertEqual(ndt.type(dt), ndt.string)
if sys.version_info < (3, 0):
dt = np.dtype(('O', [( ({'type': unicode},'vlen'), 'O' )] ))
self.assertEqual(ndt.type(dt), ndt.string)
# Should be able to roundtrip dynd -> numpy -> dynd
x = nd.array(['testing', 'one', 'two'])
self.assertEqual(nd.type_of(x), ndt.type('3 * string'))
y = nd.as_numpy(x, allow_copy=True)
self.assertEqual(y.shape, (3,))
self.assertEqual(y[0], 'testing')
self.assertEqual(y[1], 'one')
self.assertEqual(y[2], 'two')
self.assertEqual(y.dtype.kind, 'O')
if sys.version_info < (3, 0):
self.assertEqual(y.dtype.metadata, {'vlen' : unicode})
else:
self.assertEqual(y.dtype.metadata, {'vlen' : str})
z = nd.array(y)
self.assertEqual(nd.type_of(z), nd.type_of(x))
self.assertEqual(nd.as_py(z), nd.as_py(x))
def test_object_in_struct_arr(self):
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', np.int64), ('y', object)])
self.assertEqual(nd.as_py(a), [{
'x': 1,
'y': 'test'
}, {
'x': 2,
'y': 'one'
}, {
'x': 3,
'y': 'two'
}])
a = nd.empty('3 * {x: int, y: string}')
a[...] = np.array([('opposite', 4)],
dtype=[('y', object), ('x', np.int64)])
self.assertEqual(nd.as_py(a), [{'x': 4, 'y': 'opposite'}] * 3)
a = nd.empty('var * {x: int, y: string}')
a[...] = np.array([(1, 'test'), (2, u'one'), (3.0, 'two')],
dtype=[('x', object), ('y', object)])
self.assertEqual(nd.as_py(a), [{
'x': 1,
'y': 'test'
}, {
'x': 2,
'y': 'one'
}, {
'x': 3,
'y': 'two'
}])
def test_lift_ckernel(self):
# First get a ckernel from numpy
requiregil = False
ckd = _lowlevel.ckernel_deferred_from_ufunc(np.ldexp,
(np.float64, np.float64, np.int32),
requiregil)
self.assertEqual(nd.as_py(ckd.types),
[ndt.float64, ndt.float64, ndt.int32])
# Now lift it
ckd_lifted = _lowlevel.lift_ckernel_deferred(ckd,
['var * var * float64', 'strided * var * float64',
'strided * 1 * int32'])
self.assertEqual(nd.as_py(ckd_lifted.types),
[ndt.type(x) for x in ['var * var * float64',
'strided * var * float64', 'strided * 1 * int32']])
# Create some compatible arguments
out = nd.empty('var * var * float64')
in0 = nd.array([[1, 2, 3], [4, 5], [6], [7,9,10]], type='strided * var * float64')
in1 = nd.array([[-1], [10], [100], [-12]], type='strided * 1 * int32')
# Instantiate and call the kernel on these arguments
ckd_lifted.__call__(out, in0, in1)
# Verify that we got the expected result
self.assertEqual(nd.as_py(out),
[[0.5, 1.0, 1.5],
[4096.0, 5120.0],
[float(6*2**100)],
[0.001708984375, 0.002197265625, 0.00244140625]])
def test_iter(self):
# Iteration of a 1D array
a = nd.array([1, 2, 3])
self.assertEqual([nd.as_py(x) for x in a], [1, 2, 3])
# Iteration of a 2D array
a = nd.array([[1, 2, 3], [4, 5, 6]])
self.assertEqual([nd.as_py(x) for x in a], [[1, 2, 3], [4, 5, 6]])
def test_ckernel_deferred_from_pyfunc(self):
# Test wrapping make_assignment_ckernel as a deferred ckernel
def instantiate_assignment(out_ckb, ckb_offset, types, meta,
kerntype, ectx):
out_ckb = _lowlevel.CKernelBuilderStruct.from_address(out_ckb)
return _lowlevel.make_assignment_ckernel(out_ckb, ckb_offset,
types[0], meta[0],
types[1], meta[1],
'expr', kerntype, ectx)
ckd = _lowlevel.ckernel_deferred_from_pyfunc(instantiate_assignment,
[ndt.string, ndt.date])
self.assertEqual(nd.as_py(ckd.types), [ndt.string, ndt.date])
out = nd.empty(ndt.string)
in0 = nd.array('2012-11-05', ndt.date)
ckd.__call__(out, in0)
self.assertEqual(nd.as_py(out), '2012-11-05')
# Also test it as a lifted kernel
ckd_lifted = _lowlevel.lift_ckernel_deferred(ckd,
['3 * var * string', '3 * var * date'])
self.assertEqual(nd.as_py(ckd_lifted.types),
[ndt.type('3 * var * string'), ndt.type('3 * var * date')])
out = nd.empty('3 * var * string')
from datetime import date
in0 = nd.array([['2013-03-11', date(2010, 10, 10)],
[date(1999, 12, 31)],
[]], type='3 * var * date')
ckd_lifted.__call__(out, in0)
self.assertEqual(nd.as_py(out),
[['2013-03-11', '2010-10-10'],
['1999-12-31'], []])
def test_dynd(self):
self.assertEqual(nd.as_py(into(nd.array(), (1, 2, 3))),
nd.as_py(nd.array([1, 2, 3])))
self.assertEqual(into([], nd.array([1, 2])),
[1, 2])
self.assertEqual(into([], nd.array([[1, 2], [3, 4]])),
[[1, 2], [3, 4]])
def check_constructor(self, cons, value):
# Constructor from scalar type
a = cons(ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.int32)
self.assertEqual(nd.as_py(a), value)
# Constructor from type with fixed dimension
a = cons('3 * int32')
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32))
self.assertEqual(a.shape, (3, ))
self.assertEqual(nd.as_py(a), [value] * 3)
# Constructor from shape as single integer
a = cons(3, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * int32'))
self.assertEqual(a.shape, (3, ))
self.assertEqual(nd.as_py(a), [value] * 3)
# Constructor from shape as tuple
a = cons((3, 4), ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3, 4))
self.assertEqual(nd.as_py(a), [[value] * 4] * 3)
# Constructor from shape as variadic arguments
a = cons(3, 4, ndt.int32)
self.assertEqual(a.access_flags, 'readwrite')
self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32'))
self.assertEqual(a.shape, (3, 4))
self.assertEqual(nd.as_py(a), [[value] * 4] * 3)
def test_unicode_array(self):
a = nd.array([u'this', 'is', u'a', 'test'], dtype=ndt.string)
self.assertEqual(nd.type_of(a), ndt.type('4 * string'))
self.assertEqual(nd.as_py(a), ['this', 'is', 'a', 'test'])
a = nd.array([u'this', 'is', u'a', 'test'], dtype='string["U16"]')
self.assertEqual(nd.type_of(a), ndt.type('4 * string["U16"]'))
self.assertEqual(nd.as_py(a), ['this', 'is', 'a', 'test'])
def test_date(self):
lst = [date(2011, 3, 15), date(1933, 12, 25), date(1979, 3, 22)]
lststr = ['2011-03-15', '1933-12-25', '1979-03-22']
a = nd.array(lst)
self.assertEqual(nd.dtype_of(a), ndt.date)
self.assertEqual(a.shape, (3,))
self.assertEqual(nd.as_py(a), lst)
self.assertEqual(nd.as_py(a.ucast(ndt.string)), lststr)
def test_simple(self):
# Default is a count of 50. For these simple cases of integers,
# the result should be exact
self.assertEqual(nd.as_py(nd.linspace(0, 49)), list(range(50)))
self.assertEqual(nd.as_py(nd.linspace(49, 0)), list(range(49, -1, -1)))
self.assertEqual(nd.as_py(nd.linspace(0, 10, count=11)), list(range(11)))
self.assertEqual(nd.as_py(nd.linspace(1, -1, count=2)), [1, -1])
self.assertEqual(nd.as_py(nd.linspace(1j, 50j)), [i*1j for i in range(1, 51)])
def test_extend(self):
dd = HDF5(self.filename, '/data', 'a', schema='2 * int32')
dd.extend([(1, 1), (2, 2)])
results = list(dd)
self.assertEquals(nd.as_py(results[0]), [1, 1])
self.assertEquals(nd.as_py(results[1]), [2, 2])
def test_date(self):
lst = [date(2011, 3, 15), date(1933, 12, 25), date(1979, 3, 22)]
lststr = ['2011-03-15', '1933-12-25', '1979-03-22']
a = nd.array(lst)
self.assertEqual(nd.dtype_of(a), ndt.date)
self.assertEqual(a.shape, (3, ))
self.assertEqual(nd.as_py(a), lst)
self.assertEqual(nd.as_py(a.ucast(ndt.string)), lststr)
def test_unicode_array(self):
a = nd.array([u"this", "is", u"a", "test"], dtype=ndt.string)
self.assertEqual(nd.type_of(a), ndt.type("strided * string"))
self.assertEqual(nd.as_py(a), ["this", "is", "a", "test"])
a = nd.array([u"this", "is", u"a", "test"], dtype='string["U16"]')
self.assertEqual(nd.type_of(a), ndt.type('strided * string["U16"]'))
self.assertEqual(nd.as_py(a), ["this", "is", "a", "test"])
def test_extend(self):
dd = HDF5(self.filename, '/data', 'a', schema='2 * int32')
dd.extend([(1, 1), (2, 2)])
results = list(dd)
self.assertEquals(nd.as_py(results[0]), [1, 1])
self.assertEquals(nd.as_py(results[1]), [2, 2])
def test_struct_via_pep3118(self):
# Aligned struct
a = nd.array([[1, 2], [3, 4]], type='2 * {x : int32, y: int64}')
b = np.asarray(a)
self.assertEqual(b.dtype,
np.dtype([('x', np.int32), ('y', np.int64)], align=True))
self.assertEqual(nd.as_py(a.x), b['x'].tolist())
self.assertEqual(nd.as_py(a.y), b['y'].tolist())
def test_json_date_parse(self):
a = nd.parse_json('var * date', '["2012-03-17", "1922-12-30"]')
self.assertEqual(nd.as_py(a), [date(2012, 3, 17), date(1922, 12, 30)])
self.assertRaises(ValueError, nd.parse_json, 'var * date',
'["2012-03-17T17:00:15-0600", "1922-12-30 Thursday"]')
a = nd.parse_json('var * date',
'["2012-06-17T17:00:15-0600", "1921-12-30 Thursday"]',
ectx=nd.eval_context(errmode='nocheck'))
self.assertEqual(nd.as_py(a), [date(2012, 6, 17), date(1921, 12, 30)])
def test_string(self):
a = nd.array('abcdef')
self.assertEqual(nd.type_of(a), ndt.string)
self.assertEqual(type(nd.as_py(a)), unicode)
self.assertEqual(nd.as_py(a), u'abcdef')
a = nd.array(u'abcdef')
self.assertEqual(nd.type_of(a), ndt.string)
self.assertEqual(type(nd.as_py(a)), unicode)
self.assertEqual(nd.as_py(a), u'abcdef')
def test_simple(self):
self.assertEqual(nd.as_py(nd.old_range(10)), list(range(10)))
self.assertEqual(nd.as_py(nd.old_range(5, 10)), list(range(5, 10)))
self.assertEqual(nd.as_py(nd.old_range(5, 10, 3)),
list(range(5, 10, 3)))
self.assertEqual(nd.as_py(nd.old_range(10, 5, -1)),
list(range(10, 5, -1)))
self.assertEqual(nd.as_py(nd.old_range(stop=10, step=2)),
list(range(0, 10, 2)))
def test_strftime(self):
a = nd.array(date(1955,3,13))
self.assertEqual(nd.as_py(a.strftime('%Y')), '1955')
self.assertEqual(nd.as_py(a.strftime('%m/%d/%y')), '03/13/55')
self.assertEqual(nd.as_py(a.strftime('%Y and %j')), '1955 and 072')
a = nd.array([date(1931,12,12), date(2013,5,14), date(2012,12,25)])
self.assertEqual(nd.as_py(a.strftime('%Y-%m-%d %j %U %w %W')),
['1931-12-12 346 49 6 49', '2013-05-14 134 19 2 19', '2012-12-25 360 52 2 52'])
def test_iter_fixed_dim(self):
# Iteration of a 1D array
a = nd.array([1, 2, 3], type='3 * int64')
self.assertEqual(len(a), 3)
self.assertEqual([nd.as_py(x) for x in a], [1, 2, 3])
# Iteration of a 2D array
a = nd.array([[1, 2, 3], [4, 5, 6]], type='2 * 3 * int64')
self.assertEqual(len(a), 2)
self.assertEqual([nd.as_py(x) for x in a], [[1, 2, 3], [4, 5, 6]])
def test_iter_fixed_dim(self):
# Iteration of a 1D array
a = nd.array([1, 2, 3], type='3 * int64')
self.assertEqual(len(a), 3)
self.assertEqual([nd.as_py(x) for x in a], [1, 2, 3])
# Iteration of a 2D array
a = nd.array([[1, 2, 3], [4,5,6]], type='2 * 3 * int64')
self.assertEqual(len(a), 2)
self.assertEqual([nd.as_py(x) for x in a], [[1, 2, 3], [4,5,6]])
def test_date_properties(self):
a = nd.array(date(1955,3,13))
self.assertEqual(str(a), '1955-03-13')
self.assertEqual(nd.dtype_of(a.year.eval()), ndt.int32)
self.assertEqual(nd.dtype_of(a.month.eval()), ndt.int32)
self.assertEqual(nd.dtype_of(a.day.eval()), ndt.int32)
self.assertEqual(nd.as_py(a.year), 1955)
self.assertEqual(nd.as_py(a.month), 3)
self.assertEqual(nd.as_py(a.day), 13)
def test_json_date_parse(self):
a = nd.parse_json('var * date', '["2012-03-17", "1922-12-30"]')
self.assertEqual(nd.as_py(a), [date(2012, 3, 17), date(1922, 12, 30)])
self.assertRaises(ValueError, nd.parse_json, 'var * date',
'["2012-03-17T17:00:15-0600", "1922-12-30 Thursday"]')
a = nd.parse_json('var * date',
'["2012-06-17T17:00:15-0600", "1921-12-30 Thursday"]',
ectx=nd.eval_context(errmode='nocheck'))
self.assertEqual(nd.as_py(a), [date(2012, 6, 17), date(1921, 12, 30)])
def test_dynd_scalar_array(self):
a = np.array(3, dtype='int64')
n = nd.array(a)
self.assertEqual(nd.type_of(n), ndt.int64)
self.assertEqual(nd.as_py(n), 3)
self.assertEqual(n.access_flags, 'readwrite')
# Ensure it's not a view
a[...] = 4
self.assertEqual(nd.as_py(n), 3)
def test_strftime(self):
a = nd.array(date(1955,3,13))
self.assertEqual(nd.as_py(a.strftime('%Y')), '1955')
self.assertEqual(nd.as_py(a.strftime('%m/%d/%y')), '03/13/55')
self.assertEqual(nd.as_py(a.strftime('%Y and %j')), '1955 and 072')
a = nd.array([date(1931,12,12), date(2013,5,14), date(2012,12,25)])
self.assertEqual(nd.as_py(a.strftime('%Y-%m-%d %j %U %w %W')),
['1931-12-12 346 49 6 49', '2013-05-14 134 19 2 19', '2012-12-25 360 52 2 52'])
def test_dynd_scalar_array(self):
a = np.array(3, dtype='int64')
n = nd.array(a)
self.assertEqual(nd.type_of(n), ndt.int64)
self.assertEqual(nd.as_py(n), 3)
self.assertEqual(n.access_flags, 'immutable')
# Ensure it's not a view
a[...] = 4
self.assertEqual(nd.as_py(n), 3)
def test_string(self):
a = nd.array('abcdef')
self.assertEqual(nd.type_of(a), ndt.string)
self.assertEqual(type(nd.as_py(a)), unicode)
self.assertEqual(nd.as_py(a), u'abcdef')
a = nd.array(u'abcdef')
self.assertEqual(nd.type_of(a), ndt.string)
self.assertEqual(type(nd.as_py(a)), unicode)
self.assertEqual(nd.as_py(a), u'abcdef')
def test_complex_type_realimag(self):
a = nd.array(1 + 3j)
self.assertEqual(ndt.complex_float64, nd.type_of(a))
self.assertEqual(1, nd.as_py(a.real))
self.assertEqual(3, nd.as_py(a.imag))
a = nd.array([1 + 2j, 3 + 4j, 5 + 6j])
self.assertEqual(ndt.type('3 * complex[float64]'), nd.type_of(a))
self.assertEqual([1, 3, 5], nd.as_py(a.real))
self.assertEqual([2, 4, 6], nd.as_py(a.imag))
def test_date_properties(self):
a = nd.array(date(1955, 3, 13))
self.assertEqual(str(a), '1955-03-13')
self.assertEqual(nd.dtype_of(a.year.eval()), ndt.int32)
self.assertEqual(nd.dtype_of(a.month.eval()), ndt.int32)
self.assertEqual(nd.dtype_of(a.day.eval()), ndt.int32)
self.assertEqual(nd.as_py(a.year), 1955)
self.assertEqual(nd.as_py(a.month), 3)
self.assertEqual(nd.as_py(a.day), 13)
self.assertEqual(nd.as_py(a), date(1955, 3, 13))
def test_bool(self):
# Boolean true/false
a = nd.array(True)
self.assertEqual(nd.type_of(a), ndt.bool)
self.assertEqual(type(nd.as_py(a)), bool)
self.assertEqual(nd.as_py(a), True)
a = nd.array(False)
self.assertEqual(nd.type_of(a), ndt.bool)
self.assertEqual(type(nd.as_py(a)), bool)
self.assertEqual(nd.as_py(a), False)
def test_simple_strided(self):
a = nd.empty('3 * int32')
a[...] = np.int64(1)
self.assertEqual(nd.as_py(a), [1] * 3)
a[...] = np.array(2.0)
self.assertEqual(nd.as_py(a), [2] * 3)
a[...] = np.array([3], dtype=np.int8)
self.assertEqual(nd.as_py(a), [3] * 3)
a[...] = np.array([1, 2, 3])
self.assertEqual(nd.as_py(a), [1, 2, 3])
def test_float_step(self):
# Should produce the correct count for 1.0/int steps
for i in range(1, 32):
a = nd.range(1.0, step=1.0/i)
self.assertEqual(len(a), i)
self.assertEqual(nd.as_py(a[0]), 0)
# For powers of two, should be getting exact answers
for i in range(5):
a = nd.range(1.0, step=1.0/2**i)
self.assertEqual(nd.as_py(a), [float(x)/2**i for x in range(2**i)])
def test_struct_function(self):
a = nd.array(date(1955, 3, 13))
s = a.to_struct().eval()
self.assertEqual(
nd.dtype_of(s),
ndt.make_struct([ndt.int16, ndt.int8, ndt.int8],
['year', 'month', 'day']))
self.assertEqual(nd.as_py(s.year), 1955)
self.assertEqual(nd.as_py(s.month), 3)
self.assertEqual(nd.as_py(s.day), 13)
def test_struct_function(self):
a = nd.array(time(13, 25, 8, 765432))
s = a.to_struct().eval()
self.assertEqual(
nd.dtype_of(s),
ndt.make_struct([ndt.int8, ndt.int8, ndt.int8, ndt.int32],
['hour', 'minute', 'second', 'tick']))
self.assertEqual(nd.as_py(s.hour), 13)
self.assertEqual(nd.as_py(s.minute), 25)
self.assertEqual(nd.as_py(s.second), 8)
self.assertEqual(nd.as_py(s.tick), 7654320)
def test_inf(self):
# Validate nd.inf
self.assertEqual(nd.inf * 2, nd.inf)
self.assertTrue(nd.inf > 0)
self.assertTrue(-nd.inf < 0)
# as an array
a = nd.array(nd.inf)
self.assertEqual(nd.as_py(a), nd.inf)
self.assertEqual(nd.type_of(a), ndt.float64)
a = nd.array(nd.inf, type=ndt.float32)
self.assertEqual(nd.as_py(a), nd.inf)
