def test_single_struct_array(self): a = nd.array([(0,0), (3,5), (12,10)], dtype='{x:int32, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:int32, y:int32}')) self.assertEqual(nd.as_py(a.x), [0, 3, 12]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array([{'x':0,'y':0}, {'x':3,'y':5}, {'x':12,'y':10}], dtype='{x:int32, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:int32, y:int32}')) self.assertEqual(nd.as_py(a.x), [0, 3, 12]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array([[(3, 'X')], [(10, 'L'), (12, 'M')]], dtype='{count:int32, size:fixed_string[1,"A"]}') self.assertEqual(nd.type_of(a), ndt.type('2 * var * {count:int32, size:fixed_string[1,"A"]}')) self.assertEqual(nd.as_py(a.count), [[3], [10, 12]]) self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']]) a = nd.array([[{'count':3, 'size':'X'}], [{'count':10, 'size':'L'}, {'count':12, 'size':'M'}]], dtype='{count:int32, size:fixed_string[1,"A"]}') self.assertEqual(nd.type_of(a), ndt.type('2 * var * {count:int32, size:fixed_string[1,"A"]}')) self.assertEqual(nd.as_py(a.count), [[3], [10, 12]]) self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']])
def test_struct(self): a = nd.parse_json('{x:int32, y:string, z:float32}', '{"x":20, "y":"testing one two three", "z":-3.25}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:float32}')) self.assertEqual(nd.type_of(a[...]), ndt.type('{x:int32, y:string, z:float32}')) self.assertEqual(nd.type_of(a[0]), ndt.int32) self.assertEqual(nd.type_of(a[1]), ndt.string) self.assertEqual(nd.type_of(a[2]), ndt.float32) self.assertEqual(nd.type_of(a[-3]), ndt.int32) self.assertEqual(nd.type_of(a[-2]), ndt.string) self.assertEqual(nd.type_of(a[-1]), ndt.float32) self.assertEqual( nd.type_of(a[1:]), ndt.make_struct([ndt.string, ndt.float32], ['y', 'z'])) self.assertEqual(nd.type_of(a[::-2]), ndt.make_struct([ndt.float32, ndt.int32], ['z', 'x'])) self.assertEqual(nd.as_py(a[0]), 20) self.assertEqual(nd.as_py(a[1]), "testing one two three") self.assertEqual(nd.as_py(a[2]), -3.25) self.assertEqual(nd.as_py(a[1:]), { 'y': 'testing one two three', 'z': -3.25 }) self.assertEqual(nd.as_py(a[::-2]), {'x': 20, 'z': -3.25})
def test__type_from_h5py_special(self): # h5py 2.3 style "special dtype" dt = np.dtype(object, metadata={'vlen' : str}) self.assertEqual(ndt.type(dt), ndt.string) if sys.version_info < (3, 0): dt = np.dtype(object, metadata={'vlen' : unicode}) self.assertEqual(ndt.type(dt), ndt.string) # h5py 2.2 style "special dtype" dt = np.dtype(('O', [( ({'type': str},'vlen'), 'O' )] )) self.assertEqual(ndt.type(dt), ndt.string) if sys.version_info < (3, 0): dt = np.dtype(('O', [( ({'type': unicode},'vlen'), 'O' )] )) self.assertEqual(ndt.type(dt), ndt.string) # Should be able to roundtrip dynd -> numpy -> dynd x = nd.array(['testing', 'one', 'two']) self.assertEqual(nd.type_of(x), ndt.type('3 * string')) y = nd.as_numpy(x, allow_copy=True) self.assertEqual(y.shape, (3,)) self.assertEqual(y[0], 'testing') self.assertEqual(y[1], 'one') self.assertEqual(y[2], 'two') self.assertEqual(y.dtype.kind, 'O') if sys.version_info < (3, 0): self.assertEqual(y.dtype.metadata, {'vlen' : unicode}) else: self.assertEqual(y.dtype.metadata, {'vlen' : str}) z = nd.array(y) self.assertEqual(nd.type_of(z), nd.type_of(x)) self.assertEqual(nd.as_py(z), nd.as_py(x))
def test_ndt_type_from_h5py_special(self): # h5py 2.3 style "special dtype" dt = np.dtype(object, metadata={'vlen' : str}) self.assertEqual(ndt.type(dt), ndt.string) if sys.version_info < (3, 0): dt = np.dtype(object, metadata={'vlen' : unicode}) self.assertEqual(ndt.type(dt), ndt.string) # h5py 2.2 style "special dtype" dt = np.dtype(('O', [( ({'type': str},'vlen'), 'O' )] )) self.assertEqual(ndt.type(dt), ndt.string) if sys.version_info < (3, 0): dt = np.dtype(('O', [( ({'type': unicode},'vlen'), 'O' )] )) self.assertEqual(ndt.type(dt), ndt.string) # Should be able to roundtrip dynd -> numpy -> dynd x = nd.array(['testing', 'one', 'two']) self.assertEqual(nd.type_of(x), ndt.type('3 * string')) y = nd.as_numpy(x, allow_copy=True) self.assertEqual(y.dtype.kind, 'O') if sys.version_info < (3, 0): self.assertEqual(y.dtype.metadata, {'vlen' : unicode}) else: self.assertEqual(y.dtype.metadata, {'vlen' : str}) z = nd.array(y) self.assertEqual(nd.type_of(z), nd.type_of(x)) self.assertEqual(nd.as_py(z), nd.as_py(x))
def test_empty(self): # Constructor from scalar type a = nd.empty(ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.int32) # Constructor from type with fixed dimension a = nd.empty('3 * int32') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32)) self.assertEqual(a.shape, (3,)) # Constructor from shape as single integer a = nd.empty(3, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_strided_dim(ndt.int32)) self.assertEqual(a.shape, (3,)) # Constructor from shape as tuple a = nd.empty((3,4), ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('strided * strided * int32')) self.assertEqual(a.shape, (3,4)) # Constructor from shape as variadic arguments a = nd.empty(3, 4, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('strided * strided * int32')) self.assertEqual(a.shape, (3,4))
def test_dynamic_fromiter_onetype(self): # Constructing with an iterator like this uses a dynamic # array construction method. In this simple case, we # use generators that have a consistent type # bool result a = nd.array(iter([True, False])) self.assertEqual(nd.type_of(a), ndt.type('2 * bool')) self.assertEqual(nd.as_py(a), [True, False]) # int32 result a = nd.array(iter([1, 2, True, False])) self.assertEqual(nd.type_of(a), ndt.type('4 * int32')) self.assertEqual(nd.as_py(a), [1, 2, 1, 0]) # int64 result a = nd.array(iter([10000000000, 1, 2, True, False])) self.assertEqual(nd.type_of(a), ndt.type('5 * int64')) self.assertEqual(nd.as_py(a), [10000000000, 1, 2, 1, 0]) # float64 result a = nd.array(iter([3.25, 10000000000, 1, 2, True, False])) self.assertEqual(nd.type_of(a), ndt.type('6 * float64')) self.assertEqual(nd.as_py(a), [3.25, 10000000000, 1, 2, 1, 0]) # complex[float64] result a = nd.array(iter([3.25j, 3.25, 10000000000, 1, 2, True, False])) self.assertEqual(nd.type_of(a), ndt.type('7 * complex[float64]')) self.assertEqual(nd.as_py(a), [3.25j, 3.25, 10000000000, 1, 2, 1, 0]) """
def test_numpy_struct_scalar(self): # Create a NumPy struct scalar object, by indexing into # a structured array a = np.array([(10, 11, 12)], dtype='i4,i8,f8')[0] aligned_tp = ndt.type('c{f0: int32, f1: int64, f2: float64}') val = {'f0': 10, 'f1': 11, 'f2': 12} # Construct using nd.array b = nd.array(a) self.assertEqual(nd.type_of(b), aligned_tp) self.assertEqual(nd.as_py(b), val) self.assertEqual(b.access_flags, 'immutable') b = nd.array(a, access='rw') self.assertEqual(nd.type_of(b), aligned_tp) self.assertEqual(nd.as_py(b), val) self.assertEqual(b.access_flags, 'readwrite') # Construct using nd.asarray b = nd.asarray(a) self.assertEqual(nd.type_of(b), aligned_tp) self.assertEqual(nd.as_py(b), val) self.assertEqual(b.access_flags, 'immutable') b = nd.asarray(a, access='rw') self.assertEqual(nd.type_of(b), aligned_tp) self.assertEqual(nd.as_py(b), val) self.assertEqual(b.access_flags, 'readwrite') # nd.view should fail self.assertRaises(RuntimeError, nd.view, a)
def test_single_struct(self): a = nd.array([12, 'test', True], type='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True) # With dtype= parameter instead of type= a = nd.array([12, 'test', True], dtype='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True) a = nd.array({'x':12, 'y':'test', 'z':True}, type='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True) # With dtype= parameter instead of type= a = nd.array({'x':12, 'y':'test', 'z':True}, dtype='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True)
def check_constructor(self, cons, value): # Constructor from scalar type a = cons(ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.int32) self.assertEqual(nd.as_py(a), value) # Constructor from type with fixed dimension a = cons('3 * int32') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32)) self.assertEqual(a.shape, (3,)) self.assertEqual(nd.as_py(a), [value]*3) # Constructor from shape as single integer a = cons(3, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) self.assertEqual(a.shape, (3,)) self.assertEqual(nd.as_py(a), [value]*3) # Constructor from shape as tuple a = cons((3,4), ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3,4)) self.assertEqual(nd.as_py(a), [[value]*4]*3) # Constructor from shape as variadic arguments a = cons(3, 4, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3,4)) self.assertEqual(nd.as_py(a), [[value]*4]*3)
def test_single_struct(self): a = nd.array([12, "test", True], type="{x:int32, y:string, z:bool}") self.assertEqual(nd.type_of(a), ndt.type("{x:int32, y:string, z:bool}")) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), "test") self.assertEqual(nd.as_py(a[2]), True) # With dtype= parameter instead of type= a = nd.array([12, "test", True], dtype="{x:int32, y:string, z:bool}") self.assertEqual(nd.type_of(a), ndt.type("{x:int32, y:string, z:bool}")) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), "test") self.assertEqual(nd.as_py(a[2]), True) a = nd.array({"x": 12, "y": "test", "z": True}, type="{x:int32, y:string, z:bool}") self.assertEqual(nd.type_of(a), ndt.type("{x:int32, y:string, z:bool}")) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), "test") self.assertEqual(nd.as_py(a[2]), True) # With dtype= parameter instead of type= a = nd.array({"x": 12, "y": "test", "z": True}, dtype="{x:int32, y:string, z:bool}") self.assertEqual(nd.type_of(a), ndt.type("{x:int32, y:string, z:bool}")) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), "test") self.assertEqual(nd.as_py(a[2]), True)
def check_constructor(self, cons, value): # Constructor from scalar type a = cons(ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.int32) self.assertEqual(nd.as_py(a), value) # Constructor from type with fixed dimension a = cons('3 * int32') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32)) self.assertEqual(a.shape, (3, )) self.assertEqual(nd.as_py(a), [value] * 3) # Constructor from shape as single integer a = cons(3, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) self.assertEqual(a.shape, (3, )) self.assertEqual(nd.as_py(a), [value] * 3) # Constructor from shape as tuple a = cons((3, 4), ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3, 4)) self.assertEqual(nd.as_py(a), [[value] * 4] * 3) # Constructor from shape as variadic arguments a = cons(3, 4, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3, 4)) self.assertEqual(nd.as_py(a), [[value] * 4] * 3)
def test_empty_array(self): # Empty arrays default to int32 a = nd.array([]) self.assertEqual(nd.type_of(a), ndt.type('0 * int32')) self.assertEqual(a.shape, (0,)) a = nd.array([[], [], []]) self.assertEqual(nd.type_of(a), ndt.type('3 * 0 * int32')) self.assertEqual(a.shape, (3, 0))
def test_unicode_array(self): a = nd.array([u'this', 'is', u'a', 'test'], dtype=ndt.string) self.assertEqual(nd.type_of(a), ndt.type('4 * string')) self.assertEqual(nd.as_py(a), ['this', 'is', 'a', 'test']) a = nd.array([u'this', 'is', u'a', 'test'], dtype='string["U16"]') self.assertEqual(nd.type_of(a), ndt.type('4 * string["U16"]')) self.assertEqual(nd.as_py(a), ['this', 'is', 'a', 'test'])
def test_unicode_array(self): a = nd.array([u"this", "is", u"a", "test"], dtype=ndt.string) self.assertEqual(nd.type_of(a), ndt.type("strided * string")) self.assertEqual(nd.as_py(a), ["this", "is", "a", "test"]) a = nd.array([u"this", "is", u"a", "test"], dtype='string["U16"]') self.assertEqual(nd.type_of(a), ndt.type('strided * string["U16"]')) self.assertEqual(nd.as_py(a), ["this", "is", "a", "test"])
def test_empty_array(self): # Empty arrays default to int32 a = nd.array([]) self.assertEqual(nd.type_of(a), ndt.type('0 * int32')) self.assertEqual(a.shape, (0, )) a = nd.array([[], [], []]) self.assertEqual(nd.type_of(a), ndt.type('3 * 0 * int32')) self.assertEqual(a.shape, (3, 0))
def test_string(self): a = nd.array('abcdef') self.assertEqual(nd.type_of(a), ndt.string) self.assertEqual(type(nd.as_py(a)), unicode) self.assertEqual(nd.as_py(a), u'abcdef') a = nd.array(u'abcdef') self.assertEqual(nd.type_of(a), ndt.string) self.assertEqual(type(nd.as_py(a)), unicode) self.assertEqual(nd.as_py(a), u'abcdef')
def test_complex_type_realimag(self): a = nd.array(1 + 3j) self.assertEqual(ndt.complex_float64, nd.type_of(a)) self.assertEqual(1, nd.as_py(a.real)) self.assertEqual(3, nd.as_py(a.imag)) a = nd.array([1 + 2j, 3 + 4j, 5 + 6j]) self.assertEqual(ndt.type('3 * complex[float64]'), nd.type_of(a)) self.assertEqual([1, 3, 5], nd.as_py(a.real)) self.assertEqual([2, 4, 6], nd.as_py(a.imag))
def test_ragged_fromlistofiter_typepromo(self): # list of iterators vals = [[True, False], [False, 2, 3], [-10000000000], [True, 10, 3.125, 5.5j]] a = nd.array([iter(x) for x in vals]) self.assertEqual(nd.type_of(a), ndt.type("strided * var * complex[float64]")) self.assertEqual(nd.as_py(a), vals) # list of list/iterator a = nd.array([[1, 2, 3], (1.5 * x for x in range(4)), iter([-1, 1])]) self.assertEqual(nd.type_of(a), ndt.type("strided * var * float64")) self.assertEqual(nd.as_py(a), [[1, 2, 3], [1.5 * x for x in range(4)], [-1, 1]])
def test_bool(self): # Boolean true/false a = nd.array(True) self.assertEqual(nd.type_of(a), ndt.bool) self.assertEqual(type(nd.as_py(a)), bool) self.assertEqual(nd.as_py(a), True) a = nd.array(False) self.assertEqual(nd.type_of(a), ndt.bool) self.assertEqual(type(nd.as_py(a)), bool) self.assertEqual(nd.as_py(a), False)
def test_ragged_fromlistofiter_typepromo(self): # list of iterators vals = [[True, False], [False, 2, 3], [-10000000000], [True, 10, 3.125, 5.5j]] a = nd.array([iter(x) for x in vals]) self.assertEqual(nd.type_of(a), ndt.type('4 * var * complex[float64]')) self.assertEqual(nd.as_py(a), vals) # list of list/iterator a = nd.array([[1, 2, 3], (1.5 * x for x in range(4)), iter([-1, 1])]) self.assertEqual(nd.type_of(a), ndt.type('3 * var * float64')) self.assertEqual(nd.as_py(a), [[1, 2, 3], [1.5 * x for x in range(4)], [-1, 1]])
def test_simple_expr(self): a = np.array([(1,), (2,), (3,), (4,), (5,)], dtype=[('xyz', np.int32)]) b = nd.add_computed_fields(a, [('twice', np.int32, '2 * xyz'), ('onemore', np.int16, 'xyz + 1')]) self.assertEqual(nd.type_of(b).element_type.type_id, 'unary_expr') self.assertEqual(nd.type_of(b).element_type.value_type, ndt.type('c{xyz: int32, twice: int32, onemore: int16}')) self.assertEqual(nd.as_py(b.xyz), [1, 2, 3, 4, 5]) self.assertEqual(nd.as_py(b.twice), [2, 4, 6, 8, 10]) self.assertEqual(nd.as_py(b.onemore), [2, 3, 4, 5, 6])
def test_ragged_fromiter(self): # Strided array of var from list of iterators a = nd.array([(1 + x for x in range(3)), (5 * x - 10 for x in range(5)), [2, 10]], type="strided * var * int32") self.assertEqual(nd.type_of(a), ndt.type("strided * var * int32")) self.assertEqual(nd.as_py(a), [[1, 2, 3], [-10, -5, 0, 5, 10], [2, 10]]) # Var array of var from iterator of iterators a = nd.array(((2 * x for x in range(y)) for y in range(4)), type="var * var * int32") self.assertEqual(nd.type_of(a), ndt.type("var * var * int32")) self.assertEqual(nd.as_py(a), [[], [0], [0, 2], [0, 2, 4]]) # Range of ranges a = nd.array(range(i) for i in range(4)) self.assertEqual(nd.as_py(a), [list(range(i)) for i in range(4)])
def test_type_type(self): d = ndt.type('type') self.assertEqual(str(d), 'type') # Creating a dynd array out of a dtype # results in it having the dtype 'dtype' n = nd.array(d) self.assertEqual(nd.type_of(n), d) # Python float type converts to float64 n = nd.array(float) self.assertEqual(nd.type_of(n), d) self.assertEqual(nd.as_py(n), ndt.float64)
def test_full_of_struct(self): # Constructor of a cstruct type a = nd.full(3, "{x: int32, y: int32}", value=[1, 5], access="rw") self.assertEqual(a.access_flags, "readwrite") self.assertEqual(nd.type_of(a), ndt.type("strided * {x: int32, y: int32}")) self.assertEqual(a.shape, (3,)) self.assertEqual(nd.as_py(a), [{"x": 1, "y": 5}] * 3) # Constructor of a struct type a = nd.full(3, ndt.make_struct([ndt.int32] * 2, ["x", "y"]), value={"x": 3, "y": 10}, access="rw") self.assertEqual(a.access_flags, "readwrite") self.assertEqual(nd.type_of(a), ndt.make_strided_dim(ndt.make_struct([ndt.int32] * 2, ["x", "y"]))) self.assertEqual(a.shape, (3,)) self.assertEqual(nd.as_py(a), [{"x": 3, "y": 10}] * 3)
def test_dynamic_fromiter_int64typepromo(self): # Test iterator construction cases promoting from an int64 # float64 result a = nd.array(iter([10000000000, 2, 3.25])) self.assertEqual(nd.type_of(a), ndt.type('3 * float64')) self.assertEqual(nd.as_py(a), [10000000000, 2, 3.25]) # complex[float64] result a = nd.array(iter([10000000000, 2, 3.25j])) self.assertEqual(nd.type_of(a), ndt.type('3 * complex[float64]')) self.assertEqual(nd.as_py(a), [10000000000, 2, 3.25j]) # Should raise an error mixing int64 and string/bytes self.assertRaises(TypeError, nd.array, iter([10000000000, 2, "test"])) self.assertRaises(TypeError, nd.array, iter([10000000000, 2, u"test"])) self.assertRaises(TypeError, nd.array, iter([10000000000, 2, b"test"]))
def test_fixed_var(self): a = nd.array( [[(1, 2, 'a', 'b'), (3, 4, 'ab', 'cd')], [(5, 6, 'def', 'ghi')], [(7, 8, 'alpha', 'beta'), (9, 10, 'X', 'Y'), (11, 12, 'the', 'end')]], type='3 * var * {x: int32, y: int32, z: string, w: string}') # Selecting a single field b = nd.fields(a, 'x') self.assertEqual( nd.type_of(b), ndt.make_fixed_dim( 3, ndt.make_var_dim(ndt.make_struct([ndt.int32], ['x'])))) self.assertEqual(nd.as_py(b.x), nd.as_py(a.x)) # Selecting two fields b = nd.fields(a, 'z', 'y') self.assertEqual( nd.type_of(b), ndt.make_fixed_dim( 3, ndt.make_var_dim( ndt.make_struct([ndt.string, ndt.int32], ['z', 'y'])))) self.assertEqual(nd.as_py(b.z), nd.as_py(a.z)) self.assertEqual(nd.as_py(b.y), nd.as_py(a.y)) # Selecting three fields b = nd.fields(a, 'w', 'y', 'z') self.assertEqual( nd.type_of(b), ndt.make_fixed_dim( 3, ndt.make_var_dim( ndt.make_struct([ndt.string, ndt.int32, ndt.string], ['w', 'y', 'z'])))) self.assertEqual(nd.as_py(b.w), nd.as_py(a.w)) self.assertEqual(nd.as_py(b.y), nd.as_py(a.y)) self.assertEqual(nd.as_py(b.z), nd.as_py(a.z)) # Reordering all four fields b = nd.fields(a, 'w', 'y', 'x', 'z') self.assertEqual( nd.type_of(b), ndt.make_fixed_dim( 3, ndt.make_var_dim( ndt.make_struct( [ndt.string, ndt.int32, ndt.int32, ndt.string], ['w', 'y', 'x', 'z'])))) self.assertEqual(nd.as_py(b.w), nd.as_py(a.w)) self.assertEqual(nd.as_py(b.y), nd.as_py(a.y)) self.assertEqual(nd.as_py(b.x), nd.as_py(a.x)) self.assertEqual(nd.as_py(b.z), nd.as_py(a.z))
def test_strided_dim(self): a = nd.empty(100, ndt.int32) a[...] = nd.range(100) self.assertEqual(nd.type_of(a), ndt.type('A * int32')) self.assertEqual(nd.type_of(a[...]), ndt.type('A * int32')) self.assertEqual(nd.type_of(a[0]), ndt.int32) self.assertEqual(nd.type_of(a[0:1]), ndt.type('A * int32')) self.assertEqual(nd.as_py(a[0]), 0) self.assertEqual(nd.as_py(a[99]), 99) self.assertEqual(nd.as_py(a[-1]), 99) self.assertEqual(nd.as_py(a[-100]), 0) self.assertRaises(IndexError, lambda x : x[-101], a) self.assertRaises(IndexError, lambda x : x[100], a) self.assertRaises(IndexError, lambda x : x[-101:], a) self.assertRaises(IndexError, lambda x : x[-5:101:2], a)
def test_nested_struct_array(self): a = nd.array([((0, 1), 0), ((2, 2), 5), ((100, 10), 10)], type='3 * {x:{a:int16, b:int16}, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:{a:int16, b:int16}, y:int32}')) self.assertEqual(nd.as_py(a.x.a), [0, 2, 100]) self.assertEqual(nd.as_py(a.x.b), [1, 2, 10]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array([{ 'x': { 'a': 0, 'b': 1 }, 'y': 0 }, { 'x': { 'a': 2, 'b': 2 }, 'y': 5 }, { 'x': { 'a': 100, 'b': 10 }, 'y': 10 }], type='3 * {x:{a:int16, b:int16}, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:{a:int16, b:int16}, y:int32}')) self.assertEqual(nd.as_py(a.x.a), [0, 2, 100]) self.assertEqual(nd.as_py(a.x.b), [1, 2, 10]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array( [[(3, ('X', 10))], [(10, ('L', 7)), (12, ('M', 5))]], type= '2 * var * {count:int32, size:{name:fixed_string[1,"A"], id: int8}}' ) self.assertEqual( nd.type_of(a), ndt.type( '2 * var * {count:int32, size:{name:fixed_string[1,"A"], id: int8}}' )) self.assertEqual(nd.as_py(a.count), [[3], [10, 12]]) self.assertEqual(nd.as_py(a.size.name), [['X'], ['L', 'M']]) self.assertEqual(nd.as_py(a.size.id), [[10], [7, 5]])
def render_dynd_datashape_recursive(base_url, arr, indent): result = '' if isinstance(arr, ndt.type): dt = arr.value_type else: dt = nd.type_of(arr).value_type if dt.kind == 'struct': result += '{' + json_comment(base_url) for farr, fname in zip(dt.field_types, dt.field_names): farr = nd.as_py(farr) child_url = base_url + '.' + str(fname) child_result = render_dynd_datashape_recursive(child_url, farr, indent + ' ') result += (indent + ' ' + '<a href="' + child_url + '">' + str(fname) + '</a>' ': ' + child_result + ';') dt = nd.dtype_of(farr) if isinstance(farr, nd.array) else farr if dt.kind != 'struct': result += json_comment(child_url) else: result += '\n' result += (indent + '}') elif dt.kind == 'uniform_dim': if dt.type_id in ['strided_dim', 'var_dim']: if isinstance(arr, ndt.type): result += 'var, ' else: result += ('%d, ' % len(arr)) elif dt.type_id == 'fixed_dim': result += ('%d, ' % dt.fixed_dim_size) else: raise TypeError('Unrecognized DyND uniform array type ' + str(dt)) # Descend to the element type if isinstance(arr, ndt.type): arr = arr.element_type elif len(arr) == 1: # If there's only one element in the array, can # keep using the array sizes in the datashape arr = arr[0] else: arr = nd.type_of(arr).element_type result += render_dynd_datashape_recursive(base_url, arr, indent) elif dt.kind in ['bool', 'int', 'uint', 'real', 'datetime', 'json', 'string']: result += str(dt.dshape) else: raise TypeError('Unrecognized DyND type ' + str(dt)) return result
def test_empty_array_dtype(self): a = nd.array([], dtype=ndt.int64) self.assertEqual(nd.type_of(a), ndt.type("strided * int64")) self.assertEqual(a.shape, (0,)) a = nd.array([], dtype="strided * float64") self.assertEqual(nd.type_of(a), ndt.type("strided * float64")) self.assertEqual(a.shape, (0,)) a = nd.array([], dtype="var * int16") self.assertEqual(nd.type_of(a), ndt.type("var * int16")) self.assertEqual(len(a), 0) a = nd.array([], dtype="0 * int16") self.assertEqual(nd.type_of(a), ndt.type("0 * int16")) self.assertEqual(len(a), 0) a = nd.array([], dtype="3 * int16") self.assertEqual(nd.type_of(a), ndt.type("strided * 3 * int16")) self.assertEqual(a.shape, (0, 3))
def test_ragged_fromiter(self): # Strided array of var from list of iterators a = nd.array([(1 + x for x in range(3)), (5 * x - 10 for x in range(5)), [2, 10]], type='Fixed * var * int32') self.assertEqual(nd.type_of(a), ndt.type('3 * var * int32')) self.assertEqual(nd.as_py(a), [[1, 2, 3], [-10, -5, 0, 5, 10], [2, 10]]) # Var array of var from iterator of iterators a = nd.array(((2 * x for x in range(y)) for y in range(4)), type='var * var * int32') self.assertEqual(nd.type_of(a), ndt.type('var * var * int32')) self.assertEqual(nd.as_py(a), [[], [0], [0, 2], [0, 2, 4]]) # Range of ranges a = nd.array(range(i) for i in range(4)) self.assertEqual(nd.as_py(a), [list(range(i)) for i in range(4)])
def op_ckernel(self, op): deferred_ckernel = op.args[0] args = [self.values[arg] for arg in op.args[1]] dst = args[0] srcs = args[1:] dst_descriptor = dst._data src_descriptors = [src._data for src in srcs] out = dst_descriptor.dynd_arr() inputs = [desc.dynd_arr() for desc in src_descriptors] # TODO: Remove later, explicit casting necessary for now because # of BLZ/numpy interop effect. for i, (inp, tp) in enumerate(zip(inputs, deferred_ckernel.types[1:])): tp = ndt.type(tp) if nd.type_of(inp) != tp: inputs[i] = nd.array(inp, type=tp) # Execute! deferred_ckernel.__call__(out, *inputs) # Operations are rewritten to already refer to 'dst' # We are essentially a 'void' operation self.values[op] = None
def test_simple(self): a = nd.array([1, 2, 3]) self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) # Modifying 'a' shouldn't affect 'b', because it's a copy b = nd.array(a) a[1] = 10 self.assertEqual(nd.as_py(b), [1, 2, 3])
def test_array_unicode(self): a = nd.array(u"\uc548\ub155") b = nd.array([u"\uc548\ub155", u"Hello"]) self.assertEqual(nd.type_of(a), ndt.string) self.assertEqual(nd.dtype_of(b), ndt.string) # self.assertEqual(unicode(a), u"\uc548\ub155") self.assertEqual(nd.as_py(b), [u"\uc548\ub155", u"Hello"])
def test_fixed_dim(self): a = nd.empty(100, ndt.int32) a[...] = nd.range(100) b = list(range(100)) self.assertEqual(nd.type_of(a), ndt.type('100 * int32')) self.assertEqual(nd.type_of(a[...]), ndt.type('100 * int32')) self.assertEqual(nd.type_of(a[0]), ndt.int32) self.assertEqual(nd.type_of(a[0:1]), ndt.type('1 * int32')) self.assertEqual(nd.as_py(a[0]), b[0]) self.assertEqual(nd.as_py(a[99]), b[99]) self.assertEqual(nd.as_py(a[-1]), b[-1]) self.assertEqual(nd.as_py(a[-100]), b[-100]) self.assertEqual(nd.as_py(a[-101:]), b[-101:]) self.assertEqual(nd.as_py(a[-5:101:2]), b[-5:101:2]) self.assertRaises(IndexError, lambda x: x[-101], a) self.assertRaises(IndexError, lambda x: x[100], a)
def test_empty_array_dtype(self): a = nd.array([], dtype=ndt.int64) self.assertEqual(nd.type_of(a), ndt.type('0 * int64')) self.assertEqual(a.shape, (0,)) a = nd.array([], dtype='fixed * float64') self.assertEqual(nd.type_of(a), ndt.type('0 * float64')) self.assertEqual(a.shape, (0,)) a = nd.array([], dtype='var * int16') self.assertEqual(nd.type_of(a), ndt.type('var * int16')) self.assertEqual(len(a), 0) a = nd.array([], dtype='0 * int16') self.assertEqual(nd.type_of(a), ndt.type('0 * int16')) self.assertEqual(len(a), 0) a = nd.array([], dtype='3 * int16') self.assertEqual(nd.type_of(a), ndt.type('0 * 3 * int16')) self.assertEqual(a.shape, (0, 3))
def test_fixed_dim(self): a = nd.empty(100, ndt.int32) a[...] = nd.range(100) b = list(range(100)) self.assertEqual(nd.type_of(a), ndt.type('100 * int32')) self.assertEqual(nd.type_of(a[...]), ndt.type('100 * int32')) self.assertEqual(nd.type_of(a[0]), ndt.int32) self.assertEqual(nd.type_of(a[0:1]), ndt.type('1 * int32')) self.assertEqual(nd.as_py(a[0]), b[0]) self.assertEqual(nd.as_py(a[99]), b[99]) self.assertEqual(nd.as_py(a[-1]), b[-1]) self.assertEqual(nd.as_py(a[-100]), b[-100]) self.assertEqual(nd.as_py(a[-101:]), b[-101:]) self.assertEqual(nd.as_py(a[-5:101:2]), b[-5:101:2]) self.assertRaises(IndexError, lambda x : x[-101], a) self.assertRaises(IndexError, lambda x : x[100], a)
def test_simple(self): a = nd.array([1, 2, 3], access='rw') self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) # Modifying 'a' shouldn't affect 'b', because it's a copy b = nd.array(a) a[1] = 10 self.assertEqual(nd.as_py(b), [1, 2, 3])
def test_ragged_initial_empty_typepromo(self): # iterator of lists, first one is empty vals = [[], [False, 2, 3]] a = nd.array(iter(x) for x in vals) self.assertEqual(nd.type_of(a), ndt.type('2 * var * int32')) self.assertEqual(nd.as_py(a), vals)
def test_simple(self): a = nd.asarray([1, 2, 3], access='rw') self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) # Modifying 'a' should affect 'b', because it's a view b = nd.asarray(a) self.assertEqual(nd.as_py(b), [1, 2, 3]) a[1] = 10 self.assertEqual(nd.as_py(b), [1, 10, 3]) # Can take a readonly view, but still modify the original b = nd.asarray(a, access='r') self.assertEqual(nd.as_py(b), [1, 10, 3]) a[1] = 20 self.assertEqual(nd.as_py(b), [1, 20, 3]) # The readonly view we took can't be written to def assign_at(x, i, y): x[i] = y self.assertRaises(RuntimeError, assign_at, b, 1, 30) # Asking for immutable makes a copy instead of a view b = nd.asarray(a, access='immutable') self.assertEqual(nd.as_py(b), [1, 20, 3]) a[1] = 40 self.assertEqual(nd.as_py(b), [1, 20, 3]) # Asking for immutable from a non-immutable # readonly array makes a copy aprime = nd.asarray(a, access='r') b = nd.asarray(aprime, access='immutable') self.assertEqual(nd.as_py(aprime), [1, 40, 3]) self.assertEqual(nd.as_py(b), [1, 40, 3]) a[1] = 50 self.assertEqual(nd.as_py(aprime), [1, 50, 3]) self.assertEqual(nd.as_py(b), [1, 40, 3])
def test_empty_array_dtype(self): a = nd.array([], type=ndt.make_fixed_dim(0, ndt.int64)) self.assertEqual(nd.type_of(a), ndt.type('0 * int64')) self.assertEqual(a.shape, (0, )) # Todo: Need to reenable this failing test # a = nd.array([], dtype='Fixed * float64') # self.assertEqual(nd.type_of(a), ndt.type('0 * float64')) # self.assertEqual(a.shape, (0,)) a = nd.array([], type='var * int16') self.assertEqual(nd.type_of(a), ndt.type('var * int16')) self.assertEqual(len(a), 0) a = nd.array([], type='0 * int16') self.assertEqual(nd.type_of(a), ndt.type('0 * int16')) self.assertEqual(len(a), 0) a = nd.array([], type='0 * 3 * int16') self.assertEqual(nd.type_of(a), ndt.type('0 * 3 * int16')) self.assertEqual(a.shape, (0, 3))
def test_single_struct_array(self): a = nd.array([(0, 0), (3, 5), (12, 10)], type='3 * {x:int32, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:int32, y:int32}')) self.assertEqual(nd.as_py(a.x), [0, 3, 12]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array([{ 'x': 0, 'y': 0 }, { 'x': 3, 'y': 5 }, { 'x': 12, 'y': 10 }], type='3 * {x:int32, y:int32}') self.assertEqual(nd.type_of(a), ndt.type('3 * {x:int32, y:int32}')) self.assertEqual(nd.as_py(a.x), [0, 3, 12]) self.assertEqual(nd.as_py(a.y), [0, 5, 10]) a = nd.array([[(3, 'X')], [(10, 'L'), (12, 'M')]], type='2 * var * {count:int32, size:fixed_string[1,"A"]}') self.assertEqual( nd.type_of(a), ndt.type('2 * var * {count:int32, size:fixed_string[1,"A"]}')) self.assertEqual(nd.as_py(a.count), [[3], [10, 12]]) self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']]) a = nd.array([[{ 'count': 3, 'size': 'X' }], [{ 'count': 10, 'size': 'L' }, { 'count': 12, 'size': 'M' }]], type='2 * var * {count:int32, size:fixed_string[1,"A"]}') self.assertEqual( nd.type_of(a), ndt.type('2 * var * {count:int32, size:fixed_string[1,"A"]}')) self.assertEqual(nd.as_py(a.count), [[3], [10, 12]]) self.assertEqual(nd.as_py(a.size), [['X'], ['L', 'M']])
def test_empty(self): # Constructor from scalar type a = nd.empty(ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.int32) # Constructor from type with fixed dimension a = nd.empty('3 * int32') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32)) self.assertEqual(a.shape, (3, )) # Constructor from type with fixed dimension, accesskwarg a = nd.empty('3 * int32', access='rw') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.make_fixed_dim(3, ndt.int32)) self.assertEqual(a.shape, (3, )) # Constructor from shape as single integer a = nd.empty(3, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * int32')) self.assertEqual(a.shape, (3, )) # Constructor from shape as tuple a = nd.empty((3, 4), ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3, 4)) # Constructor from shape as variadic arguments a = nd.empty(3, 4, ndt.int32) self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3, 4)) # Constructor from shape as variadic arguments, access kwarg a = nd.empty(3, 4, ndt.int32, access='rw') self.assertEqual(a.access_flags, 'readwrite') self.assertEqual(nd.type_of(a), ndt.type('3 * 4 * int32')) self.assertEqual(a.shape, (3, 4))
def test_empty(self): # A fixed dimension of non-zero size gets pushed down a = nd.array([], dtype='3 * int32') self.assertEqual(nd.type_of(a), ndt.type('0 * 3 * int32')) self.assertEqual(nd.as_py(a), []) # A fixed dimension of zero size gets absorbed a = nd.array([], dtype='0 * int32') self.assertEqual(nd.type_of(a), ndt.type('0 * int32')) self.assertEqual(nd.as_py(a), []) # A symbolic fixed dimension gets absorbed # Todo: Need to reenable this failing test # a = nd.array([], dtype='Fixed * int32') # self.assertEqual(nd.type_of(a), ndt.type('0 * int32')) # self.assertEqual(nd.as_py(a), []) # A var dimension gets absorbed a = nd.array([], dtype='var * int32') self.assertEqual(nd.type_of(a), ndt.type('var * int32')) self.assertEqual(nd.as_py(a), [])
def test_dynd_scalar_asarray(self): a = np.array(3, dtype='int64') n = nd.asarray(a) self.assertEqual(nd.type_of(n), ndt.int64) self.assertEqual(nd.as_py(n), 3) self.assertEqual(n.access_flags, 'readwrite') # Ensure it's a view n[...] = 4 self.assertEqual(a[()], 4)
def test_var_dim(self): # TODO: Reenable tests below when var dim slicing is implemented properly a = nd.empty('var * int32') a[...] = nd.range(100) b = list(range(100)) self.assertEqual(nd.type_of(a), ndt.type('var * int32')) self.assertEqual(nd.type_of(a[...]), ndt.type('var * int32')) self.assertEqual(nd.type_of(a[:]), ndt.type('var * int32')) self.assertEqual(nd.type_of(a[0]), ndt.int32) # self.assertEqual(nd.type_of(a[0:1]), ndt.type('fixed * int32')) self.assertEqual(nd.as_py(a[0]), b[0]) self.assertEqual(nd.as_py(a[99]), b[99]) self.assertEqual(nd.as_py(a[-1]), b[-1]) self.assertEqual(nd.as_py(a[-100]), b[-100]) #self.assertEqual(nd.as_py(a[-101:]), b[-101:]) #self.assertEqual(nd.as_py(a[-5:101:2]), b[-5:101:2]) self.assertRaises(IndexError, lambda x: x[-101], a) self.assertRaises(IndexError, lambda x: x[100], a)
def test_int(self): # Integer that fits in 32 bits a = nd.array(10) self.assertEqual(nd.type_of(a), ndt.int32) self.assertEqual(type(nd.as_py(a)), int) self.assertEqual(nd.as_py(a), 10) a = nd.array(-2000000000) self.assertEqual(nd.type_of(a), ndt.int32) self.assertEqual(type(nd.as_py(a)), int) self.assertEqual(nd.as_py(a), -2000000000) # Integer that requires 64 bits a = nd.array(2200000000) self.assertEqual(nd.type_of(a), ndt.int64) self.assertEqual(nd.as_py(a), 2200000000) a = nd.array(-2200000000) self.assertEqual(nd.type_of(a), ndt.int64) self.assertEqual(nd.as_py(a), -2200000000)
def fromdynd(arr): """Build a DyNDArray from an nd.array object""" # Construct the DyNDArrayLayout[a] pointer type ntp = DyNDArrayLayout[_meta_type(nd.type_of(arr))] # Get the raw array pointer arr_raw_ptr = _lowlevel.get_array_ptr(arr) arr_ptr = cast(arr_raw_ptr, types.Pointer[ntp]) # Return a wrapping DyNDArray return DyNDArray(arr_ptr)
def test_single_struct(self): a = nd.array([12, 'test', True], type='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True) a = nd.array({ 'x': 12, 'y': 'test', 'z': True }, type='{x:int32, y:string, z:bool}') self.assertEqual(nd.type_of(a), ndt.type('{x:int32, y:string, z:bool}')) self.assertEqual(nd.as_py(a[0]), 12) self.assertEqual(nd.as_py(a[1]), 'test') self.assertEqual(nd.as_py(a[2]), True)
def test_array_unicode(self): a = nd.array(u"\uc548\ub155") b = nd.array([u"\uc548\ub155", u"Hello"]) self.assertEqual(nd.type_of(a), ndt.string) self.assertEqual(nd.dtype_of(b), ndt.string) self.assertEqual(unicode(a), u"\uc548\ub155") self.assertEqual(nd.as_py(b), [u"\uc548\ub155", u"Hello"]) # In Python 2, 'str' is not unicode if sys.version_info < (3, 0): self.assertRaises(UnicodeEncodeError, str, a)
def test_inf(self): # Validate nd.inf self.assertEqual(nd.inf * 2, nd.inf) self.assertTrue(nd.inf > 0) self.assertTrue(-nd.inf < 0) # as an array a = nd.array(nd.inf) self.assertEqual(nd.as_py(a), nd.inf) self.assertEqual(nd.type_of(a), ndt.float64) a = nd.array(nd.inf, type=ndt.float32) self.assertEqual(nd.as_py(a), nd.inf)
def test_blaze_array_in_list(self): class Array(object): def __init__(self, dshape, value): self.dshape = str(dshape) self.value = value def __int__(self): return self.value def __float__(self): return self.value self.assertEqual(type(Array('int32', 1)).__name__, "Array") a = nd.array([Array('int32', 3)]) self.assertEqual(nd.type_of(a), ndt.type('1 * int32')) self.assertEqual(nd.as_py(a), [3]) a = nd.array([Array('float32', 1.25)]) self.assertEqual(nd.type_of(a), ndt.type('1 * float32')) self.assertEqual(nd.as_py(a), [1.25])