def testCompoundArrayVlenStringTypeItem(self): dt_vlen = special_dtype(vlen=bytes) dt_arr = np.dtype((dt_vlen, (4, ))) dt_compound = np.dtype([('VALUE1', np.float64), ('VALUE2', np.int64), ('VALUE3', dt_arr)]) typeItem = hdf5dtype.getTypeItem(dt_compound) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 'H5T_VARIABLE') self.assertEqual(typeItem['class'], 'H5T_COMPOUND') fields = typeItem['fields'] field_a = fields[0] self.assertEqual(field_a['name'], 'VALUE1') field_a_type = field_a['type'] self.assertEqual(field_a_type['class'], 'H5T_FLOAT') self.assertEqual(field_a_type['base'], 'H5T_IEEE_F64LE') field_b = fields[1] self.assertEqual(field_b['name'], 'VALUE2') field_b_type = field_b['type'] self.assertEqual(field_b_type['class'], 'H5T_INTEGER') self.assertEqual(field_b_type['base'], 'H5T_STD_I64LE') field_c = fields[2] field_c_type = field_c['type'] self.assertEqual(field_c_type['class'], 'H5T_ARRAY') self.assertEqual(field_c_type['dims'], (4, )) field_c_base_type = field_c_type['base'] self.assertEqual(field_c_base_type['class'], 'H5T_STRING') self.assertEqual(field_c_base_type['length'], 'H5T_VARIABLE') self.assertEqual(field_c_base_type['charSet'], 'H5T_CSET_ASCII')
def testVlenDataItem(self): dt = special_dtype(vlen=np.dtype('int32')) typeItem = hdf5dtype.getTypeItem(dt) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeItem['class'], 'H5T_VLEN') self.assertEqual(typeItem['size'], 'H5T_VARIABLE') baseItem = typeItem['base'] self.assertEqual(baseItem['base'], 'H5T_STD_I32LE') self.assertEqual(typeSize, 'H5T_VARIABLE')
def testObjReferenceTypeItem(self): dt = special_dtype(ref=Reference) typeItem = hdf5dtype.getTypeItem(dt) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeItem['class'], 'H5T_REFERENCE') self.assertEqual(typeItem['base'], 'H5T_STD_REF_OBJ') # length of obj id, e.g.: # g-b2c9a750-a557-11e7-ab09-0242ac110009 self.assertEqual(typeSize, 48)
def testBaseVLenUnicodeTypeItem(self): dt = special_dtype(vlen=str) typeItem = hdf5dtype.getTypeItem(dt) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeItem['class'], 'H5T_STRING') self.assertEqual(typeItem['length'], 'H5T_VARIABLE') self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLTERM') self.assertEqual(typeItem['charSet'], 'H5T_CSET_UTF8') self.assertEqual(typeSize, 'H5T_VARIABLE')
def testBaseEnumTypeItem(self): mapping = {'RED': 0, 'GREEN': 1, 'BLUE': 2} dt = special_dtype(enum=(np.int8, mapping)) typeItem = hdf5dtype.getTypeItem(dt) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeItem['class'], 'H5T_ENUM') baseItem = typeItem['base'] self.assertEqual(baseItem['class'], 'H5T_INTEGER') self.assertEqual(baseItem['base'], 'H5T_STD_I8LE') self.assertTrue('mapping' in typeItem) self.assertEqual(typeItem['mapping']['GREEN'], 1) self.assertEqual(typeSize, 1)
def testEnumArrayTypeItem(self): mapping = {'RED': 0, 'GREEN': 1, 'BLUE': 2} dt_enum = special_dtype(enum=(np.int8, mapping)) typeItem = hdf5dtype.getTypeItem(dt_enum) dt_array = np.dtype('(2,3)' + dt_enum.str, metadata=dict(dt_enum.metadata)) typeItem = hdf5dtype.getTypeItem(dt_array) self.assertEqual(typeItem['class'], 'H5T_ARRAY') self.assertTrue("dims" in typeItem) self.assertEqual(typeItem["dims"], (2, 3)) baseItem = typeItem['base'] self.assertEqual(baseItem['class'], 'H5T_ENUM') self.assertTrue('mapping' in baseItem) self.assertEqual(baseItem['mapping']['GREEN'], 1) self.assertTrue("base" in baseItem) basePrim = baseItem["base"] self.assertEqual(basePrim["class"], 'H5T_INTEGER') self.assertEqual(basePrim['base'], 'H5T_STD_I8LE') typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 6) # one-byte for base enum type * shape of (2,3)
def testJsonToArray(self): dt = np.dtype('i4') shape = [4,] data = [0,2,4,6] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(out.shape, (4,)) for i in range(4): self.assertEqual(out[i], i*2) # compound type dt = np.dtype([('a', 'i4'), ('b', 'S5')]) shape = [2,] data = [[4, 'four'], [5, 'five']] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(out.shape, (2,)) self.assertTrue(isinstance(out[0], np.void)) e0 = out[0].tolist() self.assertEqual(e0, (4, b'four')) self.assertTrue(isinstance(out[1], np.void)) e1 = out[1].tolist() self.assertEqual(e1, (5, b'five')) shape = [1,] data = [[6, 'six'],] out = jsonToArray(shape, dt, data) e0 = out[0].tolist() self.assertEqual(e0, (6, b'six')) data = [6, 'six'] out = jsonToArray(shape, dt, data) e0 = out[0].tolist() self.assertEqual(e0, (6, b'six')) # VLEN ascii dt = special_dtype(vlen=bytes) data = [b"one", b"two", b"three", "four", b"five"] shape = [5,] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], bytes) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(out.shape, (5,)) # TBD: code does not actually enforce use of bytes vs. str, # probably not worth the effort to fix self.assertEqual(out[2], b"three") self.assertEqual(out[3], "four") # VLEN str dt = special_dtype(vlen=str) data = [['part 1 - section A', 'part 1 - section B'], ['part 2 - section A', 'part 2 - section B']] shape = [2,] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], str) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(out.shape, (2,)) self.assertEqual(out[0], tuple(data[0])) self.assertEqual(out[1], tuple(data[1])) # VLEN Scalar str dt = special_dtype(vlen=str) data = "I'm a string!" shape = [1,] out = jsonToArray(shape, dt, data) # VLEN unicode dt = special_dtype(vlen=bytes) data = ["one", "two", "three", "four", "five"] shape = [5,] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], bytes) self.assertEqual(out.dtype.kind, 'O') # TBD: this should show up as bytes, but may not be worth the effort self.assertEqual(out[2], "three") # VLEN data dt = special_dtype(vlen=np.dtype('int32')) shape = [4,] data = [[1,], [1,2], [1,2,3], [1,2,3,4]] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) self.assertEqual(out.shape, (4,)) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) for i in range(4): e = out[i] #.tolist() self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, tuple(range(1, i+2))) # VLEN 2D data dt = special_dtype(vlen=np.dtype('int32')) shape = [2,2] data = [[[0,], [1,2]], [[1,], [2,3]]] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) self.assertEqual(out.shape, (2,2)) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) for i in range(2): for j in range(2): e = out[i,j] #.tolist() self.assertTrue(isinstance(e, tuple)) # create VLEN of obj ref's ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"} vlen_type = {"class": "H5T_VLEN", "base": ref_type} dt = createDataType(vlen_type) # np datatype id0 = 'g-a4f455b2-c8cf-11e7-8b73-0242ac110009' id1 = 'g-a50af844-c8cf-11e7-8b73-0242ac110009' id2 = 'g-a5236276-c8cf-11e7-8b73-0242ac110009' data = [ [id0,], [id0,id1], [id0,id1,id2] ] shape = [3,] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) base_type = check_dtype(vlen=out.dtype) self.assertEqual(base_type.kind, 'S') self.assertEqual(base_type.itemsize, 48) self.assertEqual(out.shape, (3,)) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('S48')) e = out[0] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0,)) e = out[1] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0,id1)) e = out[2] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0,id1,id2))
def testJsonToBytes(self): # # VLEN int # dt = special_dtype(vlen=np.dtype('int32')) shape = [4,] data = [[1,], [1,2], [1,2,3], [1,2,3,4]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) self.assertEqual(check_dtype(vlen=arr.dtype), np.dtype('int32')) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 56) expected = b'\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00' self.assertEqual(buffer, expected) # convert back to array arr_copy = bytesToArray(buffer, dt, (4,)) # np.array_equal doesn't work for object arrays self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # Compound vlen # dt_str = np.dtype('O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_str)]) shape = [4,] data = [[42, "Hello"], [0,0], [0,0], [84, "Bye"]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 40) expected = b'*\x00\x00\x00\x05\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x03\x00\x00\x00Bye' self.assertEqual(buffer, expected) # convert back to array arr_copy = bytesToArray(buffer, dt, (4,)) # np.array_equal doesn't work for object arrays self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN utf with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) shape = [4,] data = [[42, ["hi","bye"]], [0,[0,0]], [0,[0,0]], [84, ["hi-hi", "bye-bye"]]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) arr_copy = bytesToArray(buffer, dt, (4,)) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN ascii with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) shape = [4,] data = [[42, [b"hi", b"bye"]], [0,[0,0]], [0,[0,0]], [84, [b"hi-hi", b"bye-bye"]]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) arr_copy = bytesToArray(buffer, dt, (4,)) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy))
def testRegionReferenceTypeItem(self): dt = special_dtype(ref=RegionReference) typeItem = hdf5dtype.getTypeItem(dt) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeItem['class'], 'H5T_REFERENCE')