def testCompoundArrayVlenStringTypeItem(self):
dt_vlen = special_dtype(vlen=bytes)
dt_arr = np.dtype((dt_vlen, (4, )))
dt_compound = np.dtype([('VALUE1', np.float64), ('VALUE2', np.int64),
('VALUE3', dt_arr)])
typeItem = hdf5dtype.getTypeItem(dt_compound)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 'H5T_VARIABLE')
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
fields = typeItem['fields']
field_a = fields[0]
self.assertEqual(field_a['name'], 'VALUE1')
field_a_type = field_a['type']
self.assertEqual(field_a_type['class'], 'H5T_FLOAT')
self.assertEqual(field_a_type['base'], 'H5T_IEEE_F64LE')
field_b = fields[1]
self.assertEqual(field_b['name'], 'VALUE2')
field_b_type = field_b['type']
self.assertEqual(field_b_type['class'], 'H5T_INTEGER')
self.assertEqual(field_b_type['base'], 'H5T_STD_I64LE')
field_c = fields[2]
field_c_type = field_c['type']
self.assertEqual(field_c_type['class'], 'H5T_ARRAY')
self.assertEqual(field_c_type['dims'], (4, ))
field_c_base_type = field_c_type['base']
self.assertEqual(field_c_base_type['class'], 'H5T_STRING')
self.assertEqual(field_c_base_type['length'], 'H5T_VARIABLE')
self.assertEqual(field_c_base_type['charSet'], 'H5T_CSET_ASCII')
def testVlenDataItem(self):
dt = special_dtype(vlen=np.dtype('int32'))
typeItem = hdf5dtype.getTypeItem(dt)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeItem['class'], 'H5T_VLEN')
self.assertEqual(typeItem['size'], 'H5T_VARIABLE')
baseItem = typeItem['base']
self.assertEqual(baseItem['base'], 'H5T_STD_I32LE')
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testObjReferenceTypeItem(self):
dt = special_dtype(ref=Reference)
typeItem = hdf5dtype.getTypeItem(dt)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeItem['class'], 'H5T_REFERENCE')
self.assertEqual(typeItem['base'], 'H5T_STD_REF_OBJ')
# length of obj id, e.g.:
# g-b2c9a750-a557-11e7-ab09-0242ac110009
self.assertEqual(typeSize, 48)
def testBaseVLenUnicodeTypeItem(self):
dt = special_dtype(vlen=str)
typeItem = hdf5dtype.getTypeItem(dt)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['length'], 'H5T_VARIABLE')
self.assertEqual(typeItem['strPad'], 'H5T_STR_NULLTERM')
self.assertEqual(typeItem['charSet'], 'H5T_CSET_UTF8')
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testBaseEnumTypeItem(self):
mapping = {'RED': 0, 'GREEN': 1, 'BLUE': 2}
dt = special_dtype(enum=(np.int8, mapping))
typeItem = hdf5dtype.getTypeItem(dt)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeItem['class'], 'H5T_ENUM')
baseItem = typeItem['base']
self.assertEqual(baseItem['class'], 'H5T_INTEGER')
self.assertEqual(baseItem['base'], 'H5T_STD_I8LE')
self.assertTrue('mapping' in typeItem)
self.assertEqual(typeItem['mapping']['GREEN'], 1)
self.assertEqual(typeSize, 1)
def testEnumArrayTypeItem(self):
mapping = {'RED': 0, 'GREEN': 1, 'BLUE': 2}
dt_enum = special_dtype(enum=(np.int8, mapping))
typeItem = hdf5dtype.getTypeItem(dt_enum)
dt_array = np.dtype('(2,3)' + dt_enum.str,
metadata=dict(dt_enum.metadata))
typeItem = hdf5dtype.getTypeItem(dt_array)
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue("dims" in typeItem)
self.assertEqual(typeItem["dims"], (2, 3))
baseItem = typeItem['base']
self.assertEqual(baseItem['class'], 'H5T_ENUM')
self.assertTrue('mapping' in baseItem)
self.assertEqual(baseItem['mapping']['GREEN'], 1)
self.assertTrue("base" in baseItem)
basePrim = baseItem["base"]
self.assertEqual(basePrim["class"], 'H5T_INTEGER')
self.assertEqual(basePrim['base'], 'H5T_STD_I8LE')
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize,
6) # one-byte for base enum type * shape of (2,3)
def testJsonToArray(self):
dt = np.dtype('i4')
shape = [4,]
data = [0,2,4,6]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (4,))
for i in range(4):
self.assertEqual(out[i], i*2)
# compound type
dt = np.dtype([('a', 'i4'), ('b', 'S5')])
shape = [2,]
data = [[4, 'four'], [5, 'five']]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (2,))
self.assertTrue(isinstance(out[0], np.void))
e0 = out[0].tolist()
self.assertEqual(e0, (4, b'four'))
self.assertTrue(isinstance(out[1], np.void))
e1 = out[1].tolist()
self.assertEqual(e1, (5, b'five'))
shape = [1,]
data = [[6, 'six'],]
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
data = [6, 'six']
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
# VLEN ascii
dt = special_dtype(vlen=bytes)
data = [b"one", b"two", b"three", "four", b"five"]
shape = [5,]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (5,))
# TBD: code does not actually enforce use of bytes vs. str,
# probably not worth the effort to fix
self.assertEqual(out[2], b"three")
self.assertEqual(out[3], "four")
# VLEN str
dt = special_dtype(vlen=str)
data = [['part 1 - section A', 'part 1 - section B'], ['part 2 - section A', 'part 2 - section B']]
shape = [2,]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], str)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (2,))
self.assertEqual(out[0], tuple(data[0]))
self.assertEqual(out[1], tuple(data[1]))
# VLEN Scalar str
dt = special_dtype(vlen=str)
data = "I'm a string!"
shape = [1,]
out = jsonToArray(shape, dt, data)
# VLEN unicode
dt = special_dtype(vlen=bytes)
data = ["one", "two", "three", "four", "five"]
shape = [5,]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
# TBD: this should show up as bytes, but may not be worth the effort
self.assertEqual(out[2], "three")
# VLEN data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [4,]
data = [[1,], [1,2], [1,2,3], [1,2,3,4]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (4,))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(4):
e = out[i] #.tolist()
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, tuple(range(1, i+2)))
# VLEN 2D data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [2,2]
data = [[[0,], [1,2]], [[1,], [2,3]]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (2,2))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(2):
for j in range(2):
e = out[i,j] #.tolist()
self.assertTrue(isinstance(e, tuple))
# create VLEN of obj ref's
ref_type = {"class": "H5T_REFERENCE",
"base": "H5T_STD_REF_OBJ"}
vlen_type = {"class": "H5T_VLEN", "base": ref_type}
dt = createDataType(vlen_type) # np datatype
id0 = 'g-a4f455b2-c8cf-11e7-8b73-0242ac110009'
id1 = 'g-a50af844-c8cf-11e7-8b73-0242ac110009'
id2 = 'g-a5236276-c8cf-11e7-8b73-0242ac110009'
data = [ [id0,], [id0,id1], [id0,id1,id2] ]
shape = [3,]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
base_type = check_dtype(vlen=out.dtype)
self.assertEqual(base_type.kind, 'S')
self.assertEqual(base_type.itemsize, 48)
self.assertEqual(out.shape, (3,))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('S48'))
e = out[0]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0,))
e = out[1]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0,id1))
e = out[2]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0,id1,id2))
def testJsonToBytes(self):
#
# VLEN int
#
dt = special_dtype(vlen=np.dtype('int32'))
shape = [4,]
data = [[1,], [1,2], [1,2,3], [1,2,3,4]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
self.assertEqual(check_dtype(vlen=arr.dtype), np.dtype('int32'))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 56)
expected = b'\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# Compound vlen
#
dt_str = np.dtype('O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_str)])
shape = [4,]
data = [[42, "Hello"], [0,0], [0,0], [84, "Bye"]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 40)
expected = b'*\x00\x00\x00\x05\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x03\x00\x00\x00Bye'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN utf with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [4,]
data = [[42, ["hi","bye"]], [0,[0,0]], [0,[0,0]], [84, ["hi-hi", "bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN ascii with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [4,]
data = [[42, [b"hi", b"bye"]], [0,[0,0]], [0,[0,0]], [84, [b"hi-hi", b"bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
def testRegionReferenceTypeItem(self):
dt = special_dtype(ref=RegionReference)
typeItem = hdf5dtype.getTypeItem(dt)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeItem['class'], 'H5T_REFERENCE')
