def testCreateCompoundType(self):
typeItem = {
'class':
'H5T_COMPOUND',
'fields': [{
'name': 'temp',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'pressure',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'location',
'type': {
'length': 'H5T_VARIABLE',
'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strPad': 'H5T_STR_NULLTERM'
}
}, {
'name': 'wind',
'type': 'H5T_STD_I16LE'
}]
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void144')
self.assertEqual(dt.kind, 'V')
self.assertEqual(len(dt.fields), 4)
dtLocation = dt[2]
self.assertEqual(dtLocation.name, 'object')
self.assertEqual(dtLocation.kind, 'O')
self.assertEqual(check_dtype(vlen=dtLocation), bytes)
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCreateEnumType(self):
typeItem = {
"class": "H5T_ENUM",
"base": {
"base": "H5T_STD_I16LE",
"class": "H5T_INTEGER"
},
"mapping": {
"GAS": 2,
"LIQUID": 1,
"PLASMA": 3,
"SOLID": 0
}
}
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 2)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'int16')
self.assertEqual(dt.kind, 'i')
mapping = check_dtype(enum=dt)
self.assertTrue(isinstance(mapping, dict))
self.assertEqual(mapping["SOLID"], 0)
self.assertEqual(mapping["LIQUID"], 1)
self.assertEqual(mapping["GAS"], 2)
self.assertEqual(mapping["PLASMA"], 3)
def testCreateVLenUTF8Type(self):
typeItem = {
'class': 'H5T_STRING',
'charSet': 'H5T_CSET_UTF8',
'length': 'H5T_VARIABLE'
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'object')
self.assertEqual(dt.kind, 'O')
self.assertEqual(check_dtype(vlen=dt), str)
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCompoundArrayType(self):
typeItem = {
"class":
"H5T_COMPOUND",
"fields": [{
"type": {
"class": "H5T_INTEGER",
"base": "H5T_STD_U64BE"
},
"name": "VALUE1"
}, {
"type": {
"class": "H5T_FLOAT",
"base": "H5T_IEEE_F64BE"
},
"name": "VALUE2"
}, {
"type": {
"class": "H5T_ARRAY",
"dims": [2],
"base": {
"class": "H5T_STRING",
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
}
},
"name": "VALUE3"
}]
}
dt = hdf5dtype.createDataType(typeItem)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 'H5T_VARIABLE')
self.assertEqual(len(dt), 3)
self.assertTrue("VALUE1" in dt.fields.keys())
self.assertTrue("VALUE2" in dt.fields.keys())
self.assertTrue("VALUE3" in dt.fields.keys())
dt3 = dt["VALUE3"]
self.assertEqual(check_dtype(vlen=dt3), bytes)
def testJsonToArray(self):
dt = np.dtype('i4')
shape = [
4,
]
data = [0, 2, 4, 6]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (4, ))
for i in range(4):
self.assertEqual(out[i], i * 2)
# compound type
dt = np.dtype([('a', 'i4'), ('b', 'S5')])
shape = [
2,
]
data = [[4, 'four'], [5, 'five']]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (2, ))
self.assertTrue(isinstance(out[0], np.void))
e0 = out[0].tolist()
self.assertEqual(e0, (4, b'four'))
self.assertTrue(isinstance(out[1], np.void))
e1 = out[1].tolist()
self.assertEqual(e1, (5, b'five'))
shape = [
1,
]
data = [
[6, 'six'],
]
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
data = [6, 'six']
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
# VLEN ascii
dt = special_dtype(vlen=bytes)
data = [b"one", b"two", b"three", "four", b"five"]
shape = [
5,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (5, ))
# TBD: code does not actually enforce use of bytes vs. str,
# probably not worth the effort to fix
self.assertEqual(out[2], b"three")
self.assertEqual(out[3], "four")
# VLEN str
dt = special_dtype(vlen=str)
data = [['part 1 - section A', 'part 1 - section B'],
['part 2 - section A', 'part 2 - section B']]
shape = [
2,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], str)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (2, ))
self.assertEqual(out[0], tuple(data[0]))
self.assertEqual(out[1], tuple(data[1]))
# VLEN Scalar str
dt = special_dtype(vlen=str)
data = "I'm a string!"
shape = [
1,
]
out = jsonToArray(shape, dt, data)
# VLEN unicode
dt = special_dtype(vlen=bytes)
data = ["one", "two", "three", "four", "five"]
shape = [
5,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
# TBD: this should show up as bytes, but may not be worth the effort
self.assertEqual(out[2], "three")
# VLEN data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [
4,
]
data = [[
1,
], [1, 2], [1, 2, 3], [1, 2, 3, 4]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (4, ))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(4):
e = out[i] #.tolist()
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, tuple(range(1, i + 2)))
# VLEN 2D data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [2, 2]
data = [[[
0,
], [1, 2]], [[
1,
], [2, 3]]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (2, 2))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(2):
for j in range(2):
e = out[i, j] #.tolist()
self.assertTrue(isinstance(e, tuple))
# create VLEN of obj ref's
ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"}
vlen_type = {"class": "H5T_VLEN", "base": ref_type}
dt = createDataType(vlen_type) # np datatype
id0 = 'g-a4f455b2-c8cf-11e7-8b73-0242ac110009'
id1 = 'g-a50af844-c8cf-11e7-8b73-0242ac110009'
id2 = 'g-a5236276-c8cf-11e7-8b73-0242ac110009'
data = [[
id0,
], [id0, id1], [id0, id1, id2]]
shape = [
3,
]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
base_type = check_dtype(vlen=out.dtype)
self.assertEqual(base_type.kind, 'S')
self.assertEqual(base_type.itemsize, 48)
self.assertEqual(out.shape, (3, ))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('S48'))
e = out[0]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, ))
e = out[1]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, id1))
e = out[2]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, id1, id2))
def testJsonToBytes(self):
#
# VLEN int
#
dt = special_dtype(vlen=np.dtype('int32'))
shape = [
4,
]
data = [[
1,
], [1, 2], [1, 2, 3], [1, 2, 3, 4]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
self.assertEqual(check_dtype(vlen=arr.dtype), np.dtype('int32'))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 56)
expected = b'\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4, ))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# Compound vlen
#
dt_str = np.dtype('O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_str)])
shape = [
4,
]
data = [[42, "Hello"], [0, 0], [0, 0], [84, "Bye"]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 40)
expected = b'*\x00\x00\x00\x05\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x03\x00\x00\x00Bye'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4, ))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN utf with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [
4,
]
data = [[42, ["hi", "bye"]], [0, [0, 0]], [0, [0, 0]],
[84, ["hi-hi", "bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4, ))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN ascii with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [
4,
]
data = [[42, [b"hi", b"bye"]], [0, [0, 0]], [0, [0, 0]],
[84, [b"hi-hi", b"bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4, ))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))