def testCreateArrayIntegerType(self):
typeItem = {
'class': 'H5T_INTEGER',
'base': 'H5T_STD_I64LE',
'dims': (3, 5)
}
try:
hdf5dtype.createDataType(typeItem)
self.assertTrue(
False) # expected exception - dims used with non-array type
except TypeError:
pass # should get exception
def testCreateCompoundArrayType(self):
typeItem = {
"class":
"H5T_COMPOUND",
"fields": [{
"type": {
"base": "H5T_STD_I8LE",
"class": "H5T_INTEGER"
},
"name": "a"
}, {
"type": {
"dims": [10],
"base": {
"length": 1,
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"strPad": "H5T_STR_NULLPAD"
},
"class": "H5T_ARRAY"
},
"name": "b"
}]
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(len(dt.fields), 2)
self.assertTrue('a' in dt.fields.keys())
self.assertTrue('b' in dt.fields.keys())
self.assertEqual(typeSize, 11)
def testCreateEnumType(self):
typeItem = {
"class": "H5T_ENUM",
"base": {
"base": "H5T_STD_I16LE",
"class": "H5T_INTEGER"
},
"mapping": {
"GAS": 2,
"LIQUID": 1,
"PLASMA": 3,
"SOLID": 0
}
}
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 2)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'int16')
self.assertEqual(dt.kind, 'i')
mapping = check_dtype(enum=dt)
self.assertTrue(isinstance(mapping, dict))
self.assertEqual(mapping["SOLID"], 0)
self.assertEqual(mapping["LIQUID"], 1)
self.assertEqual(mapping["GAS"], 2)
self.assertEqual(mapping["PLASMA"], 3)
def testCreateVLenDataType(self):
typeItem = {'class': 'H5T_VLEN', 'base': 'H5T_STD_I32BE'}
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 'H5T_VARIABLE')
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'object')
self.assertEqual(dt.kind, 'O')
def testCreateOpaqueType(self):
typeItem = {'class': 'H5T_OPAQUE', 'size': 200}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void1600')
self.assertEqual(dt.kind, 'V')
self.assertEqual(typeSize, 200)
def testCreateCompoundType(self):
typeItem = {
'class':
'H5T_COMPOUND',
'fields': [{
'name': 'temp',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'pressure',
'type': 'H5T_IEEE_F32LE'
}, {
'name': 'location',
'type': {
'length': 'H5T_VARIABLE',
'charSet': 'H5T_CSET_ASCII',
'class': 'H5T_STRING',
'strPad': 'H5T_STR_NULLTERM'
}
}, {
'name': 'wind',
'type': 'H5T_STD_I16LE'
}]
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void144')
self.assertEqual(dt.kind, 'V')
self.assertEqual(len(dt.fields), 4)
dtLocation = dt[2]
self.assertEqual(dtLocation.name, 'object')
self.assertEqual(dtLocation.kind, 'O')
self.assertEqual(check_dtype(vlen=dtLocation), bytes)
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCreateArrayType(self):
typeItem = {
'class': 'H5T_ARRAY',
'base': 'H5T_STD_I64LE',
'dims': (3, 5)
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void960')
self.assertEqual(dt.kind, 'V')
self.assertEqual(typeSize, 120)
def testCreateBaseStringType(self):
typeItem = {
'class': 'H5T_STRING',
'charSet': 'H5T_CSET_ASCII',
'length': 6
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'bytes48')
self.assertEqual(dt.kind, 'S')
self.assertEqual(typeSize, 6)
def testCreateVLenUTF8Type(self):
typeItem = {
'class': 'H5T_STRING',
'charSet': 'H5T_CSET_UTF8',
'length': 'H5T_VARIABLE'
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'object')
self.assertEqual(dt.kind, 'O')
self.assertEqual(check_dtype(vlen=dt), str)
self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCreateBaseUnicodeType(self):
typeItem = {
'class': 'H5T_STRING',
'charSet': 'H5T_CSET_UTF8',
'length': 32
}
try:
dt = hdf5dtype.createDataType(typeItem)
self.assertTrue(not dt is None)
self.assertTrue(False) # expected exception
except TypeError:
pass
def testCreateCompoundInvalidFieldName(self):
typeItem = {
'class':
'H5T_COMPOUND',
'fields': [{
'name': '\u03b1',
'type': {
'base': 'H5T_STD_I32LE',
'class': 'H5T_INTEGER'
}
}, {
'name': '\u03c9',
'type': {
'base': 'H5T_STD_I32LE',
'class': 'H5T_INTEGER'
}
}]
}
try:
hdf5dtype.createDataType(typeItem)
self.assertTrue(False)
except TypeError:
pass # expected
def testCreateBaseType(self):
dt = hdf5dtype.createDataType('H5T_STD_U32BE')
self.assertEqual(dt.name, 'uint32')
self.assertEqual(dt.byteorder, '>')
self.assertEqual(dt.kind, 'u')
dt = hdf5dtype.createDataType('H5T_STD_I16LE')
self.assertEqual(dt.name, 'int16')
self.assertEqual(dt.kind, 'i')
dt = hdf5dtype.createDataType('H5T_IEEE_F64LE')
self.assertEqual(dt.name, 'float64')
self.assertEqual(dt.kind, 'f')
dt = hdf5dtype.createDataType('H5T_IEEE_F32LE')
self.assertEqual(dt.name, 'float32')
self.assertEqual(dt.kind, 'f')
typeItem = {'class': 'H5T_INTEGER', 'base': 'H5T_STD_I32BE'}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'int32')
self.assertEqual(dt.kind, 'i')
self.assertEqual(typeSize, 4)
def testCreateBoolType(self):
typeItem = {
"class": "H5T_ENUM",
"base": {
"base": "H5T_STD_I8LE",
"class": "H5T_INTEGER"
},
"mapping": {
"TRUE": 1,
"FALSE": 0
}
}
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 1)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'bool')
self.assertEqual(dt.kind, 'b')
def testCreateCompoundTypeUnicodeFields(self):
typeItem = {
'class':
'H5T_COMPOUND',
'fields': [{
'name': u'temp',
'type': 'H5T_IEEE_F32LE'
}, {
'name': u'pressure',
'type': 'H5T_IEEE_F32LE'
}, {
'name': u'wind',
'type': 'H5T_STD_I16LE'
}]
}
typeSize = hdf5dtype.getItemSize(typeItem)
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void80')
self.assertEqual(dt.kind, 'V')
self.assertEqual(len(dt.fields), 3)
self.assertEqual(typeSize, 10)
def testCompoundArrayType(self):
typeItem = {
"class":
"H5T_COMPOUND",
"fields": [{
"type": {
"class": "H5T_INTEGER",
"base": "H5T_STD_U64BE"
},
"name": "VALUE1"
}, {
"type": {
"class": "H5T_FLOAT",
"base": "H5T_IEEE_F64BE"
},
"name": "VALUE2"
}, {
"type": {
"class": "H5T_ARRAY",
"dims": [2],
"base": {
"class": "H5T_STRING",
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
}
},
"name": "VALUE3"
}]
}
dt = hdf5dtype.createDataType(typeItem)
typeSize = hdf5dtype.getItemSize(typeItem)
self.assertEqual(typeSize, 'H5T_VARIABLE')
self.assertEqual(len(dt), 3)
self.assertTrue("VALUE1" in dt.fields.keys())
self.assertTrue("VALUE2" in dt.fields.keys())
self.assertTrue("VALUE3" in dt.fields.keys())
dt3 = dt["VALUE3"]
self.assertEqual(check_dtype(vlen=dt3), bytes)
def testCreateCompoundOfCompoundType(self):
typeItem = {
'class':
'H5T_COMPOUND',
'fields': [{
'name': 'field1',
'type': {
'class':
'H5T_COMPOUND',
'fields': [{
'name': 'x',
'type': {
'class': 'H5T_FLOAT',
'base': 'H5T_IEEE_F32LE'
}
}, {
'name': 'y',
'type': {
'class': 'H5T_FLOAT',
'base': 'H5T_IEEE_F32LE'
}
}]
}
}, {
'name': 'field2',
'type': {
'class':
'H5T_COMPOUND',
'fields': [{
'name': 'a',
'type': {
'class': 'H5T_FLOAT',
'base': 'H5T_IEEE_F32LE'
}
}, {
'name': 'b',
'type': {
'class': 'H5T_FLOAT',
'base': 'H5T_IEEE_F32LE'
}
}, {
'name': 'c',
'type': {
'class': 'H5T_FLOAT',
'base': 'H5T_IEEE_F32LE'
}
}]
}
}]
}
dt = hdf5dtype.createDataType(typeItem)
self.assertEqual(dt.name, 'void160')
self.assertEqual(dt.kind, 'V')
self.assertEqual(len(dt.fields), 2)
dt_field1 = dt[0]
self.assertEqual(dt_field1.name, 'void64')
self.assertEqual(dt_field1.kind, 'V')
self.assertEqual(len(dt_field1.fields), 2)
dt_field2 = dt[1]
self.assertEqual(dt_field2.name, 'void96')
self.assertEqual(dt_field2.kind, 'V')
self.assertEqual(len(dt_field2.fields), 3)
def testJsonToArray(self):
dt = np.dtype('i4')
shape = [
4,
]
data = [0, 2, 4, 6]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (4, ))
for i in range(4):
self.assertEqual(out[i], i * 2)
# compound type
dt = np.dtype([('a', 'i4'), ('b', 'S5')])
shape = [
2,
]
data = [[4, 'four'], [5, 'five']]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(out.shape, (2, ))
self.assertTrue(isinstance(out[0], np.void))
e0 = out[0].tolist()
self.assertEqual(e0, (4, b'four'))
self.assertTrue(isinstance(out[1], np.void))
e1 = out[1].tolist()
self.assertEqual(e1, (5, b'five'))
shape = [
1,
]
data = [
[6, 'six'],
]
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
data = [6, 'six']
out = jsonToArray(shape, dt, data)
e0 = out[0].tolist()
self.assertEqual(e0, (6, b'six'))
# VLEN ascii
dt = special_dtype(vlen=bytes)
data = [b"one", b"two", b"three", "four", b"five"]
shape = [
5,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (5, ))
# TBD: code does not actually enforce use of bytes vs. str,
# probably not worth the effort to fix
self.assertEqual(out[2], b"three")
self.assertEqual(out[3], "four")
# VLEN str
dt = special_dtype(vlen=str)
data = [['part 1 - section A', 'part 1 - section B'],
['part 2 - section A', 'part 2 - section B']]
shape = [
2,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], str)
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(out.shape, (2, ))
self.assertEqual(out[0], tuple(data[0]))
self.assertEqual(out[1], tuple(data[1]))
# VLEN Scalar str
dt = special_dtype(vlen=str)
data = "I'm a string!"
shape = [
1,
]
out = jsonToArray(shape, dt, data)
# VLEN unicode
dt = special_dtype(vlen=bytes)
data = ["one", "two", "three", "four", "five"]
shape = [
5,
]
out = jsonToArray(shape, dt, data)
self.assertTrue("vlen" in out.dtype.metadata)
self.assertEqual(out.dtype.metadata["vlen"], bytes)
self.assertEqual(out.dtype.kind, 'O')
# TBD: this should show up as bytes, but may not be worth the effort
self.assertEqual(out[2], "three")
# VLEN data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [
4,
]
data = [[
1,
], [1, 2], [1, 2, 3], [1, 2, 3, 4]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (4, ))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(4):
e = out[i] #.tolist()
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, tuple(range(1, i + 2)))
# VLEN 2D data
dt = special_dtype(vlen=np.dtype('int32'))
shape = [2, 2]
data = [[[
0,
], [1, 2]], [[
1,
], [2, 3]]]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
self.assertEqual(out.shape, (2, 2))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32'))
for i in range(2):
for j in range(2):
e = out[i, j] #.tolist()
self.assertTrue(isinstance(e, tuple))
# create VLEN of obj ref's
ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"}
vlen_type = {"class": "H5T_VLEN", "base": ref_type}
dt = createDataType(vlen_type) # np datatype
id0 = 'g-a4f455b2-c8cf-11e7-8b73-0242ac110009'
id1 = 'g-a50af844-c8cf-11e7-8b73-0242ac110009'
id2 = 'g-a5236276-c8cf-11e7-8b73-0242ac110009'
data = [[
id0,
], [id0, id1], [id0, id1, id2]]
shape = [
3,
]
out = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(out, np.ndarray))
base_type = check_dtype(vlen=out.dtype)
self.assertEqual(base_type.kind, 'S')
self.assertEqual(base_type.itemsize, 48)
self.assertEqual(out.shape, (3, ))
self.assertEqual(out.dtype.kind, 'O')
self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('S48'))
e = out[0]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, ))
e = out[1]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, id1))
e = out[2]
self.assertTrue(isinstance(e, tuple))
self.assertEqual(e, (id0, id1, id2))
def testPutVLenCompoundBinary(self):
# Test PUT value for 1d attribute with variable length int types
print("testPutVLenCompoundBinary", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req["Content-Type"] = "application/octet-stream"
headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_rsp["accept"] = "application/octet-stream"
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
count = 4
# create dataset
fixed_str8_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 8,
"strPad": "H5T_STR_NULLPAD"
}
fields = [{
"type": {
"class": "H5T_INTEGER",
"base": "H5T_STD_U64BE"
},
"name": "VALUE1"
}, {
"type": fixed_str8_type,
"name": "VALUE2"
}, {
"type": {
"class": "H5T_ARRAY",
"dims": [2],
"base": {
"class": "H5T_STRING",
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
}
},
"name": "VALUE3"
}]
datatype = {'class': 'H5T_COMPOUND', 'fields': fields}
payload = {
'type': datatype,
'shape': [
count,
]
}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
dt_compound = createDataType(datatype)
# create numpy vlen array
arr = np.zeros((count, ), dtype=dt_compound)
for i in range(count):
e = arr[i]
e['VALUE1'] = i + 1
s = ''
for j in range(i + 5):
offset = (i + j) % 26
s += chr(ord('A') + offset)
e['VALUE2'] = s
e['VALUE3'] = ["Hi! " * (i + 1), "Bye!" * (i + 1)]
# write as binary data
data = arrayToBytes(arr)
self.assertEqual(len(data), 192) # will vary based on count
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
rsp = requests.put(req, data=data, headers=headers_bin_req)
self.assertEqual(rsp.status_code, 200)
# read values from dataset as json
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), count)
# read as binary
rsp = requests.get(req, headers=headers_bin_rsp)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['Content-Type'],
"application/octet-stream")
data = rsp.content
self.assertEqual(len(data), 192)
arr = bytesToArray(data, dt_compound, [
count,
])
