def testCreateArrayIntegerType(self): typeItem = { 'class': 'H5T_INTEGER', 'base': 'H5T_STD_I64LE', 'dims': (3, 5) } try: hdf5dtype.createDataType(typeItem) self.assertTrue( False) # expected exception - dims used with non-array type except TypeError: pass # should get exception
def testCreateCompoundArrayType(self): typeItem = { "class": "H5T_COMPOUND", "fields": [{ "type": { "base": "H5T_STD_I8LE", "class": "H5T_INTEGER" }, "name": "a" }, { "type": { "dims": [10], "base": { "length": 1, "charSet": "H5T_CSET_ASCII", "class": "H5T_STRING", "strPad": "H5T_STR_NULLPAD" }, "class": "H5T_ARRAY" }, "name": "b" }] } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(len(dt.fields), 2) self.assertTrue('a' in dt.fields.keys()) self.assertTrue('b' in dt.fields.keys()) self.assertEqual(typeSize, 11)
def testCreateEnumType(self): typeItem = { "class": "H5T_ENUM", "base": { "base": "H5T_STD_I16LE", "class": "H5T_INTEGER" }, "mapping": { "GAS": 2, "LIQUID": 1, "PLASMA": 3, "SOLID": 0 } } typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 2) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'int16') self.assertEqual(dt.kind, 'i') mapping = check_dtype(enum=dt) self.assertTrue(isinstance(mapping, dict)) self.assertEqual(mapping["SOLID"], 0) self.assertEqual(mapping["LIQUID"], 1) self.assertEqual(mapping["GAS"], 2) self.assertEqual(mapping["PLASMA"], 3)
def testCreateVLenDataType(self): typeItem = {'class': 'H5T_VLEN', 'base': 'H5T_STD_I32BE'} typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 'H5T_VARIABLE') dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'object') self.assertEqual(dt.kind, 'O')
def testCreateOpaqueType(self): typeItem = {'class': 'H5T_OPAQUE', 'size': 200} typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'void1600') self.assertEqual(dt.kind, 'V') self.assertEqual(typeSize, 200)
def testCreateCompoundType(self): typeItem = { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': 'temp', 'type': 'H5T_IEEE_F32LE' }, { 'name': 'pressure', 'type': 'H5T_IEEE_F32LE' }, { 'name': 'location', 'type': { 'length': 'H5T_VARIABLE', 'charSet': 'H5T_CSET_ASCII', 'class': 'H5T_STRING', 'strPad': 'H5T_STR_NULLTERM' } }, { 'name': 'wind', 'type': 'H5T_STD_I16LE' }] } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'void144') self.assertEqual(dt.kind, 'V') self.assertEqual(len(dt.fields), 4) dtLocation = dt[2] self.assertEqual(dtLocation.name, 'object') self.assertEqual(dtLocation.kind, 'O') self.assertEqual(check_dtype(vlen=dtLocation), bytes) self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCreateArrayType(self): typeItem = { 'class': 'H5T_ARRAY', 'base': 'H5T_STD_I64LE', 'dims': (3, 5) } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'void960') self.assertEqual(dt.kind, 'V') self.assertEqual(typeSize, 120)
def testCreateBaseStringType(self): typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_ASCII', 'length': 6 } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'bytes48') self.assertEqual(dt.kind, 'S') self.assertEqual(typeSize, 6)
def testCreateVLenUTF8Type(self): typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_UTF8', 'length': 'H5T_VARIABLE' } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'object') self.assertEqual(dt.kind, 'O') self.assertEqual(check_dtype(vlen=dt), str) self.assertEqual(typeSize, 'H5T_VARIABLE')
def testCreateBaseUnicodeType(self): typeItem = { 'class': 'H5T_STRING', 'charSet': 'H5T_CSET_UTF8', 'length': 32 } try: dt = hdf5dtype.createDataType(typeItem) self.assertTrue(not dt is None) self.assertTrue(False) # expected exception except TypeError: pass
def testCreateCompoundInvalidFieldName(self): typeItem = { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': '\u03b1', 'type': { 'base': 'H5T_STD_I32LE', 'class': 'H5T_INTEGER' } }, { 'name': '\u03c9', 'type': { 'base': 'H5T_STD_I32LE', 'class': 'H5T_INTEGER' } }] } try: hdf5dtype.createDataType(typeItem) self.assertTrue(False) except TypeError: pass # expected
def testCreateBaseType(self): dt = hdf5dtype.createDataType('H5T_STD_U32BE') self.assertEqual(dt.name, 'uint32') self.assertEqual(dt.byteorder, '>') self.assertEqual(dt.kind, 'u') dt = hdf5dtype.createDataType('H5T_STD_I16LE') self.assertEqual(dt.name, 'int16') self.assertEqual(dt.kind, 'i') dt = hdf5dtype.createDataType('H5T_IEEE_F64LE') self.assertEqual(dt.name, 'float64') self.assertEqual(dt.kind, 'f') dt = hdf5dtype.createDataType('H5T_IEEE_F32LE') self.assertEqual(dt.name, 'float32') self.assertEqual(dt.kind, 'f') typeItem = {'class': 'H5T_INTEGER', 'base': 'H5T_STD_I32BE'} typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'int32') self.assertEqual(dt.kind, 'i') self.assertEqual(typeSize, 4)
def testCreateBoolType(self): typeItem = { "class": "H5T_ENUM", "base": { "base": "H5T_STD_I8LE", "class": "H5T_INTEGER" }, "mapping": { "TRUE": 1, "FALSE": 0 } } typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 1) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'bool') self.assertEqual(dt.kind, 'b')
def testCreateCompoundTypeUnicodeFields(self): typeItem = { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': u'temp', 'type': 'H5T_IEEE_F32LE' }, { 'name': u'pressure', 'type': 'H5T_IEEE_F32LE' }, { 'name': u'wind', 'type': 'H5T_STD_I16LE' }] } typeSize = hdf5dtype.getItemSize(typeItem) dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'void80') self.assertEqual(dt.kind, 'V') self.assertEqual(len(dt.fields), 3) self.assertEqual(typeSize, 10)
def testCompoundArrayType(self): typeItem = { "class": "H5T_COMPOUND", "fields": [{ "type": { "class": "H5T_INTEGER", "base": "H5T_STD_U64BE" }, "name": "VALUE1" }, { "type": { "class": "H5T_FLOAT", "base": "H5T_IEEE_F64BE" }, "name": "VALUE2" }, { "type": { "class": "H5T_ARRAY", "dims": [2], "base": { "class": "H5T_STRING", "charSet": "H5T_CSET_ASCII", "strPad": "H5T_STR_NULLTERM", "length": "H5T_VARIABLE" } }, "name": "VALUE3" }] } dt = hdf5dtype.createDataType(typeItem) typeSize = hdf5dtype.getItemSize(typeItem) self.assertEqual(typeSize, 'H5T_VARIABLE') self.assertEqual(len(dt), 3) self.assertTrue("VALUE1" in dt.fields.keys()) self.assertTrue("VALUE2" in dt.fields.keys()) self.assertTrue("VALUE3" in dt.fields.keys()) dt3 = dt["VALUE3"] self.assertEqual(check_dtype(vlen=dt3), bytes)
def testCreateCompoundOfCompoundType(self): typeItem = { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': 'field1', 'type': { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': 'x', 'type': { 'class': 'H5T_FLOAT', 'base': 'H5T_IEEE_F32LE' } }, { 'name': 'y', 'type': { 'class': 'H5T_FLOAT', 'base': 'H5T_IEEE_F32LE' } }] } }, { 'name': 'field2', 'type': { 'class': 'H5T_COMPOUND', 'fields': [{ 'name': 'a', 'type': { 'class': 'H5T_FLOAT', 'base': 'H5T_IEEE_F32LE' } }, { 'name': 'b', 'type': { 'class': 'H5T_FLOAT', 'base': 'H5T_IEEE_F32LE' } }, { 'name': 'c', 'type': { 'class': 'H5T_FLOAT', 'base': 'H5T_IEEE_F32LE' } }] } }] } dt = hdf5dtype.createDataType(typeItem) self.assertEqual(dt.name, 'void160') self.assertEqual(dt.kind, 'V') self.assertEqual(len(dt.fields), 2) dt_field1 = dt[0] self.assertEqual(dt_field1.name, 'void64') self.assertEqual(dt_field1.kind, 'V') self.assertEqual(len(dt_field1.fields), 2) dt_field2 = dt[1] self.assertEqual(dt_field2.name, 'void96') self.assertEqual(dt_field2.kind, 'V') self.assertEqual(len(dt_field2.fields), 3)
def testJsonToArray(self): dt = np.dtype('i4') shape = [ 4, ] data = [0, 2, 4, 6] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(out.shape, (4, )) for i in range(4): self.assertEqual(out[i], i * 2) # compound type dt = np.dtype([('a', 'i4'), ('b', 'S5')]) shape = [ 2, ] data = [[4, 'four'], [5, 'five']] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(out.shape, (2, )) self.assertTrue(isinstance(out[0], np.void)) e0 = out[0].tolist() self.assertEqual(e0, (4, b'four')) self.assertTrue(isinstance(out[1], np.void)) e1 = out[1].tolist() self.assertEqual(e1, (5, b'five')) shape = [ 1, ] data = [ [6, 'six'], ] out = jsonToArray(shape, dt, data) e0 = out[0].tolist() self.assertEqual(e0, (6, b'six')) data = [6, 'six'] out = jsonToArray(shape, dt, data) e0 = out[0].tolist() self.assertEqual(e0, (6, b'six')) # VLEN ascii dt = special_dtype(vlen=bytes) data = [b"one", b"two", b"three", "four", b"five"] shape = [ 5, ] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], bytes) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(out.shape, (5, )) # TBD: code does not actually enforce use of bytes vs. str, # probably not worth the effort to fix self.assertEqual(out[2], b"three") self.assertEqual(out[3], "four") # VLEN str dt = special_dtype(vlen=str) data = [['part 1 - section A', 'part 1 - section B'], ['part 2 - section A', 'part 2 - section B']] shape = [ 2, ] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], str) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(out.shape, (2, )) self.assertEqual(out[0], tuple(data[0])) self.assertEqual(out[1], tuple(data[1])) # VLEN Scalar str dt = special_dtype(vlen=str) data = "I'm a string!" shape = [ 1, ] out = jsonToArray(shape, dt, data) # VLEN unicode dt = special_dtype(vlen=bytes) data = ["one", "two", "three", "four", "five"] shape = [ 5, ] out = jsonToArray(shape, dt, data) self.assertTrue("vlen" in out.dtype.metadata) self.assertEqual(out.dtype.metadata["vlen"], bytes) self.assertEqual(out.dtype.kind, 'O') # TBD: this should show up as bytes, but may not be worth the effort self.assertEqual(out[2], "three") # VLEN data dt = special_dtype(vlen=np.dtype('int32')) shape = [ 4, ] data = [[ 1, ], [1, 2], [1, 2, 3], [1, 2, 3, 4]] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) self.assertEqual(out.shape, (4, )) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) for i in range(4): e = out[i] #.tolist() self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, tuple(range(1, i + 2))) # VLEN 2D data dt = special_dtype(vlen=np.dtype('int32')) shape = [2, 2] data = [[[ 0, ], [1, 2]], [[ 1, ], [2, 3]]] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) self.assertEqual(out.shape, (2, 2)) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('int32')) for i in range(2): for j in range(2): e = out[i, j] #.tolist() self.assertTrue(isinstance(e, tuple)) # create VLEN of obj ref's ref_type = {"class": "H5T_REFERENCE", "base": "H5T_STD_REF_OBJ"} vlen_type = {"class": "H5T_VLEN", "base": ref_type} dt = createDataType(vlen_type) # np datatype id0 = 'g-a4f455b2-c8cf-11e7-8b73-0242ac110009' id1 = 'g-a50af844-c8cf-11e7-8b73-0242ac110009' id2 = 'g-a5236276-c8cf-11e7-8b73-0242ac110009' data = [[ id0, ], [id0, id1], [id0, id1, id2]] shape = [ 3, ] out = jsonToArray(shape, dt, data) self.assertTrue(isinstance(out, np.ndarray)) base_type = check_dtype(vlen=out.dtype) self.assertEqual(base_type.kind, 'S') self.assertEqual(base_type.itemsize, 48) self.assertEqual(out.shape, (3, )) self.assertEqual(out.dtype.kind, 'O') self.assertEqual(check_dtype(vlen=out.dtype), np.dtype('S48')) e = out[0] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0, )) e = out[1] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0, id1)) e = out[2] self.assertTrue(isinstance(e, tuple)) self.assertEqual(e, (id0, id1, id2))
def testPutVLenCompoundBinary(self): # Test PUT value for 1d attribute with variable length int types print("testPutVLenCompoundBinary", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req["Content-Type"] = "application/octet-stream" headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) headers_bin_rsp["accept"] = "application/octet-stream" req = self.endpoint + '/' # Get root uuid rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) root_uuid = rspJson["root"] helper.validateId(root_uuid) count = 4 # create dataset fixed_str8_type = { "charSet": "H5T_CSET_ASCII", "class": "H5T_STRING", "length": 8, "strPad": "H5T_STR_NULLPAD" } fields = [{ "type": { "class": "H5T_INTEGER", "base": "H5T_STD_U64BE" }, "name": "VALUE1" }, { "type": fixed_str8_type, "name": "VALUE2" }, { "type": { "class": "H5T_ARRAY", "dims": [2], "base": { "class": "H5T_STRING", "charSet": "H5T_CSET_ASCII", "strPad": "H5T_STR_NULLTERM", "length": "H5T_VARIABLE" } }, "name": "VALUE3" }] datatype = {'class': 'H5T_COMPOUND', 'fields': fields} payload = { 'type': datatype, 'shape': [ count, ] } req = self.endpoint + "/datasets" rsp = requests.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset rspJson = json.loads(rsp.text) dset_uuid = rspJson['id'] self.assertTrue(helper.validateId(dset_uuid)) # link new dataset as 'dset' name = 'dset' req = self.endpoint + "/groups/" + root_uuid + "/links/" + name payload = {"id": dset_uuid} rsp = requests.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) dt_compound = createDataType(datatype) # create numpy vlen array arr = np.zeros((count, ), dtype=dt_compound) for i in range(count): e = arr[i] e['VALUE1'] = i + 1 s = '' for j in range(i + 5): offset = (i + j) % 26 s += chr(ord('A') + offset) e['VALUE2'] = s e['VALUE3'] = ["Hi! " * (i + 1), "Bye!" * (i + 1)] # write as binary data data = arrayToBytes(arr) self.assertEqual(len(data), 192) # will vary based on count req = self.endpoint + "/datasets/" + dset_uuid + "/value" rsp = requests.put(req, data=data, headers=headers_bin_req) self.assertEqual(rsp.status_code, 200) # read values from dataset as json rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) value = rspJson["value"] self.assertEqual(len(value), count) # read as binary rsp = requests.get(req, headers=headers_bin_rsp) self.assertEqual(rsp.status_code, 200) self.assertEqual(rsp.headers['Content-Type'], "application/octet-stream") data = rsp.content self.assertEqual(len(data), 192) arr = bytesToArray(data, dt_compound, [ count, ])