def testJsonToBytes(self): # # VLEN int # dt = special_dtype(vlen=np.dtype('int32')) shape = [ 4, ] data = [[ 1, ], [1, 2], [1, 2, 3], [1, 2, 3, 4]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) self.assertEqual(check_dtype(vlen=arr.dtype), np.dtype('int32')) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 56) expected = b'\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00' self.assertEqual(buffer, expected) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) # np.array_equal doesn't work for object arrays self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # Compound vlen # dt_str = np.dtype('O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_str)]) shape = [ 4, ] data = [[42, "Hello"], [0, 0], [0, 0], [84, "Bye"]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 40) expected = b'*\x00\x00\x00\x05\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x03\x00\x00\x00Bye' self.assertEqual(buffer, expected) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) # np.array_equal doesn't work for object arrays self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN utf with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) shape = [ 4, ] data = [[42, ["hi", "bye"]], [0, [0, 0]], [0, [0, 0]], [84, ["hi-hi", "bye-bye"]]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) arr_copy = bytesToArray(buffer, dt, (4, )) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN ascii with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) shape = [ 4, ] data = [[42, [b"hi", b"bye"]], [0, [0, 0]], [0, [0, 0]], [84, [b"hi-hi", b"bye-bye"]]] arr = jsonToArray(shape, dt, data) self.assertTrue(isinstance(arr, np.ndarray)) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) arr_copy = bytesToArray(buffer, dt, (4, )) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy))
def testPutVLenCompoundBinary(self): # Test PUT value for 1d attribute with variable length int types print("testPutVLenCompoundBinary", self.base_domain) headers = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req["Content-Type"] = "application/octet-stream" headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) headers_bin_rsp["accept"] = "application/octet-stream" req = self.endpoint + '/' # Get root uuid rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) root_uuid = rspJson["root"] helper.validateId(root_uuid) count = 4 # create dataset fixed_str8_type = { "charSet": "H5T_CSET_ASCII", "class": "H5T_STRING", "length": 8, "strPad": "H5T_STR_NULLPAD" } fields = [{ "type": { "class": "H5T_INTEGER", "base": "H5T_STD_U64BE" }, "name": "VALUE1" }, { "type": fixed_str8_type, "name": "VALUE2" }, { "type": { "class": "H5T_ARRAY", "dims": [2], "base": { "class": "H5T_STRING", "charSet": "H5T_CSET_ASCII", "strPad": "H5T_STR_NULLTERM", "length": "H5T_VARIABLE" } }, "name": "VALUE3" }] datatype = {'class': 'H5T_COMPOUND', 'fields': fields} payload = { 'type': datatype, 'shape': [ count, ] } req = self.endpoint + "/datasets" rsp = requests.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset rspJson = json.loads(rsp.text) dset_uuid = rspJson['id'] self.assertTrue(helper.validateId(dset_uuid)) # link new dataset as 'dset' name = 'dset' req = self.endpoint + "/groups/" + root_uuid + "/links/" + name payload = {"id": dset_uuid} rsp = requests.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) dt_compound = createDataType(datatype) # create numpy vlen array arr = np.zeros((count, ), dtype=dt_compound) for i in range(count): e = arr[i] e['VALUE1'] = i + 1 s = '' for j in range(i + 5): offset = (i + j) % 26 s += chr(ord('A') + offset) e['VALUE2'] = s e['VALUE3'] = ["Hi! " * (i + 1), "Bye!" * (i + 1)] # write as binary data data = arrayToBytes(arr) self.assertEqual(len(data), 192) # will vary based on count req = self.endpoint + "/datasets/" + dset_uuid + "/value" rsp = requests.put(req, data=data, headers=headers_bin_req) self.assertEqual(rsp.status_code, 200) # read values from dataset as json rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) value = rspJson["value"] self.assertEqual(len(value), count) # read as binary rsp = requests.get(req, headers=headers_bin_rsp) self.assertEqual(rsp.status_code, 200) self.assertEqual(rsp.headers['Content-Type'], "application/octet-stream") data = rsp.content self.assertEqual(len(data), 192) arr = bytesToArray(data, dt_compound, [ count, ])
def testToBytes(self): # Simple array dt = np.dtype("<i4") arr = np.asarray((1, 2, 3, 4), dtype=dt) buffer = arrayToBytes(arr) self.assertEqual(buffer, arr.tobytes()) # convert buffer back to arr arr_copy = bytesToArray(buffer, dt, (4, )) #print("arr_copy: {}".format(arr_copy)) self.assertTrue(np.array_equal(arr, arr_copy)) # fixed length string dt = np.dtype("S8") arr = np.asarray(("abcdefgh", "ABCDEFGH", "12345678"), dtype=dt) buffer = arrayToBytes(arr) self.assertEqual(buffer, arr.tobytes()) # convert back to arry arr_copy = bytesToArray(buffer, dt, (3, )) self.assertTrue(np.array_equal(arr, arr_copy)) # Compound non-vlen dt = np.dtype([('x', 'f8'), ('y', 'i4')]) arr = np.zeros((4, ), dtype=dt) arr[0] = (3.12, 42) arr[3] = (1.28, 69) buffer = arrayToBytes(arr) self.assertEqual(buffer, arr.tobytes()) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) self.assertTrue(np.array_equal(arr, arr_copy)) # VLEN of int32's dt = np.dtype('O', metadata={'vlen': np.dtype('int32')}) arr = np.zeros((4, ), dtype=dt) arr[0] = np.int32([ 1, ]) arr[1] = np.int32([1, 2]) arr[2] = 0 # test un-intialized value arr[3] = np.int32([1, 2, 3]) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 40) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) # np.array_equal doesn't work for object arrays self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # VLEN of strings dt = np.dtype('O', metadata={'vlen': str}) arr = np.zeros((5, ), dtype=dt) arr[0] = "one: \u4e00" arr[1] = "two: \u4e8c" arr[2] = "three: \u4e09" arr[3] = "four: \u56db" arr[4] = 0 buffer = arrayToBytes(arr) expected = b'\x08\x00\x00\x00one: \xe4\xb8\x80\x08\x00\x00\x00two: \xe4\xba\x8c\n\x00\x00\x00three: \xe4\xb8\x89\t\x00\x00\x00four: \xe5\x9b\x9b\x00\x00\x00\x00' self.assertEqual(buffer, expected) # convert back to array arr_copy = bytesToArray(buffer, dt, (5, )) self.assertTrue(np.array_equal(arr, arr_copy)) # VLEN of bytes dt = np.dtype('O', metadata={'vlen': bytes}) arr = np.zeros((5, ), dtype=dt) arr[0] = b"Parting" arr[1] = b"is such" arr[2] = b"sweet" arr[3] = b"sorrow" arr[4] = 0 buffer = arrayToBytes(arr) expected = b'\x07\x00\x00\x00Parting\x07\x00\x00\x00is such\x05\x00\x00\x00sweet\x06\x00\x00\x00sorrow\x00\x00\x00\x00' self.assertEqual(buffer, expected) # same serialization as weith str # convert back to array arr_copy = bytesToArray(buffer, dt, (5, )) self.assertTrue(np.array_equal(arr, arr_copy)) # Compound vlen dt_vstr = np.dtype('O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_vstr), ('code', 'S4')]) arr = np.zeros((4, ), dtype=dt) arr[0] = (42, "Hello", "X1") arr[3] = (84, "Bye", "XYZ") count = getByteArraySize(arr) buffer = arrayToBytes(arr) for i in range(len(buffer)): c = buffer[i] self.assertEqual(len(buffer), 56) self.assertEqual(buffer.find(b"Hello"), 8) self.assertEqual(buffer.find(b"Bye"), 49) self.assertEqual(buffer.find(b"X1"), 13) self.assertEqual(buffer.find(b"XYZ"), 52) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN utf string with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) arr = np.zeros((4, ), dtype=dt) dt_str = np.dtype('O', metadata={'vlen': str}) arr[0] = (42, np.asarray(["hi", "bye"], dtype=dt_str)) arr[3] = (84, np.asarray(["hi-hi", "bye-bye"], dtype=dt_str)) count = getByteArraySize(arr) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy)) # # VLEN ascii with array type # dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes}) dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)]) arr = np.zeros((4, ), dtype=dt) dt_str = np.dtype('O', metadata={'vlen': bytes}) arr[0] = (42, np.asarray([b"hi", b"bye"], dtype=dt_str)) arr[3] = (84, np.asarray([b"hi-hi", b"bye-bye"], dtype=dt_str)) count = getByteArraySize(arr) buffer = arrayToBytes(arr) self.assertEqual(len(buffer), 81) self.assertEqual(buffer.find(b"hi"), 8) self.assertEqual(buffer.find(b"bye"), 14) self.assertEqual(buffer.find(b"hi-hi"), 49) self.assertEqual(buffer.find(b"bye-bye"), 58) # convert back to array arr_copy = bytesToArray(buffer, dt, (4, )) self.assertEqual(arr.dtype, arr_copy.dtype) self.assertEqual(arr.shape, arr_copy.shape) for i in range(4): e = arr[i] e_copy = arr_copy[i] self.assertTrue(np.array_equal(e, e_copy))
def testPutVLenIntBinary(self): # Test PUT value for 1d attribute with variable length int types using binary transfer print("testPutVLenIntBinary", self.base_domain) count = 4 test_values = [] for i in range(count): e = [ 1, ] for j in range(0, i): e.append(j + 2) test_values.append(e) # test_values == [[1], [1,2]] headers = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req = helper.getRequestHeaders(domain=self.base_domain) headers_bin_req["Content-Type"] = "application/octet-stream" headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain) headers_bin_rsp["accept"] = "application/octet-stream" req = self.endpoint + '/' # Get root uuid rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) root_uuid = rspJson["root"] helper.validateId(root_uuid) # create dataset vlen_type = { "class": "H5T_VLEN", "base": { "class": "H5T_INTEGER", "base": "H5T_STD_I32LE" } } payload = { 'type': vlen_type, 'shape': [ count, ] } req = self.endpoint + "/datasets" rsp = requests.post(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create dataset rspJson = json.loads(rsp.text) dset_uuid = rspJson['id'] self.assertTrue(helper.validateId(dset_uuid)) # link new dataset as 'dset' name = 'dset' req = self.endpoint + "/groups/" + root_uuid + "/links/" + name payload = {"id": dset_uuid} rsp = requests.put(req, data=json.dumps(payload), headers=headers) self.assertEqual(rsp.status_code, 201) # create numpy vlen array dt = np.dtype('O', metadata={'vlen': np.dtype('int32')}) arr = np.zeros((count, ), dtype=dt) for i in range(count): arr[i] = np.int32(test_values[i]) # write as binary data data = arrayToBytes(arr) self.assertEqual(len(data), 56) req = self.endpoint + "/datasets/" + dset_uuid + "/value" rsp = requests.put(req, data=data, headers=headers_bin_req) self.assertEqual(rsp.status_code, 200) # read values from dataset with json rsp = requests.get(req, headers=headers) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) value = rspJson["value"] self.assertEqual(len(value), count) for i in range(count): self.assertEqual(value[i], test_values[i]) # read as binary rsp = requests.get(req, headers=headers_bin_rsp) self.assertEqual(rsp.status_code, 200) self.assertEqual(rsp.headers['Content-Type'], "application/octet-stream") data = rsp.content self.assertEqual(len(data), 56) arr = bytesToArray(data, dt, [ count, ]) for i in range(count): self.assertEqual(value[i], test_values[i]) # read back a selection params = {"select": "[2:3]"} rsp = requests.get(req, headers=headers, params=params) self.assertEqual(rsp.status_code, 200) rspJson = json.loads(rsp.text) self.assertTrue("hrefs" in rspJson) self.assertTrue("value" in rspJson) value = rspJson["value"] self.assertEqual(len(value), 1) self.assertEqual(value[0], [1, 2, 3])