def testJsonToBytes(self):
#
# VLEN int
#
dt = special_dtype(vlen=np.dtype('int32'))
shape = [4,]
data = [[1,], [1,2], [1,2,3], [1,2,3,4]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
self.assertEqual(check_dtype(vlen=arr.dtype), np.dtype('int32'))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 56)
expected = b'\x04\x00\x00\x00\x01\x00\x00\x00\x08\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x0c\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# Compound vlen
#
dt_str = np.dtype('O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_str)])
shape = [4,]
data = [[42, "Hello"], [0,0], [0,0], [84, "Bye"]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 40)
expected = b'*\x00\x00\x00\x05\x00\x00\x00Hello\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00T\x00\x00\x00\x03\x00\x00\x00Bye'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN utf with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [4,]
data = [[42, ["hi","bye"]], [0,[0,0]], [0,[0,0]], [84, ["hi-hi", "bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN ascii with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
shape = [4,]
data = [[42, [b"hi", b"bye"]], [0,[0,0]], [0,[0,0]], [84, [b"hi-hi", b"bye-bye"]]]
arr = jsonToArray(shape, dt, data)
self.assertTrue(isinstance(arr, np.ndarray))
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
def testPutVLenCompoundBinary(self):
# Test PUT value for 1d attribute with variable length int types
print("testPutVLenCompoundBinary", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req["Content-Type"] = "application/octet-stream"
headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_rsp["accept"] = "application/octet-stream"
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
count = 4
# create dataset
fixed_str8_type = {
"charSet": "H5T_CSET_ASCII",
"class": "H5T_STRING",
"length": 8,
"strPad": "H5T_STR_NULLPAD"
}
fields = [{
"type": {
"class": "H5T_INTEGER",
"base": "H5T_STD_U64BE"
},
"name": "VALUE1"
}, {
"type": fixed_str8_type,
"name": "VALUE2"
}, {
"type": {
"class": "H5T_ARRAY",
"dims": [2],
"base": {
"class": "H5T_STRING",
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
}
},
"name": "VALUE3"
}]
datatype = {'class': 'H5T_COMPOUND', 'fields': fields}
payload = {
'type': datatype,
'shape': [
count,
]
}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
dt_compound = createDataType(datatype)
# create numpy vlen array
arr = np.zeros((count, ), dtype=dt_compound)
for i in range(count):
e = arr[i]
e['VALUE1'] = i + 1
s = ''
for j in range(i + 5):
offset = (i + j) % 26
s += chr(ord('A') + offset)
e['VALUE2'] = s
e['VALUE3'] = ["Hi! " * (i + 1), "Bye!" * (i + 1)]
# write as binary data
data = arrayToBytes(arr)
self.assertEqual(len(data), 192) # will vary based on count
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
rsp = requests.put(req, data=data, headers=headers_bin_req)
self.assertEqual(rsp.status_code, 200)
# read values from dataset as json
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), count)
# read as binary
rsp = requests.get(req, headers=headers_bin_rsp)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['Content-Type'],
"application/octet-stream")
data = rsp.content
self.assertEqual(len(data), 192)
arr = bytesToArray(data, dt_compound, [
count,
])
def testToBytes(self):
# Simple array
dt = np.dtype("<i4")
arr = np.asarray((1,2,3,4), dtype=dt)
buffer = arrayToBytes(arr)
self.assertEqual(buffer, arr.tobytes())
# convert buffer back to arr
arr_copy = bytesToArray(buffer, dt, (4,))
#print("arr_copy: {}".format(arr_copy))
self.assertTrue(np.array_equal(arr, arr_copy))
# Compound non-vlen
dt = np.dtype([('x', 'f8'), ('y', 'i4')])
arr = np.zeros((4,), dtype=dt)
arr[0] = (3.12, 42)
arr[3] = (1.28, 69)
buffer = arrayToBytes(arr)
self.assertEqual(buffer, arr.tobytes())
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
#print("arr_copy: {}".format(arr_copy))
self.assertTrue(np.array_equal(arr, arr_copy))
# VLEN of int32's
dt = np.dtype('O', metadata={'vlen': np.dtype('int32')})
arr = np.zeros((4,), dtype=dt)
arr[0] = np.int32([1,])
arr[1] = np.int32([1,2])
arr[2] = 0 # test un-intialized value
arr[3] = np.int32([1,2,3])
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 40)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
# np.array_equal doesn't work for object arrays
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
# VLEN of strings
dt = np.dtype('O', metadata={'vlen': str})
arr = np.zeros((5,), dtype=dt)
arr[0] = "one: \u4e00"
arr[1] = "two: \u4e8c"
arr[2] = "three: \u4e09"
arr[3] = "four: \u56db"
arr[4] = 0
buffer = arrayToBytes(arr)
expected = b'\x08\x00\x00\x00one: \xe4\xb8\x80\x08\x00\x00\x00two: \xe4\xba\x8c\n\x00\x00\x00three: \xe4\xb8\x89\t\x00\x00\x00four: \xe5\x9b\x9b\x00\x00\x00\x00'
self.assertEqual(buffer, expected)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (5,))
#print("arr_copy: {}".format(arr_copy))
self.assertTrue(np.array_equal(arr, arr_copy))
# VLEN of bytes
dt = np.dtype('O', metadata={'vlen': bytes})
arr = np.zeros((5,), dtype=dt)
arr[0] = b"Parting"
arr[1] = b"is such"
arr[2] = b"sweet"
arr[3] = b"sorrow"
arr[4] = 0
buffer = arrayToBytes(arr)
expected = b'\x07\x00\x00\x00Parting\x07\x00\x00\x00is such\x05\x00\x00\x00sweet\x06\x00\x00\x00sorrow\x00\x00\x00\x00'
self.assertEqual(buffer, expected) # same serialization as weith str
# convert back to array
arr_copy = bytesToArray(buffer, dt, (5,))
#print("arr_copy: {}".format(arr_copy))
self.assertTrue(np.array_equal(arr, arr_copy))
# Compound vlen
dt_str = np.dtype('O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_str)])
arr = np.zeros((4,), dtype=dt)
arr[0] = (42, "Hello")
arr[3] = (84, "Bye")
count = getByteArraySize(arr)
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 40)
self.assertEqual(buffer.find(b"Hello"), 8)
self.assertEqual(buffer.find(b"Bye"), 37)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN utf string with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': str})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
arr = np.zeros((4,), dtype=dt)
dt_str = np.dtype('O', metadata={'vlen': str})
arr[0] = (42, np.asarray(["hi", "bye"], dtype=dt_str))
arr[3] = (84, np.asarray(["hi-hi", "bye-bye"], dtype=dt_str))
count = getByteArraySize(arr)
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
#
# VLEN ascii with array type
#
dt_arr_str = np.dtype('(2,)O', metadata={'vlen': bytes})
dt = np.dtype([('x', 'i4'), ('tag', dt_arr_str)])
arr = np.zeros((4,), dtype=dt)
dt_str = np.dtype('O', metadata={'vlen': bytes})
arr[0] = (42, np.asarray([b"hi", b"bye"], dtype=dt_str))
arr[3] = (84, np.asarray([b"hi-hi", b"bye-bye"], dtype=dt_str))
count = getByteArraySize(arr)
buffer = arrayToBytes(arr)
self.assertEqual(len(buffer), 81)
self.assertEqual(buffer.find(b"hi"), 8)
self.assertEqual(buffer.find(b"bye"), 14)
self.assertEqual(buffer.find(b"hi-hi"), 49)
self.assertEqual(buffer.find(b"bye-bye"), 58)
# convert back to array
arr_copy = bytesToArray(buffer, dt, (4,))
self.assertEqual(arr.dtype, arr_copy.dtype)
self.assertEqual(arr.shape, arr_copy.shape)
for i in range(4):
e = arr[i]
e_copy = arr_copy[i]
self.assertTrue(np.array_equal(e, e_copy))
def testPutVLenIntBinary(self):
# Test PUT value for 1d attribute with variable length int types using binary transfer
print("testPutVLenIntBinary", self.base_domain)
count = 4
test_values = []
for i in range(count):
e = [
1,
]
for j in range(0, i):
e.append(j + 2)
test_values.append(e)
# test_values == [[1], [1,2]]
headers = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_req["Content-Type"] = "application/octet-stream"
headers_bin_rsp = helper.getRequestHeaders(domain=self.base_domain)
headers_bin_rsp["accept"] = "application/octet-stream"
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# create dataset
vlen_type = {
"class": "H5T_VLEN",
"base": {
"class": "H5T_INTEGER",
"base": "H5T_STD_I32LE"
}
}
payload = {
'type': vlen_type,
'shape': [
count,
]
}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# create numpy vlen array
dt = np.dtype('O', metadata={'vlen': np.dtype('int32')})
arr = np.zeros((count, ), dtype=dt)
for i in range(count):
arr[i] = np.int32(test_values[i])
# write as binary data
data = arrayToBytes(arr)
self.assertEqual(len(data), 56)
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
rsp = requests.put(req, data=data, headers=headers_bin_req)
self.assertEqual(rsp.status_code, 200)
# read values from dataset with json
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), count)
for i in range(count):
self.assertEqual(value[i], test_values[i])
# read as binary
rsp = requests.get(req, headers=headers_bin_rsp)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['Content-Type'],
"application/octet-stream")
data = rsp.content
self.assertEqual(len(data), 56)
arr = bytesToArray(data, dt, [
count,
])
for i in range(count):
self.assertEqual(value[i], test_values[i])
# read back a selection
params = {"select": "[2:3]"}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), 1)
self.assertEqual(value[0], [1, 2, 3])