def testPutCommittedType(self):
domain = 'tall_updated.' + config.get('domain')
attr_name = 'attr_committed'
root_uuid = helper.getRootUUID(domain)
headers = {'host': domain}
# create the datatype
payload = {'type': 'H5T_IEEE_F32LE'}
req = self.endpoint + "/datatypes/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create datatype
rspJson = json.loads(rsp.text)
dtype_uuid = rspJson['id']
self.assertTrue(helper.validateId(dtype_uuid))
# link new datatype as 'dtype1'
root_uuid = helper.getRootUUID(domain)
name = 'dtype1'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {'id': dtype_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
# create the attribute using the type created above
value = []
for i in range(10):
value.append(i*0.5)
payload = {'type': dtype_uuid, 'shape': 10, 'value': value}
req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create attribute
rspJson = json.loads(rsp.text)
self.assertEqual(len(rspJson['hrefs']), 3)
def testPutCommittedType(self):
domain = 'tall_updated.' + config.get('domain')
attr_name = 'attr_committed'
root_uuid = helper.getRootUUID(domain)
headers = {'host': domain}
# create the datatype
payload = {'type': 'H5T_IEEE_F32LE'}
req = self.endpoint + "/datatypes/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create datatype
rspJson = json.loads(rsp.text)
dtype_uuid = rspJson['id']
self.assertTrue(helper.validateId(dtype_uuid))
# link new datatype as 'dtype1'
root_uuid = helper.getRootUUID(domain)
name = 'dtype1'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {'id': dtype_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
# create the attribute using the type created above
value = []
for i in range(10):
value.append(i * 0.5)
payload = {'type': dtype_uuid, 'shape': 10, 'value': value}
req = self.endpoint + "/groups/" + root_uuid + "/attributes/" + attr_name
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create attribute
rspJson = json.loads(rsp.text)
self.assertEqual(len(rspJson['hrefs']), 3)
def testQueries(self):
# use '%26' rather than '&' since otherwise it will be
# interpreted as a http query param seperator
queries = { "date == 23": 24,
"wind == 'W 5'": 3,
"temp > 61": 53,
"(date >=22) %26 (date <= 24)": 62,
"(date == 21) %26 (temp > 70)": 4,
"(wind == 'E 7') | (wind == 'S 7')": 7 }
#queries = { "(date == 21) %26 (temp >= 72)": 4 }
domain = 'compound.' + config.get('domain')
headers = {'host': domain}
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
for key in queries.keys():
query = req + "?query=" + key
rsp = requests.get(query, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('hrefs' in rspJson)
self.assertTrue('index' in rspJson)
index = rspJson['index']
self.assertTrue(len(index), queries[key])
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), queries[key])
def testDeleteAnonymous(self):
# test delete works with anonymous dataset
domain = 'tall_dset22_deleted.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
g2_uuid = helper.getUUID(domain, root_uuid, 'g2')
self.assertTrue(helper.validateId(g2_uuid))
d22_uuid = helper.getUUID(domain, g2_uuid, 'dset2.2')
self.assertTrue(helper.validateId(d22_uuid))
# delete g2, that will make dataset anonymous
req = self.endpoint + "/groups/" + g2_uuid
headers = {'host': domain}
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# verify that a GET on the dataset succeeds still
req = helper.getEndpoint() + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# delete dataset...
req = self.endpoint + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# verify that a GET on the dataset fails
req = helper.getEndpoint() + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 410)
def testPostArray(self):
domain = 'newarraydset.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
datatype = {
'class': 'H5T_ARRAY',
'base': 'H5T_STD_I64LE',
'shape': (3, 5)
}
payload = {'type': datatype, 'shape': 10}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset1'
root_uuid = helper.getRootUUID(domain)
name = 'dset1'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
def testGetArray(self):
domain = 'array_dset.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeItemBase = typeItem['base']
self.assertEqual(typeItemBase['class'], 'H5T_INTEGER')
self.assertEqual(typeItemBase['base'], 'H5T_STD_I64LE')
def testGetCompoundCommitted(self):
domain = 'compound_committed.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 72)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
self.assertTrue('fields' in typeItem)
fields = typeItem['fields']
self.assertEqual(len(fields), 3)
timeField = fields[1]
self.assertEqual(timeField['name'], 'time')
self.assertTrue('type' in timeField)
timeFieldType = timeField['type']
self.assertEqual(timeFieldType['class'], 'H5T_STRING')
self.assertEqual(timeFieldType['cset'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['order'], 'H5T_ORDER_NONE')
self.assertEqual(timeFieldType['strsize'], 6)
self.assertEqual(timeFieldType['strpad'], 'H5T_STR_NULLPAD')
tempField = fields[2]
self.assertEqual(tempField['name'], 'temp')
tempFieldType = tempField['type']
self.assertEqual(tempFieldType['class'], 'H5T_INTEGER')
self.assertEqual(tempFieldType['base'], 'H5T_STD_I32LE')
def testGetInvalidName(self):
domain = 'tall.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootUUID + "/attributes/no_attr_here"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 404)
def testPutExternalLink(self):
logging.info("LinkTest.testPutExternalLink")
domain = 'tall_updated.' + config.get('domain')
target_domain = 'external_target.' + config.get('domain')
target_path = '/dset1'
grpId = helper.createGroup(domain)
rootId = helper.getRootUUID(domain)
name = 'extlink'
req = helper.getEndpoint() + "/groups/" + rootId + "/links/" + name
payload = {"h5path": target_path, "h5domain": target_domain}
headers = {'host': domain}
# verify extlink does not exist
rsp = requests.get(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 404)
# make request
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
# verify link is created
rsp = requests.get(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# verify that it is an external link
rspJson = json.loads(rsp.text)
target = rspJson['link']
self.failUnlessEqual(target['class'], 'H5L_TYPE_EXTERNAL')
self.failUnlessEqual(target['h5domain'], target_domain)
self.failUnlessEqual(target['h5path'], target_path)
def testQueryBatch(self):
domain = 'compound.' + config.get('domain')
headers = {'host': domain}
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
start = 0
stop = 72
count = 0
count = req_count = 0
limit = 10
req += "?query=date == 23" # values where date field = 23
req += "&Limit=" + str(
limit) # return no more than 10 results at a time
for i in range(50):
sreq = req + "&select=[" + str(start) + ":" + str(stop) + "]"
rsp = requests.get(sreq, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
req_count += 1
rspJson = json.loads(rsp.text)
self.assertTrue('hrefs' in rspJson)
self.assertTrue('index' in rspJson)
index = rspJson['index']
self.assertTrue(len(index) <= limit)
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), len(index))
count += len(index)
if len(index) < limit:
break # no more results
start = index[-1] + 1 # start at next index
self.assertEqual(count, 24)
self.assertEqual(req_count, 3)
def testQueryBatch(self):
domain = 'compound.' + config.get('domain')
headers = {'host': domain}
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
start = 0
stop = 72
count = 0
count = req_count=0
limit = 10
req += "?query=date == 23" # values where date field = 23
req += "&Limit=" + str(limit) # return no more than 10 results at a time
for i in range(50):
sreq = req+"&select=[" + str(start) + ":" + str(stop) + "]"
rsp = requests.get(sreq, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
req_count += 1
rspJson = json.loads(rsp.text)
self.assertTrue('hrefs' in rspJson)
self.assertTrue('index' in rspJson)
index = rspJson['index']
self.assertTrue(len(index) <= limit)
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), len(index))
count += len(index)
if len(index) < limit:
break # no more results
start = index[-1] + 1 # start at next index
self.assertEqual(count, 24)
self.assertEqual(req_count, 3)
def testGetVlen(self):
domain = 'vlen_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 2)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_VLEN')
baseType = typeItem['base']
self.assertEqual(baseType['class'], 'H5T_INTEGER')
self.assertEqual(baseType['base'], 'H5T_STD_I32LE')
#verify data returned
value = rspJson['value']
self.assertEqual(len(value), 2)
self.assertEqual(len(value[1]), 12)
self.assertEqual(value[1][11], 144)
def testPutCompound(self):
domain = 'valueputcompound.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
headers = {'host': domain}
fields = ({'name': 'temp', 'type': 'H5T_STD_I32LE'},
{'name': 'pressure', 'type': 'H5T_IEEE_F32LE'})
datatype = {'class': 'H5T_COMPOUND', 'fields': fields }
#create 1d dataset
payload = {'type': datatype, 'shape': 2}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset1UUID = rspJson['id']
self.assertTrue(helper.validateId(dset1UUID))
# link new dataset as 'dset1'
ok = helper.linkObject(domain, dset1UUID, 'dset_compound')
self.assertTrue(ok)
value = ((55, 32.34), (59, 29.34))
payload = {'value': value}
req = self.endpoint + "/datasets/" + dset1UUID + "/value"
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 200) # write value
def testDeleteAnonymous(self):
# test delete works with anonymous dataset
domain = 'tall_dset22_deleted.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
g2_uuid = helper.getUUID(domain, root_uuid, 'g2')
self.assertTrue(helper.validateId(g2_uuid))
d22_uuid = helper.getUUID(domain, g2_uuid, 'dset2.2')
self.assertTrue(helper.validateId(d22_uuid))
# delete g2, that will make dataset anonymous
req = self.endpoint + "/groups/" + g2_uuid
headers = {'host': domain}
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# verify that a GET on the dataset succeeds still
req = helper.getEndpoint() + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# delete dataset...
req = self.endpoint + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
# verify that a GET on the dataset fails
req = helper.getEndpoint() + "/datasets/" + d22_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 410)
def testGetArray(self):
domain = 'array_dset.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeItemBase = typeItem['base']
self.assertEqual(typeItemBase['class'], 'H5T_INTEGER')
self.assertEqual(typeItemBase['base'], 'H5T_STD_I64LE')
def testPostCompoundType(self):
domain = 'compound.datasettest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
fields = ({'name': 'temp', 'type': 'H5T_STD_I32LE'},
{'name': 'pressure', 'type': 'H5T_IEEE_F32LE'})
datatype = {'class': 'H5T_COMPOUND', 'fields': fields }
payload = {'type': datatype, 'shape': 10}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link the new dataset
name = "dset"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
def testGetArray(self):
domain = 'array_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeBase = typeItem['base']
self.assertEqual(typeBase['class'], 'H5T_INTEGER')
self.assertEqual(typeBase['base'], 'H5T_STD_I64LE')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
elem = value[0] # elem should be a 3x5 array
self.assertEqual(len(elem), 3)
self.assertEqual(elem[2], [0, -2, -4, -6, -8])
def testPutDimensionScale(self):
domain = 'dim_scale_updated.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
headers = {'host': domain}
dset_uuid = helper.getUUID(domain, root_uuid, 'temperatures')
scale_x_uuid = helper.getUUID(domain, root_uuid, 'scale_x')
scale_y_uuid = helper.getUUID(domain, root_uuid, 'scale_y')
scale_z_uuid = helper.getUUID(domain, root_uuid, 'scale_z')
# attach a dimension_list attribute to temperatures dataset
reftype = {'class': 'H5T_REFERENCE', 'base': 'H5T_STD_REF_OBJ'}
attr_name = "DIMENSION_LIST"
vlen_type = {'class': 'H5T_VLEN', 'base': reftype}
value = []
for item_uuid in (scale_x_uuid, scale_y_uuid, scale_z_uuid):
obj_ref = 'datasets/' + item_uuid
vlen_item = (obj_ref, )
value.append(vlen_item)
payload = {'type': vlen_type, 'shape': 3, 'value': value}
req = self.endpoint + "/datasets/" + dset_uuid + "/attributes/" + attr_name
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create attribute
def testGetFixedString(self):
domain = 'fixed_string_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 7)
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLPAD')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
self.assertEqual(value[0], "Parting")
self.assertEqual(value[1], "is such")
self.assertEqual(value[2], "sweet")
self.assertEqual(value[3], "sorrow.")
def testGetBatch(self):
domain = 'attr1k.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootUUID + "/attributes"
headers = {'host': domain}
params = {'Limit': 50 }
names = set()
# get attributes in 20 batches of 50 links each
lastName = None
for batchno in range(20):
if lastName:
params['Marker'] = lastName
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 200)
if rsp.status_code != 200:
break
rspJson = json.loads(rsp.text)
attrs = rspJson['attributes']
self.failUnlessEqual(len(attrs) <= 50, True)
for attr in attrs:
lastName = attr['name']
names.add(lastName)
if len(attrs) == 0:
break
self.failUnlessEqual(len(names), 1000) # should get 1000 unique attributes
def testGetArray(self):
domain = 'array_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeBase = typeItem['base']
self.assertEqual(typeBase['class'], 'H5T_INTEGER')
self.assertEqual(typeBase['base'], 'H5T_STD_I64LE')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
elem = value[0] # elem should be a 3x5 array
self.assertEqual(len(elem), 3)
self.assertEqual(elem[2], [0, -2, -4, -6, -8])
def testPutExternalLink(self):
logging.info("LinkTest.testPutExternalLink")
domain = 'tall_updated.' + config.get('domain')
target_domain = 'external_target.' + config.get('domain')
target_path = '/dset1'
grpId = helper.createGroup(domain)
rootId = helper.getRootUUID(domain)
name = 'extlink'
req = helper.getEndpoint() + "/groups/" + rootId + "/links/" + name
payload = {"h5path": target_path, "h5domain": target_domain}
headers = {'host': domain}
# verify extlink does not exist
rsp = requests.get(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 404)
# make request
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# verify link is created
rsp = requests.get(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200)
# verify that it is an external link
rspJson = json.loads(rsp.text)
target = rspJson['link']
self.assertEqual(target['class'], 'H5L_TYPE_EXTERNAL')
self.assertEqual(target['h5domain'], target_domain)
self.assertEqual(target['h5path'], target_path)
def testGetBatch(self):
logging.info("LinkTest.testGetBatch")
domain = 'group1k.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + root_uuid + "/links"
headers = {'host': domain}
params = {'Limit': 50}
names = set()
# get links in 20 batches of 50 links each
lastName = None
for batchno in range(20):
if lastName:
params['Marker'] = lastName
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 200)
if rsp.status_code != 200:
break
rspJson = json.loads(rsp.text)
links = rspJson['links']
self.failUnlessEqual(len(links) <= 50, True)
for link in links:
lastName = link['name']
names.add(lastName)
if len(links) == 0:
break
self.failUnlessEqual(len(names), 1000) # should get 1000 unique links
def testGetCompoundArray(self):
for domain_name in ('compound_array_attr', ):
domain = domain_name + '.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['name'], 'A1')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SCALAR')
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
self.assertEqual(len(typeItem['fields']), 2)
fields = typeItem['fields']
field0 = fields[0]
self.assertEqual(field0['name'], 'temp')
field0Type = field0['type']
self.assertEqual(field0Type['class'], 'H5T_FLOAT')
self.assertEqual(field0Type['base'], 'H5T_IEEE_F64LE')
field1 = fields[1]
self.assertEqual(field1['name'], '2x2')
field1Type = field1['type']
self.assertEqual(field1Type['class'], 'H5T_ARRAY')
self.assertEqual(field1Type['dims'], [2, 2])
baseType = field1Type['base']
self.assertEqual(baseType['class'], 'H5T_FLOAT')
self.assertEqual(baseType['base'], 'H5T_IEEE_F32LE')
def testPostWithLink(self):
# test PUT_root
domain = 'newlinkedtype.datatypetest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
root_uuid = helper.getRootUUID(domain)
payload = {
'type': 'H5T_IEEE_F64LE',
'link': {
'id': root_uuid,
'name': 'linked_dtype'
}
}
req = self.endpoint + "/datatypes"
headers = {'host': domain}
# create a new group
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
self.failUnlessEqual(rspJson["attributeCount"], 0)
self.assertTrue(helper.validateId(rspJson["id"]))
def testGetBatch(self):
logging.info("LinkTest.testGetBatch")
domain = 'group1k.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + root_uuid + "/links"
headers = {'host': domain}
params = {'Limit': 50 }
names = set()
# get links in 20 batches of 50 links each
lastName = None
for batchno in range(20):
if lastName:
params['Marker'] = lastName
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 200)
if rsp.status_code != 200:
break
rspJson = json.loads(rsp.text)
links = rspJson['links']
self.failUnlessEqual(len(links) <= 50, True)
for link in links:
lastName = link['title']
names.add(lastName)
if len(links) == 0:
break
self.failUnlessEqual(len(names), 1000) # should get 1000 unique links
def testPostTypes(self):
domain = 'datatypes.datatypetest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
# list of types supported
datatypes = ( 'H5T_STD_I8LE', 'H5T_STD_U8LE',
'H5T_STD_I16LE', 'H5T_STD_U16LE',
'H5T_STD_I32LE', 'H5T_STD_U32LE',
'H5T_STD_I64LE', 'H5T_STD_I64LE',
'H5T_IEEE_F32LE', 'H5T_IEEE_F64LE' )
#todo: check on 'vlen_bytes', 'vlen_unicode'
for datatype in datatypes:
payload = {'type': datatype}
req = self.endpoint + "/datatypes"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create datatypes
rspJson = json.loads(rsp.text)
dtype_uuid = rspJson['id']
self.assertTrue(helper.validateId(dtype_uuid))
# link new datatype using the type name
name = datatype
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dtype_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
def testGet(self):
for domain_name in ('tall', 'tall_ro'):
domain = domain_name + '.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootUUID + "/attributes/attr1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['name'], 'attr1')
self.assertTrue('type' in rspJson)
type = rspJson['type']
self.assertEqual(type['class'], 'H5T_INTEGER')
self.assertEqual(type['base'], 'H5T_STD_I8LE')
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 10)
self.assertTrue('maxdims' not in shape)
data = rspJson['value']
self.assertEqual(len(data), 10)
# data should be the array [97, 98, 99, ..., 105, 0]
expected = range(97, 107)
expected[9] = 0
self.assertEqual(data, expected)
self.assertEqual(len(rspJson['hrefs']), 4)
def testPutDimensionScale(self):
domain = 'dim_scale_updated.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
headers = {'host': domain}
dset_uuid = helper.getUUID(domain, root_uuid, 'temperatures')
scale_x_uuid = helper.getUUID(domain, root_uuid, 'scale_x')
scale_y_uuid = helper.getUUID(domain, root_uuid, 'scale_y')
scale_z_uuid = helper.getUUID(domain, root_uuid, 'scale_z')
# attach a dimension_list attribute to temperatures dataset
reftype = {'class': 'H5T_REFERENCE', 'base': 'H5T_STD_REF_OBJ' }
attr_name = "DIMENSION_LIST"
vlen_type = {'class': 'H5T_VLEN', 'base': reftype }
value = []
for item_uuid in (scale_x_uuid, scale_y_uuid, scale_z_uuid):
obj_ref = 'datasets/' + item_uuid
vlen_item = (obj_ref,)
value.append(vlen_item)
payload = {'type': vlen_type, 'shape': 3, 'value': value}
req = self.endpoint + "/datasets/" + dset_uuid + "/attributes/" + attr_name
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create attribute
def testQueries(self):
# use '%26' rather than '&' since otherwise it will be
# interpreted as a http query param seperator
queries = {
"date == 23": 24,
"wind == 'W 5'": 3,
"temp > 61": 53,
"(date >=22) %26 (date <= 24)": 62,
"(date == 21) %26 (temp > 70)": 4,
"(wind == 'E 7') | (wind == 'S 7')": 7
}
#queries = { "(date == 21) %26 (temp >= 72)": 4 }
domain = 'compound.' + config.get('domain')
headers = {'host': domain}
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
for key in queries.keys():
query = req + "?query=" + key
rsp = requests.get(query, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('hrefs' in rspJson)
self.assertTrue('index' in rspJson)
index = rspJson['index']
self.assertTrue(len(index), queries[key])
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), queries[key])
def testGetBatch(self):
domain = 'attr1k.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootUUID + "/attributes"
headers = {'host': domain}
params = {'Limit': 50}
names = set()
# get attributes in 20 batches of 50 links each
lastName = None
for batchno in range(20):
if lastName:
params['Marker'] = lastName
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 200)
if rsp.status_code != 200:
break
rspJson = json.loads(rsp.text)
attrs = rspJson['attributes']
self.failUnlessEqual(len(attrs) <= 50, True)
for attr in attrs:
lastName = attr['name']
names.add(lastName)
if len(attrs) == 0:
break
self.failUnlessEqual(len(names),
1000) # should get 1000 unique attributes
def testPostCompoundType(self):
domain = 'compound.datatypetest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
fields = ({
'name': 'temp',
'type': 'H5T_STD_I32LE'
}, {
'name': 'pressure',
'type': 'H5T_IEEE_F32LE'
})
datatype = {'class': 'H5T_COMPOUND', 'fields': fields}
payload = {'type': datatype}
req = self.endpoint + "/datatypes"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create datatype
rspJson = json.loads(rsp.text)
dtype_uuid = rspJson['id']
self.assertTrue(helper.validateId(dtype_uuid))
# link the new datatype
name = "dtype_compound"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dtype_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
def testGetInvalidName(self):
domain = 'tall.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootUUID + "/attributes/no_attr_here"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 404)
def testGetCompoundArray(self):
for domain_name in ('compound_array_attr', ):
domain = domain_name + '.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['name'], 'A1')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SCALAR')
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
self.assertEqual(len(typeItem['fields']), 2)
fields = typeItem['fields']
field0 = fields[0]
self.assertEqual(field0['name'], 'temp')
field0Type = field0['type']
self.assertEqual(field0Type['class'], 'H5T_FLOAT')
self.assertEqual(field0Type['base'], 'H5T_IEEE_F64LE')
field1 = fields[1]
self.assertEqual(field1['name'], '2x2')
field1Type = field1['type']
self.assertEqual(field1Type['class'], 'H5T_ARRAY')
self.assertEqual(field1Type['dims'], [2, 2])
baseType = field1Type['base']
self.assertEqual(baseType['class'], 'H5T_FLOAT')
self.assertEqual(baseType['base'], 'H5T_IEEE_F32LE')
def testGetVlen(self):
domain = 'vlen_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 2)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_VLEN')
baseType = typeItem['base']
self.assertEqual(baseType['class'], 'H5T_INTEGER')
self.assertEqual(baseType['base'], 'H5T_STD_I32LE')
#verify data returned
value = rspJson['value']
self.assertEqual(len(value), 2)
self.assertEqual(len(value[1]), 12)
self.assertEqual(value[1][11], 144)
def testGet(self):
for domain_name in ('tall', 'tall_ro'):
domain = domain_name + '.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
req = helper.getEndpoint(
) + "/groups/" + rootUUID + "/attributes/attr1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['name'], 'attr1')
self.assertTrue('type' in rspJson)
type = rspJson['type']
self.assertEqual(type['class'], 'H5T_INTEGER')
self.assertEqual(type['base'], 'H5T_STD_I8LE')
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 10)
self.assertTrue('maxdims' not in shape)
data = rspJson['value']
self.assertEqual(len(data), 10)
# data should be the array [97, 98, 99, ..., 105, 0]
expected = range(97, 107)
expected[9] = 0
self.assertEqual(data, expected)
self.assertEqual(len(rspJson['hrefs']), 4)
def testGetEnum(self):
domain = 'enum_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][0], 4)
self.assertEqual(shape['dims'][1], 7)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ENUM')
baseType = typeItem['base']
self.assertEqual(baseType['class'], 'H5T_INTEGER')
self.assertEqual(baseType['base'], 'H5T_STD_I16BE')
self.assertTrue('mapping' in typeItem)
mapping = typeItem['mapping']
self.assertEqual(len(mapping), 4)
self.assertEqual(mapping['SOLID'], 0)
self.assertEqual(mapping['LIQUID'], 1)
self.assertEqual(mapping['GAS'], 2)
self.assertEqual(mapping['PLASMA'], 3)
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
self.assertEqual(value[1][2], mapping['GAS'])
def testGetCompoundCommitted(self):
domain = 'compound_committed.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 72)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
self.assertTrue('fields' in typeItem)
fields = typeItem['fields']
self.assertEqual(len(fields), 3)
timeField = fields[1]
self.assertEqual(timeField['name'], 'time')
self.assertTrue('type' in timeField)
timeFieldType = timeField['type']
self.assertEqual(timeFieldType['class'], 'H5T_STRING')
self.assertEqual(timeFieldType['cset'], 'H5T_CSET_ASCII')
self.assertEqual(timeFieldType['order'], 'H5T_ORDER_NONE')
self.assertEqual(timeFieldType['strsize'], 6)
self.assertEqual(timeFieldType['strpad'], 'H5T_STR_NULLPAD')
tempField = fields[2]
self.assertEqual(tempField['name'], 'temp')
tempFieldType = tempField['type']
self.assertEqual(tempFieldType['class'], 'H5T_INTEGER')
self.assertEqual(tempFieldType['base'], 'H5T_STD_I32LE')
def testGetArray(self):
domain = 'array_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeBase = typeItem['base']
self.assertEqual(typeBase['class'], 'H5T_INTEGER')
self.assertEqual(typeBase['base'], 'H5T_STD_I64LE')
# bug! - unable to read value from attribute with array type
# code is not returning 'value' key in this case
# h5py issue: https://github.com/h5py/h5py/issues/498
if 'value' in rspJson:
print 'remove above if check once h5py 2.4 is released'
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
elem = value[0] # elem should be a 3x5 array
self.assertEqual(elem[2], [0, -2, -4, -6, -8])
def testGetEnum(self):
domain = 'enum_dset.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][0], 4)
self.assertEqual(shape['dims'][1], 7)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ENUM')
typeBase = typeItem['base']
self.assertEqual(typeBase['class'], 'H5T_INTEGER')
self.assertEqual(typeBase['base'], 'H5T_STD_I16BE')
self.assertTrue('mapping' in typeItem)
mapping = typeItem['mapping']
self.assertEqual(len(mapping), 4)
self.assertEqual(mapping['SOLID'], 0)
self.assertEqual(mapping['LIQUID'], 1)
self.assertEqual(mapping['GAS'], 2)
self.assertEqual(mapping['PLASMA'], 3)
def testGetFixedString(self):
domain = 'fixed_string_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_STRING')
self.assertEqual(typeItem['cset'], 'H5T_CSET_ASCII')
self.assertEqual(typeItem['order'], 'H5T_ORDER_NONE')
self.assertEqual(typeItem['strsize'], 7)
self.assertEqual(typeItem['strpad'], 'H5T_STR_NULLPAD')
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
self.assertEqual(value[0], "Parting")
self.assertEqual(value[1], "is such")
self.assertEqual(value[2], "sweet")
self.assertEqual(value[3], "sorrow.")
def testPostTypes(self):
domain = 'datatypes.datatypetest.' + config.get('domain')
req = self.endpoint + "/"
headers = {'host': domain}
rsp = requests.put(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # creates domain
root_uuid = helper.getRootUUID(domain)
# list of types supported
datatypes = ('H5T_STD_I8LE', 'H5T_STD_U8LE', 'H5T_STD_I16LE',
'H5T_STD_U16LE', 'H5T_STD_I32LE', 'H5T_STD_U32LE',
'H5T_STD_I64LE', 'H5T_STD_I64LE', 'H5T_IEEE_F32LE',
'H5T_IEEE_F64LE')
#todo: check on 'vlen_bytes', 'vlen_unicode'
for datatype in datatypes:
payload = {'type': datatype}
req = self.endpoint + "/datatypes"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201) # create datatypes
rspJson = json.loads(rsp.text)
dtype_uuid = rspJson['id']
self.assertTrue(helper.validateId(dtype_uuid))
# link new datatype using the type name
name = datatype
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dtype_uuid}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 201)
def testGetArray(self):
domain = 'array_attr.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/A1"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 4)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_ARRAY')
self.assertTrue('dims' in typeItem)
typeShape = typeItem['dims']
self.assertEqual(len(typeShape), 2)
self.assertEqual(typeShape[0], 3)
self.assertEqual(typeShape[1], 5)
typeBase = typeItem['base']
self.assertEqual(typeBase['class'], 'H5T_INTEGER')
self.assertEqual(typeBase['base'], 'H5T_STD_I64LE')
# bug! - unable to read value from attribute with array type
# code is not returning 'value' key in this case
# h5py issue: https://github.com/h5py/h5py/issues/498
if 'value' in rspJson:
print 'remove above if check once h5py 2.4 is released'
self.assertTrue('value' in rspJson)
value = rspJson['value']
self.assertEqual(len(value), 4)
elem = value[0] # elem should be a 3x5 array
self.assertEqual(elem[2], [0, -2, -4, -6, -8])
def testGetDimensionScale(self):
domain = 'dim_scale.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'temperatures')
scale_x_uuid = helper.getUUID(domain, root_uuid, 'scale_x')
scale_y_uuid = helper.getUUID(domain, root_uuid, 'scale_y')
scale_z_uuid = helper.getUUID(domain, root_uuid, 'scale_z')
# now try reading the dimension list attribute
req = helper.getEndpoint(
) + "/datasets/" + dset_uuid + "/attributes/DIMENSION_LIST"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertTrue('dims' in shape)
dims = shape['dims']
self.assertEqual(len(dims), 1)
self.assertEqual(dims[0], 3)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_VLEN')
baseType = typeItem['base']
self.assertEqual(baseType['class'], 'H5T_REFERENCE')
self.assertEqual(baseType['base'], 'H5T_STD_REF_OBJ')
data = rspJson['value']
self.assertEqual(len(data), 3)
self.assertEqual(data[0], ['datasets/' + scale_x_uuid])
self.assertEqual(data[1], ['datasets/' + scale_y_uuid])
self.assertEqual(data[2], ['datasets/' + scale_z_uuid])
# read the x dimenscale and verify it refernces the temperature dataset
req = helper.getEndpoint(
) + "/datasets/" + scale_x_uuid + "/attributes/REFERENCE_LIST"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
fields = typeItem['fields']
self.assertEqual(len(fields), 2)
refType = fields[0]["type"]
self.assertEqual(refType["class"], 'H5T_REFERENCE')
intType = fields[1]["type"]
self.assertEqual(intType["class"], 'H5T_INTEGER')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertTrue('dims' in shape)
dims = shape['dims']
self.assertEqual(len(dims), 1)
self.assertEqual(dims[0], 1)
data = rspJson['value']
elem = data[0]
self.assertEqual(len(elem), 2) # two fields of a compound type
self.assertEqual(elem[0],
'datasets/' + dset_uuid) # reference primary dataset
self.assertEqual(elem[1], 0) # first dimension
def testPostInvalidType(self):
domain = 'tall.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
payload = {'type': 'badtype'}
headers = {'host': domain}
req = self.endpoint + "/datatypes"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testPostInvalidShape(self):
domain = 'tall.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
payload = {'type': 'H5T_STD_I32LE', 'shape': -5}
headers = {'host': domain}
req = self.endpoint + "/datasets/"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testGetResizable(self):
domain = 'resizable.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
resizable_1d_uuid = helper.getUUID(domain, root_uuid, 'resizable_1d')
req = helper.getEndpoint(
) + "/datasets/" + resizable_1d_uuid + "/shape"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 10)
self.assertEqual(shape['maxdims'][0], 20)
resizable_2d_uuid = helper.getUUID(domain, root_uuid, 'resizable_2d')
req = helper.getEndpoint(
) + "/datasets/" + resizable_2d_uuid + "/shape"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][1], 10)
self.assertTrue('maxdims' in shape) # is re-sizeable!
self.assertEqual(shape['maxdims'][1], 20)
unlimited_1d_uuid = helper.getUUID(domain, root_uuid, 'unlimited_1d')
req = helper.getEndpoint(
) + "/datasets/" + unlimited_1d_uuid + "/shape"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 10)
self.assertTrue('maxdims' in shape) # is re-sizeable
self.assertEqual(shape['maxdims'][0], 0)
unlimited_2d_uuid = helper.getUUID(domain, root_uuid, 'unlimited_2d')
req = helper.getEndpoint(
) + "/datasets/" + unlimited_2d_uuid + "/shape"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue('shape' in rspJson)
shape = rspJson['shape']
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][1], 10)
self.assertTrue('maxdims' in shape) # is re-sizeable
self.assertEqual(shape['maxdims'][1], 0)
def testGetBadParam(self):
logging.info("LinkTest.testGetBatchBadParam")
domain = 'tall.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + root_uuid + "/links"
headers = {'host': domain}
params = {'Limit': 'abc' }
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 400)
def testDelete(self):
domain = 'tall_updated.' + config.get('domain')
attr_name = 'attr1'
rootUUID = helper.getRootUUID(domain)
headers = {'host': domain}
req = self.endpoint + "/groups/" + rootUUID + "/attributes/" + attr_name
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200) # delete attribute
def testBadQuery(self):
domain = 'compound.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
req += "?query=foobar"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testDelete(self):
domain = 'tall_updated.' + config.get('domain')
attr_name = 'attr1'
rootUUID = helper.getRootUUID(domain)
headers = {'host': domain}
req = self.endpoint + "/groups/" + rootUUID + "/attributes/" + attr_name
rsp = requests.delete(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200) # delete attribute
def testGetBadParam(self):
logging.info("LinkTest.testGetBatchBadParam")
domain = 'tall.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + root_uuid + "/links"
headers = {'host': domain}
params = {'Limit': 'abc'}
rsp = requests.get(req, headers=headers, params=params)
self.failUnlessEqual(rsp.status_code, 400)
def testBadQuery(self):
domain = 'compound.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'dset')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
req += "?query=foobar"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testGetDimensionScale(self):
domain = 'dim_scale.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
dset_uuid = helper.getUUID(domain, root_uuid, 'temperatures')
scale_x_uuid = helper.getUUID(domain, root_uuid, 'scale_x')
scale_y_uuid = helper.getUUID(domain, root_uuid, 'scale_y')
scale_z_uuid = helper.getUUID(domain, root_uuid, 'scale_z')
# now try reading the dimension list attribute
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/attributes/DIMENSION_LIST"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertTrue('dims' in shape)
dims = shape['dims']
self.assertEqual(len(dims), 1)
self.assertEqual(dims[0], 3)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_VLEN')
baseType = typeItem['base']
self.assertEqual(baseType['class'], 'H5T_REFERENCE')
self.assertEqual(baseType['base'], 'H5T_STD_REF_OBJ')
data = rspJson['value']
self.assertEqual(len(data), 3)
self.assertEqual(data[0], ['datasets/' + scale_x_uuid])
self.assertEqual(data[1], ['datasets/' + scale_y_uuid])
self.assertEqual(data[2], ['datasets/' + scale_z_uuid])
# read the x dimenscale and verify it refernces the temperature dataset
req = helper.getEndpoint() + "/datasets/" + scale_x_uuid + "/attributes/REFERENCE_LIST"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
typeItem = rspJson['type']
self.assertEqual(typeItem['class'], 'H5T_COMPOUND')
fields = typeItem['fields']
self.assertEqual(len(fields), 2)
refType = fields[0]["type"]
self.assertEqual(refType["class"], 'H5T_REFERENCE')
intType = fields[1]["type"]
self.assertEqual(intType["class"], 'H5T_INTEGER')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertTrue('dims' in shape)
dims = shape['dims']
self.assertEqual(len(dims), 1)
self.assertEqual(dims[0], 1)
data = rspJson['value']
elem = data[0]
self.assertEqual(len(elem), 2) # two fields of a compound type
self.assertEqual(elem[0], 'datasets/' + dset_uuid) # reference primary dataset
self.assertEqual(elem[1], 0) # first dimension
def testGet(self):
for domain_name in ('tall', 'tall_ro'):
domain = domain_name + '.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
g1UUID = helper.getUUID(domain, rootUUID, 'g1')
g11UUID = helper.getUUID(domain, g1UUID, 'g1.1')
# rank 1 dataset
dset112UUID = helper.getUUID(domain, g11UUID, 'dset1.1.2')
req = helper.getEndpoint() + "/datasets/" + dset112UUID
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['id'], dset112UUID)
typeItem = rspJson['type']
self.assertEqual(typeItem['base'], 'H5T_STD_I32BE')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 20)
req = helper.getEndpoint() + "/datasets/" + dset112UUID + "/value"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
data = rspJson['value']
self.assertEqual(len(data), 20)
for i in range(20):
self.assertEqual(data[i], i)
# rank 2 dataset
dset111UUID = helper.getUUID(domain, g11UUID, 'dset1.1.1')
req = helper.getEndpoint() + "/datasets/" + dset111UUID
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['id'], dset111UUID)
typeItem = rspJson['type']
self.assertEqual(typeItem['base'], 'H5T_STD_I32BE')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][0], 10)
self.assertEqual(shape['dims'][1], 10)
req = helper.getEndpoint() + "/datasets/" + dset111UUID + "/value"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
data = rspJson['value']
self.assertEqual(len(data), 10)
for i in range(10):
arr = data[i]
self.assertEqual(len(arr), 10)
for j in range(10):
self.assertEqual(arr[j], i*j)
def testGet(self):
for domain_name in ('tall', 'tall_ro'):
domain = domain_name + '.' + config.get('domain')
rootUUID = helper.getRootUUID(domain)
g1UUID = helper.getUUID(domain, rootUUID, 'g1')
g11UUID = helper.getUUID(domain, g1UUID, 'g1.1')
# rank 1 dataset
dset112UUID = helper.getUUID(domain, g11UUID, 'dset1.1.2')
req = helper.getEndpoint() + "/datasets/" + dset112UUID
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['id'], dset112UUID)
typeItem = rspJson['type']
self.assertEqual(typeItem['base'], 'H5T_STD_I32BE')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 1)
self.assertEqual(shape['dims'][0], 20)
req = helper.getEndpoint() + "/datasets/" + dset112UUID + "/value"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
data = rspJson['value']
self.assertEqual(len(data), 20)
for i in range(20):
self.assertEqual(data[i], i)
# rank 2 dataset
dset111UUID = helper.getUUID(domain, g11UUID, 'dset1.1.1')
req = helper.getEndpoint() + "/datasets/" + dset111UUID
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson['id'], dset111UUID)
typeItem = rspJson['type']
self.assertEqual(typeItem['base'], 'H5T_STD_I32BE')
shape = rspJson['shape']
self.assertEqual(shape['class'], 'H5S_SIMPLE')
self.assertEqual(len(shape['dims']), 2)
self.assertEqual(shape['dims'][0], 10)
self.assertEqual(shape['dims'][1], 10)
req = helper.getEndpoint() + "/datasets/" + dset111UUID + "/value"
rsp = requests.get(req, headers=headers)
self.failUnlessEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
data = rspJson['value']
self.assertEqual(len(data), 10)
for i in range(10):
arr = data[i]
self.assertEqual(len(arr), 10)
for j in range(10):
self.assertEqual(arr[j], i * j)
def testPutNoName(self):
logging.info("LinkTest.testPutNoName")
domain = 'tall_updated.' + config.get('domain')
grpId = helper.createGroup(domain)
rootId = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootId + "/links/"
payload = {"id": grpId}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testPutNoName(self):
logging.info("LinkTest.testPutNoName")
domain = 'tall_updated.' + config.get('domain')
grpId = helper.createGroup(domain)
rootId = helper.getRootUUID(domain)
req = helper.getEndpoint() + "/groups/" + rootId + "/links/"
payload = {"id": grpId}
headers = {'host': domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.failUnlessEqual(rsp.status_code, 400)
def testPutInvalid(self):
domain = 'tall_updated.' + config.get('domain')
attr_name = 'attr_invalid'
rootUUID = helper.getRootUUID(domain)
headers = {'host': domain}
# attempt to pass in a string directly (which is not valid JSON)
payload = "{'type': 'H5T_IEEE_F32LE', 'shape': (0,), 'value': 3.12}"
req = self.endpoint + "/groups/" + rootUUID + "/attributes/" + attr_name
rsp = requests.put(req, data=payload, headers=headers)
self.failUnlessEqual(rsp.status_code, 400) # Bad Request
def testGetOpaque(self):
domain = 'opaque_dset.' + config.get('domain')
root_uuid = helper.getRootUUID(domain)
self.assertTrue(helper.validateId(root_uuid))
dset_uuid = helper.getUUID(domain, root_uuid, 'DS1')
req = helper.getEndpoint() + "/datasets/" + dset_uuid + "/value"
headers = {'host': domain}
rsp = requests.get(req, headers=headers)
# get for Opaque data is not supported yet. Check that the call returns 501
self.failUnlessEqual(rsp.status_code, 501)