def testDeleteFolderWithChildren(self):
folder_name = "testDeleteFolder"
domain_name = "myfile"
domain = self.base_domain + "/" + folder_name
print("testCreateFolder", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
body = {"folder": True}
rsp = requests.put(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
for k in ("owner", "acls", "created", "lastModified"):
self.assertTrue(k in rspJson)
self.assertFalse("root" in rspJson) # no root -> folder
# verify that putting the same domain again fails with a 409 error
rsp = requests.put(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 409)
# do a get on the new folder
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("owner" in rspJson)
self.assertTrue("class" in rspJson)
self.assertEqual(rspJson["class"], "folder")
# create a child domain
domain = self.base_domain + "/" + folder_name + "/" + domain_name
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.put(req, headers=headers)
self.assertEqual(rsp.status_code, 201)
# try delete the folder
domain = self.base_domain + "/" + folder_name
print("testCreateFolder", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
body = {"folder": True}
rsp = requests.delete(req, headers=headers)
# should get a09
self.assertEqual(rsp.status_code, 409)
# delete the child domain
domain = self.base_domain + "/" + folder_name + "/" + domain_name
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
# try delete the folder
domain = self.base_domain + "/" + folder_name
print("delete child domain", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
body = {"folder": True}
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
def testGroupAcl(self):
print("testPutAcl", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
test_group = config.get("test_group")
if not test_group:
print("test_group config not set, skipping testGroupAcl")
return
group_acl = "g:" + test_group # "g:" prefix is used to distinguish user from group names
username = config.get("user_name")
user2name = config.get("user2_name")
# create an ACL for with read and update access for the test_group
req = helper.getEndpoint() + '/acls/' + group_acl
perm = {"read": True, "update": True}
rsp = requests.put(req, headers=headers, data=json.dumps(perm))
self.assertEqual(rsp.status_code, 201)
# fetch the acl and verify it has been updated
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rsp_json = json.loads(rsp.text)
self.assertTrue("acl" in rsp_json)
self.assertTrue("hrefs" in rsp_json)
acl = rsp_json["acl"]
self.assertEqual(len(acl.keys()),
len(acl_keys) + 2) # acl_keys + "domain" + "username"
for k in acl_keys:
self.assertTrue(k in acl)
if k in ("read", "update"):
self.assertEqual(acl[k], True)
else:
self.assertEqual(acl[k], False)
# The ACL should not be fetchable by test_user2...
req = helper.getEndpoint() + '/acls/' + user2name
headers = helper.getRequestHeaders(domain=self.base_domain,
username=user2name)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 404) # not found
# The default ACL should be fetchable by test_user2 as well...
if config.get("default_public"):
req = helper.getEndpoint() + '/acls/default'
headers = helper.getRequestHeaders(domain=self.base_domain,
username=user2name)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200) # ok
else:
self.assertEqual(rsp.status_code, 404) # not found
# test_user2 shouldn't be able to read test_user1's ACL
username = config.get("user_name")
req = helper.getEndpoint() + '/acls/' + username
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 403) # Forbidden
def testGet(self):
domain = helper.getTestDomain("tall.h5")
headers = helper.getRequestHeaders(domain=domain)
# verify domain exists
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
if rsp.status_code != 200:
print("WARNING: Failed to get domain: {}. Is test data setup?".
format(domain))
return # abort rest of test
rspJson = json.loads(rsp.text)
grp_uuid = root_uuid = rspJson["root"]
self.assertTrue(grp_uuid.startswith("g-"))
# get the group json
req = helper.getEndpoint() + '/groups/' + grp_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
for name in ("id", "hrefs", "attributeCount", "linkCount", "domain",
"root", "created", "lastModified"):
self.assertTrue(name in rspJson)
self.assertEqual(rspJson["id"], grp_uuid)
hrefs = rspJson["hrefs"]
self.assertEqual(len(hrefs), 5)
self.assertEqual(rspJson["id"], grp_uuid)
self.assertEqual(rspJson["attributeCount"], 2)
self.assertEqual(rspJson["linkCount"], 2)
self.assertEqual(rspJson["root"], root_uuid)
self.assertEqual(rspJson["domain"], domain)
now = time.time()
# the object shouldn't have been just created or updated
self.assertTrue(rspJson["created"] < now - 10)
self.assertTrue(rspJson["lastModified"] < now - 10)
# request the group path
req = helper.getEndpoint() + '/groups/' + grp_uuid
params = {"getalias": 1}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("alias" in rspJson)
self.assertEqual(rspJson["alias"], ['/'])
# verify trying to read this group from a different domain fails
headers = helper.getRequestHeaders(domain=self.base_domain)
req = helper.getEndpoint() + '/groups/' + grp_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400)
def testPutAcl(self):
print("testPutAcl", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
# create an ACL for "test_user2" with read and update access
user2name = config.get("user2_name")
req = helper.getEndpoint() + '/acls/' + user2name
perm = {"read": True, "update": True}
rsp = requests.put(req, headers=headers, data=json.dumps(perm))
self.assertEqual(rsp.status_code, 201)
# fetch the acl and verify it has been updated
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rsp_json = json.loads(rsp.text)
self.assertTrue("acl" in rsp_json)
self.assertTrue("hrefs" in rsp_json)
acl = rsp_json["acl"]
self.assertEqual(len(acl.keys()),
len(acl_keys) + 2) # acl_keys + "domain" + "username"
for k in acl_keys:
self.assertTrue(k in acl)
if k in ("read", "update"):
self.assertEqual(acl[k], True)
else:
self.assertEqual(acl[k], False)
# The ACL should be fetchable by test_user2...
req = helper.getEndpoint() + '/acls/' + user2name
headers = helper.getRequestHeaders(domain=self.base_domain,
username=user2name)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200) # ok
# The default ACL should be fetchable by test_user2 as well...
if config.get("default_public"):
req = helper.getEndpoint() + '/acls/default'
headers = helper.getRequestHeaders(domain=self.base_domain,
username=user2name)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200) # ok
# test_user2 shouldn't be able to read test_user1's ACL
username = config.get("user_name")
req = helper.getEndpoint() + '/acls/' + username
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 403) # Forbidden
def testGetTopLevelDomains(self):
print("testGetTopLevelDomains", self.base_domain)
import os.path as op
# Either '/' or no domain should get same result
for domain in (None, '/'):
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/domains'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
self.assertTrue("domains" in rspJson)
domains = rspJson["domains"]
domain_count = len(domains)
if domain_count == 0:
# this should only happen in the very first test run
print("Expected to find more domains!")
self.assertTrue(False)
return
for item in domains:
self.assertTrue("name" in item)
name = item["name"]
self.assertEqual(name[0], '/')
self.assertTrue(name[-1] != '/')
self.assertTrue("owner" in item)
self.assertTrue("created" in item)
self.assertTrue("lastModified" in item)
self.assertFalse("size" in item)
self.assertTrue("class") in item
self.assertTrue(item["class"] in ("domain", "folder"))
def testBaseDomain(self):
# make a non-folder domain
print("testBaseDomain", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
# verify that passing domain as query string works as well
del headers["X-Hdf-domain"]
req += "?host=" + self.base_domain
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
# try using DNS-style domain name
domain = helper.getDNSDomain(self.base_domain)
params = {"host": domain}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
async def verifyDomain(domain):
""" create domain if it doesn't already exist
"""
params = {"host": domain}
headers = getRequestHeaders()
client = globals["client"]
req = getEndpoint() + '/'
root_id = None
log.info("GET " + req)
timeout = config.get("timeout")
async with client.get(req, headers=headers, timeout=timeout,
params=params) as rsp:
if rsp.status == 200:
domain_json = await rsp.json()
else:
log.info("got status: {}".format(rsp.status))
if rsp.status == 200:
root_id = domain_json["root"]
elif rsp.status == 404:
# create the domain
setupDomain(domain)
async with client.get(req,
headers=headers,
timeout=timeout,
params=params) as rsp:
if rsp.status == 200:
domain_json = await rsp.json()
root_id = domain_json["root"]
else:
log.error("got status: {} for GET req: {}".format(
rsp.status, req))
raise HttpProcessingError(code=rsp.status,
message="Service error")
globals["root"] = root_id
def testGetByPath(self):
domain = helper.getTestDomain("tall.h5")
print("testGetByPath", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
if rsp.status_code != 200:
print("WARNING: Failed to get domain: {}. Is test data setup?".format(domain))
return # abort rest of test
domainJson = json.loads(rsp.text)
self.assertTrue("root" in domainJson)
root_id = domainJson["root"]
# Get group at /g1/g1.1 by using h5path
params = {"h5path": "/g1/g1.1"}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
g11id = helper.getUUIDByPath(domain, "/g1/g1.1")
self.assertEqual(g11id, rspJson["id"])
self.assertTrue("root" in rspJson)
self.assertEqual(root_id, rspJson["root"])
# Get dataset at /g1/g1.1/dset1.1.1 by using relative h5path
params = {"h5path": "./g1/g1.1/dset1.1.1"}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
d111id = helper.getUUIDByPath(domain, "/g1/g1.1/dset1.1.1")
self.assertEqual(d111id, rspJson["id"])
self.assertTrue("root" in rspJson)
self.assertEqual(root_id, rspJson["root"])
def testCreateLinkedDomain(self):
target_domain = self.base_domain + "/target_domain.h5"
print("testCreateLinkedDomain", target_domain)
headers = helper.getRequestHeaders(domain=target_domain)
req = helper.getEndpoint() + '/'
rsp = requests.put(req, headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
for k in ("root", "owner", "acls", "created", "lastModified"):
self.assertTrue(k in rspJson)
root_id = rspJson["root"]
# do a get on the new domain
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
for k in ("root", "owner"):
self.assertTrue(k in rspJson)
# we should get the same value for root id
self.assertEqual(root_id, rspJson["root"])
# create new domain linked with the existing root
linked_domain = self.base_domain + "/linked_domain.h5"
print("testCreateLinkedDomain - linked domain", linked_domain)
headers = helper.getRequestHeaders(domain=linked_domain)
body = {"linked_domain": target_domain}
rsp = requests.put(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
for k in ("root", "owner", "acls", "created", "lastModified"):
self.assertTrue(k in rspJson)
self.assertEqual(rspJson["root"], root_id)
# delete the target domain but keep the root
headers = helper.getRequestHeaders(domain=target_domain)
body = {"keep_root": 1}
rsp = requests.delete(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 200)
# verify we can access the root group under the linked domain
headers = helper.getRequestHeaders(domain=linked_domain)
root_req = helper.getEndpoint() + "/groups/" + root_id
rsp = requests.get(root_req, headers=headers)
self.assertEqual(rsp.status_code, 200)
def testGetDomain(self):
domain = helper.getTestDomain("tall.h5")
print("testGetDomain", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
if rsp.status_code != 200:
print("WARNING: Failed to get domain: {}. Is test data setup?".
format(domain))
return # abort rest of test
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
for name in ("lastModified", "created", "hrefs", "root", "owner",
"class"):
self.assertTrue(name in rspJson)
now = time.time()
self.assertTrue(rspJson["created"] < now - 10)
self.assertTrue(rspJson["lastModified"] < now - 10)
self.assertEqual(len(rspJson["hrefs"]), 7)
self.assertTrue(rspJson["root"].startswith("g-"))
self.assertTrue(rspJson["owner"])
self.assertEqual(rspJson["class"], "domain")
self.assertFalse(
"num_groups"
in rspJson) # should only show up with the verbose param
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# verify that passing domain as query string works as well
del headers["X-Hdf-domain"]
params = {"host": domain}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
root_uuid_2 = rspJson["root"]
self.assertEqual(root_uuid, root_uuid_2)
# same deal using the "domain" param
params = {"domain": domain}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
root_uuid_3 = rspJson["root"]
self.assertEqual(root_uuid, root_uuid_3)
# verify that invalid domain fails
domain = domain[1:] # strip off the '/'
params = {"domain": domain}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 400)
def testPutVLenInt(self):
# Test PUT value for 1d attribute with variable length int types
print("testPutVLenInt", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# create dataset
vlen_type = {"class": "H5T_VLEN", "base": { "class": "H5T_INTEGER", "base": "H5T_STD_I32LE"}}
payload = {'type': vlen_type, 'shape': [4,]}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# write values to dataset
data = [[1,], [1,2], [1,2,3], [1,2,3,4]]
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
payload = { 'value': data }
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200)
# read values from dataset
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), 4)
for i in range(4):
self.assertEqual(value[i], data[i])
# read back a selection
params = {"select": "[2:3]"}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), 1)
self.assertEqual(value[0], data[2])
def testInvalidChildDomain(self):
domain = self.base_domain + "/notafolder/newdomain.h5"
# should fail assuming "notafolder" doesn't exist
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.put(req, headers=headers)
self.assertEqual(rsp.status_code, 404)
def testGetNotFound(self):
domain = self.base_domain + "/doesnotexist.h6"
print("testGetNotFound", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 404)
def testPutVLenString(self):
# Test PUT value for 1d attribute with variable length string types
print("testPutVLenString", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# create dataset
vlen_type = {
"class": "H5T_STRING",
"charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM",
"length": "H5T_VARIABLE"
}
payload = {
'type': vlen_type,
'shape': [
4,
]
}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# write values to dataset
data = ["This is", "a variable length", "string", "array"]
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
payload = {'value': data}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200)
# read values from dataset
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), 4)
for i in range(4):
self.assertEqual(value[i], data[i])
def testPutAttributeDatatype(self):
# Test creation/deletion of datatype obj
print("testPutAttributeDatatype", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# create a committed type obj
data = {"type": "H5T_IEEE_F32LE"}
req = self.endpoint + '/datatypes'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["attributeCount"], 0)
ctype_id = rspJson["id"]
self.assertTrue(helper.validateId(ctype_id))
# link the new datatype
name = "dtype_with_attribute"
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": ctype_id}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# add an attribute
attr_name = "attr"
attr_payload = {'type': 'H5T_STD_I32LE', 'value': 42}
req = self.endpoint + "/datatypes/" + ctype_id + "/attributes/" + attr_name
rsp = requests.put(req, data=json.dumps(attr_payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # created
# read back the obj
req = self.endpoint + '/datatypes/' + ctype_id
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
self.assertEqual(rspJson["id"], ctype_id)
self.assertFalse("attributes" in rspJson)
# read back the obj with attributes
params = {"include_attrs": 1}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
self.assertEqual(rspJson["id"], ctype_id)
self.assertTrue("attributes" in rspJson)
attrs = rspJson["attributes"]
self.assertTrue("attr" in attrs)
async def createGroup():
""" create a new group and link it to the parent group with
link name of group name
"""
client = globals["client"]
domain = globals["domain"]
params = {"host": domain}
base_req = getEndpoint()
headers = getRequestHeaders()
# create a new group
req = base_req + "/groups"
log.info("POST:" + req)
globals["grp_request_count"] += 1
group_name = globals["grp_request_count"]
timeout = config.get("timeout")
async with client.post(req,
headers=headers,
timeout=timeout,
params=params) as rsp:
if rsp.status != 201:
log.error("POST {} failed with status: {}, rsp: {}".format(
req, rsp.status, str(rsp)))
globals["grp_failed_posts"] += 1
raise HttpProcessingError(code=rsp.status,
message="Unexpected error")
else:
globals["group_count"] += 1
log.info("group_count: {}".format(globals["group_count"]))
group_json = await rsp.json()
group_id = group_json["id"]
# link group to parent
root_id = globals["root"]
group_name = "group_{}".format(group_name)
req = base_req + "/groups/" + root_id + "/links/" + group_name
data = {"id": group_id}
log.info("PUT " + req)
globals["lnk_request_count"] += 1
async with client.put(req,
data=json.dumps(data),
headers=headers,
timeout=timeout,
params=params) as rsp:
if rsp.status == 409:
# another task has created this link already
log.warn("got 409 in request: " + req)
elif rsp.status != 201:
globals["lnk_failed_posts"] += 1
log.error("got http error: {} for request: {}, rsp: {}".format(
rsp.status, req, rsp))
raise HttpProcessingError(code=rsp.status,
message="Unexpected error")
else:
link_created = True
return group_id
def testGetRootGroup(self):
print("testGetRootGroup", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
req = helper.getEndpoint() + '/groups/' + root_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
group_id = rspJson["id"]
helper.validateId(group_id)
self.assertTrue("root" in rspJson)
root_id = rspJson["root"]
self.assertEqual(group_id, root_id)
self.assertTrue("domain" in rspJson)
#self.assertEqual(rspJson["domain"], self.base_domain) #TBD
self.assertTrue("created" in rspJson)
self.assertTrue("lastModified" in rspJson)
self.assertTrue("linkCount" in rspJson)
self.assertTrue("attributeCount" in rspJson)
# try get with a different user (who has read permission)
headers = helper.getRequestHeaders(domain=self.base_domain,
username="******")
rsp = requests.get(req, headers=headers)
if config.get("default_public"):
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["root"], root_uuid)
else:
self.assertEqual(rsp.status_code, 403)
# try to do a GET with a different domain (should fail)
another_domain = helper.getParentDomain(self.base_domain)
headers = helper.getRequestHeaders(domain=another_domain)
req = helper.getEndpoint() + '/groups/' + root_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400)
async def createGroup(parent_group, group_name):
""" create a new group and link it to the parent group with
link name of group name
"""
client = globals["client"]
domain = globals["domain"]
params = {"host": domain}
base_req = getEndpoint()
headers = getRequestHeaders()
timeout = config.get("timeout")
# TBD - replace with atomic create & link operation?
# create a new group
req = base_req + "/groups"
log.info("POST:" + req)
globals["request_count"] += 1
async with client.post(req, headers=headers, params=params, timeout=timeout) as rsp:
if rsp.status != 201:
log.error("POST {} failed with status: {}, rsp: {}".format(req, rsp.status, str(rsp)))
raise HttpProcessingError(code=rsp.status, message="Unexpected error")
group_json = await rsp.json()
group_id = group_json["id"]
# link group to parent
req = base_req + "/groups/" + parent_group + "/links/" + group_name
data = {"id": group_id }
link_created = False
log.info("PUT " + req)
globals["request_count"] += 1
async with client.put(req, data=json.dumps(data), headers=headers, params=params, timeout=timeout) as rsp:
if rsp.status == 409:
# another task has created this link already
log.warn("got 409 in request: " + req)
elif rsp.status != 201:
log.error("got http error: {} for request: {}, rsp: {}".format(rsp.status, req, rsp))
raise HttpProcessingError(code=rsp.status, message="Unexpected error")
else:
link_created = True
if not link_created:
# fetch the existing link and return the group
log.info("GET " + req)
globals["request_count"] += 1
async with client.get(req, headers=headers, params=params, timeout=timeout) as rsp:
if rsp.status != 200:
log.warn("unexpected error (expected to find link) {} for request: {}".format(rsp.status, req))
raise HttpProcessingError(code=rsp.status, message="Unexpected error")
else:
rsp_json = await rsp.json()
link_json = rsp_json["link"]
if link_json["class"] != "H5L_TYPE_HARD":
raise ValueError("Unexpected Link type: {}".format(link_json))
group_id = link_json["id"]
return group_id
def testPutVLenCompound(self):
# Test PUT value for 1d attribute with variable length int types
print("testPutVLenCompound", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = self.endpoint + '/'
# Get root uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# create dataset
fields = [ {"type": {"class": "H5T_INTEGER", "base": "H5T_STD_U64BE"}, "name": "VALUE1"},
{"type": {"class": "H5T_FLOAT", "base": "H5T_IEEE_F64BE"}, "name": "VALUE2"},
{"type": {"class": "H5T_ARRAY", "dims": [2], "base":
{"class": "H5T_STRING", "charSet": "H5T_CSET_ASCII",
"strPad": "H5T_STR_NULLTERM", "length": "H5T_VARIABLE"}}, "name": "VALUE3"}]
datatype = {'class': 'H5T_COMPOUND', 'fields': fields }
payload = {'type': datatype, 'shape': [4,]}
req = self.endpoint + "/datasets"
rsp = requests.post(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # create dataset
rspJson = json.loads(rsp.text)
dset_uuid = rspJson['id']
self.assertTrue(helper.validateId(dset_uuid))
# link new dataset as 'dset'
name = 'dset'
req = self.endpoint + "/groups/" + root_uuid + "/links/" + name
payload = {"id": dset_uuid}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201)
# write values to dataset
data = []
for i in range(4):
e = [i+1, i+1/10.0, ["Hi! "*(i+1), "Bye!" *(i+1)]]
data.append(e)
req = self.endpoint + "/datasets/" + dset_uuid + "/value"
payload = { 'value': data }
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200)
# read values from dataset
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("value" in rspJson)
value = rspJson["value"]
self.assertEqual(len(value), 4)
def testDNSDomain(self):
# DNS domain names are in reverse order with dots as seperators...
dns_domain = helper.getDNSDomain(self.base_domain)
print("testDNSDomain", dns_domain)
# verify we can access base domain as via dns name
headers = helper.getRequestHeaders(domain=dns_domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
# can't have two consecutive dots'
domain = 'two.dots..are.bad.' + dns_domain
req = helper.getEndpoint() + '/'
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400) # 400 == bad syntax
# can't have a slash
domain = 'no/slash.' + dns_domain
req = helper.getEndpoint() + '/'
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.get(req, headers=headers)
# somehow this is showing up as a 400 in ceph and 404 in S3
self.assertTrue(rsp.status_code in (400, 404)) # 400 == bad syntax
# just a dot is no good
domain = '.' + dns_domain
req = helper.getEndpoint() + '/'
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400) # 400 == bad syntax
# dot in the front is bad
domain = '.dot.in.front.is.bad.' + dns_domain
req = helper.getEndpoint() + '/'
headers = helper.getRequestHeaders(domain=domain)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400) # 400 == bad syntax
def testGetTopLevelDomain(self):
domain = "/home"
print("testGetTopLevelDomain", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertFalse("root" in rspJson) # no root group for folder domain
self.assertTrue("owner" in rspJson)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("class" in rspJson)
self.assertEqual(rspJson["class"], "folder")
domain = "test_user1.home"
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
def testPost(self):
# test POST group
print("testPost", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = helper.getEndpoint() + '/groups'
# create a new group
rsp = requests.post(req, headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["linkCount"], 0)
self.assertEqual(rspJson["attributeCount"], 0)
group_id = rspJson["id"]
self.assertTrue(helper.validateId(group_id))
# verify we can do a get on the new group
req = helper.getEndpoint() + '/groups/' + group_id
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
self.assertEqual(rspJson["id"], group_id)
self.assertTrue("root" in rspJson)
self.assertTrue(rspJson["root"] != group_id)
self.assertTrue("domain" in rspJson)
#self.assertEqual(rspJson["domain"], domain) # TBD
# try getting the path of the group
params = {"getalias": 1}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("alias" in rspJson)
self.assertEqual(rspJson["alias"], [])
# try POST with user who doesn't have create permission on this domain
headers = helper.getRequestHeaders(domain=self.base_domain,
username="******")
req = helper.getEndpoint() + '/groups'
rsp = requests.post(req, headers=headers)
self.assertEqual(rsp.status_code, 403) # forbidden
def testExternalLink(self):
domain = self.base_domain + "/testExternalLink.h5"
print("testExternalLink", domain)
helper.setupDomain(domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
root_id = rspJson["root"]
# get root group and check it has no links
req = helper.getEndpoint() + "/groups/" + root_id
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["linkCount"], 0) # no links
# create external link
target_domain = self.base_domain + '/external_target.h5'
target_path = 'somewhere'
link_title = 'external_link'
req = helper.getEndpoint(
) + "/groups/" + root_id + "/links/" + link_title
payload = {"h5path": target_path, "h5domain": target_domain}
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 201) # created
# get root group and check it has one link
req = helper.getEndpoint() + "/groups/" + root_id
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["linkCount"], 1) # no links
# get the link
req = helper.getEndpoint(
) + "/groups/" + root_id + "/links/" + link_title
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200) # should get the softlink
rspJson = json.loads(rsp.text)
self.assertTrue("created" in rspJson)
self.assertTrue("lastModified" in rspJson)
self.assertTrue("hrefs" in rspJson)
self.assertTrue("link" in rspJson)
rspLink = rspJson["link"]
self.assertEqual(rspLink["title"], link_title)
self.assertEqual(rspLink["class"], "H5L_TYPE_EXTERNAL")
self.assertEqual(rspLink["h5path"], target_path)
self.assertEqual(rspLink["h5domain"], target_domain)
def testGetInvalidUUID(self):
print("testGetInvalidUUID", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
req = helper.getEndpoint() + '/'
invalid_uuid = "foobar"
req = helper.getEndpoint() + "/groups/" + invalid_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 400)
import uuid
bad_uuid = "g-" + str(uuid.uuid1())
req = helper.getEndpoint() + "/groups/" + bad_uuid
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 404)
def testGetDomainVerbose(self):
domain = helper.getTestDomain("tall.h5")
print("testGetDomainVerbose", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
params = {"verbose": 1}
rsp = requests.get(req, params=params, headers=headers)
if rsp.status_code == 404:
print("WARNING: Failed to get domain: {}. Is test data setup?".
format(domain))
return # abort rest of test
self.assertEqual(rsp.status_code, 200)
self.assertEqual(rsp.headers['content-type'],
'application/json; charset=utf-8')
rspJson = json.loads(rsp.text)
for name in ("lastModified", "created", "hrefs", "root", "owner",
"class"):
self.assertTrue(name in rspJson)
now = time.time()
self.assertTrue(rspJson["created"] < now - 10)
self.assertTrue(rspJson["lastModified"] < now - 10)
self.assertEqual(len(rspJson["hrefs"]), 7)
self.assertTrue(rspJson["root"].startswith("g-"))
self.assertTrue(rspJson["owner"])
self.assertEqual(rspJson["class"], "domain")
root_uuid = rspJson["root"]
helper.validateId(root_uuid)
# restore when sqlite changes are complete
self.assertTrue("num_groups" in rspJson)
self.assertEqual(rspJson["num_groups"], 6)
self.assertTrue("num_datasets" in rspJson)
self.assertEqual(rspJson["num_datasets"], 4)
self.assertTrue("num_datatypes" in rspJson)
self.assertEqual(rspJson["num_datatypes"], 0)
self.assertTrue("allocated_bytes" in rspJson)
# test that allocated_bytes falls in a given range
self.assertEqual(rspJson["allocated_bytes"], 580)
# total_size may change slightly based on specifics of JSON serialization
self.assertTrue(rspJson["total_size"] > 5000)
self.assertTrue(rspJson["total_size"] < 7000)
# TODO - num_chunks should be present
self.assertTrue("num_objects" in rspJson)
self.assertTrue(rspJson["num_objects"], 14)
def testCreateFolder(self):
domain = self.base_domain + "/newfolder"
print("testCreateFolder", domain)
headers = helper.getRequestHeaders(domain=domain)
req = helper.getEndpoint() + '/'
body = {"folder": True}
rsp = requests.put(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
for k in ("owner", "acls", "created", "lastModified"):
self.assertTrue(k in rspJson)
self.assertFalse("root" in rspJson) # no root -> folder
# verify that putting the same domain again fails with a 409 error
rsp = requests.put(req, data=json.dumps(body), headers=headers)
self.assertEqual(rsp.status_code, 409)
# do a get on the new folder
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("owner" in rspJson)
self.assertTrue("class" in rspJson)
self.assertEqual(rspJson["class"], "folder")
# try doing a un-authenticated request
if config.get("test_noauth") and config.get("default_public"):
headers = helper.getRequestHeaders()
req = helper.getEndpoint() + "/?host=" + domain
# do a get on the folder with a query arg for host
rsp = requests.get(req)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
for k in ("class", "owner"):
self.assertTrue(k in rspJson)
self.assertFalse("root" in rspJson)
async def verifyGroupPath(h5path):
""" create any groups along the path that doesn't exist
"""
#print("current task: ", asyncio.Task.current_task())
client = globals["client"]
domain = globals["domain"]
h5path_cache = globals["h5path_cache"]
params = {"host": domain}
parent_group = h5path_cache['/'] # start with root
group_names = h5path.split('/')
headers = getRequestHeaders()
base_req = getEndpoint() + '/groups/'
next_path = '/'
timeout = config.get("timeout")
for group_name in group_names:
if not group_name:
continue # skip empty names
next_path += group_name
if not next_path.endswith('/'):
next_path += '/' # prep for next roundtrips
if next_path in h5path_cache:
# we already have the group id
parent_group = h5path_cache[next_path]
continue
req = base_req + parent_group + "/links/" + group_name
log.info("GET " + req)
globals["request_count"] += 1
async with client.get(req, headers=headers, params=params, timeout=timeout) as rsp:
if rsp.status == 404:
parent_group = await createGroup(parent_group, group_name)
elif rsp.status != 200:
raise HttpProcessingError(code=rsp.status, message="Unexpected error")
else:
rsp_json = await rsp.json()
link_json = rsp_json["link"]
if link_json["class"] != "H5L_TYPE_HARD":
raise ValueError("Unexpected Link type: {}".format(link_json))
parent_group = link_json["id"]
h5path_cache[next_path] = parent_group
return parent_group
def testScalarDataset(self):
print("testScalarDataset:", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
# get domain
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
rspJson = json.loads(rsp.text)
self.assertTrue("root" in rspJson)
root_uuid = rspJson["root"]
# create a dataset obj
data = {"type": "H5T_IEEE_F32LE"}
req = self.endpoint + '/datasets'
rsp = requests.post(req, data=json.dumps(data), headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
self.assertEqual(rspJson["attributeCount"], 0)
dset_id = rspJson["id"]
self.assertTrue(helper.validateId(dset_id))
# write to the dset
data = [
42,
]
payload = {'value': data}
req = self.endpoint + "/datasets/" + dset_id + "/value"
rsp = requests.put(req, data=json.dumps(payload), headers=headers)
self.assertEqual(rsp.status_code, 200)
points = [
0,
]
body = {"points": points}
# read selected points
rsp = requests.post(req, data=json.dumps(body), headers=headers)
# point select not supported on zero-dimensional datasets
self.assertEqual(rsp.status_code, 400)
def testGetByPath(self):
domain = helper.getTestDomain("tall.h5")
print("testGetByPath", domain)
headers = helper.getRequestHeaders(domain=domain)
# verify domain exists
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
if rsp.status_code != 200:
print("WARNING: Failed to get domain: {}. Is test data setup?".
format(domain))
return # abort rest of test
rspJson = json.loads(rsp.text)
root_uuid = rspJson["root"]
# get the group at "/g1/g1.1"
h5path = "/g1/g1.1"
req = helper.getEndpoint() + "/groups/"
params = {"h5path": h5path}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
for name in ("id", "hrefs", "attributeCount", "linkCount", "domain",
"root", "created", "lastModified"):
self.assertTrue(name in rspJson)
# verify we get the same id when following the path via service calls
g11id = helper.getUUIDByPath(domain, "/g1/g1.1")
self.assertEqual(g11id, rspJson["id"])
# Try with a trailing slash
h5path = "/g1/g1.1/"
req = helper.getEndpoint() + "/groups/"
params = {"h5path": h5path}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(g11id, rspJson["id"])
# try relative h5path
g1id = helper.getUUIDByPath(domain, "/g1/")
h5path = "./g1.1"
req = helper.getEndpoint() + "/groups/" + g1id
params = {"h5path": h5path}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertEqual(g11id, rspJson["id"])
# try a invalid link and verify a 404 is returened
h5path = "/g1/foobar"
req = helper.getEndpoint() + "/groups/"
params = {"h5path": h5path}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 404)
# try passing a path to a dataset and verify we get 404
h5path = "/g1/g1.1/dset1.1.1"
req = helper.getEndpoint() + "/groups/"
params = {"h5path": h5path}
rsp = requests.get(req, headers=headers, params=params)
self.assertEqual(rsp.status_code, 404)
# try getting the path of the group
req = helper.getEndpoint() + "/groups/" + g11id
params = {"getalias": 1}
rsp = requests.get(req, params=params, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("alias" in rspJson)
self.assertEqual(rspJson["alias"], [
'/g1/g1.1',
])
def testDelete(self):
# test Delete
print("testDelete", self.base_domain)
headers = helper.getRequestHeaders(domain=self.base_domain)
# get domain
req = helper.getEndpoint() + '/'
rsp = requests.get(req, headers=headers)
rspJson = json.loads(rsp.text)
self.assertTrue("root" in rspJson)
root_id = rspJson["root"]
req = helper.getEndpoint() + '/groups'
# create a new group
rsp = requests.post(req, headers=headers)
self.assertEqual(rsp.status_code, 201)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
group_id = rspJson["id"]
self.assertTrue(helper.validateId(group_id))
# verify we can do a get on the new group
req = helper.getEndpoint() + '/groups/' + group_id
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue("id" in rspJson)
self.assertEqual(rspJson["id"], group_id)
self.assertTrue("root" in rspJson)
self.assertTrue(rspJson["root"] != group_id)
self.assertTrue("domain" in rspJson)
#self.assertEqual(rspJson["domain"], self.base_domain) #TBD
# try DELETE with user who doesn't have create permission on this domain
headers = helper.getRequestHeaders(domain=self.base_domain,
username="******")
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 403) # forbidden
# try to do a DELETE with a different domain (should fail)
another_domain = helper.getParentDomain(self.base_domain)
headers = helper.getRequestHeaders(domain=another_domain)
req = helper.getEndpoint() + '/groups/' + group_id
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 400)
# delete the new group
headers = helper.getRequestHeaders(domain=self.base_domain)
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 200)
rspJson = json.loads(rsp.text)
self.assertTrue(rspJson is not None)
# a get for the group should now return 410 (GONE)
rsp = requests.get(req, headers=headers)
self.assertEqual(rsp.status_code, 410)
# try deleting the root group
req = helper.getEndpoint() + '/groups/' + root_id
rsp = requests.delete(req, headers=headers)
self.assertEqual(rsp.status_code, 403) # Forbidden
