def get_domain(request, body=None):
""" Extract domain and validate """
app = request.app
params = request.rel_url.query
domain = None
if "domain" in params:
domain = params["domain"]
log.debug(f"got domain param: {domain}")
elif body and "domain" in body:
domain = body["domain"]
if not domain:
msg = "No domain provided"
log.error(msg)
raise HTTPInternalServerError()
if not isValidDomain(domain):
msg = f"Expected valid domain for [{domain}]"
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, domain)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
return domain
async def check_metadata_obj(app, obj_id):
""" Return False is obj does not exist
"""
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = "{} has been deleted".format(obj_id)
log.info(msg)
return False
meta_cache = app['meta_cache']
if obj_id in meta_cache:
found = True
else:
# Not in chache, check s3 obj exists
s3_key = getS3Key(obj_id)
log.debug("check_metadata_obj({})".format(s3_key))
# does key exist?
found = await isS3Obj(app, s3_key)
return found
async def DELETE_Link(request):
"""HTTP method to delete a link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "delete")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
rsp_json = await http_delete(app, req)
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def check_metadata_obj(app, obj_id, bucket=None):
""" Return False is obj does not exist
"""
validateObjId(obj_id, bucket)
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = f"{obj_id} has been deleted"
log.info(msg)
return False
meta_cache = app['meta_cache']
if obj_id in meta_cache:
found = True
else:
# Not in chache, check s3 obj exists
s3_key = getS3Key(obj_id)
log.debug(f"check_metadata_obj({s3_key})")
# does key exist?
found = await isS3Obj(app, s3_key, bucket=bucket)
return found
async def DELETE_Group(request):
"""HTTP method to delete a group resource"""
log.request(request)
app = request.app
meta_cache = app['meta_cache']
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, "Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, group_id, username, "delete")
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
if group_id == domain_json["root"]:
msg = "Forbidden - deletion of root group is not allowed - delete domain first"
log.warn(msg)
raise HTTPForbidden()
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id
params = {}
if bucket:
params["bucket"] = bucket
log.debug(f"http_delete req: {req} params: {params}")
await http_delete(app, req, params=params)
if group_id in meta_cache:
del meta_cache[group_id] # remove from cache
resp = await jsonResponse(request, {})
log.response(request, resp=resp)
return resp
async def delete_metadata_obj(app,
obj_id,
notify=True,
root_id=None,
bucket=None):
""" Delete the given object """
meta_cache = app['meta_cache']
dirty_ids = app["dirty_ids"]
log.info(f"delete_meta_data_obj: {obj_id} notify: {notify}")
validateObjId(obj_id, bucket)
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
try:
validateInPartition(app, obj_id)
except KeyError:
log.error(f"obj: {obj_id} not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
log.warn(f"{obj_id} has already been deleted")
else:
log.debug(f"adding {obj_id} to deleted ids")
deleted_ids.add(obj_id)
if obj_id in meta_cache:
log.debug(f"removing {obj_id} from meta_cache")
del meta_cache[obj_id]
if obj_id in dirty_ids:
log.debug(f"removing dirty_ids for: {obj_id}")
del dirty_ids[obj_id]
# remove from S3 (if present)
s3key = getS3Key(obj_id)
if await isS3Obj(app, s3key, bucket=bucket):
await deleteS3Obj(app, s3key, bucket=bucket)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
if isValidUuid(obj_id) and isSchema2Id(obj_id):
if isRootObjId(obj_id):
# add to gc ids so sub-objects will be deleted
gc_ids = app["gc_ids"]
log.info(f"adding root id: {obj_id} for GC cleanup")
gc_ids.add(obj_id)
elif notify:
root_id = getRootObjId(obj_id)
await notify_root(app, root_id, bucket=bucket)
# no notify for domain deletes since the root group is being deleted
log.debug(f"delete_metadata_obj for {obj_id} done")
async def DELETE_Attribute(request):
"""HTTP method to delete a attribute resource"""
log.request(request)
app = request.app
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
log.debug(f"Attribute name: [{attr_name}]")
validateAttributeName(attr_name)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "delete")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.info("PUT Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
rsp_json = await http_delete(app, req, params=params)
log.info(f"PUT Attribute resp: {rsp_json}")
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
resp = await jsonResponse(request, req_rsp)
log.response(request, resp=resp)
return resp
async def GET_DatasetShape(request):
"""HTTP method to return JSON for dataset's shape"""
log.request(request)
app = request.app
dset_id = request.match_info.get('id')
if not dset_id:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(dset_id, "Dataset"):
msg = f"Invalid dataset id: {dset_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app, dset_id, refresh=True, bucket=bucket)
await validateAction(app, domain, dset_id, username, "read")
hrefs = []
dset_uri = '/datasets/' + dset_id
self_uri = dset_uri + "/shape"
hrefs.append({'rel': 'self', 'href': getHref(request, self_uri)})
dset_uri = '/datasets/' + dset_id
hrefs.append({'rel': 'owner', 'href': getHref(request, dset_uri)})
root_uri = '/groups/' + dset_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
resp_json = {}
resp_json["shape"] = dset_json["shape"]
resp_json["hrefs"] = hrefs
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def DELETE_Datatype(request):
"""HTTP method to delete a committed type resource"""
log.request(request)
app = request.app
meta_cache = app['meta_cache']
ctype_id = request.match_info.get('id')
if not ctype_id:
msg = "Missing committed type id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(ctype_id, "Type"):
msg = f"Invalid committed type id: {ctype_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
params = {}
if bucket:
params["bucket"] = bucket
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, ctype_id, username, "delete")
req = getDataNodeUrl(app, ctype_id) + "/datatypes/" + ctype_id
await http_delete(app, req, params=params)
if ctype_id in meta_cache:
del meta_cache[ctype_id] # remove from cache
resp = await jsonResponse(request, {})
log.response(request, resp=resp)
return resp
async def DELETE_Dataset(request):
"""HTTP method to delete a dataset resource"""
log.request(request)
app = request.app
meta_cache = app['meta_cache']
dset_id = request.match_info.get('id')
if not dset_id:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(dset_id, "Dataset"):
msg = "Invalid dataset id: {}".format(dset_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error("Expected root key for domain: {}".format(domain))
raise HTTPBadRequest(reason="Unexpected Error")
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, dset_id, username, "delete")
req = getDataNodeUrl(app, dset_id) + "/datasets/" + dset_id
await http_delete(app, req)
if dset_id in meta_cache:
del meta_cache[dset_id] # remove from cache
resp = await jsonResponse(request, {})
log.response(request, resp=resp)
return resp
def validateObjId(obj_id, bucket):
"""
Verifies the passed in is what we are expecting.
For uuids, obj_id should be an actual uuid and bucket should be non-null
For domains, obj_id should include the bucket prefix and bucket should be empty
e.g. obj_id="mybucket/home/bob/myfile.h5"
"""
if isValidDomain(obj_id):
if obj_id[0] == '/':
# bucket name should always be prefixed
# (so the obj_id is cannonical)
msg = f"bucket not included for domain: {obj_id}"
log.error(msg)
raise HTTPInternalServerError()
if bucket:
msg = f"bucket param should not be used with obj_id for domain: {obj_id}"
log.error(msg)
raise HTTPInternalServerError()
elif not isValidUuid(obj_id):
msg = f"Invalid obj id: {obj_id}"
log.error(msg)
raise HTTPInternalServerError()
async def PUT_Attribute(request):
"""HTTP method to create a new attribute"""
log.request(request)
app = request.app
collection = getRequestCollectionName(request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
log.debug(f"Attribute name: [{attr_name}]")
validateAttributeName(attr_name)
log.info(f"PUT Attribute id: {obj_id} name: {attr_name}")
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Attribute with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
root_id = domain_json["root"]
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "create")
if "type" not in body:
msg = "PUT attribute with no type in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str) and datatype.startswith("t-"):
# Committed type - fetch type json from DN
ctype_id = datatype
log.debug(f"got ctypeid: {ctype_id}")
ctype_json = await getObjectJson(app, ctype_id, bucket=bucket)
log.debug(f"ctype {ctype_id}: {ctype_json}")
if ctype_json["root"] != root_id:
msg = "Referenced committed datatype must belong in same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = ctype_json["type"]
# add the ctype_id to type type
datatype["id"] = ctype_id
elif isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug(f"got datatype: {datatype}")
except TypeError:
msg = "PUT attribute with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
dims = None
shape_json = {}
if "shape" in body:
shape_body = body["shape"]
shape_class = None
if isinstance(shape_body, dict) and "class" in shape_body:
shape_class = shape_body["class"]
elif isinstance(shape_body, str):
shape_class = shape_body
if shape_class:
if shape_class == "H5S_NULL":
shape_json["class"] = "H5S_NULL"
if isinstance(shape_body, dict) and "dims" in shape_body:
msg = "can't include dims with null shape"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if isinstance(shape_body, dict) and "value" in body:
msg = "can't have H5S_NULL shape with value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif shape_class == "H5S_SCALAR":
shape_json["class"] = "H5S_SCALAR"
dims = getShapeDims(shape_body)
if len(dims) != 1 or dims[0] != 1:
msg = "dimensions aren't valid for scalar attribute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif shape_class == "H5S_SIMPLE":
shape_json["class"] = "H5S_SIMPLE"
dims = getShapeDims(shape_body)
shape_json["dims"] = dims
else:
msg = f"Unknown shape class: {shape_class}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
# no class, interpet shape value as dimensions and
# use H5S_SIMPLE as class
if isinstance(shape_body, list) and len(shape_body) == 0:
shape_json["class"] = "H5S_SCALAR"
dims = [1,]
else:
shape_json["class"] = "H5S_SIMPLE"
dims = getShapeDims(shape_body)
shape_json["dims"] = dims
else:
shape_json["class"] = "H5S_SCALAR"
dims = [1,]
if "value" in body:
if dims is None:
msg = "Bad Request: data can not be included with H5S_NULL space"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
value = body["value"]
# validate that the value agrees with type/shape
arr_dtype = createDataType(datatype) # np datatype
if len(dims) == 0:
np_dims = [1,]
else:
np_dims = dims
log.debug(f"attribute dims: {np_dims}")
log.debug(f"attribute value: {value}")
try:
arr = jsonToArray(np_dims, arr_dtype, value)
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"Got: {arr.size} array elements")
else:
value = None
# ready to add attribute now
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.info("PUT Attribute: " + req)
attr_json = {}
attr_json["type"] = datatype
attr_json["shape"] = shape_json
if value is not None:
attr_json["value"] = value
params = {}
if bucket:
params["bucket"] = bucket
put_rsp = await http_put(app, req, params=params, data=attr_json)
log.info(f"PUT Attribute resp: {put_rsp}")
hrefs = [] # TBD
req_rsp = { "hrefs": hrefs }
# attribute creation successful
resp = await jsonResponse(request, req_rsp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Attributes(request):
"""HTTP method to return JSON for attribute collection"""
log.request(request)
app = request.app
params = request.rel_url.query
collection = getRequestCollectionName(request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = "Invalid obj id: {}".format(obj_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
include_data = False
if "IncludeData" in params and params["IncludeData"]:
include_data = True
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes"
params = {}
if limit is not None:
params["Limit"] = str(limit)
if marker is not None:
params["Marker"] = marker
if include_data:
params["IncludeData"] = '1'
if bucket:
params["bucket"] = bucket
log.debug(f"get attributes: {req}")
dn_json = await http_get(app, req, params=params)
log.debug(f"got attributes json from dn for obj_id: {obj_id}")
attributes = dn_json["attributes"]
# mixin hrefs
for attribute in attributes:
attr_name = attribute["name"]
attr_href = '/' + collection + '/' + obj_id + '/attributes/' + attr_name
attribute["href"] = getHref(request, attr_href)
resp_json = {}
resp_json["attributes"] = attributes
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
hrefs.append({'rel': 'self', 'href': getHref(request, obj_uri + '/attributes')})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Links(request):
"""HTTP method to return JSON for link collection"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links"
query_sep = '?'
if limit is not None:
req += query_sep + "Limit=" + str(limit)
query_sep = '&'
if marker is not None:
req += query_sep + "Marker=" + marker
log.debug("get LINKS: " + req)
links_json = await http_get(app, req)
log.debug("got links json from dn for group_id: {}".format(group_id))
links = links_json["links"]
# mix in collection key, target and hrefs
for link in links:
if link["class"] == "H5L_TYPE_HARD":
collection_name = getCollectionForId(link["id"])
link["collection"] = collection_name
target_uri = '/' + collection_name + '/' + link["id"]
link["target"] = getHref(request, target_uri)
link_uri = '/groups/' + group_id + '/links/' + link['title']
link["href"] = getHref(request, link_uri)
resp_json = {}
resp_json["links"] = links
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links')
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def PUT_Link(request):
"""HTTP method to create a new link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
log.info("PUT Link_title: [{}]".format(link_title))
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Link with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
link_json = {}
if "id" in body:
if not isValidUuid(body["id"]):
msg = "PUT Link with invalid id in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_json["id"] = body["id"]
link_json["class"] = "H5L_TYPE_HARD"
elif "h5path" in body:
link_json["h5path"] = body["h5path"]
# could be hard or soft link
if "h5domain" in body:
link_json["h5domain"] = body["h5domain"]
link_json["class"] = "H5L_TYPE_EXTERNAL"
else:
# soft link
link_json["class"] = "H5L_TYPE_SOFT"
else:
msg = "PUT Link with no id or h5path keys"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "create")
# for hard links, verify that the referenced id exists and is in this domain
if "id" in body:
ref_id = body["id"]
ref_json = await getObjectJson(app, ref_id)
group_json = await getObjectJson(app, group_id)
if ref_json["root"] != group_json["root"]:
msg = "Hard link must reference an object in the same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# ready to add link now
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("PUT link - getting group: " + req)
put_rsp = await http_put(app, req, data=link_json)
log.debug("PUT Link resp: " + str(put_rsp))
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
# link creation successful
resp = await jsonResponse(request, req_rsp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Link(request):
"""HTTP method to return JSON for a group link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("get LINK: " + req)
link_json = await http_get(app, req)
log.debug("got link_json: " + str(link_json))
resp_link = {}
resp_link["title"] = link_title
resp_link["class"] = link_json["class"]
if link_json["class"] == "H5L_TYPE_HARD":
resp_link["id"] = link_json["id"]
resp_link["collection"] = getCollectionForId(link_json["id"])
elif link_json["class"] == "H5L_TYPE_SOFT":
resp_link["h5path"] = link_json["h5path"]
elif link_json["class"] == "H5L_TYPE_EXTERNAL":
resp_link["h5path"] = link_json["h5path"]
resp_link["h5domain"] = link_json["h5domain"]
else:
log.warn("Unexpected link class: {}".format(link_json["class"]))
resp_json = {}
resp_json["link"] = resp_link
resp_json["created"] = link_json["created"]
# links don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = link_json["created"]
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links/' + link_title)
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
if link_json["class"] == "H5L_TYPE_HARD":
target = '/' + resp_link["collection"] + '/' + resp_link["id"]
hrefs.append({'rel': 'target', 'href': getHref(request, target)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Group(request):
"""HTTP method to return JSON for group"""
log.request(request)
app = request.app
params = request.rel_url.query
h5path = None
getAlias = False
include_links = False
include_attrs = False
group_id = request.match_info.get('id')
if not group_id and "h5path" not in params:
# no id, or path provided, so bad request
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if group_id:
log.info(f"GET_Group, id: {group_id}")
# is the id a group id and not something else?
if not isValidUuid(group_id, "Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
if "h5path" in params:
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute if no parent id is provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Group, h5path: {h5path}")
if "include_links" in params and params["include_links"]:
include_links = True
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
if h5path and h5path[0] == '/':
# ignore the request path id (if given) and start
# from root group for absolute paths
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
if h5path:
group_id = await getObjectIdByPath(app,
group_id,
h5path,
bucket=bucket
) # throws 404 if not found
if not isValidUuid(group_id, "Group"):
msg = f"No group exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get group_id: {group_id} from h5path: {h5path}")
# verify authorization to read the group
await validateAction(app, domain, group_id, username, "read")
# get authoritative state for group from DN (even if it's in the meta_cache).
group_json = await getObjectJson(app,
group_id,
refresh=True,
include_links=include_links,
include_attrs=include_attrs,
bucket=bucket)
log.debug(f"domain from request: {domain}")
group_json["domain"] = getPathForDomain(domain)
if bucket:
group_json["bucket"] = bucket
if getAlias:
root_id = group_json["root"]
alias = []
if group_id == root_id:
alias.append('/')
else:
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=group_id,
bucket=bucket)
if h5path:
alias.append(h5path)
group_json["alias"] = alias
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri)})
hrefs.append({
'rel': 'links',
'href': getHref(request, group_uri + '/links')
})
root_uri = '/groups/' + group_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, group_uri + '/attributes')
})
group_json["hrefs"] = hrefs
resp = await jsonResponse(request, group_json)
log.response(request, resp=resp)
return resp
async def GET_Dataset(request):
"""HTTP method to return JSON description of a dataset"""
log.request(request)
app = request.app
params = request.rel_url.query
h5path = None
getAlias = False
dset_id = request.match_info.get('id')
if not dset_id and "h5path" not in params:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if dset_id:
if not isValidUuid(dset_id, "Dataset"):
msg = "Invalid dataset id: {}".format(dset_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
else:
group_id = None
if "grpid" in params:
group_id = params["grpid"]
if not isValidUuid(group_id, "Group"):
msg = "Invalid parent group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "h5path" not in params:
msg = "Expecting either ctype id or h5path url param"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info("GET_Dataset, h5path: {}".format(h5path))
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if h5path:
if group_id is None:
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = "Expected root key for domain: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
dset_id = await getObjectIdByPath(app, group_id,
h5path) # throws 404 if not found
if not isValidUuid(dset_id, "Dataset"):
msg = "No dataset exist with the path: {}".format(h5path)
log.warn(msg)
raise HTTPNotFound()
log.info("get dataset_id: {} from h5path: {}".format(dset_id, h5path))
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app, dset_id, refresh=True)
# check that we have permissions to read the object
await validateAction(app, domain, dset_id, username, "read")
log.debug("got dset_json: {}".format(dset_json))
resp_json = {}
resp_json["id"] = dset_json["id"]
resp_json["root"] = dset_json["root"]
resp_json["shape"] = dset_json["shape"]
resp_json["type"] = dset_json["type"]
if "creationProperties" in dset_json:
resp_json["creationProperties"] = dset_json["creationProperties"]
else:
resp_json["creationProperties"] = {}
if "layout" in dset_json:
resp_json["layout"] = dset_json["layout"]
resp_json["attributeCount"] = dset_json["attributeCount"]
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["domain"] = domain
if getAlias:
root_id = dset_json["root"]
alias = []
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=dset_id)
if h5path:
alias.append(h5path)
resp_json["alias"] = alias
hrefs = []
dset_uri = '/datasets/' + dset_id
hrefs.append({'rel': 'self', 'href': getHref(request, dset_uri)})
root_uri = '/groups/' + dset_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, dset_uri + '/attributes')
})
# provide a value link if the dataset is relatively small,
# otherwise create a preview link that shows a limited number of data values
dset_shape = dset_json["shape"]
if dset_shape["class"] != 'H5S_NULL':
count = 1
if dset_shape["class"] == 'H5S_SIMPLE':
dims = dset_shape["dims"]
count = getNumElements(dims)
if count <= 100:
# small number of values, provide link to entire dataset
hrefs.append({
'rel': 'data',
'href': getHref(request, dset_uri + '/value')
})
else:
# large number of values, create preview link
previewQuery = getPreviewQuery(dset_shape["dims"])
hrefs.append({
'rel':
'preview',
'href':
getHref(request, dset_uri + '/value', query=previewQuery)
})
resp_json["hrefs"] = hrefs
if verbose:
# get allocated size and num_chunks for the dataset if available
dset_detail = await getDatasetDetails(app, dset_id, dset_json["root"])
if dset_detail is not None:
if "num_chunks" in dset_detail:
resp_json["num_chunks"] = dset_detail["num_chunks"]
if "allocated_bytes" in dset_detail:
resp_json["allocated_size"] = dset_detail["allocated_bytes"]
if "lastModified" in dset_detail:
resp_json["lastModified"] = dset_detail["lastModified"]
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_AttributeValue(request):
"""HTTP method to return an attribute value"""
log.request(request)
app = request.app
log.info("GET_AttributeValue")
collection = getRequestCollectionName(request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain value: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(dn_json))
attr_shape = dn_json["shape"]
log.debug(f"attribute shape: {attr_shape}")
if attr_shape["class"] == 'H5S_NULL':
msg = "Null space attributes can not be read"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
accept_type = getAcceptType(request)
response_type = accept_type # will adjust later if binary not possible
type_json = dn_json["type"]
shape_json = dn_json["shape"]
item_size = getItemSize(type_json)
if item_size == 'H5T_VARIABLE' and accept_type != "json":
msg = "Client requested binary, but only JSON is supported for variable length data types"
log.info(msg)
response_type = "json"
if response_type == "binary":
arr_dtype = createDataType(type_json) # np datatype
np_shape = getShapeDims(shape_json)
try:
arr = jsonToArray(np_shape, arr_dtype, dn_json["value"])
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
output_data = arr.tobytes()
log.debug(f"GET AttributeValue - returning {len(output_data)} bytes binary data")
# write response
try:
resp = StreamResponse()
resp.content_type = "application/octet-stream"
resp.content_length = len(output_data)
# allow CORS
resp.headers['Access-Control-Allow-Origin'] = '*'
resp.headers['Access-Control-Allow-Methods'] = "GET, POST, DELETE, PUT, OPTIONS"
resp.headers['Access-Control-Allow-Headers'] = "Content-Type, api_key, Authorization"
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Got exception: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
else:
resp_json = {}
if "value" in dn_json:
resp_json["value"] = dn_json["value"]
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
attr_uri = obj_uri + '/attributes/' + attr_name
hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def POST_Datatype(request):
"""HTTP method to create new committed datatype object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "POST Datatype with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
if "type" not in body:
msg = "POST Datatype has no type key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug(f"got datatype: {datatype}")
except TypeError:
msg = "POST Dataset with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if "link" in body:
link_body = body["link"]
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username, "create")
root_id = domain_json["root"]
ctype_id = createObjId("datatypes", rootid=root_id)
log.debug(f"new type id: {ctype_id}")
ctype_json = {"id": ctype_id, "root": root_id, "type": datatype}
log.debug("create named type, body: " + json.dumps(ctype_json))
req = getDataNodeUrl(app, ctype_id) + "/datatypes"
params = {}
if bucket:
params["bucket"] = bucket
type_json = await http_post(app, req, data=ctype_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = ctype_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_rsp = await http_put(app, link_req, data=link_json, params=params)
log.debug(f"PUT Link resp: {put_rsp}")
# datatype creation successful
resp = await jsonResponse(request, type_json, status=201)
log.response(request, resp=resp)
return resp
async def PUT_AttributeValue(request):
"""HTTP method to update an attributes data"""
log.request(request)
log.info("PUT_AttributeValue")
app = request.app
collection = getRequestCollectionName(request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
log.debug(f"Attribute name: [{attr_name}]")
validateAttributeName(attr_name)
log.info(f"PUT Attribute Value id: {obj_id} name: {attr_name}")
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT AttributeValue with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPInternalServerError()
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "update")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(obj_id))
log.debug(f"got dn_json: {dn_json}")
attr_shape = dn_json["shape"]
if attr_shape["class"] == 'H5S_NULL':
msg = "Null space attributes can not be updated"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
np_shape = getShapeDims(attr_shape)
type_json = dn_json["type"]
np_dtype = createDataType(type_json) # np datatype
request_type = "json"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type not in ("application/json", "application/octet-stream"):
msg = f"Unknown content_type: {content_type}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if content_type == "application/octet-stream":
log.debug("PUT AttributeValue - request_type is binary")
request_type = "binary"
else:
log.debug("PUT AttribueValue - request type is json")
binary_data = None
if request_type == "binary":
item_size = getItemSize(type_json)
if item_size == 'H5T_VARIABLE':
msg = "Only JSON is supported for variable length data types"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# read binary data
binary_data = await request.read()
if len(binary_data) != request.content_length:
msg = f"Read {len(binary_data)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
arr = None # np array to hold request data
if binary_data:
npoints = getNumElements(np_shape)
if npoints*item_size != len(binary_data):
msg = "Expected: " + str(npoints*item_size) + " bytes, but got: " + str(len(binary_data))
log.warn(msg)
raise HTTPBadRequest(reason=msg)
arr = np.fromstring(binary_data, dtype=np_dtype)
arr = arr.reshape(np_shape) # conform to selection shape
# convert to JSON for transmission to DN
data = arr.tolist()
value = bytesArrayToList(data)
else:
body = await request.json()
if "value" not in body:
msg = "PUT attribute value with no value in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
value = body["value"]
# validate that the value agrees with type/shape
try:
arr = jsonToArray(np_shape, np_dtype, value)
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"Got: {arr.size} array elements")
# ready to add attribute now
attr_json = {}
attr_json["type"] = type_json
attr_json["shape"] = attr_shape
attr_json["value"] = value
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.info(f"PUT Attribute Value: {req}")
dn_json["value"] = value
params = {}
params = {"replace": 1} # let the DN know we can overwrite the attribute
if bucket:
params["bucket"] = bucket
put_rsp = await http_put(app, req, params=params, data=attr_json)
log.info(f"PUT Attribute Value resp: {put_rsp}")
hrefs = [] # TBD
req_rsp = { "hrefs": hrefs }
# attribute creation successful
resp = await jsonResponse(request, req_rsp)
log.response(request, resp=resp)
return resp
async def GET_Attribute(request):
"""HTTP method to return JSON for an attribute"""
log.request(request)
app = request.app
collection = getRequestCollectionName(request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(obj_id))
resp_json = {}
resp_json["name"] = attr_name
resp_json["type"] = dn_json["type"]
resp_json["shape"] = dn_json["shape"]
if "value" in dn_json:
resp_json["value"] = dn_json["value"]
resp_json["created"] = dn_json["created"]
# attributes don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = dn_json["created"]
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
attr_uri = obj_uri + '/attributes/' + attr_name
hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def POST_Dataset(request):
"""HTTP method to create a new dataset object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "POST Datasets with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
# get domain, check authorization
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain_json = await getDomainJson(app, domain, reload=True)
root_id = domain_json["root"]
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = "Expected root key for domain: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
#
# validate type input
#
if "type" not in body:
msg = "POST Dataset has no type key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str) and datatype.startswith("t-"):
# Committed type - fetch type json from DN
ctype_id = datatype
log.debug("got ctypeid: {}".format(ctype_id))
ctype_json = await getObjectJson(app, ctype_id)
log.debug("ctype: {}".format(ctype_json))
if ctype_json["root"] != root_id:
msg = "Referenced committed datatype must belong in same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = ctype_json["type"]
# add the ctype_id to type type
datatype["id"] = ctype_id
elif isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug("got datatype: {}".format(datatype))
except TypeError:
msg = "POST Dataset with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
item_size = getItemSize(datatype)
#
# Validate shape input
#
dims = None
shape_json = {}
if "shape" not in body:
shape_json["class"] = "H5S_SCALAR"
else:
shape = body["shape"]
if isinstance(shape, int):
shape_json["class"] = "H5S_SIMPLE"
dims = [
shape,
]
shape_json["dims"] = dims
elif isinstance(shape, str):
# only valid string value is H5S_NULL
if shape != "H5S_NULL":
msg = "POST Datset with invalid shape value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["class"] = "H5S_NULL"
elif isinstance(shape, list):
if len(shape) == 0:
shape_json["class"] = "H5S_SCALAR"
else:
shape_json["class"] = "H5S_SIMPLE"
shape_json["dims"] = shape
dims = shape
else:
msg = "Bad Request: shape is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if dims is not None:
for i in range(len(dims)):
extent = dims[i]
if not isinstance(extent, int):
msg = "Invalid shape type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "shape dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = None
if "maxdims" in body:
if dims is None:
msg = "Maxdims cannot be supplied if space is NULL"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = body["maxdims"]
if isinstance(maxdims, int):
dim1 = maxdims
maxdims = [dim1]
elif isinstance(maxdims, list):
pass # can use as is
else:
msg = "Bad Request: maxdims is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(dims) != len(maxdims):
msg = "Maxdims rank doesn't match Shape"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if maxdims is not None:
for extent in maxdims:
if not isinstance(extent, int):
msg = "Invalid maxdims type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "maxdims dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(maxdims) != len(dims):
msg = "Bad Request: maxdims array length must equal shape array length"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["maxdims"] = []
for i in range(len(dims)):
maxextent = maxdims[i]
if not isinstance(maxextent, int):
msg = "Bad Request: maxdims must be integer type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif maxextent == 0:
# unlimited dimension
shape_json["maxdims"].append(0)
elif maxextent < dims[i]:
msg = "Bad Request: maxdims extent can't be smaller than shape extent"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
shape_json["maxdims"].append(maxextent)
layout = None
min_chunk_size = int(config.get("min_chunk_size"))
max_chunk_size = int(config.get("max_chunk_size"))
if 'creationProperties' in body:
creationProperties = body["creationProperties"]
if 'layout' in creationProperties:
layout = creationProperties["layout"]
validateChunkLayout(shape_json, item_size, layout)
if layout is None and shape_json["class"] != "H5S_NULL":
# default to chunked layout
layout = {"class": "H5D_CHUNKED"}
if layout and layout["class"] == 'H5D_CONTIGUOUS_REF':
chunk_dims = getContiguousLayout(shape_json,
item_size,
chunk_min=min_chunk_size,
chunk_max=max_chunk_size)
layout["dims"] = chunk_dims
log.debug(f"autoContiguous layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED' and "dims" not in layout:
# do autochunking
chunk_dims = guessChunk(shape_json, item_size)
layout["dims"] = chunk_dims
log.debug(f"initial autochunk layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED':
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug("chunk_size: {}, min: {}, max: {}".format(
chunk_size, min_chunk_size, max_chunk_size))
# adjust the chunk shape if chunk size is too small or too big
adjusted_chunk_dims = None
if chunk_size < min_chunk_size:
log.debug(
"chunk size: {} less than min size: {}, expanding".format(
chunk_size, min_chunk_size))
adjusted_chunk_dims = expandChunk(chunk_dims,
item_size,
shape_json,
chunk_min=min_chunk_size,
layout_class=layout["class"])
elif chunk_size > max_chunk_size:
log.debug(
"chunk size: {} greater than max size: {}, expanding".format(
chunk_size, max_chunk_size, layout_class=layout["class"]))
adjusted_chunk_dims = shrinkChunk(chunk_dims,
item_size,
chunk_max=max_chunk_size)
if adjusted_chunk_dims:
log.debug(
f"requested chunk_dimensions: {chunk_dims} modified dimensions: {adjusted_chunk_dims}"
)
layout["dims"] = adjusted_chunk_dims
if layout and layout["class"] in ('H5D_CHUNKED_REF',
'H5D_CHUNKED_REF_INDIRECT'):
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug("chunk_size: {}, min: {}, max: {}".format(
chunk_size, min_chunk_size, max_chunk_size))
# adjust the chunk shape if chunk size is too small or too big
if chunk_size < min_chunk_size:
log.warn(
"chunk size: {} less than min size: {} for H5D_CHUNKED_REF dataset"
.format(chunk_size, min_chunk_size))
elif chunk_size > max_chunk_size:
log.warn(
"chunk size: {} greater than max size: {}, for H5D_CHUNKED_REF dataset"
.format(chunk_size,
max_chunk_size,
layout_class=layout["class"]))
link_id = None
link_title = None
if "link" in body:
link_body = body["link"]
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.info("link id: {}".format(link_id))
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username, "create")
dset_id = createObjId("datasets", rootid=root_id)
log.info("new dataset id: {}".format(dset_id))
dataset_json = {
"id": dset_id,
"root": root_id,
"type": datatype,
"shape": shape_json
}
if "creationProperties" in body:
# TBD - validate all creationProperties
creationProperties = body["creationProperties"]
if "fillValue" in creationProperties:
# validate fill value compatible with type
dt = createDataType(datatype)
fill_value = creationProperties["fillValue"]
if isinstance(fill_value, list):
fill_value = tuple(fill_value)
try:
np.asarray(fill_value, dtype=dt)
except (TypeError, ValueError):
msg = "Fill value {} not compatible with dataset type: {}".format(
fill_value, datatype)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dataset_json["creationProperties"] = creationProperties
if layout is not None:
dataset_json["layout"] = layout
log.debug("create dataset: " + json.dumps(dataset_json))
req = getDataNodeUrl(app, dset_id) + "/datasets"
post_json = await http_post(app, req, data=dataset_json)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = dset_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.info("PUT link - : " + link_req)
put_rsp = await http_put(app, link_req, data=link_json)
log.debug("PUT Link resp: {}".format(put_rsp))
# dataset creation successful
resp = await jsonResponse(request, post_json, status=201)
log.response(request, resp=resp)
return resp
async def PUT_DatasetShape(request):
"""HTTP method to update dataset's shape"""
log.request(request)
app = request.app
shape_update = None
extend = 0
extend_dim = 0
dset_id = request.match_info.get('id')
if not dset_id:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(dset_id, "Dataset"):
msg = "Invalid dataset id: {}".format(dset_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
# validate request
if not request.has_body:
msg = "PUT shape with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
data = await request.json()
if "shape" not in data and "extend" not in data:
msg = "PUT shape has no shape or extend key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "shape" in data:
shape_update = data["shape"]
if isinstance(shape_update, int):
# convert to a list
shape_update = [
shape_update,
]
log.debug("shape_update: {}".format(shape_update))
if "extend" in data:
try:
extend = int(data["extend"])
except ValueError:
msg = "extend value must be integer"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extend <= 0:
msg = "extend value must be positive"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "extend_dim" in data:
try:
extend_dim = int(data["extend_dim"])
except ValueError:
msg = "extend_dim value must be integer"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extend_dim < 0:
msg = "extend_dim value must be non-negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# verify the user has permission to update shape
await validateAction(app, domain, dset_id, username, "update")
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app, dset_id, refresh=True)
shape_orig = dset_json["shape"]
log.debug("shape_orig: {}".format(shape_orig))
# verify that the extend request is valid
if shape_orig["class"] != "H5S_SIMPLE":
msg = "Unable to extend shape of datasets who are not H5S_SIMPLE"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "maxdims" not in shape_orig:
msg = "Dataset is not extensible"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dims = shape_orig["dims"]
rank = len(dims)
maxdims = shape_orig["maxdims"]
if shape_update and len(shape_update) != rank:
msg = "Extent of update shape request does not match dataset sahpe"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
if shape_update and shape_update[i] < dims[i]:
msg = "Dataspace can not be made smaller"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if shape_update and maxdims[i] != 0 and shape_update[i] > maxdims[i]:
msg = "Database can not be extended past max extent"
log.warn(msg)
raise HTTPConflict()
if extend_dim < 0 or extend_dim >= rank:
msg = "Extension dimension must be less than rank and non-negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# send request onto DN
req = getDataNodeUrl(app, dset_id) + "/datasets/" + dset_id + "/shape"
json_resp = {"hrefs": []}
if extend:
data = {"extend": extend, "extend_dim": extend_dim}
else:
data = {"shape": shape_update}
try:
put_rsp = await http_put(app, req, data=data)
log.info(f"got shape put rsp: {put_rsp}")
if "selection" in put_rsp:
json_resp["selection"] = put_rsp["selection"]
except HTTPConflict:
log.warn("got 409 extending dataspace")
raise
resp = await jsonResponse(request, json_resp, status=201)
log.response(request, resp=resp)
return resp
async def get_metadata_obj(app, obj_id, bucket=None):
""" Get object from metadata cache (if present).
Otherwise fetch from S3 and add to cache
"""
log.info(f"get_metadata_obj: {obj_id} bucket: {bucket}")
validateObjId(obj_id, bucket) # throws internal server error if invalid
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
"""
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
"""
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = f"{obj_id} has been deleted"
log.warn(msg)
raise HTTPGone()
meta_cache = app['meta_cache']
obj_json = None
if obj_id in meta_cache:
log.debug(f"{obj_id} found in meta cache")
obj_json = meta_cache[obj_id]
else:
s3_key = getS3Key(obj_id)
pending_s3_read = app["pending_s3_read"]
if obj_id in pending_s3_read:
# already a read in progress, wait for it to complete
read_start_time = pending_s3_read[obj_id]
log.info(
f"s3 read request for {s3_key} was requested at: {read_start_time}"
)
while time.time() - read_start_time < 2.0:
log.debug("waiting for pending s3 read, sleeping")
await asyncio.sleep(1) # sleep for sub-second?
if obj_id in meta_cache:
log.info(f"object {obj_id} has arrived!")
obj_json = meta_cache[obj_id]
break
if not obj_json:
log.warn(
f"s3 read for object {s3_key} timed-out, initiaiting a new read"
)
# invoke S3 read unless the object has just come in from pending read
if not obj_json:
log.debug(f"getS3JSONObj({s3_key}, bucket={bucket})")
if obj_id not in pending_s3_read:
pending_s3_read[obj_id] = time.time()
# read S3 object as JSON
obj_json = await getS3JSONObj(app, s3_key, bucket=bucket)
if obj_id in pending_s3_read:
# read complete - remove from pending map
elapsed_time = time.time() - pending_s3_read[obj_id]
log.info(f"s3 read for {s3_key} took {elapsed_time}")
del pending_s3_read[obj_id]
meta_cache[obj_id] = obj_json # add to cache
return obj_json
async def POST_Group(request):
"""HTTP method to create new Group object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if request.has_body:
body = await request.json()
log.info(f"POST Group body: {body}")
if body:
if "link" in body:
link_body = body["link"]
log.debug(f"link_body: {link_body}")
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username,
"create")
if not link_id or not link_title:
log.warn(f"POST Group body with no link: {body}")
domain_json = await getDomainJson(
app, domain) # get again in case cache was invalidated
root_id = domain_json["root"]
group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
log.debug("create group, body: " + json.dumps(group_json))
req = getDataNodeUrl(app, group_id) + "/groups"
params = {}
if bucket:
params["bucket"] = bucket
group_json = await http_post(app, req, data=group_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = group_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_json_rsp = await http_put(app,
link_req,
data=link_json,
params=params)
log.debug(f"PUT Link resp: {put_json_rsp}")
log.debug("returning resp")
# group creation successful
resp = await jsonResponse(request, group_json, status=201)
log.response(request, resp=resp)
return resp
async def delete_metadata_obj(app, obj_id, notify=True, root_id=None):
""" Delete the given object """
meta_cache = app['meta_cache']
dirty_ids = app["dirty_ids"]
log.info("delete_meta_data_obj: {} notify: {}".format(obj_id, notify))
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error(f"obj: {obj_id} not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
log.warn("{} has already been deleted".format(obj_id))
else:
deleted_ids.add(obj_id)
if obj_id in meta_cache:
log.debug(f"removing {obj_id} from meta_cache")
del meta_cache[obj_id]
if obj_id in dirty_ids:
del dirty_ids[obj_id]
# remove from S3 (if present)
s3key = getS3Key(obj_id)
if await isS3Obj(app, s3key):
await deleteS3Obj(app, s3key)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
if notify:
an_url = getAsyncNodeUrl(app)
if obj_id.startswith("/"):
# domain delete
req = an_url + "/domain"
params = {"domain": obj_id}
try:
log.info("ASync DELETE notify: {} params: {}".format(
req, params))
await http_delete(app, req, params=params)
except ClientError as ce:
log.error(f"got error notifying async node: {ce}")
except HTTPInternalServerError as hse:
log.error(f"got HTTPInternalServerError: {hse}")
else:
req = an_url + "/object/" + obj_id
try:
log.info(f"ASync DELETE notify: {req}")
await http_delete(app, req)
except ClientError as ce:
log.error(f"got ClientError notifying async node: {ce}")
except HTTPInternalServerError as ise:
log.error(
f"got HTTPInternalServerError notifying async node: {ise}")
log.debug(f"delete_metadata_obj for {obj_id} done")
async def GET_Datatype(request):
"""HTTP method to return JSON for committed datatype"""
log.request(request)
app = request.app
params = request.rel_url.query
include_attrs = False
h5path = None
getAlias = False
ctype_id = request.match_info.get('id')
if not ctype_id and "h5path" not in params:
msg = "Missing type id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
if ctype_id:
if not isValidUuid(ctype_id, "Type"):
msg = f"Invalid type id: {ctype_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
else:
group_id = None
if "grpid" in params:
group_id = params["grpid"]
if not isValidUuid(group_id, "Group"):
msg = f"Invalid parent group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "h5path" not in params:
msg = "Expecting either ctype id or h5path url param"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
h5path = params["h5path"]
if h5path[0] != '/' and group_id is None:
msg = "h5paths must be absolute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Datatype, h5path: {h5path}")
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
if h5path:
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if group_id is None:
group_id = domain_json["root"]
ctype_id = await getObjectIdByPath(app,
group_id,
h5path,
bucket=bucket
) # throws 404 if not found
if not isValidUuid(ctype_id, "Datatype"):
msg = f"No datatype exist with the path: {h5path}"
log.warn(msg)
raise HTTPGone()
log.info(f"got ctype_id: {ctype_id} from h5path: {h5path}")
await validateAction(app, domain, ctype_id, username, "read")
# get authoritative state for ctype from DN (even if it's in the meta_cache).
type_json = await getObjectJson(app,
ctype_id,
bucket=bucket,
refresh=True,
include_attrs=include_attrs)
type_json["domain"] = domain
if getAlias:
root_id = type_json["root"]
alias = []
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=ctype_id,
bucket=bucket)
if h5path:
alias.append(h5path)
type_json["alias"] = alias
hrefs = []
ctype_uri = '/datatypes/' + ctype_id
hrefs.append({'rel': 'self', 'href': getHref(request, ctype_uri)})
root_uri = '/groups/' + type_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, ctype_uri + '/attributes')
})
type_json["hrefs"] = hrefs
resp = await jsonResponse(request, type_json)
log.response(request, resp=resp)
return resp
async def write_s3_obj(app, obj_id, bucket=None):
""" writes the given object to s3 """
s3key = getS3Key(obj_id)
log.info(
f"write_s3_obj for obj_id: {obj_id} / s3_key: {s3key} bucket: {bucket}"
)
pending_s3_write = app["pending_s3_write"]
pending_s3_write_tasks = app["pending_s3_write_tasks"]
dirty_ids = app["dirty_ids"]
chunk_cache = app['chunk_cache']
meta_cache = app['meta_cache']
deflate_map = app['deflate_map']
shuffle_map = app['shuffle_map']
notify_objs = app["root_notify_ids"]
deleted_ids = app['deleted_ids']
success = False
if isValidDomain(obj_id):
domain_bucket = getBucketForDomain(obj_id)
if bucket and bucket != domain_bucket:
log.error(
f"expected bucket for domain: {obj_id} to match what wsas passed to write_s3_obj"
)
else:
bucket = domain_bucket
if s3key in pending_s3_write:
msg = f"write_s3_key - not expected for key {s3key} to be in pending_s3_write map"
log.error(msg)
raise KeyError(msg)
if obj_id not in pending_s3_write_tasks:
# don't allow reentrant write
log.debug(f"write_s3_obj for {obj_id} not s3sync task")
if obj_id in deleted_ids and isValidUuid(obj_id):
# if this objid has been deleted (and its unique since this is not a domain id)
# cancel any pending task and return
log.warn(f"Canceling write for {obj_id} since it has been deleted")
if obj_id in pending_s3_write_tasks:
log.info(f"removing pending s3 write task for {obj_id}")
task = pending_s3_write_tasks[obj_id]
task.cancel()
del pending_s3_write_tasks[obj_id]
return None
now = time.time()
last_update_time = now
if obj_id in dirty_ids:
last_update_time = dirty_ids[obj_id][
0] # timestamp is first element of two-tuple
if last_update_time > now:
msg = f"last_update time {last_update_time} is in the future for obj_id: {obj_id}"
log.error(msg)
raise ValueError(msg)
pending_s3_write[s3key] = now
# do the following in the try block so we can always remove the pending_s3_write at the end
try:
if isValidChunkId(obj_id):
if obj_id not in chunk_cache:
log.error(f"expected to find obj_id: {obj_id} in chunk cache")
raise KeyError(f"{obj_id} not found in chunk cache")
if not chunk_cache.isDirty(obj_id):
log.error(f"expected chunk cache obj {obj_id} to be dirty")
raise ValueError("bad dirty state for obj")
chunk_arr = chunk_cache[obj_id]
chunk_bytes = arrayToBytes(chunk_arr)
dset_id = getDatasetId(obj_id)
deflate_level = None
shuffle = 0
if dset_id in shuffle_map:
shuffle = shuffle_map[dset_id]
if dset_id in deflate_map:
deflate_level = deflate_map[dset_id]
log.debug(
f"got deflate_level: {deflate_level} for dset: {dset_id}")
if dset_id in shuffle_map:
shuffle = shuffle_map[dset_id]
log.debug(f"got shuffle size: {shuffle} for dset: {dset_id}")
await putS3Bytes(app,
s3key,
chunk_bytes,
shuffle=shuffle,
deflate_level=deflate_level,
bucket=bucket)
success = True
# if chunk has been evicted from cache something has gone wrong
if obj_id not in chunk_cache:
msg = f"expected to find {obj_id} in chunk_cache"
log.error(msg)
elif obj_id in dirty_ids and dirty_ids[obj_id][
0] > last_update_time:
log.info(
f"write_s3_obj {obj_id} got updated while s3 write was in progress"
)
else:
# no new write, can clear dirty
chunk_cache.clearDirty(obj_id) # allow eviction from cache
log.debug(
"putS3Bytes Chunk cache utilization: {} per, dirty_count: {}"
.format(chunk_cache.cacheUtilizationPercent,
chunk_cache.dirtyCount))
else:
# meta data update
# check for object in meta cache
if obj_id not in meta_cache:
log.error(f"expected to find obj_id: {obj_id} in meta cache")
raise KeyError(f"{obj_id} not found in meta cache")
if not meta_cache.isDirty(obj_id):
log.error(f"expected meta cache obj {obj_id} to be dirty")
raise ValueError("bad dirty state for obj")
obj_json = meta_cache[obj_id]
await putS3JSONObj(app, s3key, obj_json, bucket=bucket)
success = True
# should still be in meta_cache...
if obj_id in deleted_ids:
log.info(
f"obj {obj_id} has been deleted while write was in progress"
)
elif obj_id not in meta_cache:
msg = f"expected to find {obj_id} in meta_cache"
log.error(msg)
elif obj_id in dirty_ids and dirty_ids[obj_id][
0] > last_update_time:
log.info(
f"write_s3_obj {obj_id} got updated while s3 write was in progress"
)
else:
meta_cache.clearDirty(obj_id) # allow eviction from cache
finally:
# clear pending_s3_write item
log.debug(f"write_s3_obj finally block, success={success}")
if s3key not in pending_s3_write:
msg = f"write s3 obj: Expected to find {s3key} in pending_s3_write map"
log.error(msg)
else:
if pending_s3_write[s3key] != now:
msg = f"pending_s3_write timestamp got updated unexpectedly for {s3key}"
log.error(msg)
del pending_s3_write[s3key]
# clear task
if obj_id not in pending_s3_write_tasks:
log.debug(f"no pending s3 write task for {obj_id}")
else:
log.debug(f"removing pending s3 write task for {obj_id}")
del pending_s3_write_tasks[obj_id]
# clear dirty flag
if obj_id in dirty_ids and dirty_ids[obj_id][0] == last_update_time:
log.debug(f"clearing dirty flag for {obj_id}")
del dirty_ids[obj_id]
# add to map so that root can be notified about changed objects
if isValidUuid(obj_id) and isSchema2Id(obj_id):
root_id = getRootObjId(obj_id)
notify_objs[root_id] = bucket
# calculate time to do the write
elapsed_time = time.time() - now
log.info(f"s3 write for {s3key} took {elapsed_time:.3f}s")
return obj_id
async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
log.info(f"PUT domain: {domain}, bucket: {bucket}")
body = None
if request.has_body:
body = await request.json()
log.debug(f"PUT domain with body: {body}")
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug(f"got domain_json: {domain_json}")
if domain_json is None:
log.warn(f"domain: {domain} not found")
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"], bucket=bucket)
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
linked_bucket = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
if not isValidDomain(linked_domain):
msg = f"linked_domain: {linked_domain} is not valid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "linked_bucket" in body:
linked_bucket = body["linked_bucket"]
elif bucket:
linked_bucket = bucket
elif "bucket_name" in request.app and request.app["bucket_name"]:
linked_bucket = request.app["bucket_name"]
else:
linked_bucket = None
if not linked_bucket:
msg = "Could not determine bucket for linked domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug(f"Parent domain: [{parent_domain}]")
if not parent_domain or getPathForDomain(parent_domain) == '/':
is_toplevel = True
else:
is_toplevel = False
if is_toplevel and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if is_toplevel and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if not is_toplevel:
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = f"Parent domain: {parent_domain} not found"
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = f"Parent domain: {parent_domain} removed"
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug(f"parent_json {parent_domain}: {parent_json}")
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app,
linked_bucket + linked_domain,
reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug(f"new root group id: {root_id}")
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
params = {}
bucket = getBucketForDomain(domain)
if bucket:
params["bucket"] = bucket
try:
group_json = await http_post(app,
req,
data=group_json,
params=params)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug(f"adding default perm for domain: {domain}")
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug(f"creating domain: {domain} with body: {body}")
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp