async def GET_Group(request):
"""HTTP method to return JSON for group"""
log.request(request)
app = request.app
params = request.rel_url.query
h5path = None
getAlias = False
include_links = False
include_attrs = False
group_id = request.match_info.get('id')
if not group_id and "h5path" not in params:
# no id, or path provided, so bad request
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if group_id:
log.info(f"GET_Group, id: {group_id}")
# is the id a group id and not something else?
if not isValidUuid(group_id, "Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
if "h5path" in params:
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute if no parent id is provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Group, h5path: {h5path}")
if "include_links" in params and params["include_links"]:
include_links = True
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
if h5path and h5path[0] == '/':
# ignore the request path id (if given) and start
# from root group for absolute paths
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
if h5path:
group_id = await getObjectIdByPath(app,
group_id,
h5path,
bucket=bucket
) # throws 404 if not found
if not isValidUuid(group_id, "Group"):
msg = f"No group exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get group_id: {group_id} from h5path: {h5path}")
# verify authorization to read the group
await validateAction(app, domain, group_id, username, "read")
# get authoritative state for group from DN (even if it's in the meta_cache).
group_json = await getObjectJson(app,
group_id,
refresh=True,
include_links=include_links,
include_attrs=include_attrs,
bucket=bucket)
log.debug(f"domain from request: {domain}")
group_json["domain"] = getPathForDomain(domain)
if bucket:
group_json["bucket"] = bucket
if getAlias:
root_id = group_json["root"]
alias = []
if group_id == root_id:
alias.append('/')
else:
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=group_id,
bucket=bucket)
if h5path:
alias.append(h5path)
group_json["alias"] = alias
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri)})
hrefs.append({
'rel': 'links',
'href': getHref(request, group_uri + '/links')
})
root_uri = '/groups/' + group_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, group_uri + '/attributes')
})
group_json["hrefs"] = hrefs
resp = await jsonResponse(request, group_json)
log.response(request, resp=resp)
return resp
async def DELETE_Domain(request):
"""HTTP method to delete a domain resource"""
log.request(request)
app = request.app
params = request.rel_url.query
meta_only = False # if True, just delete the meta cache value
keep_root = False
if request.has_body:
body = await request.json()
if "meta_only" in body:
meta_only = body["meta_only"]
if "keep_root" in body:
keep_root = body["keep_root"]
else:
if "meta_only" in params:
meta_only = params["meta_only"]
if "keep_root" in params:
keep_root = params["keep_root"]
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn(f"Invalid domain: {domain}")
raise HTTPBadRequest(reason="Invalid domain name")
bucket = getBucketForDomain(domain)
log.debug(f"GET_Domain domain: {domain}")
if not domain:
msg = "No domain given"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"meta_only domain delete: {meta_only}")
if meta_only:
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
log.info(f"deleting {domain} from domain_cache")
del domain_cache[domain]
resp = await jsonResponse(request, {})
return resp
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
parent_domain = getParentDomain(domain)
if not parent_domain or getPathForDomain(parent_domain) == '/':
is_toplevel = True
else:
is_toplevel = False
if is_toplevel and username != "admin":
msg = "Deletion of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
log.warn("domain not found")
raise HTTPNotFound()
elif ce.code == 410:
log.warn("domain has been removed")
raise HTTPGone()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
aclCheck(domain_json, "delete",
username) # throws exception if not allowed
# check for sub-objects if this is a folder
if "root" not in domain_json:
index = domain.find('/')
s3prefix = domain[(index + 1):] + '/'
log.info(f"checking s3key with prefix: {s3prefix} in bucket: {bucket}")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/',
bucket=bucket)
for s3key in s3keys:
if s3key.endswith("/"):
log.warn(f"attempt to delete folder {domain} with sub-items")
log.debug(f"got prefix: {s3keys[0]}")
raise HTTPConflict(reason="folder has sub-items")
req = getDataNodeUrl(app, domain)
req += "/domains"
params = {} # for http_delete requests to DN nodes
params["domain"] = domain
rsp_json = await http_delete(app, req, params=params)
if "root" in domain_json and not keep_root:
# delete the root group
root_id = domain_json["root"]
req = getDataNodeUrl(app, root_id)
req += "/groups/" + root_id
await http_delete(app, req, params=params)
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
del domain_cache[domain]
# delete domain cache from other sn_urls
sn_urls = app["sn_urls"]
params = {}
params["domain"] = getPathForDomain(domain)
params["bucket"] = getBucketForDomain(domain)
params[
"meta_only"] = 1 # can't pass booleans as params, so use 1 instead of True
for node_no in sn_urls:
if node_no == app["node_number"]:
continue # don't send to ourselves
sn_url = sn_urls[node_no]
req = sn_url + "/"
log.info(f"sending sn request: {req}")
try:
sn_rsp = await http_delete(app, req, params=params)
log.info(f"{req} response: {sn_rsp}")
except ClientResponseError as ce:
log.warn(f"got error for sn_delete: {ce}")
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_Domain(request):
"""HTTP method to return JSON for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn(f"Invalid domain: {domain}")
raise HTTPBadRequest(reason="Invalid domain name")
bucket = getBucketForDomain(domain)
log.debug(f"GET_Domain domain: {domain} bucket: {bucket}")
if not bucket and not config.get("bucket_name"):
# no bucket defined, raise 400
msg = "Bucket not provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if not domain:
log.info("no domain passed in, returning all top-level domains")
# no domain passed in, return top-level domains for this request
domains = await get_domains(request)
rsp_json = {"domains": domains}
rsp_json["hrefs"] = []
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
log.info(f"got domain: {domain}")
domain_json = await getDomainJson(app, domain, reload=True)
if domain_json is None:
log.warn(f"domain: {domain} not found")
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug(f"got domain_json: {domain_json}")
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
if "h5path" in params:
# if h5path is passed in, return object info for that path
# (if exists)
h5path = params["h5path"]
root_id = domain_json["root"]
obj_id = await getObjectIdByPath(app, root_id, h5path, bucket=bucket
) # throws 404 if not found
log.info(f"get obj_id: {obj_id} from h5path: {h5path}")
# get authoritative state for object from DN (even if it's in the meta_cache).
obj_json = await getObjectJson(app,
obj_id,
refresh=True,
bucket=bucket)
obj_json["domain"] = domain
# Not bothering with hrefs for h5path lookups...
resp = await jsonResponse(request, obj_json)
log.response(request, resp=resp)
return resp
# return just the keys as per the REST API
rsp_json = await get_domain_response(app,
domain_json,
bucket=bucket,
verbose=verbose)
# include domain objects if requested
if "getobjs" in params and params["getobjs"] and "root" in domain_json:
root_id = domain_json["root"]
include_attrs = False
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
domain_objs = await getDomainObjects(app,
root_id,
include_attrs=include_attrs,
bucket=bucket)
rsp_json["domain_objs"] = domain_objs
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'database',
'href': getHref(request, '/datasets')
})
hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')})
hrefs.append({
'rel': 'typebase',
'href': getHref(request, '/datatypes')
})
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')})
parent_domain = getParentDomain(domain)
if not parent_domain or getPathForDomain(parent_domain) == '/':
is_toplevel = True
else:
is_toplevel = False
log.debug(f"href parent domain: {parent_domain}")
if not is_toplevel:
hrefs.append({
'rel': 'parent',
'href': getHref(request, '/', domain=parent_domain)
})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
log.info(f"PUT domain: {domain}, bucket: {bucket}")
body = None
if request.has_body:
body = await request.json()
log.debug(f"PUT domain with body: {body}")
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug(f"got domain_json: {domain_json}")
if domain_json is None:
log.warn(f"domain: {domain} not found")
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"], bucket=bucket)
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
linked_bucket = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
if not isValidDomain(linked_domain):
msg = f"linked_domain: {linked_domain} is not valid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "linked_bucket" in body:
linked_bucket = body["linked_bucket"]
elif bucket:
linked_bucket = bucket
elif "bucket_name" in request.app and request.app["bucket_name"]:
linked_bucket = request.app["bucket_name"]
else:
linked_bucket = None
if not linked_bucket:
msg = "Could not determine bucket for linked domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug(f"Parent domain: [{parent_domain}]")
if not parent_domain or getPathForDomain(parent_domain) == '/':
is_toplevel = True
else:
is_toplevel = False
if is_toplevel and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if is_toplevel and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if not is_toplevel:
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = f"Parent domain: {parent_domain} not found"
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = f"Parent domain: {parent_domain} removed"
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug(f"parent_json {parent_domain}: {parent_json}")
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app,
linked_bucket + linked_domain,
reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug(f"new root group id: {root_id}")
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
params = {}
bucket = getBucketForDomain(domain)
if bucket:
params["bucket"] = bucket
try:
group_json = await http_post(app,
req,
data=group_json,
params=params)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug(f"adding default perm for domain: {domain}")
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug(f"creating domain: {domain} with body: {body}")
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp
async def GET_Dataset(request):
"""HTTP method to return JSON description of a dataset"""
log.request(request)
app = request.app
params = request.rel_url.query
include_attrs = False
h5path = None
getAlias = False
dset_id = request.match_info.get('id')
if not dset_id and "h5path" not in params:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
if dset_id:
if not isValidUuid(dset_id, "Dataset"):
msg = f"Invalid dataset id: {dset_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
else:
group_id = None
if "grpid" in params:
group_id = params["grpid"]
if not isValidUuid(group_id, "Group"):
msg = f"Invalid parent group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "h5path" not in params:
msg = "Expecting either ctype id or h5path url param"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Dataset, h5path: {h5path}")
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if h5path:
if group_id is None:
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
dset_id = await getObjectIdByPath(app, group_id, h5path, bucket=bucket
) # throws 404 if not found
if not isValidUuid(dset_id, "Dataset"):
msg = f"No dataset exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get dataset_id: {dset_id} from h5path: {h5path}")
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app,
dset_id,
refresh=True,
include_attrs=include_attrs,
bucket=bucket)
# check that we have permissions to read the object
await validateAction(app, domain, dset_id, username, "read")
log.debug(f"got dset_json: {dset_json}")
resp_json = {}
resp_json["id"] = dset_json["id"]
resp_json["root"] = dset_json["root"]
resp_json["shape"] = dset_json["shape"]
resp_json["type"] = dset_json["type"]
if "creationProperties" in dset_json:
resp_json["creationProperties"] = dset_json["creationProperties"]
else:
resp_json["creationProperties"] = {}
if "layout" in dset_json:
resp_json["layout"] = dset_json["layout"]
resp_json["attributeCount"] = dset_json["attributeCount"]
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["domain"] = getPathForDomain(domain)
if getAlias:
root_id = dset_json["root"]
alias = []
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=dset_id,
bucket=bucket)
if h5path:
alias.append(h5path)
resp_json["alias"] = alias
if include_attrs:
resp_json["attributes"] = dset_json["attributes"]
hrefs = []
dset_uri = '/datasets/' + dset_id
hrefs.append({'rel': 'self', 'href': getHref(request, dset_uri)})
root_uri = '/groups/' + dset_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, dset_uri + '/attributes')
})
# provide a value link if the dataset is relatively small,
# otherwise create a preview link that shows a limited number of data values
dset_shape = dset_json["shape"]
if dset_shape["class"] != 'H5S_NULL':
count = 1
if dset_shape["class"] == 'H5S_SIMPLE':
dims = dset_shape["dims"]
count = getNumElements(dims)
if count <= 100:
# small number of values, provide link to entire dataset
hrefs.append({
'rel': 'data',
'href': getHref(request, dset_uri + '/value')
})
else:
# large number of values, create preview link
previewQuery = getPreviewQuery(dset_shape["dims"])
hrefs.append({
'rel':
'preview',
'href':
getHref(request, dset_uri + '/value', query=previewQuery)
})
resp_json["hrefs"] = hrefs
if verbose:
# get allocated size and num_chunks for the dataset if available
dset_detail = await getDatasetDetails(app,
dset_id,
dset_json["root"],
bucket=bucket)
if dset_detail is not None:
if "num_chunks" in dset_detail:
resp_json["num_chunks"] = dset_detail["num_chunks"]
if "allocated_bytes" in dset_detail:
resp_json["allocated_size"] = dset_detail["allocated_bytes"]
if "lastModified" in dset_detail:
resp_json["lastModified"] = dset_detail["lastModified"]
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp