async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info("PUT domain: {}, username: {}".format(domain, username))
body = None
if request.has_body:
body = await request.json()
log.debug("PUT domain with body: {}".format(body))
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug("got domain_json: {}".format(domain_json))
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"])
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
log.info(f"linking to domain: {linked_domain}")
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug("Parent domain: [{}]".format(parent_domain))
if (not parent_domain or parent_domain == '/') and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if parent_domain and parent_domain != '/':
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = "Parent domain: {} not found".format(parent_domain)
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = "Parent domain: {} removed".format(parent_domain)
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug("parent_json {}: {}".format(parent_domain, parent_json))
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app, linked_domain, reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug("new root group id: {}".format(root_id))
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
try:
group_json = await http_post(app, req, data=group_json)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug("adding default perm for domain: {}".format(domain))
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug("creating domain: {} with body: {}".format(domain, body))
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp
async def POST_Dataset(request):
"""HTTP method to create a new dataset object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "POST Datasets with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
# get domain, check authorization
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain_json = await getDomainJson(app, domain, reload=True)
root_id = domain_json["root"]
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = "Expected root key for domain: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
#
# validate type input
#
if "type" not in body:
msg = "POST Dataset has no type key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str) and datatype.startswith("t-"):
# Committed type - fetch type json from DN
ctype_id = datatype
log.debug("got ctypeid: {}".format(ctype_id))
ctype_json = await getObjectJson(app, ctype_id)
log.debug("ctype: {}".format(ctype_json))
if ctype_json["root"] != root_id:
msg = "Referenced committed datatype must belong in same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = ctype_json["type"]
# add the ctype_id to type type
datatype["id"] = ctype_id
elif isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug("got datatype: {}".format(datatype))
except TypeError:
msg = "POST Dataset with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
item_size = getItemSize(datatype)
#
# Validate shape input
#
dims = None
shape_json = {}
if "shape" not in body:
shape_json["class"] = "H5S_SCALAR"
else:
shape = body["shape"]
if isinstance(shape, int):
shape_json["class"] = "H5S_SIMPLE"
dims = [
shape,
]
shape_json["dims"] = dims
elif isinstance(shape, str):
# only valid string value is H5S_NULL
if shape != "H5S_NULL":
msg = "POST Datset with invalid shape value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["class"] = "H5S_NULL"
elif isinstance(shape, list):
if len(shape) == 0:
shape_json["class"] = "H5S_SCALAR"
else:
shape_json["class"] = "H5S_SIMPLE"
shape_json["dims"] = shape
dims = shape
else:
msg = "Bad Request: shape is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if dims is not None:
for i in range(len(dims)):
extent = dims[i]
if not isinstance(extent, int):
msg = "Invalid shape type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "shape dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = None
if "maxdims" in body:
if dims is None:
msg = "Maxdims cannot be supplied if space is NULL"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = body["maxdims"]
if isinstance(maxdims, int):
dim1 = maxdims
maxdims = [dim1]
elif isinstance(maxdims, list):
pass # can use as is
else:
msg = "Bad Request: maxdims is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(dims) != len(maxdims):
msg = "Maxdims rank doesn't match Shape"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if maxdims is not None:
for extent in maxdims:
if not isinstance(extent, int):
msg = "Invalid maxdims type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "maxdims dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(maxdims) != len(dims):
msg = "Bad Request: maxdims array length must equal shape array length"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["maxdims"] = []
for i in range(len(dims)):
maxextent = maxdims[i]
if not isinstance(maxextent, int):
msg = "Bad Request: maxdims must be integer type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif maxextent == 0:
# unlimited dimension
shape_json["maxdims"].append(0)
elif maxextent < dims[i]:
msg = "Bad Request: maxdims extent can't be smaller than shape extent"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
shape_json["maxdims"].append(maxextent)
layout = None
min_chunk_size = int(config.get("min_chunk_size"))
max_chunk_size = int(config.get("max_chunk_size"))
if 'creationProperties' in body:
creationProperties = body["creationProperties"]
if 'layout' in creationProperties:
layout = creationProperties["layout"]
validateChunkLayout(shape_json, item_size, layout)
if layout is None and shape_json["class"] != "H5S_NULL":
# default to chunked layout
layout = {"class": "H5D_CHUNKED"}
if layout and layout["class"] == 'H5D_CONTIGUOUS_REF':
chunk_dims = getContiguousLayout(shape_json,
item_size,
chunk_min=min_chunk_size,
chunk_max=max_chunk_size)
layout["dims"] = chunk_dims
log.debug(f"autoContiguous layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED' and "dims" not in layout:
# do autochunking
chunk_dims = guessChunk(shape_json, item_size)
layout["dims"] = chunk_dims
log.debug(f"initial autochunk layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED':
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug("chunk_size: {}, min: {}, max: {}".format(
chunk_size, min_chunk_size, max_chunk_size))
# adjust the chunk shape if chunk size is too small or too big
adjusted_chunk_dims = None
if chunk_size < min_chunk_size:
log.debug(
"chunk size: {} less than min size: {}, expanding".format(
chunk_size, min_chunk_size))
adjusted_chunk_dims = expandChunk(chunk_dims,
item_size,
shape_json,
chunk_min=min_chunk_size,
layout_class=layout["class"])
elif chunk_size > max_chunk_size:
log.debug(
"chunk size: {} greater than max size: {}, expanding".format(
chunk_size, max_chunk_size, layout_class=layout["class"]))
adjusted_chunk_dims = shrinkChunk(chunk_dims,
item_size,
chunk_max=max_chunk_size)
if adjusted_chunk_dims:
log.debug(
f"requested chunk_dimensions: {chunk_dims} modified dimensions: {adjusted_chunk_dims}"
)
layout["dims"] = adjusted_chunk_dims
if layout and layout["class"] in ('H5D_CHUNKED_REF',
'H5D_CHUNKED_REF_INDIRECT'):
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug("chunk_size: {}, min: {}, max: {}".format(
chunk_size, min_chunk_size, max_chunk_size))
# adjust the chunk shape if chunk size is too small or too big
if chunk_size < min_chunk_size:
log.warn(
"chunk size: {} less than min size: {} for H5D_CHUNKED_REF dataset"
.format(chunk_size, min_chunk_size))
elif chunk_size > max_chunk_size:
log.warn(
"chunk size: {} greater than max size: {}, for H5D_CHUNKED_REF dataset"
.format(chunk_size,
max_chunk_size,
layout_class=layout["class"]))
link_id = None
link_title = None
if "link" in body:
link_body = body["link"]
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.info("link id: {}".format(link_id))
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username, "create")
dset_id = createObjId("datasets", rootid=root_id)
log.info("new dataset id: {}".format(dset_id))
dataset_json = {
"id": dset_id,
"root": root_id,
"type": datatype,
"shape": shape_json
}
if "creationProperties" in body:
# TBD - validate all creationProperties
creationProperties = body["creationProperties"]
if "fillValue" in creationProperties:
# validate fill value compatible with type
dt = createDataType(datatype)
fill_value = creationProperties["fillValue"]
if isinstance(fill_value, list):
fill_value = tuple(fill_value)
try:
np.asarray(fill_value, dtype=dt)
except (TypeError, ValueError):
msg = "Fill value {} not compatible with dataset type: {}".format(
fill_value, datatype)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dataset_json["creationProperties"] = creationProperties
if layout is not None:
dataset_json["layout"] = layout
log.debug("create dataset: " + json.dumps(dataset_json))
req = getDataNodeUrl(app, dset_id) + "/datasets"
post_json = await http_post(app, req, data=dataset_json)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = dset_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.info("PUT link - : " + link_req)
put_rsp = await http_put(app, link_req, data=link_json)
log.debug("PUT Link resp: {}".format(put_rsp))
# dataset creation successful
resp = await jsonResponse(request, post_json, status=201)
log.response(request, resp=resp)
return resp
async def POST_Group(request):
"""HTTP method to create new Group object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if request.has_body:
body = await request.json()
log.info(f"POST Group body: {body}")
if body:
if "link" in body:
link_body = body["link"]
log.debug(f"link_body: {link_body}")
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username,
"create")
if not link_id or not link_title:
log.warn(f"POST Group body with no link: {body}")
domain_json = await getDomainJson(
app, domain) # get again in case cache was invalidated
root_id = domain_json["root"]
group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
log.debug("create group, body: " + json.dumps(group_json))
req = getDataNodeUrl(app, group_id) + "/groups"
params = {}
if bucket:
params["bucket"] = bucket
group_json = await http_post(app, req, data=group_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = group_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_json_rsp = await http_put(app,
link_req,
data=link_json,
params=params)
log.debug(f"PUT Link resp: {put_json_rsp}")
log.debug("returning resp")
# group creation successful
resp = await jsonResponse(request, group_json, status=201)
log.response(request, resp=resp)
return resp
async def POST_Datatype(request):
"""HTTP method to create new committed datatype object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "POST Datatype with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
if "type" not in body:
msg = "POST Datatype has no type key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug(f"got datatype: {datatype}")
except TypeError:
msg = "POST Dataset with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if "link" in body:
link_body = body["link"]
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username, "create")
root_id = domain_json["root"]
ctype_id = createObjId("datatypes", rootid=root_id)
log.debug(f"new type id: {ctype_id}")
ctype_json = {"id": ctype_id, "root": root_id, "type": datatype}
log.debug("create named type, body: " + json.dumps(ctype_json))
req = getDataNodeUrl(app, ctype_id) + "/datatypes"
params = {}
if bucket:
params["bucket"] = bucket
type_json = await http_post(app, req, data=ctype_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = ctype_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_rsp = await http_put(app, link_req, data=link_json, params=params)
log.debug(f"PUT Link resp: {put_rsp}")
# datatype creation successful
resp = await jsonResponse(request, type_json, status=201)
log.response(request, resp=resp)
return resp