async def validateAction(app, domain, obj_id, username, action): """ check that the given object belongs in the domain and that the requested action (create, read, update, delete, readACL, udpateACL) is permitted for the requesting user. """ meta_cache = app['meta_cache'] log.info( f"validateAction(domain={domain}, obj_id={obj_id}, username={username}, action={action})" ) # get domain JSON domain_json = await getDomainJson(app, domain) if "root" not in domain_json: msg = f"Expected root key for domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) obj_json = None if obj_id in meta_cache: obj_json = meta_cache[obj_id] else: # fetch from DN collection = getCollectionForId(obj_id) req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id bucket = getBucketForDomain(domain) params = {} if bucket: params["bucket"] = bucket obj_json = await http_get(app, req, params=params) meta_cache[obj_id] = obj_json log.debug("obj_json[root]: {} domain_json[root]: {}".format( obj_json["root"], domain_json["root"])) if obj_json["root"] != domain_json["root"]: log.info("unexpected root, reloading domain") domain_json = await getDomainJson(app, domain, reload=True) if "root" not in domain_json or obj_json["root"] != domain_json["root"]: msg = "Object id is not a member of the given domain" log.warn(msg) raise HTTPBadRequest(reason=msg) if action not in ("create", "read", "update", "delete", "readACL", "updateACL"): log.error(f"unexpected action: {action}") raise HTTPInternalServerError() reload = False try: aclCheck(domain_json, action, username) # throws exception if not allowed except HTTPForbidden: log.info( f"got HttpProcessing error on validate action for domain: {domain}, reloading..." ) # just in case the ACL was recently updated, refetch the domain reload = True if reload: domain_json = await getDomainJson(app, domain, reload=True) aclCheck(domain_json, action, username)
async def GET_ACLs(request): """HTTP method to return JSON for domain/ACLs""" log.request(request) app = request.app (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(message=msg) # use reload to get authoritative domain json try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError: log.warn("domain not found") log.warn(msg) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() acls = domain_json["acls"] log.debug("got domain_json: {}".format(domain_json)) # validate that the requesting user has permission to read this domain aclCheck(domain_json, "readACL", username) # throws exception if not authorized acl_list = [] acl_usernames = list(acls.keys()) acl_usernames.sort() for acl_username in acl_usernames: entry = {"userName": acl_username} acl = acls[acl_username] for k in acl.keys(): entry[k] = acl[k] acl_list.append(entry) # return just the keys as per the REST API rsp_json = {} rsp_json["acls"] = acl_list hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')}) if "root" in domain_json: hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + domain_json["root"]) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, '/')}) rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def GET_ACL(request): """HTTP method to return JSON for given domain/ACL""" log.request(request) app = request.app acl_username = request.match_info.get('username') if not acl_username: msg = "Missing username for ACL" log.warn(msg) raise HTTPBadRequest(reason=msg) (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) # use reload to get authoritative domain json try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code in (404, 410): msg = "domain not found" log.warn(msg) raise HTTPNotFound() else: log.error(f"unexpected error: {ce.code}") raise HTTPInternalServerError() # validate that the requesting user has permission to read ACLs in this domain if acl_username in (username, "default"): # allow read access for a users on ACL, or default aclCheck(domain_json, "read", username) # throws exception if not authorized else: aclCheck(domain_json, "readACL", username) # throws exception if not authorized if 'owner' not in domain_json: log.warn("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.warn("No acls key found in domain") raise HTTPInternalServerError() acls = domain_json["acls"] log.debug("got domain_json: {}".format(domain_json)) if acl_username not in acls: msg = "acl for username: [{}] not found".format(acl_username) log.warn(msg) raise HTTPNotFound() acl = acls[acl_username] acl_rsp = {} for k in acl.keys(): acl_rsp[k] = acl[k] acl_rsp["userName"] = acl_username # return just the keys as per the REST API rsp_json = {} rsp_json["acl"] = acl_rsp hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')}) if "root" in domain_json: hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + domain_json["root"]) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, '/')}) rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def DELETE_Domain(request): """HTTP method to delete a domain resource""" log.request(request) app = request.app params = request.rel_url.query domain = None meta_only = False # if True, just delete the meta cache value keep_root = False if request.has_body: body = await request.json() if "domain" in body: domain = body["domain"] else: msg = "No domain in request body" log.warn(msg) raise HTTPBadRequest(reason=msg) if "meta_only" in body: meta_only = body["meta_only"] if "keep_root" in body: keep_root = body["keep_root"] else: # get domain from request uri try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) if "keep_root" in params: keep_root = params["keep_root"] log.info("meta_only domain delete: {}".format(meta_only)) if meta_only: # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: log.info("deleting {} from domain_cache".format(domain)) del domain_cache[domain] resp = await jsonResponse(request, {}) return resp username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) parent_domain = getParentDomain(domain) if (not parent_domain or parent_domain == '/') and username != "admin": msg = "Deletion of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code == 404: log.warn("domain not found") raise HTTPNotFound() elif ce.code == 410: log.warn("domain has been removed") raise HTTPGone() else: log.error(f"unexpected error: {ce.code}") raise HTTPInternalServerError() aclCheck(domain_json, "delete", username) # throws exception if not allowed # check for sub-objects if this is a folder if "root" not in domain_json: s3prefix = domain[1:] + '/' log.info(f"checking kets with prefix: {s3prefix} ") s3keys = await getS3Keys(app, include_stats=False, prefix=s3prefix, deliminator='/') for s3key in s3keys: if s3key.endswith("/"): log.warn(f"attempt to delete folder {domain} with sub-items") log.debug(f"got prefix: {s3keys[0]}") raise HTTPConflict(reason="folder has sub-items") req = getDataNodeUrl(app, domain) req += "/domains" body = {"domain": domain} rsp_json = await http_delete(app, req, data=body) if "root" in domain_json and not keep_root: # delete the root group root_id = domain_json["root"] req = getDataNodeUrl(app, root_id) req += "/groups/" + root_id await http_delete(app, req) # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: del domain_cache[domain] # delete domain cache from other sn_urls sn_urls = app["sn_urls"] body["meta_only"] = True for node_no in sn_urls: if node_no == app["node_number"]: continue # don't send to ourselves sn_url = sn_urls[node_no] req = sn_url + "/" log.info("sending sn request: {}".format(req)) try: sn_rsp = await http_delete(app, req, data=body) log.info("{} response: {}".format(req, sn_rsp)) except ClientResponseError as ce: log.warn("got error for sn_delete: {}".format(ce)) resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def PUT_Domain(request): """HTTP method to create a new domain""" log.request(request) app = request.app params = request.rel_url.query # verify username, password username, pswd = getUserPasswordFromRequest( request) # throws exception if user/password is not valid await validateUserPassword(app, username, pswd) # inital perms for owner and default owner_perm = { 'create': True, 'read': True, 'update': True, 'delete': True, 'readACL': True, 'updateACL': True } default_perm = { 'create': False, 'read': True, 'update': False, 'delete': False, 'readACL': False, 'updateACL': False } try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info("PUT domain: {}, username: {}".format(domain, username)) body = None if request.has_body: body = await request.json() log.debug("PUT domain with body: {}".format(body)) if ("flush" in params and params["flush"]) or (body and "flush" in body and body["flush"]): # flush domain - update existing domain rather than create a new resource domain_json = await getDomainJson(app, domain, reload=True) log.debug("got domain_json: {}".format(domain_json)) if domain_json is None: log.warn("domain: {} not found".format(domain)) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() aclCheck(domain_json, "update", username) # throws exception if not allowed if "root" in domain_json: # nothing to do for folder objects await doFlush(app, domain_json["root"]) # flush successful resp = await jsonResponse(request, None, status=204) log.response(request, resp=resp) return resp is_folder = False owner = username linked_domain = None root_id = None if body and "folder" in body: if body["folder"]: is_folder = True if body and "owner" in body: owner = body["owner"] if body and "linked_domain" in body: if is_folder: msg = "Folder domains can not be used for links" log.warn(msg) raise HTTPBadRequest(reason=msg) linked_domain = body["linked_domain"] log.info(f"linking to domain: {linked_domain}") if owner != username and username != "admin": log.warn("Only admin users are allowed to set owner for new domains") raise HTTPForbidden() parent_domain = getParentDomain(domain) log.debug("Parent domain: [{}]".format(parent_domain)) if (not parent_domain or parent_domain == '/') and not is_folder: msg = "Only folder domains can be created at the top-level" log.warn(msg) raise HTTPBadRequest(reason=msg) if (not parent_domain or parent_domain == '/') and username != "admin": msg = "creation of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() parent_json = None if parent_domain and parent_domain != '/': try: parent_json = await getDomainJson(app, parent_domain, reload=True) except ClientResponseError as ce: if ce.code == 404: msg = "Parent domain: {} not found".format(parent_domain) log.warn(msg) raise HTTPNotFound() elif ce.code == 410: msg = "Parent domain: {} removed".format(parent_domain) log.warn(msg) raise HTTPGone() else: log.error(f"Unexpected error: {ce.code}") raise HTTPInternalServerError() log.debug("parent_json {}: {}".format(parent_domain, parent_json)) if "root" in parent_json and parent_json["root"]: msg = "Parent domain must be a folder" log.warn(msg) raise HTTPBadRequest(reason=msg) if parent_json: aclCheck(parent_json, "create", username) # throws exception if not allowed if linked_domain: linked_json = await getDomainJson(app, linked_domain, reload=True) log.debug(f"got linked json: {linked_json}") if "root" not in linked_json: msg = "Folder domains cannot ber used as link target" log.warn(msg) raise HTTPBadRequest(reason=msg) root_id = linked_json["root"] aclCheck(linked_json, "read", username) aclCheck(linked_json, "delete", username) else: linked_json = None if not is_folder and not linked_json: # create a root group for the new domain root_id = createObjId("roots") log.debug("new root group id: {}".format(root_id)) group_json = {"id": root_id, "root": root_id, "domain": domain} log.debug("create group for domain, body: " + json.dumps(group_json)) # create root group req = getDataNodeUrl(app, root_id) + "/groups" try: group_json = await http_post(app, req, data=group_json) except ClientResponseError as ce: msg = "Error creating root group for domain -- " + str(ce) log.error(msg) raise HTTPInternalServerError() else: log.debug("no root group, creating folder") domain_json = {} domain_acls = {} # owner gets full control domain_acls[owner] = owner_perm if config.get("default_public") or is_folder: # this will make the domain public readable log.debug("adding default perm for domain: {}".format(domain)) domain_acls["default"] = default_perm # construct dn request to create new domain req = getDataNodeUrl(app, domain) req += "/domains" body = {"owner": owner, "domain": domain} body["acls"] = domain_acls if root_id: body["root"] = root_id log.debug("creating domain: {} with body: {}".format(domain, body)) try: domain_json = await http_put(app, req, data=body) except ClientResponseError as ce: msg = "Error creating domain state -- " + str(ce) log.error(msg) raise HTTPInternalServerError() # domain creation successful # maxin limits domain_json["limits"] = getLimits() domain_json["version"] = getVersion() resp = await jsonResponse(request, domain_json, status=201) log.response(request, resp=resp) return resp
async def GET_Domain(request): """HTTP method to return JSON for given domain""" log.request(request) app = request.app params = request.rel_url.query (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = None try: domain = getDomainFromRequest(request) except ValueError: log.warn("Invalid domain") raise HTTPBadRequest(reason="Invalid domain name") verbose = False if "verbose" in params and params["verbose"]: verbose = True if not domain: log.info("no domain passed in, returning all top-level domains") # no domain passed in, return top-level domains for this request domains = await get_domains(request) rsp_json = {"domains": domains} rsp_json["hrefs"] = [] resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp log.info("got domain: {}".format(domain)) domain_json = await getDomainJson(app, domain, reload=True) if domain_json is None: log.warn("domain: {} not found".format(domain)) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() log.debug("got domain_json: {}".format(domain_json)) # validate that the requesting user has permission to read this domain aclCheck(domain_json, "read", username) # throws exception if not authorized if "h5path" in params: # if h5path is passed in, return object info for that path # (if exists) h5path = params["h5path"] root_id = domain_json["root"] obj_id = await getObjectIdByPath(app, root_id, h5path) # throws 404 if not found log.info("get obj_id: {} from h5path: {}".format(obj_id, h5path)) # get authoritative state for object from DN (even if it's in the meta_cache). obj_json = await getObjectJson(app, obj_id, refresh=True) obj_json["domain"] = domain # Not bothering with hrefs for h5path lookups... resp = await jsonResponse(request, obj_json) log.response(request, resp=resp) return resp # return just the keys as per the REST API rsp_json = await get_domain_response(app, domain_json, verbose=verbose) # include domain objects if requested if "getobjs" in params and params["getobjs"] and "root" in domain_json: root_id = domain_json["root"] include_attrs = False if "include_attrs" in params and params["include_attrs"]: include_attrs = True domain_objs = await getDomainObjects(app, root_id, include_attrs=include_attrs) rsp_json["domain_objs"] = domain_objs hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/')}) if "root" in domain_json: root_uuid = domain_json["root"] hrefs.append({ 'rel': 'database', 'href': getHref(request, '/datasets') }) hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')}) hrefs.append({ 'rel': 'typebase', 'href': getHref(request, '/datatypes') }) hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + root_uuid) }) hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')}) parent_domain = getParentDomain(domain) log.debug("href parent domain: {}".format(parent_domain)) if parent_domain: hrefs.append({ 'rel': 'parent', 'href': getHref(request, '/', domain=parent_domain) }) rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def GET_Datatypes(request): """HTTP method to return datatype collection for given domain""" log.request(request) app = request.app params = request.rel_url.query (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) # use reload to get authoritative domain json try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code in (404, 410): msg = "domain not found" log.warn(msg) raise HTTPNotFound() else: log.error(f"Unexpected Error: {ce.code})") raise HTTPInternalServerError() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() log.debug("got domain_json: {}".format(domain_json)) # validate that the requesting user has permission to read this domain aclCheck(domain_json, "read", username) # throws exception if not authorized limit = None if "Limit" in params: try: limit = int(params["Limit"]) except ValueError: msg = "Bad Request: Expected int type for limit" log.warn(msg) raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] # get the datatype collection list obj_ids = [] if "root" in domain_json or domain_json["root"]: # get the groups collection list collections = await get_collections(app, domain_json["root"]) objs = collections["datatypes"] obj_ids = getIdList(objs, marker=marker, limit=limit) # create hrefs hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/datatypes')}) if "root" in domain_json: root_uuid = domain_json["root"] hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + root_uuid) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) # return obj ids and hrefs rsp_json = {} rsp_json["datatypes"] = obj_ids rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def POST_Group(request): """HTTP method to create new Group object""" log.request(request) app = request.app username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) domain_json = await getDomainJson(app, domain, reload=True) aclCheck(domain_json, "create", username) # throws exception if not allowed if "root" not in domain_json: msg = f"Expected root key for domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) link_id = None link_title = None if request.has_body: body = await request.json() log.info(f"POST Group body: {body}") if body: if "link" in body: link_body = body["link"] log.debug(f"link_body: {link_body}") if "id" in link_body: link_id = link_body["id"] if "name" in link_body: link_title = link_body["name"] if link_id and link_title: log.debug(f"link id: {link_id}") # verify that the referenced id exists and is in this domain # and that the requestor has permissions to create a link await validateAction(app, domain, link_id, username, "create") if not link_id or not link_title: log.warn(f"POST Group body with no link: {body}") domain_json = await getDomainJson( app, domain) # get again in case cache was invalidated root_id = domain_json["root"] group_id = createObjId("groups", rootid=root_id) log.info(f"new group id: {group_id}") group_json = {"id": group_id, "root": root_id} log.debug("create group, body: " + json.dumps(group_json)) req = getDataNodeUrl(app, group_id) + "/groups" params = {} if bucket: params["bucket"] = bucket group_json = await http_post(app, req, data=group_json, params=params) # create link if requested if link_id and link_title: link_json = {} link_json["id"] = group_id link_json["class"] = "H5L_TYPE_HARD" link_req = getDataNodeUrl(app, link_id) link_req += "/groups/" + link_id + "/links/" + link_title log.debug("PUT link - : " + link_req) put_json_rsp = await http_put(app, link_req, data=link_json, params=params) log.debug(f"PUT Link resp: {put_json_rsp}") log.debug("returning resp") # group creation successful resp = await jsonResponse(request, group_json, status=201) log.response(request, resp=resp) return resp
async def DELETE_Domain(request): """HTTP method to delete a domain resource""" log.request(request) app = request.app params = request.rel_url.query meta_only = False # if True, just delete the meta cache value keep_root = False if request.has_body: body = await request.json() if "meta_only" in body: meta_only = body["meta_only"] if "keep_root" in body: keep_root = body["keep_root"] else: if "meta_only" in params: meta_only = params["meta_only"] if "keep_root" in params: keep_root = params["keep_root"] domain = None try: domain = getDomainFromRequest(request) except ValueError: log.warn(f"Invalid domain: {domain}") raise HTTPBadRequest(reason="Invalid domain name") bucket = getBucketForDomain(domain) log.debug(f"GET_Domain domain: {domain}") if not domain: msg = "No domain given" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info(f"meta_only domain delete: {meta_only}") if meta_only: # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: log.info(f"deleting {domain} from domain_cache") del domain_cache[domain] resp = await jsonResponse(request, {}) return resp username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) parent_domain = getParentDomain(domain) if not parent_domain or getPathForDomain(parent_domain) == '/': is_toplevel = True else: is_toplevel = False if is_toplevel and username != "admin": msg = "Deletion of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code == 404: log.warn("domain not found") raise HTTPNotFound() elif ce.code == 410: log.warn("domain has been removed") raise HTTPGone() else: log.error(f"unexpected error: {ce.code}") raise HTTPInternalServerError() aclCheck(domain_json, "delete", username) # throws exception if not allowed # check for sub-objects if this is a folder if "root" not in domain_json: index = domain.find('/') s3prefix = domain[(index + 1):] + '/' log.info(f"checking s3key with prefix: {s3prefix} in bucket: {bucket}") s3keys = await getS3Keys(app, include_stats=False, prefix=s3prefix, deliminator='/', bucket=bucket) for s3key in s3keys: if s3key.endswith("/"): log.warn(f"attempt to delete folder {domain} with sub-items") log.debug(f"got prefix: {s3keys[0]}") raise HTTPConflict(reason="folder has sub-items") req = getDataNodeUrl(app, domain) req += "/domains" params = {} # for http_delete requests to DN nodes params["domain"] = domain rsp_json = await http_delete(app, req, params=params) if "root" in domain_json and not keep_root: # delete the root group root_id = domain_json["root"] req = getDataNodeUrl(app, root_id) req += "/groups/" + root_id await http_delete(app, req, params=params) # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: del domain_cache[domain] # delete domain cache from other sn_urls sn_urls = app["sn_urls"] params = {} params["domain"] = getPathForDomain(domain) params["bucket"] = getBucketForDomain(domain) params[ "meta_only"] = 1 # can't pass booleans as params, so use 1 instead of True for node_no in sn_urls: if node_no == app["node_number"]: continue # don't send to ourselves sn_url = sn_urls[node_no] req = sn_url + "/" log.info(f"sending sn request: {req}") try: sn_rsp = await http_delete(app, req, params=params) log.info(f"{req} response: {sn_rsp}") except ClientResponseError as ce: log.warn(f"got error for sn_delete: {ce}") resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def POST_Dataset(request): """HTTP method to create a new dataset object""" log.request(request) app = request.app username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "POST Datasets with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() # get domain, check authorization domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = "Invalid host value: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) domain_json = await getDomainJson(app, domain, reload=True) root_id = domain_json["root"] aclCheck(domain_json, "create", username) # throws exception if not allowed if "root" not in domain_json: msg = "Expected root key for domain: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) # # validate type input # if "type" not in body: msg = "POST Dataset has no type key in body" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = body["type"] if isinstance(datatype, str) and datatype.startswith("t-"): # Committed type - fetch type json from DN ctype_id = datatype log.debug("got ctypeid: {}".format(ctype_id)) ctype_json = await getObjectJson(app, ctype_id) log.debug("ctype: {}".format(ctype_json)) if ctype_json["root"] != root_id: msg = "Referenced committed datatype must belong in same domain" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = ctype_json["type"] # add the ctype_id to type type datatype["id"] = ctype_id elif isinstance(datatype, str): try: # convert predefined type string (e.g. "H5T_STD_I32LE") to # corresponding json representation datatype = getBaseTypeJson(datatype) log.debug("got datatype: {}".format(datatype)) except TypeError: msg = "POST Dataset with invalid predefined type" log.warn(msg) raise HTTPBadRequest(reason=msg) validateTypeItem(datatype) item_size = getItemSize(datatype) # # Validate shape input # dims = None shape_json = {} if "shape" not in body: shape_json["class"] = "H5S_SCALAR" else: shape = body["shape"] if isinstance(shape, int): shape_json["class"] = "H5S_SIMPLE" dims = [ shape, ] shape_json["dims"] = dims elif isinstance(shape, str): # only valid string value is H5S_NULL if shape != "H5S_NULL": msg = "POST Datset with invalid shape value" log.warn(msg) raise HTTPBadRequest(reason=msg) shape_json["class"] = "H5S_NULL" elif isinstance(shape, list): if len(shape) == 0: shape_json["class"] = "H5S_SCALAR" else: shape_json["class"] = "H5S_SIMPLE" shape_json["dims"] = shape dims = shape else: msg = "Bad Request: shape is invalid" log.warn(msg) raise HTTPBadRequest(reason=msg) if dims is not None: for i in range(len(dims)): extent = dims[i] if not isinstance(extent, int): msg = "Invalid shape type" log.warn(msg) raise HTTPBadRequest(reason=msg) if extent < 0: msg = "shape dimension is negative" log.warn(msg) raise HTTPBadRequest(reason=msg) maxdims = None if "maxdims" in body: if dims is None: msg = "Maxdims cannot be supplied if space is NULL" log.warn(msg) raise HTTPBadRequest(reason=msg) maxdims = body["maxdims"] if isinstance(maxdims, int): dim1 = maxdims maxdims = [dim1] elif isinstance(maxdims, list): pass # can use as is else: msg = "Bad Request: maxdims is invalid" log.warn(msg) raise HTTPBadRequest(reason=msg) if len(dims) != len(maxdims): msg = "Maxdims rank doesn't match Shape" log.warn(msg) raise HTTPBadRequest(reason=msg) if maxdims is not None: for extent in maxdims: if not isinstance(extent, int): msg = "Invalid maxdims type" log.warn(msg) raise HTTPBadRequest(reason=msg) if extent < 0: msg = "maxdims dimension is negative" log.warn(msg) raise HTTPBadRequest(reason=msg) if len(maxdims) != len(dims): msg = "Bad Request: maxdims array length must equal shape array length" log.warn(msg) raise HTTPBadRequest(reason=msg) shape_json["maxdims"] = [] for i in range(len(dims)): maxextent = maxdims[i] if not isinstance(maxextent, int): msg = "Bad Request: maxdims must be integer type" log.warn(msg) raise HTTPBadRequest(reason=msg) elif maxextent == 0: # unlimited dimension shape_json["maxdims"].append(0) elif maxextent < dims[i]: msg = "Bad Request: maxdims extent can't be smaller than shape extent" log.warn(msg) raise HTTPBadRequest(reason=msg) else: shape_json["maxdims"].append(maxextent) layout = None min_chunk_size = int(config.get("min_chunk_size")) max_chunk_size = int(config.get("max_chunk_size")) if 'creationProperties' in body: creationProperties = body["creationProperties"] if 'layout' in creationProperties: layout = creationProperties["layout"] validateChunkLayout(shape_json, item_size, layout) if layout is None and shape_json["class"] != "H5S_NULL": # default to chunked layout layout = {"class": "H5D_CHUNKED"} if layout and layout["class"] == 'H5D_CONTIGUOUS_REF': chunk_dims = getContiguousLayout(shape_json, item_size, chunk_min=min_chunk_size, chunk_max=max_chunk_size) layout["dims"] = chunk_dims log.debug(f"autoContiguous layout: {layout}") if layout and layout["class"] == 'H5D_CHUNKED' and "dims" not in layout: # do autochunking chunk_dims = guessChunk(shape_json, item_size) layout["dims"] = chunk_dims log.debug(f"initial autochunk layout: {layout}") if layout and layout["class"] == 'H5D_CHUNKED': chunk_dims = layout["dims"] chunk_size = getChunkSize(chunk_dims, item_size) log.debug("chunk_size: {}, min: {}, max: {}".format( chunk_size, min_chunk_size, max_chunk_size)) # adjust the chunk shape if chunk size is too small or too big adjusted_chunk_dims = None if chunk_size < min_chunk_size: log.debug( "chunk size: {} less than min size: {}, expanding".format( chunk_size, min_chunk_size)) adjusted_chunk_dims = expandChunk(chunk_dims, item_size, shape_json, chunk_min=min_chunk_size, layout_class=layout["class"]) elif chunk_size > max_chunk_size: log.debug( "chunk size: {} greater than max size: {}, expanding".format( chunk_size, max_chunk_size, layout_class=layout["class"])) adjusted_chunk_dims = shrinkChunk(chunk_dims, item_size, chunk_max=max_chunk_size) if adjusted_chunk_dims: log.debug( f"requested chunk_dimensions: {chunk_dims} modified dimensions: {adjusted_chunk_dims}" ) layout["dims"] = adjusted_chunk_dims if layout and layout["class"] in ('H5D_CHUNKED_REF', 'H5D_CHUNKED_REF_INDIRECT'): chunk_dims = layout["dims"] chunk_size = getChunkSize(chunk_dims, item_size) log.debug("chunk_size: {}, min: {}, max: {}".format( chunk_size, min_chunk_size, max_chunk_size)) # adjust the chunk shape if chunk size is too small or too big if chunk_size < min_chunk_size: log.warn( "chunk size: {} less than min size: {} for H5D_CHUNKED_REF dataset" .format(chunk_size, min_chunk_size)) elif chunk_size > max_chunk_size: log.warn( "chunk size: {} greater than max size: {}, for H5D_CHUNKED_REF dataset" .format(chunk_size, max_chunk_size, layout_class=layout["class"])) link_id = None link_title = None if "link" in body: link_body = body["link"] if "id" in link_body: link_id = link_body["id"] if "name" in link_body: link_title = link_body["name"] if link_id and link_title: log.info("link id: {}".format(link_id)) # verify that the referenced id exists and is in this domain # and that the requestor has permissions to create a link await validateAction(app, domain, link_id, username, "create") dset_id = createObjId("datasets", rootid=root_id) log.info("new dataset id: {}".format(dset_id)) dataset_json = { "id": dset_id, "root": root_id, "type": datatype, "shape": shape_json } if "creationProperties" in body: # TBD - validate all creationProperties creationProperties = body["creationProperties"] if "fillValue" in creationProperties: # validate fill value compatible with type dt = createDataType(datatype) fill_value = creationProperties["fillValue"] if isinstance(fill_value, list): fill_value = tuple(fill_value) try: np.asarray(fill_value, dtype=dt) except (TypeError, ValueError): msg = "Fill value {} not compatible with dataset type: {}".format( fill_value, datatype) log.warn(msg) raise HTTPBadRequest(reason=msg) dataset_json["creationProperties"] = creationProperties if layout is not None: dataset_json["layout"] = layout log.debug("create dataset: " + json.dumps(dataset_json)) req = getDataNodeUrl(app, dset_id) + "/datasets" post_json = await http_post(app, req, data=dataset_json) # create link if requested if link_id and link_title: link_json = {} link_json["id"] = dset_id link_json["class"] = "H5L_TYPE_HARD" link_req = getDataNodeUrl(app, link_id) link_req += "/groups/" + link_id + "/links/" + link_title log.info("PUT link - : " + link_req) put_rsp = await http_put(app, link_req, data=link_json) log.debug("PUT Link resp: {}".format(put_rsp)) # dataset creation successful resp = await jsonResponse(request, post_json, status=201) log.response(request, resp=resp) return resp
async def POST_Datatype(request): """HTTP method to create new committed datatype object""" log.request(request) app = request.app username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "POST Datatype with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() if "type" not in body: msg = "POST Datatype has no type key in body" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = body["type"] if isinstance(datatype, str): try: # convert predefined type string (e.g. "H5T_STD_I32LE") to # corresponding json representation datatype = getBaseTypeJson(datatype) log.debug(f"got datatype: {datatype}") except TypeError: msg = "POST Dataset with invalid predefined type" log.warn(msg) raise HTTPBadRequest(reason=msg) validateTypeItem(datatype) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) domain_json = await getDomainJson(app, domain, reload=True) aclCheck(domain_json, "create", username) # throws exception if not allowed if "root" not in domain_json: msg = f"Expected root key for domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) link_id = None link_title = None if "link" in body: link_body = body["link"] if "id" in link_body: link_id = link_body["id"] if "name" in link_body: link_title = link_body["name"] if link_id and link_title: log.debug(f"link id: {link_id}") # verify that the referenced id exists and is in this domain # and that the requestor has permissions to create a link await validateAction(app, domain, link_id, username, "create") root_id = domain_json["root"] ctype_id = createObjId("datatypes", rootid=root_id) log.debug(f"new type id: {ctype_id}") ctype_json = {"id": ctype_id, "root": root_id, "type": datatype} log.debug("create named type, body: " + json.dumps(ctype_json)) req = getDataNodeUrl(app, ctype_id) + "/datatypes" params = {} if bucket: params["bucket"] = bucket type_json = await http_post(app, req, data=ctype_json, params=params) # create link if requested if link_id and link_title: link_json = {} link_json["id"] = ctype_id link_json["class"] = "H5L_TYPE_HARD" link_req = getDataNodeUrl(app, link_id) link_req += "/groups/" + link_id + "/links/" + link_title log.debug("PUT link - : " + link_req) put_rsp = await http_put(app, link_req, data=link_json, params=params) log.debug(f"PUT Link resp: {put_rsp}") # datatype creation successful resp = await jsonResponse(request, type_json, status=201) log.response(request, resp=resp) return resp