async def createDomain(app, domain, domain_json):
try:
s3_key = getS3Key(domain)
domain_exists = await isS3Obj(app, s3_key)
if domain_exists:
raise ValueError("Domain already exists")
parent_domain = getParentDomain(domain)
if parent_domain is None:
raise ValueError("Domain must have a parent")
log.info("writing domain")
await putS3JSONObj(app, s3_key, domain_json)
print("domain created! s3_key: {} domain_json: {}".format(s3_key, domain_json))
except ValueError as ve:
print("Got ValueError exception: {}".format(str(ve)))
except ClientOSError as coe:
print("Got S3 error: {}".format(str(coe)))
async def DELETE_Domain(request):
"""HTTP method to delete a domain resource"""
log.request(request)
app = request.app
params = request.rel_url.query
domain = None
meta_only = False # if True, just delete the meta cache value
keep_root = False
if request.has_body:
body = await request.json()
if "domain" in body:
domain = body["domain"]
else:
msg = "No domain in request body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "meta_only" in body:
meta_only = body["meta_only"]
if "keep_root" in body:
keep_root = body["keep_root"]
else:
# get domain from request uri
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "keep_root" in params:
keep_root = params["keep_root"]
log.info("meta_only domain delete: {}".format(meta_only))
if meta_only:
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
log.info("deleting {} from domain_cache".format(domain))
del domain_cache[domain]
resp = await jsonResponse(request, {})
return resp
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
parent_domain = getParentDomain(domain)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "Deletion of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
log.warn("domain not found")
raise HTTPNotFound()
elif ce.code == 410:
log.warn("domain has been removed")
raise HTTPGone()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
aclCheck(domain_json, "delete",
username) # throws exception if not allowed
# check for sub-objects if this is a folder
if "root" not in domain_json:
s3prefix = domain[1:] + '/'
log.info(f"checking kets with prefix: {s3prefix} ")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/')
for s3key in s3keys:
if s3key.endswith("/"):
log.warn(f"attempt to delete folder {domain} with sub-items")
log.debug(f"got prefix: {s3keys[0]}")
raise HTTPConflict(reason="folder has sub-items")
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"domain": domain}
rsp_json = await http_delete(app, req, data=body)
if "root" in domain_json and not keep_root:
# delete the root group
root_id = domain_json["root"]
req = getDataNodeUrl(app, root_id)
req += "/groups/" + root_id
await http_delete(app, req)
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
del domain_cache[domain]
# delete domain cache from other sn_urls
sn_urls = app["sn_urls"]
body["meta_only"] = True
for node_no in sn_urls:
if node_no == app["node_number"]:
continue # don't send to ourselves
sn_url = sn_urls[node_no]
req = sn_url + "/"
log.info("sending sn request: {}".format(req))
try:
sn_rsp = await http_delete(app, req, data=body)
log.info("{} response: {}".format(req, sn_rsp))
except ClientResponseError as ce:
log.warn("got error for sn_delete: {}".format(ce))
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_Domain(request):
"""HTTP method to return JSON for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn("Invalid domain")
raise HTTPBadRequest(reason="Invalid domain name")
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if not domain:
log.info("no domain passed in, returning all top-level domains")
# no domain passed in, return top-level domains for this request
domains = await get_domains(request)
rsp_json = {"domains": domains}
rsp_json["hrefs"] = []
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
log.info("got domain: {}".format(domain))
domain_json = await getDomainJson(app, domain, reload=True)
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
if "h5path" in params:
# if h5path is passed in, return object info for that path
# (if exists)
h5path = params["h5path"]
root_id = domain_json["root"]
obj_id = await getObjectIdByPath(app, root_id,
h5path) # throws 404 if not found
log.info("get obj_id: {} from h5path: {}".format(obj_id, h5path))
# get authoritative state for object from DN (even if it's in the meta_cache).
obj_json = await getObjectJson(app, obj_id, refresh=True)
obj_json["domain"] = domain
# Not bothering with hrefs for h5path lookups...
resp = await jsonResponse(request, obj_json)
log.response(request, resp=resp)
return resp
# return just the keys as per the REST API
rsp_json = await get_domain_response(app, domain_json, verbose=verbose)
# include domain objects if requested
if "getobjs" in params and params["getobjs"] and "root" in domain_json:
root_id = domain_json["root"]
include_attrs = False
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
domain_objs = await getDomainObjects(app,
root_id,
include_attrs=include_attrs)
rsp_json["domain_objs"] = domain_objs
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'database',
'href': getHref(request, '/datasets')
})
hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')})
hrefs.append({
'rel': 'typebase',
'href': getHref(request, '/datatypes')
})
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')})
parent_domain = getParentDomain(domain)
log.debug("href parent domain: {}".format(parent_domain))
if parent_domain:
hrefs.append({
'rel': 'parent',
'href': getHref(request, '/', domain=parent_domain)
})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info("PUT domain: {}, username: {}".format(domain, username))
body = None
if request.has_body:
body = await request.json()
log.debug("PUT domain with body: {}".format(body))
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug("got domain_json: {}".format(domain_json))
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"])
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
log.info(f"linking to domain: {linked_domain}")
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug("Parent domain: [{}]".format(parent_domain))
if (not parent_domain or parent_domain == '/') and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if parent_domain and parent_domain != '/':
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = "Parent domain: {} not found".format(parent_domain)
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = "Parent domain: {} removed".format(parent_domain)
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug("parent_json {}: {}".format(parent_domain, parent_json))
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app, linked_domain, reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug("new root group id: {}".format(root_id))
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
try:
group_json = await http_post(app, req, data=group_json)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug("adding default perm for domain: {}".format(domain))
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug("creating domain: {} with body: {}".format(domain, body))
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp
async def DELETE_Domain(request):
"""HTTP method to delete a domain resource"""
log.request(request)
app = request.app
params = request.rel_url.query
meta_only = False # if True, just delete the meta cache value
keep_root = False
if request.has_body:
body = await request.json()
if "meta_only" in body:
meta_only = body["meta_only"]
if "keep_root" in body:
keep_root = body["keep_root"]
else:
if "meta_only" in params:
meta_only = params["meta_only"]
if "keep_root" in params:
keep_root = params["keep_root"]
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn(f"Invalid domain: {domain}")
raise HTTPBadRequest(reason="Invalid domain name")
bucket = getBucketForDomain(domain)
log.debug(f"GET_Domain domain: {domain}")
if not domain:
msg = "No domain given"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"meta_only domain delete: {meta_only}")
if meta_only:
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
log.info(f"deleting {domain} from domain_cache")
del domain_cache[domain]
resp = await jsonResponse(request, {})
return resp
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
parent_domain = getParentDomain(domain)
if not parent_domain or getPathForDomain(parent_domain) == '/':
is_toplevel = True
else:
is_toplevel = False
if is_toplevel and username != "admin":
msg = "Deletion of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
log.warn("domain not found")
raise HTTPNotFound()
elif ce.code == 410:
log.warn("domain has been removed")
raise HTTPGone()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
aclCheck(domain_json, "delete",
username) # throws exception if not allowed
# check for sub-objects if this is a folder
if "root" not in domain_json:
index = domain.find('/')
s3prefix = domain[(index + 1):] + '/'
log.info(f"checking s3key with prefix: {s3prefix} in bucket: {bucket}")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/',
bucket=bucket)
for s3key in s3keys:
if s3key.endswith("/"):
log.warn(f"attempt to delete folder {domain} with sub-items")
log.debug(f"got prefix: {s3keys[0]}")
raise HTTPConflict(reason="folder has sub-items")
req = getDataNodeUrl(app, domain)
req += "/domains"
params = {} # for http_delete requests to DN nodes
params["domain"] = domain
rsp_json = await http_delete(app, req, params=params)
if "root" in domain_json and not keep_root:
# delete the root group
root_id = domain_json["root"]
req = getDataNodeUrl(app, root_id)
req += "/groups/" + root_id
await http_delete(app, req, params=params)
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
del domain_cache[domain]
# delete domain cache from other sn_urls
sn_urls = app["sn_urls"]
params = {}
params["domain"] = getPathForDomain(domain)
params["bucket"] = getBucketForDomain(domain)
params[
"meta_only"] = 1 # can't pass booleans as params, so use 1 instead of True
for node_no in sn_urls:
if node_no == app["node_number"]:
continue # don't send to ourselves
sn_url = sn_urls[node_no]
req = sn_url + "/"
log.info(f"sending sn request: {req}")
try:
sn_rsp = await http_delete(app, req, params=params)
log.info(f"{req} response: {sn_rsp}")
except ClientResponseError as ce:
log.warn(f"got error for sn_delete: {ce}")
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
