async def about(request):
""" HTTP Method to return general info about the service """
log.request(request)
app = request.app
(username, pswd) = getUserPasswordFromRequest(request)
if username:
await validateUserPassword(app, username, pswd)
answer = {}
answer['start_time'] = app["start_time"]
answer['state'] = app['node_state']
answer["hsds_version"] = getVersion()
answer["name"] = config.get("server_name")
answer["greeting"] = config.get("greeting")
answer["about"] = config.get("about")
resp = await jsonResponse(request, answer)
log.response(request, resp=resp)
return resp
async def GET_Domain(request):
"""HTTP GET method to return JSON for /domains/
"""
log.request(request)
app = request.app
domain = get_domain(request)
log.debug(f"get domain: {domain}")
bucket = getBucketForDomain(domain)
if not bucket:
log.error(f"expected bucket to be used in domain: {domain}")
raise HTTPInternalServerError()
log.debug(f"using bucket: {bucket}")
domain_json = await get_metadata_obj(app, domain)
log.debug(f"returning domain_json: {domain_json}")
resp = json_response(domain_json)
log.response(request, resp=resp)
return resp
async def DELETE_Attribute(request):
"""HTTP DELETE method for /(obj)/<id>/attributes/<name>
"""
log.request(request)
app = request.app
params = request.rel_url.query
obj_id = get_obj_id(request)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
attr_name = request.match_info.get('name')
log.info(f"DELETE attribute {attr_name} in {obj_id} bucket: {bucket}")
validateAttributeName(attr_name)
obj_json = await get_metadata_obj(app, obj_id, bucket=bucket)
log.debug(f"DELETE attribute obj_id: {obj_id} got json")
if "attributes" not in obj_json:
msg = f"unexpected data for obj id: {obj_id}"
msg.error(msg)
raise HTTPInternalServerError()
# return a list of attributes based on sorted dictionary keys
attributes = obj_json["attributes"]
if attr_name not in attributes:
msg = f"Attribute {attr_name} not found in objid: {obj_id}"
log.warn(msg)
raise HTTPNotFound()
del attributes[attr_name]
await save_metadata_obj(app, obj_id, obj_json, bucket=bucket)
resp_json = { }
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def DELETE_Chunk(request):
"""HTTP DELETE method for /chunks/
Note: clients (i.e. SN nodes) don't directly delete chunks. This method should
only be called by the AN node.
"""
log.request(request)
app = request.app
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info("DELETE chunk: {}".format(chunk_id))
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
chunk_cache = app['chunk_cache']
s3_key = getS3Key(chunk_id)
log.debug("DELETE_Chunk s3_key: {}".format(s3_key))
if chunk_id in chunk_cache:
del chunk_cache[chunk_id]
deflate_map = app["deflate_map"]
dset_id = getDatasetId(chunk_id)
if dset_id in deflate_map:
# The only reason chunks are ever deleted is if the dataset is being deleted,
# so it should be save to remove this entry now
log.info("Removing deflate_map entry for {}".format(dset_id))
del deflate_map[dset_id]
resp_json = { }
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def PUT_Domain(request):
"""HTTP method to get object s3 state """
log.request(request)
app = request.app
pending_set = app["pending"]
params = request.rel_url.query
if "domain" not in params:
msg = "No domain provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = params["domain"]
if not domain.startswith("/"):
msg = "Domain expected to start with /"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(domain) < 2:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "root" in params:
rootid = params["root"]
if not isValidUuid(rootid):
log.warn(f"Invalid id: {rootid}")
raise HTTPBadRequest()
log.debug(f"new rootid: {rootid} for domain: {domain}")
if isSchema2Id(rootid):
log.info(f"Adding root: {rootid} to pending for PUT domain: {domain}")
pending_set.add(rootid)
resp_json = {}
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def info(request):
"""HTTP Method to return node state to caller"""
log.request(request)
app = request.app
resp = StreamResponse()
resp.headers['Content-Type'] = 'application/json'
answer = {}
# copy relevant entries from state dictionary to response
answer['id'] = request.app['id']
answer['start_time'] = unixTimeToUTC(app['start_time'])
answer['last_health_check'] = unixTimeToUTC(app['last_health_check'])
answer['up_time'] = elapsedTime(app['start_time'])
answer['cluster_state'] = app['cluster_state']
answer['bucket_name'] = app['bucket_name']
answer['target_sn_count'] = getTargetNodeCount(app, "sn")
answer['active_sn_count'] = getActiveNodeCount(app, "sn")
answer['target_dn_count'] = getTargetNodeCount(app, "dn")
answer['active_dn_count'] = getActiveNodeCount(app, "dn")
resp = json_response(answer)
log.response(request, resp=resp)
return resp
async def DELETE_Link(request):
"""HTTP method to delete a link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "delete")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
params = {}
if bucket:
params["bucket"] = bucket
rsp_json = await http_delete(app, req, params=params)
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def DELETE_Attribute(request):
"""HTTP DELETE method for /(obj)/<id>/attributes/<name>
"""
log.request(request)
app = request.app
obj_id = get_obj_id(request)
attr_name = request.match_info.get('name')
log.info("DELETE attribute {} in {}".format(attr_name, obj_id))
validateAttributeName(attr_name)
obj_json = await get_metadata_obj(app, obj_id)
log.debug("DELETE attribute obj_id: {} got json".format(obj_id))
if "attributes" not in obj_json:
msg = "unexpected data for obj id: {}".format(obj_id)
msg.error(msg)
raise HTTPInternalServerError()
# return a list of attributes based on sorted dictionary keys
attributes = obj_json["attributes"]
if attr_name not in attributes:
msg = "Attribute {} not found in id: {}".format(attr_name, obj_id)
log.warn(msg)
raise HTTPNotFound()
del attributes[attr_name]
await save_metadata_obj(app, obj_id, obj_json)
resp_json = { }
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def GET_Link(request):
"""HTTP GET method to return JSON for a link
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
log.info(f"GET link: {group_id}")
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
link_title = request.match_info.get('title')
validateLinkName(link_title)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
group_json = await get_metadata_obj(app, group_id, bucket=bucket)
log.info(f"for id: {group_id} got group json: {group_json}")
if "links" not in group_json:
log.error(f"unexpected group data for id: {group_id}")
raise HTTPInternalServerError()
links = group_json["links"]
if link_title not in links:
log.warn(f"Link name {link_title} not found in group: {group_id}")
raise HTTPNotFound()
link_json = links[link_title]
resp = json_response(link_json)
log.response(request, resp=resp)
return resp
async def GET_Dataset(request):
"""HTTP GET method to return JSON for /groups/
"""
log.request(request)
app = request.app
params = request.rel_url.query
dset_id = get_obj_id(request)
if not isValidUuid(dset_id, obj_class="dataset"):
log.error( "Unexpected type_id: {}".format(dset_id))
raise HTTPInternalServerError()
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
resp_json = { }
resp_json["id"] = dset_json["id"]
resp_json["root"] = dset_json["root"]
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["type"] = dset_json["type"]
resp_json["shape"] = dset_json["shape"]
resp_json["attributeCount"] = len(dset_json["attributes"])
if "creationProperties" in dset_json:
resp_json["creationProperties"] = dset_json["creationProperties"]
if "layout" in dset_json:
resp_json["layout"] = dset_json["layout"]
if "include_attrs" in params and params["include_attrs"]:
resp_json["attributes"] = dset_json["attributes"]
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def DELETE_Group(request):
"""HTTP DELETE method for /groups/
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
log.info(f"DELETE group: {group_id} bucket: {bucket}")
# verify the id exist
obj_found = await check_metadata_obj(app, group_id, bucket=bucket)
if not obj_found:
log.debug(f"delete called on non-exsistet obj: {group_id}")
raise HTTPNotFound()
log.debug("deleting group: {}".format(group_id))
notify = True
if "Notify" in params and not params["Notify"]:
notify = False
await delete_metadata_obj(app, group_id, bucket=bucket, notify=notify)
resp_json = {}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def PUT_Objects(request):
"""HTTP method to notify creation/update of objid"""
log.request(request)
app = request.app
pending_set = app["pending"]
log.info("PUT_Objects")
if not request.has_body:
msg = "PUT objects with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
log.debug("Got PUT Objects body: {}".format(body))
if "objs" not in body:
msg = "expected to find objs key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
objs = body["objs"]
for objid in objs:
log.debug("PUT_Objects, objid: {}".format(objid))
if not isValidUuid(objid):
log.warn(f"Invalid id: {objid}, ignoring")
continue
if not isSchema2Id(objid):
log.info(f"PUT_Objects ignoring v1 id: {objid}")
continue
rootid = getRootObjId(objid)
log.debug(f"adding root: {rootid} to pending queue for objid: {objid}")
pending_set.add(rootid)
resp_json = { }
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def DELETE_Domain(request):
log.request(request)
#app = request.app
params = request.rel_url.query
if "domain" not in params:
msg = "No domain provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = params["domain"]
if not domain.startswith("/"):
msg = "Domain expected to start with /"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(domain) < 2:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "root" in params:
rootid = params["root"]
log.debug(f"delete rootid: {rootid} for domain: {domain}")
if not isValidUuid(rootid):
log.warn(f"Invalid id: {rootid}")
raise HTTPBadRequest()
if isSchema2Id(rootid):
# TBD: schedule root collection for deletion
pass
resp_json = {}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def GET_Attribute(request):
"""HTTP GET method to return JSON for /(obj)/<id>/attributes/<name>
"""
log.request(request)
app = request.app
params = request.rel_url.query
obj_id = get_obj_id(request)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
obj_json = await get_metadata_obj(app, obj_id, bucket=bucket)
log.info(
f"GET attribute obj_id: {obj_id} name: {attr_name} bucket: {bucket}")
log.debug(f"got obj_json: {obj_json}")
if "attributes" not in obj_json:
log.error(f"unexpected obj data for id: {obj_id}")
raise HTTPInternalServerError()
attributes = obj_json["attributes"]
if attr_name not in attributes:
msg = f"Attribute '{attr_name}' not found for id: {obj_id}"
log.warn(msg)
raise HTTPNotFound()
attr_json = attributes[attr_name]
resp = json_response(attr_json)
log.response(request, resp=resp)
return resp
async def GET_Datatypes(request):
"""HTTP method to return datatype collection for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code in (404, 410):
msg = "domain not found"
log.warn(msg)
raise HTTPNotFound()
else:
log.error(f"Unexpected Error: {ce.code})")
raise HTTPInternalServerError()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
# get the datatype collection list
obj_ids = []
if "root" in domain_json or domain_json["root"]:
# get the groups collection list
collections = await get_collections(app, domain_json["root"])
objs = collections["datatypes"]
obj_ids = getIdList(objs, marker=marker, limit=limit)
# create hrefs
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/datatypes')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
# return obj ids and hrefs
rsp_json = {}
rsp_json["datatypes"] = obj_ids
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
type_json = dset_json["type"]
dims = getChunkLayout(dset_json)
log.debug("got dims: {}".format(dims))
rank = len(dims)
# get chunk selection from query params
if "select" in params:
log.debug("select: {}".format(params["select"]))
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug("got selection: {}".format(selection))
dt = createDataType(type_json)
log.debug("dtype: {}".format(dt))
rank = len(dims)
if rank == 0:
msg = "No dimension passed to GET chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug("selection[{}]: {}".format(i, s))
s3path = None
s3offset = 0
s3size = 0
if "s3path" in params:
s3path = params["s3path"]
log.debug(f"GET_Chunk - useing s3path: {s3path}")
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
chunk_arr = await getChunk(app, chunk_id, dset_json, s3path=s3path, s3offset=s3offset, s3size=s3size)
if chunk_arr is None:
# return a 404
msg = "Chunk {} does not exist".format(chunk_id)
log.info(msg)
raise HTTPNotFound()
resp = None
if "query" in params:
# do query selection
query = params["query"]
log.info("query: {}".format(query))
if rank != 1:
msg = "Query selection only supported for one dimensional arrays"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = 0
if "Limit" in params:
limit = int(params["Limit"])
values = []
indices = []
field_names = []
if dt.fields:
field_names = list(dt.fields.keys())
x = chunk_arr[selection]
log.debug("x: {}".format(x))
eval_str = getEvalStr(query, "x", field_names)
log.debug("eval_str: {}".format(eval_str))
where_result = np.where(eval(eval_str))
log.debug("where_result: {}".format(where_result))
where_result_index = where_result[0]
log.debug("whare_result index: {}".format(where_result_index))
log.debug("boolean selection: {}".format(x[where_result_index]))
s = selection[0]
count = 0
for index in where_result_index:
log.debug("index: {}".format(index))
value = x[index].tolist()
log.debug("value: {}".format(value))
json_val = bytesArrayToList(value)
log.debug("json_value: {}".format(json_val))
json_index = index.tolist() * s.step + s.start # adjust for selection
indices.append(json_index)
values.append(json_val)
count += 1
if limit > 0 and count >= limit:
log.info("got limit items")
break
query_result = {}
query_result["index"] = indices
query_result["value"] = values
log.info(f"query_result retiurning: {len(indices)} rows")
log.debug(f"query_result: {query_result}")
resp = json_response(query_result)
else:
# get requested data
output_arr = chunk_arr[selection]
output_data = arrayToBytes(output_arr)
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def POST_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
put_points = False
num_points = 0
if "count" in params:
num_points = int(params["count"])
if "action" in params and params["action"] == "put":
log.info("POST Chunk put points, num_points: {}".format(num_points))
put_points = True
else:
log.info("POST Chunk get points")
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info("POST chunk_id: {}".format(chunk_id))
chunk_index = getChunkIndex(chunk_id)
log.debug("chunk_index: {}".format(chunk_index))
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
chunk_layout = getChunkLayout(dset_json)
chunk_coord = getChunkCoordinate(chunk_id, chunk_layout)
log.debug("chunk_coord: {}".format(chunk_coord))
if not request.has_body:
msg = "POST Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = "Unexpected content_type: {}".format(content_type)
log.error(msg)
raise HTTPBadRequest(reason=msg)
type_json = dset_json["type"]
dset_dtype = createDataType(type_json)
log.debug("dtype: {}".format(dset_dtype))
dims = getChunkLayout(dset_json)
log.debug("got dims: {}".format(dims))
rank = len(dims)
if rank == 0:
msg = "POST chunk request with no dimensions"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming points
input_bytes = await request_read(request)
if len(input_bytes) != request.content_length:
msg = "Read {} bytes, expecting: {}".format(len(input_bytes), request.content_length)
log.error(msg)
raise HTTPInternalServerError()
# get chunk from cache/s3. If not found init a new chunk if this is a write request
chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=put_points)
if put_points:
# writing point data
# create a numpy array with the following type:
# (coord1, coord2, ...) | dset_dtype
if rank == 1:
coord_type_str = "uint64"
else:
coord_type_str = "({},)uint64".format(rank)
comp_dtype = np.dtype([("coord", np.dtype(coord_type_str)), ("value", dset_dtype)])
point_arr = np.fromstring(input_bytes, dtype=comp_dtype)
if len(point_arr) != num_points:
msg = "Unexpected size of point array, got: {} expected: {}".format(len(point_arr), num_points)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(num_points):
elem = point_arr[i]
if rank == 1:
coord = int(elem[0])
else:
coord = tuple(elem[0]) # index to update
val = elem[1] # value
chunk_arr[coord] = val # update the point
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = now
log.info("set {} to dirty".format(chunk_id))
else:
# reading point data
point_dt = np.dtype('uint64') # use unsigned long for point index
point_arr = np.fromstring(input_bytes, dtype=point_dt) # read points as unsigned longs
if len(point_arr) % rank != 0:
msg = "Unexpected size of point array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
num_points = len(point_arr) // rank
log.debug("got {} points".format(num_points))
point_arr = point_arr.reshape((num_points, rank))
output_arr = np.zeros((num_points,), dtype=dset_dtype)
for i in range(num_points):
point = point_arr[i,:]
tr_point = getChunkRelativePoint(chunk_coord, point)
val = chunk_arr[tuple(tr_point)]
output_arr[i] = val
if put_points:
# write empty response
resp = json_response({})
else:
# get data
output_data = output_arr.tobytes()
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def POST_Dataset(request):
""" Handler for POST /datasets"""
log.request(request)
app = request.app
if not request.has_body:
msg = "POST_Dataset with no body"
log.error(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
log.info("POST_Dataset, body: {}".format(body))
dset_id = get_obj_id(request, body=body)
if not isValidUuid(dset_id, obj_class="dataset"):
log.error("Unexpected dataset_id: {}".format(dset_id))
raise HTTPInternalServerError()
# verify the id doesn't already exist
obj_found = await check_metadata_obj(app, dset_id)
if obj_found:
log.error("Post with existing dset_id: {}".format(dset_id))
raise HTTPInternalServerError()
if "root" not in body:
msg = "POST_Dataset with no root"
log.error(msg)
raise HTTPInternalServerError()
root_id = body["root"]
try:
validateUuid(root_id, "group")
except ValueError:
msg = "Invalid root_id: " + root_id
log.error(msg)
raise HTTPInternalServerError()
if "type" not in body:
msg = "POST_Dataset with no type"
log.error(msg)
raise HTTPInternalServerError()
type_json = body["type"]
if "shape" not in body:
msg = "POST_Dataset with no shape"
log.error(msg)
raise HTTPInternalServerError()
shape_json = body["shape"]
layout = None
if "layout" in body:
layout = body["layout"] # client specified chunk layout
# ok - all set, create committed type obj
now = int(time.time())
log.debug("POST_dataset typejson: {}, shapejson: {}".format(
type_json, shape_json))
dset_json = {
"id": dset_id,
"root": root_id,
"created": now,
"lastModified": now,
"type": type_json,
"shape": shape_json,
"attributes": {}
}
if "creationProperties" in body:
dset_json["creationProperties"] = body["creationProperties"]
if layout is not None:
dset_json["layout"] = layout
await save_metadata_obj(app, dset_id, dset_json, notify=True, flush=True)
resp_json = {}
resp_json["id"] = dset_id
resp_json["root"] = root_id
resp_json["created"] = dset_json["created"]
resp_json["type"] = type_json
resp_json["shape"] = shape_json
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["attributeCount"] = 0
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def PUT_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "PUT Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = "Unexpected content_type: {}".format(content_type)
log.error(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
dims = getChunkLayout(dset_json)
if "root" not in dset_json:
msg = "expected root key in dset_json"
log.error(msg)
raise KeyError(msg)
rank = len(dims)
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug("got selection: {}".format(selection))
type_json = dset_json["type"]
itemsize = 'H5T_VARIABLE'
if "size" in type_json:
itemsize = type_json["size"]
dt = createDataType(type_json)
log.debug("dtype: {}".format(dt))
if rank == 0:
msg = "No dimension passed to PUT chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug("selection[{}]: {}".format(i, s))
mshape = getSelectionShape(selection)
log.debug(f"mshape: {mshape}")
num_elements = 1
for extent in mshape:
num_elements *= extent
# check that the content_length is what we expect
if itemsize != 'H5T_VARIABLE':
log.debug("expect content_length: {}".format(num_elements*itemsize))
log.debug("actual content_length: {}".format(request.content_length))
if itemsize != 'H5T_VARIABLE' and (num_elements * itemsize) != request.content_length:
msg = "Expected content_length of: {}, but got: {}".format(num_elements*itemsize, request.content_length)
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming data
input_bytes = await request_read(request) # TBD - will it cause problems when failures are raised before reading data?
if len(input_bytes) != request.content_length:
msg = "Read {} bytes, expecting: {}".format(len(input_bytes), request.content_length)
log.error(msg)
raise HTTPInternalServerError()
input_arr = bytesToArray(input_bytes, dt, mshape)
chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=True)
# update chunk array
chunk_arr[selection] = input_arr
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}")
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = now
# chunk update successful
resp = json_response({}, status=201)
log.response(request, resp=resp)
return resp
async def PUT_DatasetShape(request):
"""HTTP method to update dataset's shape"""
log.request(request)
app = request.app
dset_id = request.match_info.get('id')
if not isValidUuid(dset_id, obj_class="dataset"):
log.error("Unexpected type_id: {}".format(dset_id))
raise HTTPInternalServerError()
body = await request.json()
log.info("PUT datasetshape: {}, body: {}".format(dset_id, body))
if "shape" not in body and "extend" not in body:
log.error("Expected shape or extend keys")
raise HTTPInternalServerError()
dset_json = await get_metadata_obj(app, dset_id)
shape_orig = dset_json["shape"]
log.debug("shape_orig: {}".format(shape_orig))
if "maxdims" not in shape_orig:
log.error("expected maxdims in dataset json")
raise HTTPInternalServerError()
dims = shape_orig["dims"]
maxdims = shape_orig["maxdims"]
resp_json = {}
if "extend" in body:
# extend the shape by the give value and return the
# newly extended area
extension = body["extend"]
extend_dim = 0
if "extend_dim" in body:
extend_dim = body["extend_dim"]
log.info(f"datashape extend: {extension} dim: {extend_dim}")
selection = "["
for i in range(len(dims)):
if i == extend_dim:
lb = dims[i]
ub = lb + extension
if maxdims[extend_dim] != 0 and ub > maxdims[extend_dim]:
msg = "maximum extent exceeded"
log.warn(msg)
raise HTTPConflict()
selection += f"{lb}:{ub}"
dims[i] = ub
else:
if dims[i] == 0:
dims[i] = 1 # each dimension must be non-zero
selection += ":"
if i < len(dims) - 1:
selection += ","
selection += "]"
resp_json["selection"] = selection
else:
# verify that the extend request is still valid
# e.g. another client has already extended the shape since the SN
# verified it
shape_update = body["shape"]
log.debug("shape_update: {}".format(shape_update))
for i in range(len(dims)):
if shape_update[i] < dims[i]:
msg = "Dataspace can not be made smaller"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# Update the shape!
for i in range(len(dims)):
dims[i] = shape_update[i]
# write back to S3, save to metadata cache
log.info(f"Updated dimensions: {dims}")
await save_metadata_obj(app, dset_id, dset_json)
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def GET_ACLs(request):
"""HTTP method to return JSON for domain/ACLs"""
log.request(request)
app = request.app
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(message=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError:
log.warn("domain not found")
log.warn(msg)
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
acls = domain_json["acls"]
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "readACL",
username) # throws exception if not authorized
acl_list = []
acl_usernames = list(acls.keys())
acl_usernames.sort()
for acl_username in acl_usernames:
entry = {"userName": acl_username}
acl = acls[acl_username]
for k in acl.keys():
entry[k] = acl[k]
acl_list.append(entry)
# return just the keys as per the REST API
rsp_json = {}
rsp_json["acls"] = acl_list
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')})
if "root" in domain_json:
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + domain_json["root"])
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, '/')})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_ACL(request):
"""HTTP method to return JSON for given domain/ACL"""
log.request(request)
app = request.app
acl_username = request.match_info.get('username')
if not acl_username:
msg = "Missing username for ACL"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code in (404, 410):
msg = "domain not found"
log.warn(msg)
raise HTTPNotFound()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
# validate that the requesting user has permission to read ACLs in this domain
if acl_username in (username, "default"):
# allow read access for a users on ACL, or default
aclCheck(domain_json, "read",
username) # throws exception if not authorized
else:
aclCheck(domain_json, "readACL",
username) # throws exception if not authorized
if 'owner' not in domain_json:
log.warn("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.warn("No acls key found in domain")
raise HTTPInternalServerError()
acls = domain_json["acls"]
log.debug("got domain_json: {}".format(domain_json))
if acl_username not in acls:
msg = "acl for username: [{}] not found".format(acl_username)
log.warn(msg)
raise HTTPNotFound()
acl = acls[acl_username]
acl_rsp = {}
for k in acl.keys():
acl_rsp[k] = acl[k]
acl_rsp["userName"] = acl_username
# return just the keys as per the REST API
rsp_json = {}
rsp_json["acl"] = acl_rsp
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')})
if "root" in domain_json:
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + domain_json["root"])
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, '/')})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_Link(request):
"""HTTP method to return JSON for a group link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("get LINK: " + req)
link_json = await http_get(app, req)
log.debug("got link_json: " + str(link_json))
resp_link = {}
resp_link["title"] = link_title
resp_link["class"] = link_json["class"]
if link_json["class"] == "H5L_TYPE_HARD":
resp_link["id"] = link_json["id"]
resp_link["collection"] = getCollectionForId(link_json["id"])
elif link_json["class"] == "H5L_TYPE_SOFT":
resp_link["h5path"] = link_json["h5path"]
elif link_json["class"] == "H5L_TYPE_EXTERNAL":
resp_link["h5path"] = link_json["h5path"]
resp_link["h5domain"] = link_json["h5domain"]
else:
log.warn("Unexpected link class: {}".format(link_json["class"]))
resp_json = {}
resp_json["link"] = resp_link
resp_json["created"] = link_json["created"]
# links don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = link_json["created"]
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links/' + link_title)
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
if link_json["class"] == "H5L_TYPE_HARD":
target = '/' + resp_link["collection"] + '/' + resp_link["id"]
hrefs.append({'rel': 'target', 'href': getHref(request, target)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Links(request):
"""HTTP method to return JSON for link collection"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links"
query_sep = '?'
if limit is not None:
req += query_sep + "Limit=" + str(limit)
query_sep = '&'
if marker is not None:
req += query_sep + "Marker=" + marker
log.debug("get LINKS: " + req)
links_json = await http_get(app, req)
log.debug("got links json from dn for group_id: {}".format(group_id))
links = links_json["links"]
# mix in collection key, target and hrefs
for link in links:
if link["class"] == "H5L_TYPE_HARD":
collection_name = getCollectionForId(link["id"])
link["collection"] = collection_name
target_uri = '/' + collection_name + '/' + link["id"]
link["target"] = getHref(request, target_uri)
link_uri = '/groups/' + group_id + '/links/' + link['title']
link["href"] = getHref(request, link_uri)
resp_json = {}
resp_json["links"] = links
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links')
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def PUT_Link(request):
"""HTTP method to create a new link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = "Invalid group id: {}".format(group_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
log.info("PUT Link_title: [{}]".format(link_title))
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Link with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
link_json = {}
if "id" in body:
if not isValidUuid(body["id"]):
msg = "PUT Link with invalid id in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_json["id"] = body["id"]
link_json["class"] = "H5L_TYPE_HARD"
elif "h5path" in body:
link_json["h5path"] = body["h5path"]
# could be hard or soft link
if "h5domain" in body:
link_json["h5domain"] = body["h5domain"]
link_json["class"] = "H5L_TYPE_EXTERNAL"
else:
# soft link
link_json["class"] = "H5L_TYPE_SOFT"
else:
msg = "PUT Link with no id or h5path keys"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = "Invalid host value: {}".format(domain)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
await validateAction(app, domain, group_id, username, "create")
# for hard links, verify that the referenced id exists and is in this domain
if "id" in body:
ref_id = body["id"]
ref_json = await getObjectJson(app, ref_id)
group_json = await getObjectJson(app, group_id)
if ref_json["root"] != group_json["root"]:
msg = "Hard link must reference an object in the same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# ready to add link now
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("PUT link - getting group: " + req)
put_rsp = await http_put(app, req, data=link_json)
log.debug("PUT Link resp: " + str(put_rsp))
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
# link creation successful
resp = await jsonResponse(request, req_rsp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Domain(request):
"""HTTP method to return JSON for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn("Invalid domain")
raise HTTPBadRequest(reason="Invalid domain name")
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if not domain:
log.info("no domain passed in, returning all top-level domains")
# no domain passed in, return top-level domains for this request
domains = await get_domains(request)
rsp_json = {"domains": domains}
rsp_json["hrefs"] = []
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
log.info("got domain: {}".format(domain))
domain_json = await getDomainJson(app, domain, reload=True)
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
if "h5path" in params:
# if h5path is passed in, return object info for that path
# (if exists)
h5path = params["h5path"]
root_id = domain_json["root"]
obj_id = await getObjectIdByPath(app, root_id,
h5path) # throws 404 if not found
log.info("get obj_id: {} from h5path: {}".format(obj_id, h5path))
# get authoritative state for object from DN (even if it's in the meta_cache).
obj_json = await getObjectJson(app, obj_id, refresh=True)
obj_json["domain"] = domain
# Not bothering with hrefs for h5path lookups...
resp = await jsonResponse(request, obj_json)
log.response(request, resp=resp)
return resp
# return just the keys as per the REST API
rsp_json = await get_domain_response(app, domain_json, verbose=verbose)
# include domain objects if requested
if "getobjs" in params and params["getobjs"] and "root" in domain_json:
root_id = domain_json["root"]
include_attrs = False
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
domain_objs = await getDomainObjects(app,
root_id,
include_attrs=include_attrs)
rsp_json["domain_objs"] = domain_objs
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'database',
'href': getHref(request, '/datasets')
})
hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')})
hrefs.append({
'rel': 'typebase',
'href': getHref(request, '/datatypes')
})
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')})
parent_domain = getParentDomain(domain)
log.debug("href parent domain: {}".format(parent_domain))
if parent_domain:
hrefs.append({
'rel': 'parent',
'href': getHref(request, '/', domain=parent_domain)
})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def PUT_Group(request):
""" Handler for PUT /groups"""
""" Used to flush all objects under a root group to S3 """
FLUSH_TIME_OUT = 10.0 # TBD make config
FLUSH_SLEEP_INTERVAL = 0.1 # TBD make config
log.request(request)
app = request.app
params = request.rel_url.query
root_id = request.match_info.get('id')
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
log.info(f"PUT group (flush): {root_id} bucket: {bucket}")
# don't really need bucket param since the dirty ids know which bucket they should write too
if not isValidUuid(root_id, obj_class="group"):
log.error(f"Unexpected group_id: {root_id}")
raise HTTPInternalServerError()
schema2 = isSchema2Id(root_id)
if schema2 and not isRootObjId(root_id):
log.error(f"Expected root id for flush but got: {root_id}")
raise HTTPInternalServerError()
flush_start = time.time()
flush_set = set()
dirty_ids = app["dirty_ids"]
for obj_id in dirty_ids:
if schema2:
if isValidUuid(obj_id) and getRootObjId(obj_id) == root_id:
flush_set.add(obj_id)
else:
# for schema1 not easy to determine if a given id is in a domain,
# so just wait on all of them
flush_set.add(obj_id)
log.debug(f"flushop - waiting on {len(flush_set)} items")
while time.time() - flush_start < FLUSH_TIME_OUT:
# check to see if the items in our flush set are still there
remaining_set = set()
for obj_id in flush_set:
if not obj_id in dirty_ids:
log.debug(f"flush - {obj_id} has been written")
elif dirty_ids[obj_id][0] > flush_start:
log.debug(
f"flush - {obj_id} has been updated after flush start")
else:
log.debug(f"flush - {obj_id} still pending")
remaining_set.add(obj_id)
flush_set = remaining_set
if len(flush_set) == 0:
log.debug("flush op - all objects have been written")
break
log.debug(
f"flushop - {len(flush_set)} item remaining, sleeping for {FLUSH_SLEEP_INTERVAL}"
)
await asyncio.sleep(FLUSH_SLEEP_INTERVAL)
if len(flush_set) > 0:
log.warn(
f"flushop - {len(flush_set)} items not updated after {FLUSH_TIME_OUT}"
)
raise HTTPServiceUnavailable()
resp = json_response(None, status=204) # NO Content response
log.response(request, resp=resp)
return resp
async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info("PUT domain: {}, username: {}".format(domain, username))
body = None
if request.has_body:
body = await request.json()
log.debug("PUT domain with body: {}".format(body))
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug("got domain_json: {}".format(domain_json))
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"])
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
log.info(f"linking to domain: {linked_domain}")
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug("Parent domain: [{}]".format(parent_domain))
if (not parent_domain or parent_domain == '/') and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if parent_domain and parent_domain != '/':
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = "Parent domain: {} not found".format(parent_domain)
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = "Parent domain: {} removed".format(parent_domain)
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug("parent_json {}: {}".format(parent_domain, parent_json))
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app, linked_domain, reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug("new root group id: {}".format(root_id))
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
try:
group_json = await http_post(app, req, data=group_json)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug("adding default perm for domain: {}".format(domain))
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug("creating domain: {} with body: {}".format(domain, body))
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp
async def POST_Group(request):
""" Handler for POST /groups"""
log.request(request)
app = request.app
params = request.rel_url.query
if not request.has_body:
msg = "POST_Group with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
if "bucket" in params:
bucket = params["bucket"]
elif "bucket" in body:
bucket = params["bucket"]
else:
bucket = None
group_id = get_obj_id(request, body=body)
log.info(f"POST group: {group_id} bucket: {bucket}")
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
if "root" not in body:
msg = "POST_Group with no root"
log.error(msg)
raise HTTPInternalServerError()
# verify the id doesn't already exist
obj_found = await check_metadata_obj(app, group_id, bucket=bucket)
if obj_found:
log.error(f"Post with existing group_id: {group_id}")
raise HTTPInternalServerError()
root_id = body["root"]
if not isValidUuid(root_id, obj_class="group"):
msg = "Invalid root_id: " + root_id
log.error(msg)
raise HTTPInternalServerError()
# ok - all set, create group obj
now = time.time()
group_json = {
"id": group_id,
"root": root_id,
"created": now,
"lastModified": now,
"links": {},
"attributes": {}
}
await save_metadata_obj(app,
group_id,
group_json,
bucket=bucket,
notify=True,
flush=True)
# formulate response
resp_json = {}
resp_json["id"] = group_id
resp_json["root"] = root_id
resp_json["created"] = group_json["created"]
resp_json["lastModified"] = group_json["lastModified"]
resp_json["linkCount"] = 0
resp_json["attributeCount"] = 0
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def DELETE_Domain(request):
"""HTTP method to delete a domain resource"""
log.request(request)
app = request.app
params = request.rel_url.query
domain = None
meta_only = False # if True, just delete the meta cache value
keep_root = False
if request.has_body:
body = await request.json()
if "domain" in body:
domain = body["domain"]
else:
msg = "No domain in request body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "meta_only" in body:
meta_only = body["meta_only"]
if "keep_root" in body:
keep_root = body["keep_root"]
else:
# get domain from request uri
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "keep_root" in params:
keep_root = params["keep_root"]
log.info("meta_only domain delete: {}".format(meta_only))
if meta_only:
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
log.info("deleting {} from domain_cache".format(domain))
del domain_cache[domain]
resp = await jsonResponse(request, {})
return resp
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
parent_domain = getParentDomain(domain)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "Deletion of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
log.warn("domain not found")
raise HTTPNotFound()
elif ce.code == 410:
log.warn("domain has been removed")
raise HTTPGone()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
aclCheck(domain_json, "delete",
username) # throws exception if not allowed
# check for sub-objects if this is a folder
if "root" not in domain_json:
s3prefix = domain[1:] + '/'
log.info(f"checking kets with prefix: {s3prefix} ")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/')
for s3key in s3keys:
if s3key.endswith("/"):
log.warn(f"attempt to delete folder {domain} with sub-items")
log.debug(f"got prefix: {s3keys[0]}")
raise HTTPConflict(reason="folder has sub-items")
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"domain": domain}
rsp_json = await http_delete(app, req, data=body)
if "root" in domain_json and not keep_root:
# delete the root group
root_id = domain_json["root"]
req = getDataNodeUrl(app, root_id)
req += "/groups/" + root_id
await http_delete(app, req)
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
del domain_cache[domain]
# delete domain cache from other sn_urls
sn_urls = app["sn_urls"]
body["meta_only"] = True
for node_no in sn_urls:
if node_no == app["node_number"]:
continue # don't send to ourselves
sn_url = sn_urls[node_no]
req = sn_url + "/"
log.info("sending sn request: {}".format(req))
try:
sn_rsp = await http_delete(app, req, data=body)
log.info("{} response: {}".format(req, sn_rsp))
except ClientResponseError as ce:
log.warn("got error for sn_delete: {}".format(ce))
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp