async def GET_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug(f"request params: {params.keys()}")
s3path = None
s3offset = 0
s3size = 0
bucket = None
if "s3path" in params:
s3path = params["s3path"]
log.debug(f"GET_Chunk - using s3path: {s3path}")
elif "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
if "dset" in params:
msg = "Unexpected dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
type_json = dset_json["type"]
dims = getChunkLayout(dset_json)
log.debug(f"got dims: {dims}")
rank = len(dims)
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug(f"got selection: {selection}")
dt = createDataType(type_json)
log.debug(f"dtype: {dt}")
rank = len(dims)
if rank == 0:
msg = "No dimension passed to GET chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug(f"selection[{i}]: {s}")
chunk_arr = await getChunk(app,
chunk_id,
dset_json,
bucket=bucket,
s3path=s3path,
s3offset=s3offset,
s3size=s3size)
if chunk_arr is None:
# return a 404
msg = f"Chunk {chunk_id} does not exist"
log.info(msg)
raise HTTPNotFound()
resp = None
if "query" in params:
# do query selection
query = params["query"]
log.info(f"query: {query}")
if rank != 1:
msg = "Query selection only supported for one dimensional arrays"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = 0
if "Limit" in params:
limit = int(params["Limit"])
values = []
indices = []
field_names = []
if dt.fields:
field_names = list(dt.fields.keys())
x = chunk_arr[selection]
log.debug(f"x: {x}")
eval_str = getEvalStr(query, "x", field_names)
log.debug(f"eval_str: {eval_str}")
where_result = np.where(eval(eval_str))
log.debug(f"where_result: {where_result}")
where_result_index = where_result[0]
log.debug(f"whare_result index: {where_result_index}")
log.debug(f"boolean selection: {x[where_result_index]}")
s = selection[0]
count = 0
for index in where_result_index:
log.debug(f"index: {index}")
value = x[index].tolist()
log.debug(f"value: {value}")
json_val = bytesArrayToList(value)
log.debug(f"json_value: {json_val}")
json_index = index.tolist(
) * s.step + s.start # adjust for selection
indices.append(json_index)
values.append(json_val)
count += 1
if limit > 0 and count >= limit:
log.info("got limit items")
break
query_result = {}
query_result["index"] = indices
query_result["value"] = values
log.info(f"query_result retiurning: {len(indices)} rows")
log.debug(f"query_result: {query_result}")
resp = json_response(query_result)
else:
# get requested data
output_arr = chunk_arr[selection]
output_data = arrayToBytes(output_arr)
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def GET_Group(request):
"""HTTP method to return JSON for group"""
log.request(request)
app = request.app
params = request.rel_url.query
h5path = None
getAlias = False
include_links = False
include_attrs = False
group_id = request.match_info.get('id')
if not group_id and "h5path" not in params:
# no id, or path provided, so bad request
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if group_id:
log.info(f"GET_Group, id: {group_id}")
# is the id a group id and not something else?
if not isValidUuid(group_id, "Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
if "h5path" in params:
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute if no parent id is provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Group, h5path: {h5path}")
if "include_links" in params and params["include_links"]:
include_links = True
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
if h5path and h5path[0] == '/':
# ignore the request path id (if given) and start
# from root group for absolute paths
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
if h5path:
group_id = await getObjectIdByPath(app,
group_id,
h5path,
bucket=bucket
) # throws 404 if not found
if not isValidUuid(group_id, "Group"):
msg = f"No group exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get group_id: {group_id} from h5path: {h5path}")
# verify authorization to read the group
await validateAction(app, domain, group_id, username, "read")
# get authoritative state for group from DN (even if it's in the meta_cache).
group_json = await getObjectJson(app,
group_id,
refresh=True,
include_links=include_links,
include_attrs=include_attrs,
bucket=bucket)
log.debug(f"domain from request: {domain}")
group_json["domain"] = getPathForDomain(domain)
if bucket:
group_json["bucket"] = bucket
if getAlias:
root_id = group_json["root"]
alias = []
if group_id == root_id:
alias.append('/')
else:
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=group_id,
bucket=bucket)
if h5path:
alias.append(h5path)
group_json["alias"] = alias
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri)})
hrefs.append({
'rel': 'links',
'href': getHref(request, group_uri + '/links')
})
root_uri = '/groups/' + group_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, group_uri + '/attributes')
})
group_json["hrefs"] = hrefs
resp = await jsonResponse(request, group_json)
log.response(request, resp=resp)
return resp
def sig_handler(sig, frame):
log.warn("Caught signal: {}".format(str(sig)))
print_results()
sys.exit()
async def GET_Links(request):
"""HTTP GET method to return JSON for a link collection
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
log.info(f"GET links: {group_id}")
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
log.info(f"GET_Links - using Limit: {limit}")
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.error(msg) # should be validated by SN
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
log.info(f"GET_Links - using Marker: {marker}")
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
group_json = await get_metadata_obj(app, group_id, bucket=bucket)
log.info(f"for id: {group_id} got group json: {group_json}")
if "links" not in group_json:
msg.error(f"unexpected group data for id: {group_id}")
raise HTTPInternalServerError()
# return a list of links based on sorted dictionary keys
link_dict = group_json["links"]
titles = list(link_dict.keys())
titles.sort() # sort by key
# TBD: provide an option to sort by create date
start_index = 0
if marker is not None:
start_index = index(titles, marker) + 1
if start_index == 0:
# marker not found, return 404
msg = f"Link marker: {marker}, not found"
log.warn(msg)
raise HTTPNotFound()
end_index = len(titles)
if limit is not None and (end_index - start_index) > limit:
end_index = start_index + limit
link_list = []
for i in range(start_index, end_index):
title = titles[i]
link = copy(link_dict[title])
link["title"] = title
link_list.append(link)
resp_json = {"links": link_list}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def healthCheck(app):
""" Periodic method that either registers with headnode (if state in INITIALIZING) or
calls headnode to verify vitals about this node (otherwise)"""
# let the server event loop startup before sarting the health check
await asyncio.sleep(1)
log.info("health check start")
sleep_secs = config.get("node_sleep_time")
while True:
print("node_state:", app["node_state"])
if "oio_proxy" in app:
# for OIO post registration request every time interval
await oio_register(app)
elif "is_k8s" in app:
await k8s_register(app)
elif app["node_state"] == "INITIALIZING" or (
app["node_state"] == "WAITING" and app["node_number"] < 0):
# startup docker registration
await register(app)
else:
# check in with the head node and make sure we are still active
head_url = getHeadUrl(app)
req_node = "{}/nodestate".format(head_url)
log.debug("health check req {}".format(req_node))
try:
rsp_json = await http_get(app, req_node)
if rsp_json is None or not isinstance(rsp_json, dict):
log.warn(
"invalid health check response: type: {} text: {}".
format(type(rsp_json), rsp_json))
else:
log.debug("cluster_state: {}".format(
rsp_json["cluster_state"]))
if rsp_json["cluster_state"] != "READY" and app[
"node_state"] == "READY":
log.info("changing node_state to WAITING")
app["node_state"] = "WAITING"
# save the url's to each of the active nodes'
sn_urls = {}
dn_urls = {}
# or rsp_json["host"] is None or rsp_json["id"] != app["id"]
this_node = None
for node in rsp_json["nodes"]:
if node["node_type"] == app["node_type"] and node[
"node_number"] == app["node_number"]:
# this should be this node
if node["id"] != app["id"]:
# flag - to re-register
log.warn(
"mis-match node ids, app: {} vs head: {} - re-initializing"
.format(node["id"], app["id"]))
app["node_state"] == "INITIALIZING"
app["node_number"] = -1
break
if not node["host"]:
# flag - to re-register
log.warn(
"host not set for this node - re-initializing"
.format(node["id"], app["id"]))
app["node_state"] == "INITIALIZING"
app["node_number"] = -1
break
if not node["host"]:
continue # not online
this_node = copy(node)
url = "http://" + node["host"] + ":" + str(
node["port"])
node_number = node["node_number"]
if node["node_type"] == "dn":
dn_urls[node_number] = url
elif node["node_type"] == "sn":
sn_urls[node_number] = url
else:
log.error(
"Unexpected node_type for node: {}".format(
node))
app["sn_urls"] = sn_urls
log.debug(f"sn_urls: {sn_urls}")
app["dn_urls"] = dn_urls
log.debug(f"dn_urls: {dn_urls}")
if this_node is None and rsp_json[
"cluster_state"] != "READY":
log.warn("this node not found, re-initialize")
app["node_state"] == "INITIALIZING"
app["node_number"] = -1
if app["node_state"] == "WAITING" and rsp_json[
"cluster_state"] == "READY" and app[
"node_number"] >= 0:
log.info(
"setting node_state to READY, node_number: {}".
format(app["node_number"]))
app["node_state"] = "READY"
log.info("health check ok")
except ClientError as ce:
log.warn(f"ClientError: {ce} for health check")
except HTTPInternalServerError as he:
log.warn(
f"HTTPInternalServiceError <{he.code}> for health check")
except HTTPNotFound as hnf:
log.warn(f"HTTPNotFound <{hnf.code}> for health check")
except HTTPGone as hg:
log.warn(f"HTTPGone <{hg.code}> for health heck")
svmem = psutil.virtual_memory()
num_tasks = len(asyncio.Task.all_tasks())
active_tasks = len(
[task for task in asyncio.Task.all_tasks() if not task.done()])
log.debug(
f"health check sleep: {sleep_secs}, vm: {svmem.percent} num tasks: {num_tasks} active tasks: {active_tasks}"
)
await asyncio.sleep(sleep_secs)
async def GET_ACL(request):
"""HTTP method to return JSON for given domain/ACL"""
log.request(request)
app = request.app
acl_username = request.match_info.get('username')
if not acl_username:
msg = "Missing username for ACL"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code in (404, 410):
msg = "domain not found"
log.warn(msg)
raise HTTPNotFound()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
# validate that the requesting user has permission to read ACLs in this domain
if acl_username in (username, "default"):
# allow read access for a users on ACL, or default
aclCheck(domain_json, "read",
username) # throws exception if not authorized
else:
aclCheck(domain_json, "readACL",
username) # throws exception if not authorized
if 'owner' not in domain_json:
log.warn("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.warn("No acls key found in domain")
raise HTTPInternalServerError()
acls = domain_json["acls"]
log.debug("got domain_json: {}".format(domain_json))
if acl_username not in acls:
msg = "acl for username: [{}] not found".format(acl_username)
log.warn(msg)
raise HTTPNotFound()
acl = acls[acl_username]
acl_rsp = {}
for k in acl.keys():
acl_rsp[k] = acl[k]
acl_rsp["userName"] = acl_username
# return just the keys as per the REST API
rsp_json = {}
rsp_json["acl"] = acl_rsp
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')})
if "root" in domain_json:
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + domain_json["root"])
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, '/')})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def healthCheck(app):
""" Periodic method that pings each active node and verifies it is still healthy.
If node doesn't respond, free up the node slot (the node can re-register if it comes back)'"""
app["last_health_check"] = int(time.time())
nodes = app["nodes"]
while True:
# sleep for a bit
sleep_secs = config.get("head_sleep_time")
await asyncio.sleep(sleep_secs)
now = int(time.time())
log.info("health check {}".format(unixTimeToUTC(now)))
fail_count = 0
HEALTH_CHECK_RETRY_COUNT = 1 # times to try before calling a node dead
for node in nodes:
if node["host"] is None:
fail_count += 1
continue
url = getUrl(node["host"], node["port"]) + "/info"
try:
rsp_json = await http_get(app, url)
if "node" not in rsp_json:
log.error("Unexpected response from node")
fail_count += 1
continue
node_state = rsp_json["node"]
node_id = node_state["id"]
if node_id != node['id']:
log.warn("unexpected node_id: {} (expecting: {})".format(node_id, node['id']))
node['host'] = None
node['id'] = None
fail_count += 1
continue
if 'number' in node_state and node_state['number'] != node['node_number']:
msg = "unexpected node_number got {} (expecting: {})"
log.warn(msg.format(node_state["number"], node['node_number']))
node['host'] = None
node['id'] = None
fail_count += 1
continue
# save off other useful info from the node
app_node_stats = app["node_stats"]
node_stats = {}
for k in NODE_STAT_KEYS:
node_stats[k] = rsp_json[k]
app_node_stats[node_id] = node_stats
# mark the last time we got a response from this node
node["healthcheck"] = unixTimeToUTC(int(time.time()))
node["failcount"] = 0 # rest
except OSError as ose:
log.warn("OSError for req: {}: {}".format(url, str(ose)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("node {}:{} not responding".format(node["host"], node["port"]))
fail_count += 1
except HTTPInternalServerError as hpe:
log.warn("HTTPInternalServerError for req: {}: {}".format(url, str(hpe)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("removing {}:{} from active list".format(node["host"], node["port"]))
fail_count += 1
except TimeoutError as toe:
log.warn("Timeout error for req: {}: {}".format(url, str(toe)))
# node has gone away?
node["failcount"] += 1
if node["failcount"] >= HEALTH_CHECK_RETRY_COUNT:
log.warn("removing {}:{} from active list".format(node["host"], node["port"]))
fail_count += 1
log.info("node health check fail_count: {}".format(fail_count))
if fail_count > 0:
if app["cluster_state"] == "READY":
# go back to INITIALIZING state until another node is registered
log.warn("Fail_count > 0, Setting cluster_state from READY to INITIALIZING")
app["cluster_state"] = "INITIALIZING"
elif fail_count == 0 and app["cluster_state"] != "READY":
log.info("All nodes healthy, changing cluster state to READY")
app["cluster_state"] = "READY"
async def put_object(self, key, data, bucket=None):
""" Write data to given key.
Returns client specific dict on success
"""
if not bucket:
log.error("put_object - bucket not set")
raise HTTPInternalServerError()
start_time = time.time()
log.debug(
f"azureBlobClient.put_object({bucket}/{key} start: {start_time}")
try:
async with self._client.get_blob_client(container=bucket,
blob=key) as blob_client:
blob_rsp = await blob_client.upload_blob(data,
blob_type='BlockBlob',
overwrite=True)
finish_time = time.time()
ETag = blob_rsp["etag"]
lastModified = int(blob_rsp["last_modified"].timestamp())
data_size = len(data)
rsp = {
"ETag": ETag,
"size": data_size,
"LastModified": lastModified
}
log.debug(f"put_object {key} returning: {rsp}")
log.info(
f"azureBlobClient.put_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}"
)
except CancelledError as cle:
self._azure_stats_increment("error_count")
msg = f"azureBlobClient.CancelledError for put_object {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
if isinstance(e, AzureError):
if e.status_code == 404:
msg = f"azureBlobClient.key: {key} not found "
log.warn(msg)
raise HTTPNotFound()
elif e.status_code in (401, 403):
msg = f"azureBlobClient.access denied for get key: {key}"
log.info(msg)
raise HTTPForbidden()
else:
self._azure_stats_increment("error_count")
log.error(
f"azureBlobClient.got unexpected AzureError for get_object {key}: {e.message}"
)
raise HTTPInternalServerError()
else:
log.error(
f"azureBlobClient.Unexpected exception for put_object {key}: {e}"
)
raise HTTPInternalServerError()
if data and len(data) > 0:
self._azure_stats_increment("bytes_out", inc=len(data))
log.debug(f"azureBlobClient.put_object {key} complete, rsp: {rsp}")
return rsp
async def list_keys(self,
prefix='',
deliminator='',
suffix='',
include_stats=False,
callback=None,
bucket=None,
limit=None):
""" return keys matching the arguments
"""
if not bucket:
log.error("list_keys - bucket not set")
raise HTTPInternalServerError()
log.info(
f"list_keys('{prefix}','{deliminator}','{suffix}', include_stats={include_stats}"
)
if include_stats:
# use a dictionary to hold return values
key_names = {}
else:
# just use a list
key_names = []
continuation_token = None
page_result_count = 1000 # compatible with what S3 uses by default
if prefix == '':
prefix = None # azure sdk expects None for no prefix
try:
async with self._client.get_container_client(
container=bucket) as container_client:
while True:
log.info(
f"list_blobs: {prefix} continuation_token: {continuation_token}"
)
keyList = container_client.walk_blobs(
name_starts_with=prefix,
delimiter=deliminator,
results_per_page=page_result_count).by_page(
continuation_token)
async for key in await keyList.__anext__():
key_name = key["name"]
if include_stats:
ETag = key["etag"]
lastModified = int(
key["last_modified"].timestamp())
data_size = key["size"]
key_names[key_name] = {
"ETag": ETag,
"Size": data_size,
"LastModified": lastModified
}
else:
key_names.append(key_name)
if callback:
if iscoroutinefunction(callback):
await callback(self._app, key_names)
else:
callback(self._app, key_names)
if not keyList.continuation_token or (
limit and len(key_names) >= limit):
# got all the keys (or as many as requested)
break
else:
# keep going
continuation_token = keyList.continuation_token
except CancelledError as cle:
self._azure_stats_increment("error_count")
msg = f"azureBlobClient.CancelledError for list_keys: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
if isinstance(e, AzureError):
if e.status_code == 404:
msg = f"azureBlobClient not found error for list_keys"
log.warn(msg)
raise HTTPNotFound()
elif e.status_code in (401, 403):
msg = f"azureBlobClient.access denied for list_keys"
log.info(msg)
raise HTTPForbidden()
else:
self._azure_stats_increment("error_count")
log.error(
f"azureBlobClient.got unexpected AzureError for list_keys: {e.message}"
)
raise HTTPInternalServerError()
else:
log.error(
f"azureBlobClient.Unexpected exception for list_keys: {e}")
raise HTTPInternalServerError()
log.info(f"list_keys done, got {len(key_names)} keys")
if limit and len(key_names) > limit:
# return requested number of keys
if include_stats:
keys = list(key_names.keys())
keys.sort()
for k in keys[limit:]:
del key_names[k]
else:
key_names = key_names[:limit]
return key_names
async def PUT_Link(request):
"""HTTP method to create a new link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
log.info(f"PUT Link_title: [{link_title}]")
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Link with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
link_json = {}
if "id" in body:
if not isValidUuid(body["id"]):
msg = "PUT Link with invalid id in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_json["id"] = body["id"]
link_json["class"] = "H5L_TYPE_HARD"
elif "h5path" in body:
link_json["h5path"] = body["h5path"]
# could be hard or soft link
if "h5domain" in body:
link_json["h5domain"] = body["h5domain"]
link_json["class"] = "H5L_TYPE_EXTERNAL"
else:
# soft link
link_json["class"] = "H5L_TYPE_SOFT"
else:
msg = "PUT Link with no id or h5path keys"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "create")
# for hard links, verify that the referenced id exists and is in this domain
if "id" in body:
ref_id = body["id"]
ref_json = await getObjectJson(app, ref_id, bucket=bucket)
group_json = await getObjectJson(app, group_id, bucket=bucket)
if ref_json["root"] != group_json["root"]:
msg = "Hard link must reference an object in the same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# ready to add link now
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("PUT link - getting group: " + req)
params = {}
if bucket:
params["bucket"] = bucket
put_rsp = await http_put(app, req, data=link_json, params=params)
log.debug("PUT Link resp: " + str(put_rsp))
hrefs = [] # TBD
req_rsp = { "hrefs": hrefs }
# link creation successful
resp = await jsonResponse(request, req_rsp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Links(request):
"""HTTP method to return JSON for link collection"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links"
query_sep = '?'
if limit is not None:
req += query_sep + "Limit=" + str(limit)
query_sep = '&'
if marker is not None:
req += query_sep + "Marker=" + marker
log.debug("get LINKS: " + req)
params = {}
if bucket:
params["bucket"] = bucket
links_json = await http_get(app, req, params=params)
log.debug(f"got links json from dn for group_id: {group_id}")
links = links_json["links"]
# mix in collection key, target and hrefs
for link in links:
if link["class"] == "H5L_TYPE_HARD":
collection_name = getCollectionForId(link["id"])
link["collection"] = collection_name
target_uri = '/' + collection_name + '/' + link["id"]
link["target"] = getHref(request, target_uri)
link_uri = '/groups/' + group_id + '/links/' + link['title']
link["href"] = getHref(request, link_uri)
resp_json = {}
resp_json["links"] = links
hrefs = []
group_uri = '/groups/'+group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri+'/links')})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Link(request):
"""HTTP method to return JSON for a group link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("get LINK: " + req)
params = {}
if bucket:
params["bucket"] = bucket
link_json = await http_get(app, req, params=params)
log.debug("got link_json: " + str(link_json))
resp_link = {}
resp_link["title"] = link_title
link_class = link_json["class"]
resp_link["class"] = link_class
if link_class == "H5L_TYPE_HARD":
resp_link["id"] = link_json["id"]
resp_link["collection"] = getCollectionForId(link_json["id"])
elif link_class == "H5L_TYPE_SOFT":
resp_link["h5path"] = link_json["h5path"]
elif link_class == "H5L_TYPE_EXTERNAL":
resp_link["h5path"] = link_json["h5path"]
resp_link["h5domain"] = link_json["h5domain"]
else:
log.warn(f"Unexpected link class: {link_class}")
resp_json = {}
resp_json["link"] = resp_link
resp_json["created"] = link_json["created"]
# links don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = link_json["created"]
hrefs = []
group_uri = '/groups/'+group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri+'/links/'+link_title)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
if link_json["class"] == "H5L_TYPE_HARD":
target = '/' + resp_link["collection"] + '/' + resp_link["id"]
hrefs.append({'rel': 'target', 'href': getHref(request, target)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def getChunk(app,
chunk_id,
dset_json,
bucket=None,
s3path=None,
s3offset=0,
s3size=0,
chunk_init=False):
# if the chunk cache has too many dirty items, wait till items get flushed to S3
MAX_WAIT_TIME = 10.0 # TBD - make this a config
chunk_cache = app['chunk_cache']
if chunk_init and s3offset > 0:
log.error(
f"unable to initiale chunk {chunk_id} for reference layouts ")
raise HTTPInternalServerError()
log.debug(
f"getChunk cache utilization: {chunk_cache.cacheUtilizationPercent} per, dirty_count: {chunk_cache.dirtyCount}, mem_dirty: {chunk_cache.memDirty}"
)
chunk_arr = None
dset_id = getDatasetId(chunk_id)
dims = getChunkLayout(dset_json)
type_json = dset_json["type"]
dt = createDataType(type_json)
# note - officially we should follow the order in which the filters are defined in the filter_list,
# but since we currently have just deflate and shuffle we will always apply deflate then shuffle on read,
# and shuffle then deflate on write
# also note - get deflate and shuffle will update the deflate and shuffle map so that the s3sync will do the right thing
deflate_level = getDeflate(app, dset_id, dset_json)
shuffle = getShuffle(app, dset_id, dset_json)
s3key = None
if s3path:
if not s3path.startswith("s3://"):
# TBD - verify these at dataset creation time?
log.error(f"unexpected s3path for getChunk: {s3path}")
raise HTTPInternalServerError()
path = s3path[5:]
index = path.find('/') # split bucket and key
if index < 1:
log.error(f"s3path is invalid: {s3path}")
raise HTTPInternalServerError()
bucket = path[:index]
s3key = path[(index + 1):]
log.debug(f"Using s3path bucket: {bucket} and s3key: {s3key}")
else:
s3key = getS3Key(chunk_id)
log.debug(f"getChunk chunkid: chunk_id bucket: {bucket}")
if chunk_id in chunk_cache:
chunk_arr = chunk_cache[chunk_id]
else:
if s3path and s3size == 0:
obj_exists = False
else:
obj_exists = await isStorObj(app, s3key, bucket=bucket)
# TBD - potential race condition?
if obj_exists:
pending_s3_read = app["pending_s3_read"]
if chunk_id in pending_s3_read:
# already a read in progress, wait for it to complete
read_start_time = pending_s3_read[chunk_id]
log.info(
f"s3 read request for {chunk_id} was requested at: {read_start_time}"
)
while time.time() - read_start_time < 2.0:
log.debug("waiting for pending s3 read, sleeping")
await asyncio.sleep(1) # sleep for sub-second?
if chunk_id in chunk_cache:
log.info(f"Chunk {chunk_id} has arrived!")
chunk_arr = chunk_cache[chunk_id]
break
if chunk_arr is None:
log.warn(
f"s3 read for chunk {chunk_id} timed-out, initiaiting a new read"
)
if chunk_arr is None:
if chunk_id not in pending_s3_read:
pending_s3_read[chunk_id] = time.time()
log.debug(f"Reading chunk {s3key} from S3")
chunk_bytes = await getStorBytes(app,
s3key,
shuffle=shuffle,
deflate_level=deflate_level,
s3offset=s3offset,
s3size=s3size,
bucket=bucket)
if chunk_id in pending_s3_read:
# read complete - remove from pending map
elapsed_time = time.time() - pending_s3_read[chunk_id]
log.info(f"s3 read for {s3key} took {elapsed_time}")
del pending_s3_read[chunk_id]
else:
log.warn(
f"expected to find {chunk_id} in pending_s3_read map")
chunk_arr = np.fromstring(chunk_bytes, dtype=dt)
chunk_arr = chunk_arr.reshape(dims)
log.debug(f"chunk size: {chunk_arr.size}")
elif chunk_init:
log.debug(f"Initializing chunk {chunk_id}")
fill_value = getFillValue(dset_json)
if fill_value:
# need to convert list to tuples for numpy broadcast
if isinstance(fill_value, list):
fill_value = tuple(fill_value)
chunk_arr = np.empty(dims, dtype=dt, order='C')
chunk_arr[...] = fill_value
else:
chunk_arr = np.zeros(dims, dtype=dt, order='C')
else:
log.debug(f"Chunk {chunk_id} not found")
if chunk_arr is not None:
# check that there's room in the cache before adding it
if chunk_cache.memTarget - chunk_cache.memDirty < chunk_arr.size:
# no room in the cache, wait till space is freed by the s3sync task
wait_start = time.time()
while chunk_cache.memTarget - chunk_cache.memDirty < chunk_arr.size:
log.warn(
f"getChunk, cache utilization: {chunk_cache.cacheUtilizationPercent}, sleeping till items are flushed"
)
if time.time() - wait_start > MAX_WAIT_TIME:
log.error(
f"unable to save updated chunk {chunk_id} to cache returning 503 error"
)
raise HTTPServiceUnavailable()
await asyncio.sleep(1)
chunk_cache[chunk_id] = chunk_arr # store in cache
return chunk_arr
async def POST_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
put_points = False
num_points = 0
if "count" in params:
num_points = int(params["count"])
if "action" in params and params["action"] == "put":
log.info(f"POST Chunk put points, num_points: {num_points}")
put_points = True
else:
log.info("POST Chunk get points")
s3path = None
s3offset = 0
s3size = 0
if "s3path" in params:
if put_points:
log.error("s3path can not be used with put points POST request")
raise HTTPBadRequest()
s3path = params["s3path"]
log.debug(f"GET_Chunk - using s3path: {s3path}")
bucket = None
elif "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"POST chunk_id: {chunk_id}")
chunk_index = getChunkIndex(chunk_id)
log.debug(f"chunk_index: {chunk_index}")
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug(f"request params: {list(params.keys())}")
if "dset" in params:
msg = "Unexpected dset in POST request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
chunk_layout = getChunkLayout(dset_json)
chunk_coord = getChunkCoordinate(chunk_id, chunk_layout)
log.debug(f"chunk_coord: {chunk_coord}")
if not request.has_body:
msg = "POST Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
type_json = dset_json["type"]
dset_dtype = createDataType(type_json)
log.debug(f"dtype: {dset_dtype}")
dims = getChunkLayout(dset_json)
log.debug(f"got dims: {dims}")
rank = len(dims)
if rank == 0:
msg = "POST chunk request with no dimensions"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming points
input_bytes = await request_read(request)
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
# get chunk from cache/s3. If not found init a new chunk if this is a write request
chunk_arr = await getChunk(app,
chunk_id,
dset_json,
bucket=bucket,
s3path=s3path,
s3offset=s3offset,
s3size=s3size,
chunk_init=put_points)
if chunk_arr is None:
if put_points:
log.error("no array returned for put_points")
raise HTTPInternalServerError()
else:
# get points on a non-existent S3 objects?
log.warn("S3 object not found for get points")
raise HTTPNotFound()
log.debug(f"chunk_arr.shape: {chunk_arr.shape}")
if put_points:
# writing point data
# create a numpy array with the following type:
# (coord1, coord2, ...) | dset_dtype
if rank == 1:
coord_type_str = "uint64"
else:
coord_type_str = f"({rank},)uint64"
comp_dtype = np.dtype([("coord", np.dtype(coord_type_str)),
("value", dset_dtype)])
point_arr = np.fromstring(input_bytes, dtype=comp_dtype)
if len(point_arr) != num_points:
msg = f"Unexpected size of point array, got: {len(point_arr)} expected: {num_points}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(num_points):
elem = point_arr[i]
log.debug(f"non-relative coordinate: {elem}")
if rank == 1:
coord = int(elem[0])
coord = coord % chunk_layout[0] # adjust to chunk relative
else:
coord = elem[0] # index to update
for dim in range(rank):
# adjust to chunk relative
coord[dim] = int(coord[dim]) % chunk_layout[dim]
coord = tuple(coord) # need to convert to a tuple
log.debug(f"relative coordinate: {coord}")
val = elem[1] # value
try:
chunk_arr[coord] = val # update the point
except IndexError:
msg = "Out of bounds point index for POST Chunk"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = (now, bucket)
log.info(f"set {chunk_id} to dirty")
else:
# reading point data
point_dt = np.dtype('uint64') # use unsigned long for point index
point_arr = np.fromstring(
input_bytes, dtype=point_dt) # read points as unsigned longs
if len(point_arr) % rank != 0:
msg = "Unexpected size of point array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
num_points = len(point_arr) // rank
log.debug(f"got {num_points} points")
point_arr = point_arr.reshape((num_points, rank))
output_arr = np.zeros((num_points, ), dtype=dset_dtype)
for i in range(num_points):
point = point_arr[i, :]
tr_point = getChunkRelativePoint(chunk_coord, point)
val = chunk_arr[tuple(tr_point)]
output_arr[i] = val
if put_points:
# write empty response
resp = json_response({})
else:
# get data
output_data = output_arr.tobytes()
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def PUT_Domain(request):
"""HTTP method to create a new domain"""
log.request(request)
app = request.app
params = request.rel_url.query
# verify username, password
username, pswd = getUserPasswordFromRequest(
request) # throws exception if user/password is not valid
await validateUserPassword(app, username, pswd)
# inital perms for owner and default
owner_perm = {
'create': True,
'read': True,
'update': True,
'delete': True,
'readACL': True,
'updateACL': True
}
default_perm = {
'create': False,
'read': True,
'update': False,
'delete': False,
'readACL': False,
'updateACL': False
}
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info("PUT domain: {}, username: {}".format(domain, username))
body = None
if request.has_body:
body = await request.json()
log.debug("PUT domain with body: {}".format(body))
if ("flush" in params and params["flush"]) or (body and "flush" in body
and body["flush"]):
# flush domain - update existing domain rather than create a new resource
domain_json = await getDomainJson(app, domain, reload=True)
log.debug("got domain_json: {}".format(domain_json))
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
aclCheck(domain_json, "update",
username) # throws exception if not allowed
if "root" in domain_json:
# nothing to do for folder objects
await doFlush(app, domain_json["root"])
# flush successful
resp = await jsonResponse(request, None, status=204)
log.response(request, resp=resp)
return resp
is_folder = False
owner = username
linked_domain = None
root_id = None
if body and "folder" in body:
if body["folder"]:
is_folder = True
if body and "owner" in body:
owner = body["owner"]
if body and "linked_domain" in body:
if is_folder:
msg = "Folder domains can not be used for links"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
linked_domain = body["linked_domain"]
log.info(f"linking to domain: {linked_domain}")
if owner != username and username != "admin":
log.warn("Only admin users are allowed to set owner for new domains")
raise HTTPForbidden()
parent_domain = getParentDomain(domain)
log.debug("Parent domain: [{}]".format(parent_domain))
if (not parent_domain or parent_domain == '/') and not is_folder:
msg = "Only folder domains can be created at the top-level"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "creation of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
parent_json = None
if parent_domain and parent_domain != '/':
try:
parent_json = await getDomainJson(app, parent_domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = "Parent domain: {} not found".format(parent_domain)
log.warn(msg)
raise HTTPNotFound()
elif ce.code == 410:
msg = "Parent domain: {} removed".format(parent_domain)
log.warn(msg)
raise HTTPGone()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
log.debug("parent_json {}: {}".format(parent_domain, parent_json))
if "root" in parent_json and parent_json["root"]:
msg = "Parent domain must be a folder"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if parent_json:
aclCheck(parent_json, "create",
username) # throws exception if not allowed
if linked_domain:
linked_json = await getDomainJson(app, linked_domain, reload=True)
log.debug(f"got linked json: {linked_json}")
if "root" not in linked_json:
msg = "Folder domains cannot ber used as link target"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
root_id = linked_json["root"]
aclCheck(linked_json, "read", username)
aclCheck(linked_json, "delete", username)
else:
linked_json = None
if not is_folder and not linked_json:
# create a root group for the new domain
root_id = createObjId("roots")
log.debug("new root group id: {}".format(root_id))
group_json = {"id": root_id, "root": root_id, "domain": domain}
log.debug("create group for domain, body: " + json.dumps(group_json))
# create root group
req = getDataNodeUrl(app, root_id) + "/groups"
try:
group_json = await http_post(app, req, data=group_json)
except ClientResponseError as ce:
msg = "Error creating root group for domain -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
else:
log.debug("no root group, creating folder")
domain_json = {}
domain_acls = {}
# owner gets full control
domain_acls[owner] = owner_perm
if config.get("default_public") or is_folder:
# this will make the domain public readable
log.debug("adding default perm for domain: {}".format(domain))
domain_acls["default"] = default_perm
# construct dn request to create new domain
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"owner": owner, "domain": domain}
body["acls"] = domain_acls
if root_id:
body["root"] = root_id
log.debug("creating domain: {} with body: {}".format(domain, body))
try:
domain_json = await http_put(app, req, data=body)
except ClientResponseError as ce:
msg = "Error creating domain state -- " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
# domain creation successful
# maxin limits
domain_json["limits"] = getLimits()
domain_json["version"] = getVersion()
resp = await jsonResponse(request, domain_json, status=201)
log.response(request, resp=resp)
return resp
async def save_metadata_obj(app, obj_id, obj_json, notify=False, flush=False):
""" Persist the given object """
log.info(f"save_metadata_obj {obj_id} notify={notify} flush={flush}")
if notify and not flush:
log.error("notify not valid when flush is false")
raise HTTPInternalServerError()
if not obj_id.startswith('/') and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
if not isinstance(obj_json, dict):
log.error("Passed non-dict obj to save_metadata_obj")
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
dirty_ids = app["dirty_ids"]
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
if isValidUuid(obj_id):
# domain objects may be re-created, but shouldn't see repeats of
# deleted uuids
log.warn("{} has been deleted".format(obj_id))
raise HTTPInternalServerError()
elif obj_id in deleted_ids:
deleted_ids.remove(obj_id) # un-gone the domain id
# update meta cache
meta_cache = app['meta_cache']
log.debug("save: {} to cache".format(obj_id))
meta_cache[obj_id] = obj_json
meta_cache.setDirty(obj_id)
now = int(time.time())
if flush:
# write to S3 immediately
if isValidChunkId(obj_id):
log.warn("flush not supported for save_metadata_obj with chunks")
raise HTTPBadRequest()
try:
await write_s3_obj(app, obj_id)
except KeyError as ke:
log.error(f"s3 sync got key error: {ke}")
raise HTTPInternalServerError()
except HTTPInternalServerError:
log.warn(f" failed to write {obj_id}")
raise # re-throw
if obj_id in dirty_ids:
log.warn(
f"save_metadata_obj flush - object {obj_id} is still dirty")
else:
# flag to write to S3
dirty_ids[obj_id] = now
# message AN immediately if notify flag is set
# otherwise AN will be notified at next S3 sync
if notify:
an_url = getAsyncNodeUrl(app)
if obj_id.startswith("/"):
# domain update
req = an_url + "/domain"
params = {"domain": obj_id}
if "root" in obj_json:
params["root"] = obj_json["root"]
if "owner" in obj_json:
params["owner"] = obj_json["owner"]
try:
log.info("ASync PUT notify: {} params: {}".format(req, params))
await http_put(app, req, params=params)
except HTTPInternalServerError as hpe:
log.error(f"got error notifying async node: {hpe}")
log.error(msg)
else:
req = an_url + "/object/" + obj_id
try:
log.info("ASync PUT notify: {}".format(req))
await http_put(app, req)
except HTTPInternalServerError:
log.error(f"got error notifying async node")
async def DELETE_Domain(request):
"""HTTP method to delete a domain resource"""
log.request(request)
app = request.app
params = request.rel_url.query
domain = None
meta_only = False # if True, just delete the meta cache value
keep_root = False
if request.has_body:
body = await request.json()
if "domain" in body:
domain = body["domain"]
else:
msg = "No domain in request body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "meta_only" in body:
meta_only = body["meta_only"]
if "keep_root" in body:
keep_root = body["keep_root"]
else:
# get domain from request uri
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "keep_root" in params:
keep_root = params["keep_root"]
log.info("meta_only domain delete: {}".format(meta_only))
if meta_only:
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
log.info("deleting {} from domain_cache".format(domain))
del domain_cache[domain]
resp = await jsonResponse(request, {})
return resp
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
parent_domain = getParentDomain(domain)
if (not parent_domain or parent_domain == '/') and username != "admin":
msg = "Deletion of top-level domains is only supported by admin users"
log.warn(msg)
raise HTTPForbidden()
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
log.warn("domain not found")
raise HTTPNotFound()
elif ce.code == 410:
log.warn("domain has been removed")
raise HTTPGone()
else:
log.error(f"unexpected error: {ce.code}")
raise HTTPInternalServerError()
aclCheck(domain_json, "delete",
username) # throws exception if not allowed
# check for sub-objects if this is a folder
if "root" not in domain_json:
s3prefix = domain[1:] + '/'
log.info(f"checking kets with prefix: {s3prefix} ")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/')
for s3key in s3keys:
if s3key.endswith("/"):
log.warn(f"attempt to delete folder {domain} with sub-items")
log.debug(f"got prefix: {s3keys[0]}")
raise HTTPConflict(reason="folder has sub-items")
req = getDataNodeUrl(app, domain)
req += "/domains"
body = {"domain": domain}
rsp_json = await http_delete(app, req, data=body)
if "root" in domain_json and not keep_root:
# delete the root group
root_id = domain_json["root"]
req = getDataNodeUrl(app, root_id)
req += "/groups/" + root_id
await http_delete(app, req)
# remove from domain cache if present
domain_cache = app["domain_cache"]
if domain in domain_cache:
del domain_cache[domain]
# delete domain cache from other sn_urls
sn_urls = app["sn_urls"]
body["meta_only"] = True
for node_no in sn_urls:
if node_no == app["node_number"]:
continue # don't send to ourselves
sn_url = sn_urls[node_no]
req = sn_url + "/"
log.info("sending sn request: {}".format(req))
try:
sn_rsp = await http_delete(app, req, data=body)
log.info("{} response: {}".format(req, sn_rsp))
except ClientResponseError as ce:
log.warn("got error for sn_delete: {}".format(ce))
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def delete_metadata_obj(app, obj_id, notify=True, root_id=None):
""" Delete the given object """
meta_cache = app['meta_cache']
dirty_ids = app["dirty_ids"]
log.info("delete_meta_data_obj: {} notify: {}".format(obj_id, notify))
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error(f"obj: {obj_id} not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
log.warn("{} has already been deleted".format(obj_id))
else:
deleted_ids.add(obj_id)
if obj_id in meta_cache:
log.debug(f"removing {obj_id} from meta_cache")
del meta_cache[obj_id]
if obj_id in dirty_ids:
del dirty_ids[obj_id]
# remove from S3 (if present)
s3key = getS3Key(obj_id)
if await isS3Obj(app, s3key):
await deleteS3Obj(app, s3key)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
if notify:
an_url = getAsyncNodeUrl(app)
if obj_id.startswith("/"):
# domain delete
req = an_url + "/domain"
params = {"domain": obj_id}
try:
log.info("ASync DELETE notify: {} params: {}".format(
req, params))
await http_delete(app, req, params=params)
except ClientError as ce:
log.error(f"got error notifying async node: {ce}")
except HTTPInternalServerError as hse:
log.error(f"got HTTPInternalServerError: {hse}")
else:
req = an_url + "/object/" + obj_id
try:
log.info(f"ASync DELETE notify: {req}")
await http_delete(app, req)
except ClientError as ce:
log.error(f"got ClientError notifying async node: {ce}")
except HTTPInternalServerError as ise:
log.error(
f"got HTTPInternalServerError notifying async node: {ise}")
log.debug(f"delete_metadata_obj for {obj_id} done")
async def GET_ACLs(request):
"""HTTP method to return JSON for domain/ACLs"""
log.request(request)
app = request.app
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(message=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError:
log.warn("domain not found")
log.warn(msg)
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
acls = domain_json["acls"]
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "readACL",
username) # throws exception if not authorized
acl_list = []
acl_usernames = list(acls.keys())
acl_usernames.sort()
for acl_username in acl_usernames:
entry = {"userName": acl_username}
acl = acls[acl_username]
for k in acl.keys():
entry[k] = acl[k]
acl_list.append(entry)
# return just the keys as per the REST API
rsp_json = {}
rsp_json["acls"] = acl_list
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')})
if "root" in domain_json:
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + domain_json["root"])
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, '/')})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def write_s3_obj(app, obj_id):
""" writes the given object to s3 """
s3key = getS3Key(obj_id)
log.info(f"write_s3_obj for obj_id: {obj_id} / s3_key: {s3key}")
pending_s3_write = app["pending_s3_write"]
pending_s3_write_tasks = app["pending_s3_write_tasks"]
dirty_ids = app["dirty_ids"]
chunk_cache = app['chunk_cache']
meta_cache = app['meta_cache']
deflate_map = app['deflate_map']
notify_objs = app["an_notify_objs"]
deleted_ids = app['deleted_ids']
success = False
if s3key in pending_s3_write:
msg = f"write_s3_key - not expected for key {s3key} to be in pending_s3_write map"
log.error(msg)
raise KeyError(msg)
if obj_id not in pending_s3_write_tasks:
# don't allow reentrant write
log.debug(f"write_s3_obj for {obj_id} not s3sync task")
if obj_id in deleted_ids and isValidUuid(obj_id):
# if this objid has been deleted (and its unique since this is not a domain id)
# cancel any pending task and return
log.warn(f"Canceling wrfite for {obj_id} since it has been deleted")
if obj_id in pending_s3_write_tasks:
log.info(f"removing pending s3 write task for {obj_id}")
task = pending_s3_write_tasks[obj_id]
task.cancel()
del pending_s3_write_tasks[obj_id]
return None
now = time.time()
last_update_time = now
if obj_id in dirty_ids:
last_update_time = dirty_ids[obj_id]
if last_update_time > now:
msg = f"last_update time {last_update_time} is in the future for obj_id: {obj_id}"
log.error(msg)
raise ValueError(msg)
pending_s3_write[s3key] = now
# do the following in the try block so we can always remove the pending_s3_write at the end
try:
if isValidChunkId(obj_id):
if obj_id not in chunk_cache:
log.error("expected to find obj_id: {} in chunk cache".format(
obj_id))
raise KeyError(f"{obj_id} not found in chunk cache")
if not chunk_cache.isDirty(obj_id):
log.error(f"expected chunk cache obj {obj_id} to be dirty")
raise ValueError("bad dirty state for obj")
chunk_arr = chunk_cache[obj_id]
chunk_bytes = arrayToBytes(chunk_arr)
dset_id = getDatasetId(obj_id)
deflate_level = None
if dset_id in deflate_map:
deflate_level = deflate_map[dset_id]
log.debug("got deflate_level: {} for dset: {}".format(
deflate_level, dset_id))
await putS3Bytes(app,
s3key,
chunk_bytes,
deflate_level=deflate_level)
success = True
# if chunk has been evicted from cache something has gone wrong
if obj_id not in chunk_cache:
msg = f"expected to find {obj_id} in chunk_cache"
log.error(msg)
elif obj_id in dirty_ids and dirty_ids[obj_id] > last_update_time:
log.info(
f"write_s3_obj {obj_id} got updated while s3 write was in progress"
)
else:
# no new write, can clear dirty
chunk_cache.clearDirty(obj_id) # allow eviction from cache
log.debug(
"putS3Bytes Chunk cache utilization: {} per, dirty_count: {}"
.format(chunk_cache.cacheUtilizationPercent,
chunk_cache.dirtyCount))
else:
# meta data update
# check for object in meta cache
if obj_id not in meta_cache:
log.error(
"expected to find obj_id: {} in meta cache".format(obj_id))
raise KeyError(f"{obj_id} not found in chunk cache")
if not meta_cache.isDirty(obj_id):
log.error(f"expected meta cache obj {obj_id} to be dirty")
raise ValueError("bad dirty state for obj")
obj_json = meta_cache[obj_id]
await putS3JSONObj(app, s3key, obj_json)
success = True
# should still be in meta_cache...
if obj_id not in meta_cache:
msg = f"expected to find {obj_id} in meta_cache"
log.error(msg)
elif obj_id in dirty_ids and dirty_ids[obj_id] > last_update_time:
log.info(
f"write_s3_obj {obj_id} got updated while s3 write was in progress"
)
else:
meta_cache.clearDirty(obj_id) # allow eviction from cache
finally:
# clear pending_s3_write item
log.debug(f"write_s3_obj finally block, success={success}")
if s3key not in pending_s3_write:
msg = f"write s3 obj: Expected to find {s3key} in pending_s3_write map"
log.error(msg)
else:
if pending_s3_write[s3key] != now:
msg = f"pending_s3_write timestamp got updated unexpectedly for {s3key}"
log.error(msg)
del pending_s3_write[s3key]
# clear task
if obj_id not in pending_s3_write_tasks:
log.debug(f"no pending s3 write task for {obj_id}")
else:
log.debug(f"removing pending s3 write task for {obj_id}")
del pending_s3_write_tasks[obj_id]
# clear dirty flag
if obj_id in dirty_ids and dirty_ids[obj_id] == last_update_time:
log.debug(f"clearing dirty flag for {obj_id}")
del dirty_ids[obj_id]
# add to set so that AN can be notified about changed objects
notify_objs.add(obj_id)
# calculate time to do the write
elapsed_time = time.time() - now
log.info(f"s3 write for {s3key} took {elapsed_time:.3f}s")
return obj_id
async def PUT_Link(request):
""" Handler creating a new link"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
log.info(f"PUT link: {group_id}")
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
link_title = request.match_info.get('title')
validateLinkName(link_title)
log.info(f"link_title: {link_title}")
if not request.has_body:
msg = "PUT Link with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
if "class" not in body:
msg = "PUT Link with no class key body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_class = body["class"]
link_json = {}
link_json["class"] = link_class
if "id" in body:
link_json["id"] = body["id"]
if "h5path" in body:
link_json["h5path"] = body["h5path"]
if "h5domain" in body:
link_json["h5domain"] = body["h5domain"]
if "bucket" in params:
bucket = params["bucket"]
elif "bucket" in body:
bucket = params["bucket"]
else:
bucket = None
group_json = await get_metadata_obj(app, group_id, bucket=bucket)
if "links" not in group_json:
log.error(f"unexpected group data for id: {group_id}")
raise HTTPInternalServerError()
links = group_json["links"]
if link_title in links:
msg = f"Link name {link_title} already found in group: {group_id}"
log.warn(msg)
raise HTTPConflict()
now = time.time()
link_json["created"] = now
# add the link
links[link_title] = link_json
# update the group lastModified
group_json["lastModified"] = now
# write back to S3, save to metadata cache
await save_metadata_obj(app, group_id, group_json, bucket=bucket)
resp_json = { }
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def s3sync(app):
""" Periodic method that writes dirty objects in the metadata cache to S3"""
MAX_PENDING_WRITE_REQUESTS = 20
dirty_ids = app["dirty_ids"]
pending_s3_write = app["pending_s3_write"]
pending_s3_write_tasks = app["pending_s3_write_tasks"]
s3_sync_interval = config.get("s3_sync_interval")
dirty_count = len(dirty_ids)
if not dirty_count:
log.info("s3sync nothing to update")
return 0
log.info(
f"s3sync update - dirtyid count: {dirty_count}, active write tasks: {len(pending_s3_write_tasks)}/{MAX_PENDING_WRITE_REQUESTS}"
)
log.debug(f"s3sync dirty_ids: {dirty_ids}")
log.debug(f"sesync pending write s3keys: {list(pending_s3_write.keys())}")
log.debug(f"s3sync write tasks: {list(pending_s3_write_tasks.keys())}")
def callback(future):
try:
obj_id = future.result() # returns a objid
log.info(f"write_s3_obj callback result: {obj_id}")
except HTTPInternalServerError as hse:
log.error(f"write_s3_obj callback got 500: {hse}")
except Exception as e:
log.error(f"write_s3_obj callback unexpected exception: {e}")
update_count = 0
s3sync_start = time.time()
for obj_id in dirty_ids:
s3key = getS3Key(obj_id)
log.debug(f"s3sync dirty id: {obj_id}, s3key: {s3key}")
create_task = True
if s3key in pending_s3_write:
log.debug(
f"key {s3key} has been pending for {s3sync_start - pending_s3_write[s3key]}"
)
if s3sync_start - pending_s3_write[s3key] > s3_sync_interval * 2:
log.warn(
f"obj {obj_id} has been in pending_s3_write for {s3sync_start - pending_s3_write[s3key]} seconds, restarting"
)
del pending_s3_write[s3key]
if obj_id not in pending_s3_write_tasks:
log.error(f"Expected to find write task for {obj_id}")
else:
task = pending_s3_write_tasks[obj_id]
task.cancel()
del pending_s3_write_tasks[obj_id]
else:
log.debug(f"key {s3key} has a pending write task")
create_task = False
if obj_id not in pending_s3_write_tasks:
log.error(
f"expected to find {obj_id} in pending_s3_write_tasks")
if create_task and len(
pending_s3_write_tasks) < MAX_PENDING_WRITE_REQUESTS:
# create a task to write this object
log.debug(f"s3sync - ensure future for {obj_id}")
task = asyncio.ensure_future(write_s3_obj(app, obj_id))
task.add_done_callback(callback)
pending_s3_write_tasks[obj_id] = task
update_count += 1
# notify AN of key updates
an_url = getAsyncNodeUrl(app)
notify_objs = app["an_notify_objs"]
if len(notify_objs) > 0:
log.info(f"Notifying AN for {len(notify_objs)} S3 Updates")
body = {"objs": list(notify_objs)}
notify_objs.clear()
req = an_url + "/objects"
try:
log.info("ASync PUT notify: {} body: {}".format(req, body))
await http_put(app, req, data=body)
except HTTPInternalServerError as hpe:
msg = "got error notifying async node: {}".format(hpe)
log.error(msg)
# return number of objects written
return update_count
async def oio_register(app):
""" register with oio conscience
"""
log.info("oio_register")
oio_proxy = app["oio_proxy"]
host_ip = app["host_ip"]
if not host_ip:
log.error("host ip not set")
return
node_type = app["node_type"]
if node_type not in ("sn", "dn"):
log.error("unexpected node type")
return
service_name = "hdf" + node_type
req = oio_proxy + "/v3.0/OPENIO/conscience/register"
body = {
"addr": host_ip + ":" + str(app["node_port"]),
"tags": {
"stat.cpu": 100,
"tag.up": True
},
"type": service_name
}
log.debug(f"conscience register: body: {body}")
try:
await http_post(app, req, data=body)
except ClientError as client_exception:
log.error(
f"got ClientError registering with oio_proxy: {client_exception}")
return
except CancelledError as cancelled_exception:
log.error(
f"got CanceeledError registering with oio_proxy: {cancelled_exception}"
)
return
log.info("oio registration successful")
# get list of DN containers
req = oio_proxy + "/v3.0/OPENIO/conscience/list?type=hdfdn"
try:
dn_node_list = await http_get(app, req)
except ClientError as client_exception:
log.error(
f"got ClientError listing dn nodes with oio_proxy: {client_exception}"
)
return
except CancelledError as cancelled_exception:
log.error(
f"got CanceeledError listing dn nodes with oio_proxy: {cancelled_exception}"
)
return
log.info(f"got {len(dn_node_list)} conscience list items")
# create map keyed by dn addr
dn_node_map = {}
for dn_node in dn_node_list:
log.debug(f"checking dn conscience list item: {dn_node}")
if "addr" not in dn_node:
log.warn(f"conscience list item with no addr: {dn_node}")
continue
addr = dn_node["addr"]
if "score" not in dn_node:
log.warn(f'conscience list item with no score key: {dn_node}')
continue
if dn_node["score"] <= 0:
log.debug(f"zero score - skipping conscience list addr: {addr}")
continue
if addr in dn_node_map:
# shouldn't ever get this?
log.warn(f"duplicate entry for node: {dn_node}")
continue
# send an info request to the node
info_rsp = await get_info(app, "http://" + addr)
if not info_rsp:
# timeout or other failure
continue
if "node" not in info_rsp:
log.error("expecteed to find node key in info resp")
continue
info_node = info_rsp["node"]
log.debug(f"got info resp: {info_node}")
for key in ("type", "id", "node_number", "node_count"):
if key not in info_node:
log.error(
f"unexpected node type in node state, expected to find key: {key}"
)
continue
if info_node["type"] != "dn":
log.error(f"expecteed node_type to be dn")
continue
# mix in node id, node number, node_count to the conscience info
dn_node["node_id"] = info_node["id"]
dn_node["node_number"] = info_node["node_number"]
dn_node["node_count"] = info_node["node_count"]
dn_node_map[addr] = dn_node
log.info(f"done with dn_node_list, got: {len(dn_node_map)} active nodes")
if len(dn_node_map) == 0:
if app["node_state"] != "INITIALIZING":
log.info(
"no active DN nodes, setting cluster state to INITIALIZING")
app["node_state"] = "INITIALIZING"
return
# sort map by address
addrs = list(dn_node_map.keys())
addrs.sort()
# check that node number is set and is the expected value for each node key
invalid_count = 0
node_index = 0
node_count = len(addrs)
dn_urls = {}
this_node_found = False
this_node_id = app["id"]
for addr in addrs:
dn_node = dn_node_map[addr]
log.debug(f"dn_node for index {node_index}: {dn_node}")
node_id = dn_node["node_id"]
if node_id == this_node_id:
this_node_found = True
node_number = dn_node["node_number"]
dn_urls[node_number] = "http://" + dn_node["addr"]
if node_index != node_number or dn_node["node_count"] != node_count:
if node_number == -1:
log.info(f"node {node_index} not yet initialized")
elif node_index != node_number:
log.warn(
f"node_id {node_id}, expected node_number of {node_index} but found {node_number}"
)
invalid_count += 1
if node_id == app["id"]:
# this is us, update our node_number, node_count
if app["node_number"] != node_index:
# TBD - clean cache items
log.info(
f"setting node_number for this node to: {node_index}")
app["node_number"] = node_index
if app["node_count"] != node_count:
# TBD - clean cache items
log.info(
f"setting node_count for this node to: {node_count}")
app["node_count"] = node_count
invalid_count += 1
else:
log.debug(f"node {node_id} node number is correct")
node_index += 1
if invalid_count == 0:
log.debug("no invalid nodes!")
if app["node_state"] != "READY":
if app["node_type"] == "dn" and not this_node_found:
# don't go to READY unless this node shows up
log.info(
f"node {this_node_id} not yet showing in proxy list, stay in INITIALIZING"
)
else:
log.info("setting node state to READY")
app["node_state"] = "READY"
if app["node_type"] == "sn" and app["node_number"] == -1:
# node number shouldn't matter for SN nodes, so set to 1
app["node_number"] = 1
if app["node_count"] != node_count:
log.info(f"setting node_count to: {node_count}")
app["node_count"] = node_count
app["dn_urls"] = dn_urls
else:
log.debug(f"number invalid nodes: {invalid_count}")
if app["node_state"] == "READY":
log.warn("invalid nodes found, setting node state to INITIALIZING")
app["node_state"] = "INITIALIZING"
log.info("oio_register done")
async def get_metadata_obj(app, obj_id):
""" Get object from metadata cache (if present).
Otherwise fetch from S3 and add to cache
"""
log.info("get_metadata_obj: {}".format(obj_id))
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = "{} has been deleted".format(obj_id)
log.warn(msg)
raise HTTPGone()
meta_cache = app['meta_cache']
obj_json = None
if obj_id in meta_cache:
log.debug("{} found in meta cache".format(obj_id))
obj_json = meta_cache[obj_id]
else:
s3_key = getS3Key(obj_id)
pending_s3_read = app["pending_s3_read"]
if obj_id in pending_s3_read:
# already a read in progress, wait for it to complete
read_start_time = pending_s3_read[obj_id]
log.info(
f"s3 read request for {s3_key} was requested at: {read_start_time}"
)
while time.time() - read_start_time < 2.0:
log.debug("waiting for pending s3 read, sleeping")
await asyncio.sleep(1) # sleep for sub-second?
if obj_id in meta_cache:
log.info(f"object {obj_id} has arrived!")
obj_json = meta_cache[obj_id]
break
if not obj_json:
log.warn(
f"s3 read for object {s3_key} timed-out, initiaiting a new read"
)
# invoke S3 read unless the object has just come in from pending read
if not obj_json:
log.debug("getS3JSONObj({})".format(s3_key))
if obj_id not in pending_s3_read:
pending_s3_read[obj_id] = time.time()
# read S3 object as JSON
obj_json = await getS3JSONObj(app, s3_key)
if obj_id in pending_s3_read:
# read complete - remove from pending map
elapsed_time = time.time() - pending_s3_read[obj_id]
log.info(f"s3 read for {s3_key} took {elapsed_time}")
del pending_s3_read[obj_id]
meta_cache[obj_id] = obj_json # add to cache
return obj_json
async def POST_Group(request):
"""HTTP method to create new Group object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if request.has_body:
body = await request.json()
log.info(f"POST Group body: {body}")
if body:
if "link" in body:
link_body = body["link"]
log.debug(f"link_body: {link_body}")
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username,
"create")
if not link_id or not link_title:
log.warn(f"POST Group body with no link: {body}")
domain_json = await getDomainJson(
app, domain) # get again in case cache was invalidated
root_id = domain_json["root"]
group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
log.debug("create group, body: " + json.dumps(group_json))
req = getDataNodeUrl(app, group_id) + "/groups"
params = {}
if bucket:
params["bucket"] = bucket
group_json = await http_post(app, req, data=group_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = group_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_json_rsp = await http_put(app,
link_req,
data=link_json,
params=params)
log.debug(f"PUT Link resp: {put_json_rsp}")
log.debug("returning resp")
# group creation successful
resp = await jsonResponse(request, group_json, status=201)
log.response(request, resp=resp)
return resp
async def get_domains(request):
""" This method is called by GET_Domains and GET_Domain """
app = request.app
# if there is no domain passed in, get a list of top level domains
if "domain" not in request.rel_url.query:
prefix = '/'
else:
prefix = request.rel_url.query["domain"]
log.info(f"get_domains for: {prefix}")
if not prefix.startswith('/'):
msg = "Prefix must start with '/'"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# always use "verbose" to pull extra info
if "verbose" in request.rel_url.query and request.rel_url.query["verbose"]:
verbose = True
else:
verbose = False
limit = None
if "Limit" in request.rel_url.query:
try:
limit = int(request.rel_url.query["Limit"])
log.debug(f"GET_Domains - using Limit: {limit}")
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in request.rel_url.query:
marker = request.rel_url.query["Marker"]
log.debug(f"got Marker request param: {marker}")
# list the S3 keys for this prefix
domainNames = []
if prefix == "/":
# search for domains from the top_level domains config
domainNames = config.get("top_level_domains")
else:
s3prefix = prefix[1:]
log.debug(f"listing S3 keys for {s3prefix}")
s3keys = await getS3Keys(app,
include_stats=False,
prefix=s3prefix,
deliminator='/')
log.debug(f"getS3Keys returned: {len(s3keys)} keys")
log.debug(f"s3keys {s3keys}")
for s3key in s3keys:
if s3key[-1] != '/':
log.debug(f"ignoring key: {s3key}")
continue
log.debug(f"got s3key: {s3key}")
domain = "/" + s3key[:-1]
if marker:
if marker == domain:
marker = None
continue
log.debug(f"adding domain: {domain} to domain list")
domainNames.append(domain)
if limit and len(domainNames) == limit:
# got to requested limit
break
# get domain info for each domain
domains = []
for domain in domainNames:
try:
# query DN's for domain json
# TBD - multicast to DN nodes
log.debug(f"getDomainJson for {domain}")
domain_json = await getDomainJson(app, domain, reload=True)
if domain_json:
domain_rsp = await get_domain_response(app,
domain_json,
verbose=verbose)
# mixin domain anme
domain_rsp["name"] = domain
domains.append(domain_rsp)
except HTTPNotFound:
# One of the dmains not found, but continue through the list
log.debug(f"not found error for: {domain}")
return domains
async def import_line(line):
domain = globals["domain"]
params = {"host": domain}
headers = getRequestHeaders()
client = globals["client"]
globals["lines_read"] += 1
task_log = {}
task_log["line"] = line
task_log["start"] = time.time()
fields = line.split(',')
if len(fields) != 8:
log.warn("unexpected number of fields in line: [()]".format(line))
return
station = fields[0]
if len(station) != 11:
log.warn("unexpected station length line: [()]".format(line))
return
date = fields[1]
if len(date) != 8:
log.warn("unexpected station length line: [()]".format(line))
return
obstype = fields[2]
if len(obstype) != 4:
log.warn("unexpected obstype length line: [()]".format(line))
return
value = 0
try:
value = int(fields[3])
except ValueError:
log.warn("unexpected value in line: [()]".format(line))
return
# TBD - do something with other fields
log.info("data: {} {} {} {}".format(station, obstype, date, value))
h5path = "/data/" + station + "/" + obstype
task_log["h5path"] = h5path
task_log["state"] = "INPROGRESS"
globals["tasks"].append(task_log) # add before the first await
try:
grp_id = await verifyGroupPath(h5path)
except HttpProcessingError as hpe:
log.error("failed to verifyGroupPath: {}, err: {}".format(h5path, str(hpe)))
globals["failed_line_updates"] += 1
task_log["state"] = "COMPLETE"
return
# create the attribute
data = {'type': 'H5T_STD_I32LE', 'value': value}
req = getEndpoint() + "/groups/" + grp_id + "/attributes/" + date
log.info("PUT " + req)
globals["request_count"] += 1
task_log["req"] = req
timeout = config.get("timeout")
async with client.put(req, headers=headers, data=json.dumps(data), params=params, timeout=timeout) as rsp:
task_log["stop"] = time.time()
task_log["state"] = "COMPLETE"
task_log["status"] = rsp.status
if rsp.status == 409:
log.warn("409 for req: " + req)
elif rsp.status != 201:
log.error("got status: {} for req: {}".format(rsp.status, req))
globals["failed_line_updates"] += 1
else:
globals["attribute_count"] += 1
async def GET_Domain(request):
"""HTTP method to return JSON for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = None
try:
domain = getDomainFromRequest(request)
except ValueError:
log.warn("Invalid domain")
raise HTTPBadRequest(reason="Invalid domain name")
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if not domain:
log.info("no domain passed in, returning all top-level domains")
# no domain passed in, return top-level domains for this request
domains = await get_domains(request)
rsp_json = {"domains": domains}
rsp_json["hrefs"] = []
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
log.info("got domain: {}".format(domain))
domain_json = await getDomainJson(app, domain, reload=True)
if domain_json is None:
log.warn("domain: {} not found".format(domain))
raise HTTPNotFound()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
if "h5path" in params:
# if h5path is passed in, return object info for that path
# (if exists)
h5path = params["h5path"]
root_id = domain_json["root"]
obj_id = await getObjectIdByPath(app, root_id,
h5path) # throws 404 if not found
log.info("get obj_id: {} from h5path: {}".format(obj_id, h5path))
# get authoritative state for object from DN (even if it's in the meta_cache).
obj_json = await getObjectJson(app, obj_id, refresh=True)
obj_json["domain"] = domain
# Not bothering with hrefs for h5path lookups...
resp = await jsonResponse(request, obj_json)
log.response(request, resp=resp)
return resp
# return just the keys as per the REST API
rsp_json = await get_domain_response(app, domain_json, verbose=verbose)
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'database',
'href': getHref(request, '/datasets')
})
hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')})
hrefs.append({
'rel': 'typebase',
'href': getHref(request, '/datatypes')
})
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')})
parent_domain = getParentDomain(domain)
log.debug("href parent domain: {}".format(parent_domain))
if parent_domain:
hrefs.append({
'rel': 'parent',
'href': getHref(request, '/', domain=parent_domain)
})
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def GET_Groups(request):
"""HTTP method to return groups collection for given domain"""
log.request(request)
app = request.app
params = request.rel_url.query
(username, pswd) = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
try:
domain = getDomainFromRequest(request)
except ValueError:
msg = "Invalid domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# use reload to get authoritative domain json
try:
domain_json = await getDomainJson(app, domain, reload=True)
except ClientResponseError as ce:
if ce.code == 404:
msg = "domain not found"
log.warn(msg)
raise HTTPNotFound()
else:
log.error(f"Unexpected error: {ce.code}")
raise HTTPInternalServerError()
if 'owner' not in domain_json:
log.error("No owner key found in domain")
raise HTTPInternalServerError()
if 'acls' not in domain_json:
log.error("No acls key found in domain")
raise HTTPInternalServerError()
log.debug("got domain_json: {}".format(domain_json))
# validate that the requesting user has permission to read this domain
aclCheck(domain_json, "read",
username) # throws exception if not authorized
# get the groups collection list
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
obj_ids = []
if "root" in domain_json or domain_json["root"]:
# get the groups collection list
collections = await get_collections(app, domain_json["root"])
objs = collections["groups"]
obj_ids = getIdList(objs, marker=marker, limit=limit)
# create hrefs
hrefs = []
hrefs.append({'rel': 'self', 'href': getHref(request, '/groups')})
if "root" in domain_json:
root_uuid = domain_json["root"]
hrefs.append({
'rel': 'root',
'href': getHref(request, '/groups/' + root_uuid)
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
# return obj ids and hrefs
rsp_json = {}
rsp_json["groups"] = obj_ids
rsp_json["hrefs"] = hrefs
resp = await jsonResponse(request, rsp_json)
log.response(request, resp=resp)
return resp
async def PUT_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
query = None
if "query" in params:
query = params["query"]
log.info(f"PUT_Chunk query: {query}")
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "PUT Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "bucket" in params:
bucket = params["bucket"]
log.debug(f"PUT_Chunk using bucket: {bucket}")
else:
bucket = None
if query:
expected_content_type = "text/plain; charset=utf-8"
else:
expected_content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != expected_content_type:
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
if "dset" in params:
msg = "Unexpected param dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.debug(f"PUT_Chunk - id: {chunk_id}")
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
dims = getChunkLayout(dset_json)
if "root" not in dset_json:
msg = "expected root key in dset_json"
log.error(msg)
raise KeyError(msg)
rank = len(dims)
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug(f"got selection: {selection}")
type_json = dset_json["type"]
itemsize = 'H5T_VARIABLE'
if "size" in type_json:
itemsize = type_json["size"]
dt = createDataType(type_json)
log.debug(f"dtype: {dt}")
if rank == 0:
msg = "No dimension passed to PUT chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug(f"selection[{i}]: {s}")
mshape = getSelectionShape(selection)
log.debug(f"mshape: {mshape}")
num_elements = 1
for extent in mshape:
num_elements *= extent
resp = {}
query_update = None
limit = 0
chunk_init = True
input_arr = None
if query:
if not dt.fields:
log.error("expected compound dtype for PUT query")
raise HTTPInternalServerError()
query_update = await request.json()
log.debug(f"query_update: {query_update}")
if "Limit" in params:
limit = int(params["Limit"])
chunk_init = False
else:
# regular chunk update
# check that the content_length is what we expect
if itemsize != 'H5T_VARIABLE':
log.debug(f"expect content_length: {num_elements*itemsize}")
log.debug(f"actual content_length: {request.content_length}")
if itemsize != 'H5T_VARIABLE' and (num_elements *
itemsize) != request.content_length:
msg = f"Expected content_length of: {num_elements*itemsize}, but got: {request.content_length}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming data
input_bytes = await request_read(
request
) # TBD - will it cause problems when failures are raised before reading data?
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
input_arr = bytesToArray(input_bytes, dt, mshape)
# TBD: Skip read if the input shape is the entire chunk?
chunk_arr = await getChunk(app,
chunk_id,
dset_json,
chunk_init=chunk_init,
bucket=bucket)
is_dirty = False
if query:
values = []
indices = []
if chunk_arr is not None:
# do query selection
limit = 0
if "Limit" in params:
limit = int(params["Limit"])
field_names = list(dt.fields.keys())
replace_mask = [
None,
] * len(field_names)
for i in range(len(field_names)):
field_name = field_names[i]
if field_name in query_update:
replace_mask[i] = query_update[field_name]
log.debug(f"replace_mask: {replace_mask}")
x = chunk_arr[selection]
log.debug(f"put_query - x: {x}")
eval_str = getEvalStr(query, "x", field_names)
log.debug(f"put_query - eval_str: {eval_str}")
where_result = np.where(eval(eval_str))
log.debug(f"put_query - where_result: {where_result}")
where_result_index = where_result[0]
log.debug(f"put_query - whare_result index: {where_result_index}")
log.debug(
f"put_query - boolean selection: {x[where_result_index]}")
s = selection[0]
count = 0
for index in where_result_index:
log.debug(f"put_query - index: {index}")
value = x[index]
log.debug(f"put_query - original value: {value}")
for i in range(len(field_names)):
if replace_mask[i] is not None:
value[i] = replace_mask[i]
log.debug(f"put_query - modified value: {value}")
x[index] = value
json_val = bytesArrayToList(value)
log.debug(f"put_query - json_value: {json_val}")
json_index = index.tolist(
) * s.step + s.start # adjust for selection
indices.append(json_index)
values.append(json_val)
count += 1
is_dirty = True
if limit > 0 and count >= limit:
log.info("put_query - got limit items")
break
query_result = {}
query_result["index"] = indices
query_result["value"] = values
log.info(f"query_result retiurning: {len(indices)} rows")
log.debug(f"query_result: {query_result}")
resp = json_response(query_result)
else:
# update chunk array
chunk_arr[selection] = input_arr
is_dirty = True
resp = json_response({}, status=201)
if is_dirty:
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}")
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = (now, bucket)
# chunk update successful
log.response(request, resp=resp)
return resp
