async def check_metadata_obj(app, obj_id):
""" Return False is obj does not exist
"""
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = "{} has been deleted".format(obj_id)
log.info(msg)
return False
meta_cache = app['meta_cache']
if obj_id in meta_cache:
found = True
else:
# Not in chache, check s3 obj exists
s3_key = getS3Key(obj_id)
log.debug("check_metadata_obj({})".format(s3_key))
# does key exist?
found = await isS3Obj(app, s3_key)
return found
def get_obj_id(request, body=None):
""" Get object id from request
Raise HTTPException on errors.
"""
obj_id = None
collection = None
app = request.app
if body and "id" in body:
obj_id = body["id"]
else:
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.error(msg)
raise HTTPInternalServerError()
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid obj id: {obj_id}"
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
return obj_id
def get_domain(request, body=None):
""" Extract domain and validate """
app = request.app
params = request.rel_url.query
domain = None
if "domain" in params:
domain = params["domain"]
log.debug(f"got domain param: {domain}")
elif body and "domain" in body:
domain = body["domain"]
if not domain:
msg = "No domain provided"
log.error(msg)
raise HTTPInternalServerError()
if not isValidDomain(domain):
msg = f"Expected valid domain for [{domain}]"
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, domain)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
return domain
async def check_metadata_obj(app, obj_id, bucket=None):
""" Return False is obj does not exist
"""
validateObjId(obj_id, bucket)
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = f"{obj_id} has been deleted"
log.info(msg)
return False
meta_cache = app['meta_cache']
if obj_id in meta_cache:
found = True
else:
# Not in chache, check s3 obj exists
s3_key = getS3Key(obj_id)
log.debug(f"check_metadata_obj({s3_key})")
# does key exist?
found = await isS3Obj(app, s3_key, bucket=bucket)
return found
async def DELETE_Chunk(request):
"""HTTP DELETE method for /chunks/
Note: clients (i.e. SN nodes) don't directly delete chunks. This method should
only be called by the AN node.
"""
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"DELETE chunk: {chunk_id}")
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
validateInPartition(app, chunk_id)
chunk_cache = app['chunk_cache']
s3key = getS3Key(chunk_id)
log.debug(f"DELETE_Chunk s3_key: {s3key}")
if chunk_id in chunk_cache:
del chunk_cache[chunk_id]
deflate_map = app["deflate_map"]
shuffle_map = app["shuffle_map"]
dset_id = getDatasetId(chunk_id)
if dset_id in deflate_map:
# The only reason chunks are ever deleted is if the dataset is being deleted,
# so it should be safe to remove this entry now
log.info(f"Removing deflate_map entry for {dset_id}")
del deflate_map[dset_id]
if dset_id in shuffle_map:
log.info(f"Removing shuffle_map entry for {dset_id}")
del shuffle_map[dset_id]
if await isStorObj(app, s3key, bucket=bucket):
await deleteStorObj(app, s3key, bucket=bucket)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
resp_json = {}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def delete_metadata_obj(app,
obj_id,
notify=True,
root_id=None,
bucket=None):
""" Delete the given object """
meta_cache = app['meta_cache']
dirty_ids = app["dirty_ids"]
log.info(f"delete_meta_data_obj: {obj_id} notify: {notify}")
validateObjId(obj_id, bucket)
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
try:
validateInPartition(app, obj_id)
except KeyError:
log.error(f"obj: {obj_id} not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
log.warn(f"{obj_id} has already been deleted")
else:
log.debug(f"adding {obj_id} to deleted ids")
deleted_ids.add(obj_id)
if obj_id in meta_cache:
log.debug(f"removing {obj_id} from meta_cache")
del meta_cache[obj_id]
if obj_id in dirty_ids:
log.debug(f"removing dirty_ids for: {obj_id}")
del dirty_ids[obj_id]
# remove from S3 (if present)
s3key = getS3Key(obj_id)
if await isS3Obj(app, s3key, bucket=bucket):
await deleteS3Obj(app, s3key, bucket=bucket)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
if isValidUuid(obj_id) and isSchema2Id(obj_id):
if isRootObjId(obj_id):
# add to gc ids so sub-objects will be deleted
gc_ids = app["gc_ids"]
log.info(f"adding root id: {obj_id} for GC cleanup")
gc_ids.add(obj_id)
elif notify:
root_id = getRootObjId(obj_id)
await notify_root(app, root_id, bucket=bucket)
# no notify for domain deletes since the root group is being deleted
log.debug(f"delete_metadata_obj for {obj_id} done")
def save_chunk(app, chunk_id, bucket=None):
""" Persist the given object """
log.info(f"save_chunk {chunk_id} bucket={bucket}")
try:
validateInPartition(app, chunk_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
log.info(f"chunk cache dirty count: {chunk_cache.dirtyCount}")
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = (now, bucket)
async def DELETE_Chunk(request):
"""HTTP DELETE method for /chunks/
Note: clients (i.e. SN nodes) don't directly delete chunks. This method should
only be called by the AN node.
"""
log.request(request)
app = request.app
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info("DELETE chunk: {}".format(chunk_id))
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
chunk_cache = app['chunk_cache']
s3_key = getS3Key(chunk_id)
log.debug("DELETE_Chunk s3_key: {}".format(s3_key))
if chunk_id in chunk_cache:
del chunk_cache[chunk_id]
deflate_map = app["deflate_map"]
dset_id = getDatasetId(chunk_id)
if dset_id in deflate_map:
# The only reason chunks are ever deleted is if the dataset is being deleted,
# so it should be save to remove this entry now
log.info("Removing deflate_map entry for {}".format(dset_id))
del deflate_map[dset_id]
resp_json = { }
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def save_metadata_obj(app,
obj_id,
obj_json,
bucket=None,
notify=False,
flush=False):
""" Persist the given object """
log.info(
f"save_metadata_obj {obj_id} bucket={bucket} notify={notify} flush={flush}"
)
if notify and not flush:
log.error("notify not valid when flush is false")
raise HTTPInternalServerError()
validateObjId(obj_id, bucket)
if not isinstance(obj_json, dict):
log.error("Passed non-dict obj to save_metadata_obj")
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
dirty_ids = app["dirty_ids"]
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
if isValidUuid(obj_id):
# domain objects may be re-created, but shouldn't see repeats of
# deleted uuids
log.warn(f"{obj_id} has been deleted")
raise HTTPInternalServerError()
elif obj_id in deleted_ids:
deleted_ids.remove(obj_id) # un-gone the domain id
# update meta cache
meta_cache = app['meta_cache']
log.debug(f"save: {obj_id} to cache")
meta_cache[obj_id] = obj_json
meta_cache.setDirty(obj_id)
now = int(time.time())
if flush:
# write to S3 immediately
if isValidChunkId(obj_id):
log.warn("flush not supported for save_metadata_obj with chunks")
raise HTTPBadRequest()
try:
await write_s3_obj(app, obj_id, bucket=bucket)
except KeyError as ke:
log.error(f"s3 sync got key error: {ke}")
raise HTTPInternalServerError()
except HTTPInternalServerError:
log.warn(f" failed to write {obj_id}")
raise # re-throw
if obj_id in dirty_ids:
log.warn(
f"save_metadata_obj flush - object {obj_id} is still dirty")
# message AN immediately if notify flag is set
# otherwise AN will be notified at next S3 sync
if notify:
if isValidUuid(obj_id) and isSchema2Id(obj_id):
root_id = getRootObjId(obj_id)
await notify_root(app, root_id, bucket=bucket)
else:
log.debug(f"setting dirty_ids[{obj_id}] = ({now}, {bucket})")
if isValidUuid(obj_id) and not bucket:
log.warn(f"bucket is not defined for save_metadata_obj: {obj_id}")
dirty_ids[obj_id] = (now, bucket)
async def POST_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
put_points = False
num_points = 0
if "count" in params:
num_points = int(params["count"])
if "action" in params and params["action"] == "put":
log.info("POST Chunk put points, num_points: {}".format(num_points))
put_points = True
else:
log.info("POST Chunk get points")
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info("POST chunk_id: {}".format(chunk_id))
chunk_index = getChunkIndex(chunk_id)
log.debug("chunk_index: {}".format(chunk_index))
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
chunk_layout = getChunkLayout(dset_json)
chunk_coord = getChunkCoordinate(chunk_id, chunk_layout)
log.debug("chunk_coord: {}".format(chunk_coord))
if not request.has_body:
msg = "POST Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = "Unexpected content_type: {}".format(content_type)
log.error(msg)
raise HTTPBadRequest(reason=msg)
type_json = dset_json["type"]
dset_dtype = createDataType(type_json)
log.debug("dtype: {}".format(dset_dtype))
dims = getChunkLayout(dset_json)
log.debug("got dims: {}".format(dims))
rank = len(dims)
if rank == 0:
msg = "POST chunk request with no dimensions"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming points
input_bytes = await request_read(request)
if len(input_bytes) != request.content_length:
msg = "Read {} bytes, expecting: {}".format(len(input_bytes), request.content_length)
log.error(msg)
raise HTTPInternalServerError()
# get chunk from cache/s3. If not found init a new chunk if this is a write request
chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=put_points)
if put_points:
# writing point data
# create a numpy array with the following type:
# (coord1, coord2, ...) | dset_dtype
if rank == 1:
coord_type_str = "uint64"
else:
coord_type_str = "({},)uint64".format(rank)
comp_dtype = np.dtype([("coord", np.dtype(coord_type_str)), ("value", dset_dtype)])
point_arr = np.fromstring(input_bytes, dtype=comp_dtype)
if len(point_arr) != num_points:
msg = "Unexpected size of point array, got: {} expected: {}".format(len(point_arr), num_points)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(num_points):
elem = point_arr[i]
if rank == 1:
coord = int(elem[0])
else:
coord = tuple(elem[0]) # index to update
val = elem[1] # value
chunk_arr[coord] = val # update the point
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = now
log.info("set {} to dirty".format(chunk_id))
else:
# reading point data
point_dt = np.dtype('uint64') # use unsigned long for point index
point_arr = np.fromstring(input_bytes, dtype=point_dt) # read points as unsigned longs
if len(point_arr) % rank != 0:
msg = "Unexpected size of point array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
num_points = len(point_arr) // rank
log.debug("got {} points".format(num_points))
point_arr = point_arr.reshape((num_points, rank))
output_arr = np.zeros((num_points,), dtype=dset_dtype)
for i in range(num_points):
point = point_arr[i,:]
tr_point = getChunkRelativePoint(chunk_coord, point)
val = chunk_arr[tuple(tr_point)]
output_arr[i] = val
if put_points:
# write empty response
resp = json_response({})
else:
# get data
output_data = output_arr.tobytes()
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def GET_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
type_json = dset_json["type"]
dims = getChunkLayout(dset_json)
log.debug("got dims: {}".format(dims))
rank = len(dims)
# get chunk selection from query params
if "select" in params:
log.debug("select: {}".format(params["select"]))
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug("got selection: {}".format(selection))
dt = createDataType(type_json)
log.debug("dtype: {}".format(dt))
rank = len(dims)
if rank == 0:
msg = "No dimension passed to GET chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug("selection[{}]: {}".format(i, s))
s3path = None
s3offset = 0
s3size = 0
if "s3path" in params:
s3path = params["s3path"]
log.debug(f"GET_Chunk - useing s3path: {s3path}")
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
chunk_arr = await getChunk(app, chunk_id, dset_json, s3path=s3path, s3offset=s3offset, s3size=s3size)
if chunk_arr is None:
# return a 404
msg = "Chunk {} does not exist".format(chunk_id)
log.info(msg)
raise HTTPNotFound()
resp = None
if "query" in params:
# do query selection
query = params["query"]
log.info("query: {}".format(query))
if rank != 1:
msg = "Query selection only supported for one dimensional arrays"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = 0
if "Limit" in params:
limit = int(params["Limit"])
values = []
indices = []
field_names = []
if dt.fields:
field_names = list(dt.fields.keys())
x = chunk_arr[selection]
log.debug("x: {}".format(x))
eval_str = getEvalStr(query, "x", field_names)
log.debug("eval_str: {}".format(eval_str))
where_result = np.where(eval(eval_str))
log.debug("where_result: {}".format(where_result))
where_result_index = where_result[0]
log.debug("whare_result index: {}".format(where_result_index))
log.debug("boolean selection: {}".format(x[where_result_index]))
s = selection[0]
count = 0
for index in where_result_index:
log.debug("index: {}".format(index))
value = x[index].tolist()
log.debug("value: {}".format(value))
json_val = bytesArrayToList(value)
log.debug("json_value: {}".format(json_val))
json_index = index.tolist() * s.step + s.start # adjust for selection
indices.append(json_index)
values.append(json_val)
count += 1
if limit > 0 and count >= limit:
log.info("got limit items")
break
query_result = {}
query_result["index"] = indices
query_result["value"] = values
log.info(f"query_result retiurning: {len(indices)} rows")
log.debug(f"query_result: {query_result}")
resp = json_response(query_result)
else:
# get requested data
output_arr = chunk_arr[selection]
output_data = arrayToBytes(output_arr)
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def PUT_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = "Invalid chunk id: {}".format(chunk_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "PUT Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = "Unexpected content_type: {}".format(content_type)
log.error(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug("request params: {}".format(list(params.keys())))
if "dset" not in params:
msg = "Missing dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_json = json.loads(params["dset"])
log.debug("dset_json: {}".format(dset_json))
dims = getChunkLayout(dset_json)
if "root" not in dset_json:
msg = "expected root key in dset_json"
log.error(msg)
raise KeyError(msg)
rank = len(dims)
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug("got selection: {}".format(selection))
type_json = dset_json["type"]
itemsize = 'H5T_VARIABLE'
if "size" in type_json:
itemsize = type_json["size"]
dt = createDataType(type_json)
log.debug("dtype: {}".format(dt))
if rank == 0:
msg = "No dimension passed to PUT chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug("selection[{}]: {}".format(i, s))
mshape = getSelectionShape(selection)
log.debug(f"mshape: {mshape}")
num_elements = 1
for extent in mshape:
num_elements *= extent
# check that the content_length is what we expect
if itemsize != 'H5T_VARIABLE':
log.debug("expect content_length: {}".format(num_elements*itemsize))
log.debug("actual content_length: {}".format(request.content_length))
if itemsize != 'H5T_VARIABLE' and (num_elements * itemsize) != request.content_length:
msg = "Expected content_length of: {}, but got: {}".format(num_elements*itemsize, request.content_length)
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming data
input_bytes = await request_read(request) # TBD - will it cause problems when failures are raised before reading data?
if len(input_bytes) != request.content_length:
msg = "Read {} bytes, expecting: {}".format(len(input_bytes), request.content_length)
log.error(msg)
raise HTTPInternalServerError()
input_arr = bytesToArray(input_bytes, dt, mshape)
chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=True)
# update chunk array
chunk_arr[selection] = input_arr
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}")
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = now
# chunk update successful
resp = json_response({}, status=201)
log.response(request, resp=resp)
return resp
async def PUT_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
query = None
query_update = None
limit = 0
bucket = None
input_arr = None
if "query" in params:
query = params["query"]
log.info(f"PUT_Chunk query: {query}")
if "Limit" in params:
limit = int(params["Limit"])
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "PUT Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "bucket" in params:
bucket = params["bucket"]
log.debug(f"PUT_Chunk using bucket: {bucket}")
else:
bucket = None
if query:
expected_content_type = "text/plain; charset=utf-8"
chunk_init = False # don't initalize new chunks on query update
else:
expected_content_type = "application/octet-stream"
chunk_init = True
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != expected_content_type:
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
if "dset" in params:
msg = "Unexpected param dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.debug(f"PUT_Chunk - id: {chunk_id}")
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
# TBD - does this work with linked datasets?
dims = getChunkLayout(dset_json)
log.debug(f"got dims: {dims}")
rank = len(dims)
type_json = dset_json["type"]
dt = createDataType(type_json)
log.debug(f"dtype: {dt}")
itemsize = 'H5T_VARIABLE'
if "size" in type_json:
itemsize = type_json["size"]
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug(f"got selection: {selection}")
mshape = getSelectionShape(selection)
log.debug(f"mshape: {mshape}")
num_elements = 1
for extent in mshape:
num_elements *= extent
chunk_arr = await get_chunk(app, chunk_id, dset_json, bucket=bucket, chunk_init=chunk_init)
is_dirty = False
if chunk_arr is None:
if chunk_init:
log.error(f"failed to create numpy array")
raise HTTPInternalServerError()
else:
log.warn(f"chunk {chunk_id} not found")
raise HTTPNotFound()
if query:
if not dt.fields:
log.error("expected compound dtype for PUT query")
raise HTTPInternalServerError()
if rank != 1:
log.error("expected one-dimensional array for PUT query")
raise HTTPInternalServerError()
query_update = await request.json()
log.debug(f"query_update: {query_update}")
# TBD - send back binary response to SN node
try:
resp = chunkQuery(chunk_id=chunk_id, chunk_layout=dims, chunk_arr=chunk_arr, slices=selection,
query=query, query_update=query_update, limit=limit, return_json=True)
except TypeError as te:
log.warn(f"chunkQuery - TypeError: {te}")
raise HTTPBadRequest()
except ValueError as ve:
log.warn(f"chunkQuery - ValueError: {ve}")
raise HTTPBadRequest()
if query_update and resp is not None:
is_dirty = True
else:
# regular chunk update
# check that the content_length is what we expect
if itemsize != 'H5T_VARIABLE':
log.debug(f"expect content_length: {num_elements*itemsize}")
log.debug(f"actual content_length: {request.content_length}")
if itemsize != 'H5T_VARIABLE' and (num_elements * itemsize) != request.content_length:
msg = f"Expected content_length of: {num_elements*itemsize}, but got: {request.content_length}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming data
input_bytes = await request_read(request) # TBD - will it cause problems when failures are raised before reading data?
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
input_arr = bytesToArray(input_bytes, dt, mshape)
chunkWriteSelection(chunk_arr=chunk_arr, slices=selection, data=input_arr)
is_dirty = True
# chunk update successful
resp = {}
if is_dirty:
save_chunk(app, chunk_id, bucket=bucket)
resp = json_response(resp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug(f"request params: {params.keys()}")
bucket = None
s3path = None
s3offset = None
s3size = None
query = None
limit = 0
if "s3path" in params:
s3path = params["s3path"]
log.debug(f"GET_Chunk - using s3path: {s3path}")
elif "bucket" in params:
bucket = params["bucket"]
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
if "query" in params:
query = params["query"]
if "Limit" in params:
limit = int(params["Limit"])
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
dims = getChunkLayout(dset_json)
log.debug(f"got dims: {dims}")
rank = len(dims)
log.debug(f"dset_json: {dset_json}")
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug(f"got selection: {selection}")
chunk_arr = await get_chunk(app, chunk_id, dset_json, bucket=bucket, s3path=s3path, s3offset=s3offset, s3size=s3size, chunk_init=False)
if chunk_arr is None:
msg = f"chunk {chunk_id} not found"
log.warn(msg)
raise HTTPNotFound()
if query:
# run given query
try:
read_resp = chunkQuery(chunk_id=chunk_id, chunk_layout=dims, chunk_arr=chunk_arr, slices=selection,
query=query, limit=limit, return_json=True)
except TypeError as te:
log.warn(f"chunkQuery - TypeError: {te}")
raise HTTPBadRequest()
except ValueError as ve:
log.warn(f"chunkQuery - ValueError: {ve}")
raise HTTPBadRequest()
else:
# read selected data from chunk
output_arr = chunkReadSelection(chunk_arr, slices=selection)
read_resp = arrayToBytes(output_arr)
# write response
if isinstance(read_resp, bytes):
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(read_resp)
await resp.prepare(request)
await resp.write(read_resp)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
else:
# JSON response
resp = json_response(read_resp)
return resp
async def delete_metadata_obj(app, obj_id, notify=True, root_id=None):
""" Delete the given object """
meta_cache = app['meta_cache']
dirty_ids = app["dirty_ids"]
log.info("delete_meta_data_obj: {} notify: {}".format(obj_id, notify))
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error(f"obj: {obj_id} not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
log.warn("{} has already been deleted".format(obj_id))
else:
deleted_ids.add(obj_id)
if obj_id in meta_cache:
log.debug(f"removing {obj_id} from meta_cache")
del meta_cache[obj_id]
if obj_id in dirty_ids:
del dirty_ids[obj_id]
# remove from S3 (if present)
s3key = getS3Key(obj_id)
if await isS3Obj(app, s3key):
await deleteS3Obj(app, s3key)
else:
log.info(
f"delete_metadata_obj - key {s3key} not found (never written)?")
if notify:
an_url = getAsyncNodeUrl(app)
if obj_id.startswith("/"):
# domain delete
req = an_url + "/domain"
params = {"domain": obj_id}
try:
log.info("ASync DELETE notify: {} params: {}".format(
req, params))
await http_delete(app, req, params=params)
except ClientError as ce:
log.error(f"got error notifying async node: {ce}")
except HTTPInternalServerError as hse:
log.error(f"got HTTPInternalServerError: {hse}")
else:
req = an_url + "/object/" + obj_id
try:
log.info(f"ASync DELETE notify: {req}")
await http_delete(app, req)
except ClientError as ce:
log.error(f"got ClientError notifying async node: {ce}")
except HTTPInternalServerError as ise:
log.error(
f"got HTTPInternalServerError notifying async node: {ise}")
log.debug(f"delete_metadata_obj for {obj_id} done")
async def save_metadata_obj(app, obj_id, obj_json, notify=False, flush=False):
""" Persist the given object """
log.info(f"save_metadata_obj {obj_id} notify={notify} flush={flush}")
if notify and not flush:
log.error("notify not valid when flush is false")
raise HTTPInternalServerError()
if not obj_id.startswith('/') and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
if not isinstance(obj_json, dict):
log.error("Passed non-dict obj to save_metadata_obj")
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
dirty_ids = app["dirty_ids"]
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
if isValidUuid(obj_id):
# domain objects may be re-created, but shouldn't see repeats of
# deleted uuids
log.warn("{} has been deleted".format(obj_id))
raise HTTPInternalServerError()
elif obj_id in deleted_ids:
deleted_ids.remove(obj_id) # un-gone the domain id
# update meta cache
meta_cache = app['meta_cache']
log.debug("save: {} to cache".format(obj_id))
meta_cache[obj_id] = obj_json
meta_cache.setDirty(obj_id)
now = int(time.time())
if flush:
# write to S3 immediately
if isValidChunkId(obj_id):
log.warn("flush not supported for save_metadata_obj with chunks")
raise HTTPBadRequest()
try:
await write_s3_obj(app, obj_id)
except KeyError as ke:
log.error(f"s3 sync got key error: {ke}")
raise HTTPInternalServerError()
except HTTPInternalServerError:
log.warn(f" failed to write {obj_id}")
raise # re-throw
if obj_id in dirty_ids:
log.warn(
f"save_metadata_obj flush - object {obj_id} is still dirty")
else:
# flag to write to S3
dirty_ids[obj_id] = now
# message AN immediately if notify flag is set
# otherwise AN will be notified at next S3 sync
if notify:
an_url = getAsyncNodeUrl(app)
if obj_id.startswith("/"):
# domain update
req = an_url + "/domain"
params = {"domain": obj_id}
if "root" in obj_json:
params["root"] = obj_json["root"]
if "owner" in obj_json:
params["owner"] = obj_json["owner"]
try:
log.info("ASync PUT notify: {} params: {}".format(req, params))
await http_put(app, req, params=params)
except HTTPInternalServerError as hpe:
log.error(f"got error notifying async node: {hpe}")
log.error(msg)
else:
req = an_url + "/object/" + obj_id
try:
log.info("ASync PUT notify: {}".format(req))
await http_put(app, req)
except HTTPInternalServerError:
log.error(f"got error notifying async node")
async def POST_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
put_points = False
num_points = 0
if "count" in params:
num_points = int(params["count"])
if "action" in params and params["action"] == "put":
log.info(f"POST Chunk put points, num_points: {num_points}")
put_points = True
else:
log.info("POST Chunk get points")
s3path = None
s3offset = 0
s3size = 0
if "s3path" in params:
if put_points:
log.error("s3path can not be used with put points POST request")
raise HTTPBadRequest()
s3path = params["s3path"]
log.debug(f"GET_Chunk - using s3path: {s3path}")
bucket = None
elif "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"POST chunk_id: {chunk_id}")
chunk_index = getChunkIndex(chunk_id)
log.debug(f"chunk_index: {chunk_index}")
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug(f"request params: {list(params.keys())}")
if "dset" in params:
msg = "Unexpected dset in POST request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
chunk_layout = getChunkLayout(dset_json)
chunk_coord = getChunkCoordinate(chunk_id, chunk_layout)
log.debug(f"chunk_coord: {chunk_coord}")
if not request.has_body:
msg = "POST Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
type_json = dset_json["type"]
dset_dtype = createDataType(type_json)
log.debug(f"dtype: {dset_dtype}")
dims = getChunkLayout(dset_json)
log.debug(f"got dims: {dims}")
rank = len(dims)
if rank == 0:
msg = "POST chunk request with no dimensions"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming points
input_bytes = await request_read(request)
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
# get chunk from cache/s3. If not found init a new chunk if this is a write request
chunk_arr = await getChunk(app,
chunk_id,
dset_json,
bucket=bucket,
s3path=s3path,
s3offset=s3offset,
s3size=s3size,
chunk_init=put_points)
if chunk_arr is None:
if put_points:
log.error("no array returned for put_points")
raise HTTPInternalServerError()
else:
# get points on a non-existent S3 objects?
log.warn("S3 object not found for get points")
raise HTTPNotFound()
log.debug(f"chunk_arr.shape: {chunk_arr.shape}")
if put_points:
# writing point data
# create a numpy array with the following type:
# (coord1, coord2, ...) | dset_dtype
if rank == 1:
coord_type_str = "uint64"
else:
coord_type_str = f"({rank},)uint64"
comp_dtype = np.dtype([("coord", np.dtype(coord_type_str)),
("value", dset_dtype)])
point_arr = np.fromstring(input_bytes, dtype=comp_dtype)
if len(point_arr) != num_points:
msg = f"Unexpected size of point array, got: {len(point_arr)} expected: {num_points}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(num_points):
elem = point_arr[i]
log.debug(f"non-relative coordinate: {elem}")
if rank == 1:
coord = int(elem[0])
coord = coord % chunk_layout[0] # adjust to chunk relative
else:
coord = elem[0] # index to update
for dim in range(rank):
# adjust to chunk relative
coord[dim] = int(coord[dim]) % chunk_layout[dim]
coord = tuple(coord) # need to convert to a tuple
log.debug(f"relative coordinate: {coord}")
val = elem[1] # value
try:
chunk_arr[coord] = val # update the point
except IndexError:
msg = "Out of bounds point index for POST Chunk"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = (now, bucket)
log.info(f"set {chunk_id} to dirty")
else:
# reading point data
point_dt = np.dtype('uint64') # use unsigned long for point index
point_arr = np.fromstring(
input_bytes, dtype=point_dt) # read points as unsigned longs
if len(point_arr) % rank != 0:
msg = "Unexpected size of point array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
num_points = len(point_arr) // rank
log.debug(f"got {num_points} points")
point_arr = point_arr.reshape((num_points, rank))
output_arr = np.zeros((num_points, ), dtype=dset_dtype)
for i in range(num_points):
point = point_arr[i, :]
tr_point = getChunkRelativePoint(chunk_coord, point)
val = chunk_arr[tuple(tr_point)]
output_arr[i] = val
if put_points:
# write empty response
resp = json_response({})
else:
# get data
output_data = output_arr.tobytes()
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp
async def PUT_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
query = None
if "query" in params:
query = params["query"]
log.info(f"PUT_Chunk query: {query}")
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "PUT Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "bucket" in params:
bucket = params["bucket"]
log.debug(f"PUT_Chunk using bucket: {bucket}")
else:
bucket = None
if query:
expected_content_type = "text/plain; charset=utf-8"
else:
expected_content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != expected_content_type:
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
if "dset" in params:
msg = "Unexpected param dset in GET request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.debug(f"PUT_Chunk - id: {chunk_id}")
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
log.debug(f"dset_json: {dset_json}")
dims = getChunkLayout(dset_json)
if "root" not in dset_json:
msg = "expected root key in dset_json"
log.error(msg)
raise KeyError(msg)
rank = len(dims)
# get chunk selection from query params
selection = []
for i in range(rank):
dim_slice = getSliceQueryParam(request, i, dims[i])
selection.append(dim_slice)
selection = tuple(selection)
log.debug(f"got selection: {selection}")
type_json = dset_json["type"]
itemsize = 'H5T_VARIABLE'
if "size" in type_json:
itemsize = type_json["size"]
dt = createDataType(type_json)
log.debug(f"dtype: {dt}")
if rank == 0:
msg = "No dimension passed to PUT chunk request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if len(selection) != rank:
msg = "Selection rank does not match shape rank"
log.error(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
s = selection[i]
log.debug(f"selection[{i}]: {s}")
mshape = getSelectionShape(selection)
log.debug(f"mshape: {mshape}")
num_elements = 1
for extent in mshape:
num_elements *= extent
resp = {}
query_update = None
limit = 0
chunk_init = True
input_arr = None
if query:
if not dt.fields:
log.error("expected compound dtype for PUT query")
raise HTTPInternalServerError()
query_update = await request.json()
log.debug(f"query_update: {query_update}")
if "Limit" in params:
limit = int(params["Limit"])
chunk_init = False
else:
# regular chunk update
# check that the content_length is what we expect
if itemsize != 'H5T_VARIABLE':
log.debug(f"expect content_length: {num_elements*itemsize}")
log.debug(f"actual content_length: {request.content_length}")
if itemsize != 'H5T_VARIABLE' and (num_elements *
itemsize) != request.content_length:
msg = f"Expected content_length of: {num_elements*itemsize}, but got: {request.content_length}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
# create a numpy array for incoming data
input_bytes = await request_read(
request
) # TBD - will it cause problems when failures are raised before reading data?
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
input_arr = bytesToArray(input_bytes, dt, mshape)
# TBD: Skip read if the input shape is the entire chunk?
chunk_arr = await getChunk(app,
chunk_id,
dset_json,
chunk_init=chunk_init,
bucket=bucket)
is_dirty = False
if query:
values = []
indices = []
if chunk_arr is not None:
# do query selection
limit = 0
if "Limit" in params:
limit = int(params["Limit"])
field_names = list(dt.fields.keys())
replace_mask = [
None,
] * len(field_names)
for i in range(len(field_names)):
field_name = field_names[i]
if field_name in query_update:
replace_mask[i] = query_update[field_name]
log.debug(f"replace_mask: {replace_mask}")
x = chunk_arr[selection]
log.debug(f"put_query - x: {x}")
eval_str = getEvalStr(query, "x", field_names)
log.debug(f"put_query - eval_str: {eval_str}")
where_result = np.where(eval(eval_str))
log.debug(f"put_query - where_result: {where_result}")
where_result_index = where_result[0]
log.debug(f"put_query - whare_result index: {where_result_index}")
log.debug(
f"put_query - boolean selection: {x[where_result_index]}")
s = selection[0]
count = 0
for index in where_result_index:
log.debug(f"put_query - index: {index}")
value = x[index]
log.debug(f"put_query - original value: {value}")
for i in range(len(field_names)):
if replace_mask[i] is not None:
value[i] = replace_mask[i]
log.debug(f"put_query - modified value: {value}")
x[index] = value
json_val = bytesArrayToList(value)
log.debug(f"put_query - json_value: {json_val}")
json_index = index.tolist(
) * s.step + s.start # adjust for selection
indices.append(json_index)
values.append(json_val)
count += 1
is_dirty = True
if limit > 0 and count >= limit:
log.info("put_query - got limit items")
break
query_result = {}
query_result["index"] = indices
query_result["value"] = values
log.info(f"query_result retiurning: {len(indices)} rows")
log.debug(f"query_result: {query_result}")
resp = json_response(query_result)
else:
# update chunk array
chunk_arr[selection] = input_arr
is_dirty = True
resp = json_response({}, status=201)
if is_dirty:
chunk_cache = app["chunk_cache"]
chunk_cache.setDirty(chunk_id)
log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}")
# async write to S3
dirty_ids = app["dirty_ids"]
now = int(time.time())
dirty_ids[chunk_id] = (now, bucket)
# chunk update successful
log.response(request, resp=resp)
return resp
async def get_metadata_obj(app, obj_id):
""" Get object from metadata cache (if present).
Otherwise fetch from S3 and add to cache
"""
log.info("get_metadata_obj: {}".format(obj_id))
if not isValidDomain(obj_id) and not isValidUuid(obj_id):
msg = "Invalid obj id: {}".format(obj_id)
log.error(msg)
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = "{} has been deleted".format(obj_id)
log.warn(msg)
raise HTTPGone()
meta_cache = app['meta_cache']
obj_json = None
if obj_id in meta_cache:
log.debug("{} found in meta cache".format(obj_id))
obj_json = meta_cache[obj_id]
else:
s3_key = getS3Key(obj_id)
pending_s3_read = app["pending_s3_read"]
if obj_id in pending_s3_read:
# already a read in progress, wait for it to complete
read_start_time = pending_s3_read[obj_id]
log.info(
f"s3 read request for {s3_key} was requested at: {read_start_time}"
)
while time.time() - read_start_time < 2.0:
log.debug("waiting for pending s3 read, sleeping")
await asyncio.sleep(1) # sleep for sub-second?
if obj_id in meta_cache:
log.info(f"object {obj_id} has arrived!")
obj_json = meta_cache[obj_id]
break
if not obj_json:
log.warn(
f"s3 read for object {s3_key} timed-out, initiaiting a new read"
)
# invoke S3 read unless the object has just come in from pending read
if not obj_json:
log.debug("getS3JSONObj({})".format(s3_key))
if obj_id not in pending_s3_read:
pending_s3_read[obj_id] = time.time()
# read S3 object as JSON
obj_json = await getS3JSONObj(app, s3_key)
if obj_id in pending_s3_read:
# read complete - remove from pending map
elapsed_time = time.time() - pending_s3_read[obj_id]
log.info(f"s3 read for {s3_key} took {elapsed_time}")
del pending_s3_read[obj_id]
meta_cache[obj_id] = obj_json # add to cache
return obj_json
async def POST_Chunk(request):
log.request(request)
app = request.app
params = request.rel_url.query
put_points = False
num_points = 0
if "count" not in params:
log.warn("expected count param")
raise HTTPBadRequest()
if "count" in params:
num_points = int(params["count"])
if "action" in params and params["action"] == "put":
log.info(f"POST Chunk put points - num_points: {num_points}")
put_points = True
else:
log.info(f"POST Chunk get points - num_points: {num_points}")
s3path = None
s3offset = 0
s3size = 0
if "s3path" in params:
if put_points:
log.error("s3path can not be used with put points POST request")
raise HTTPBadRequest()
s3path = params["s3path"]
log.debug(f"GET_Chunk - using s3path: {s3path}")
bucket = None
elif "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
if "s3offset" in params:
try:
s3offset = int(params["s3offset"])
except ValueError:
log.error(f"invalid s3offset params: {params['s3offset']}")
raise HTTPBadRequest()
if "s3size" in params:
try:
s3size = int(params["s3size"])
except ValueError:
log.error(f"invalid s3size params: {params['s3sieze']}")
raise HTTPBadRequest()
chunk_id = request.match_info.get('id')
if not chunk_id:
msg = "Missing chunk id"
log.error(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"POST chunk_id: {chunk_id}")
chunk_index = getChunkIndex(chunk_id)
log.debug(f"chunk_index: {chunk_index}")
if not isValidUuid(chunk_id, "Chunk"):
msg = f"Invalid chunk id: {chunk_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateInPartition(app, chunk_id)
log.debug(f"request params: {list(params.keys())}")
if "dset" in params:
msg = "Unexpected dset in POST request"
log.error(msg)
raise HTTPBadRequest(reason=msg)
if not request.has_body:
msg = "POST Value with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
content_type = "application/octet-stream"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type != "application/octet-stream":
msg = f"Unexpected content_type: {content_type}"
log.error(msg)
raise HTTPBadRequest(reason=msg)
dset_id = getDatasetId(chunk_id)
dset_json = await get_metadata_obj(app, dset_id, bucket=bucket)
dims = getChunkLayout(dset_json)
rank = len(dims)
type_json = dset_json["type"]
dset_dtype = createDataType(type_json)
# create a numpy array for incoming points
input_bytes = await request_read(request)
if len(input_bytes) != request.content_length:
msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
if rank == 1:
coord_type_str = "uint64"
else:
coord_type_str = f"({rank},)uint64"
if put_points:
# create a numpy array with the following type:
# (coord1, coord2, ...) | dset_dtype
point_dt = np.dtype([("coord", np.dtype(coord_type_str)), ("value", dset_dtype)])
point_shape = (num_points,)
chunk_init = True
else:
point_dt = np.dtype('uint64')
point_shape = (num_points, rank)
chunk_init = False
point_arr = bytesToArray(input_bytes, point_dt, point_shape)
chunk_arr = await get_chunk(app, chunk_id, dset_json, bucket=bucket, s3path=s3path, s3offset=s3offset, s3size=s3size, chunk_init=chunk_init)
if chunk_arr is None:
log.warn(f"chunk {chunk_id} not found")
raise HTTPNotFound()
if put_points:
# writing point data
try:
chunkWritePoints(chunk_id=chunk_id, chunk_layout=dims, chunk_arr=chunk_arr, point_arr=point_arr)
except ValueError as ve:
log.warn(f"got value error from chunkWritePoints: {ve}")
raise HTTPBadRequest()
# write empty response
resp = json_response({})
save_chunk(app, chunk_id, bucket=bucket) # lazily write chunk to storage
else:
# read points
try:
output_arr = chunkReadPoints(chunk_id=chunk_id, chunk_layout=dims, chunk_arr=chunk_arr, point_arr=point_arr)
except ValueError as ve:
log.warn(f"got value error from chunkReadPoints: {ve}")
raise HTTPBadRequest()
output_data = arrayToBytes(output_arr)
# write response
try:
resp = StreamResponse()
resp.headers['Content-Type'] = "application/octet-stream"
resp.content_length = len(output_data)
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Exception during binary data write: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
return resp