async def get_key_stats(self, key, bucket=None):
""" Get ETag, size, and last modified time for given objecct
"""
start_time = time.time()
key_stats = {}
try:
async with self._client.get_blob_client(container=bucket,
blob=key) as blob_client:
blob_props = await blob_client.get_blob_properties()
finish_time = time.time()
except CancelledError as cle:
self._azure_stats_increment("error_count")
msg = f"azureBlobClient.CancelledError get_blob_properties {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
if isinstance(e, AzureError):
if e.status_code == 404:
msg = f"storage key: {key} not found "
log.warn(msg)
raise HTTPNotFound()
elif e.status_code in (401, 403):
msg = f"azureBlobClient.access denied for get_blob_properties key: {key}"
log.info(msg)
raise HTTPForbidden()
else:
self._azure_stats_increment("error_count")
log.error(
f"azureBlobClient.got unexpected AzureError for get_blob_properties {key}: {e.message}"
)
raise HTTPInternalServerError()
else:
log.error(
f"azureBlobClient.Unexpected exception for get_blob_properties {key}: {e}"
)
raise HTTPInternalServerError()
last_modified_dt = blob_props.last_modified
if not isinstance(last_modified_dt, datetime.datetime):
msg = "azureBlobClient.get_key_stats, expected datetime object in head data"
log.error(msg)
raise HTTPInternalServerError()
key_stats = {}
key_stats["Size"] = blob_props.size
key_stats["ETag"] = blob_props.etag
key_stats["LastModified"] = datetime.datetime.timestamp(
last_modified_dt)
log.info(
f"azureBlobClient.get_key_stats({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f}"
)
return key_stats
async def get_key_stats(self, key, bucket=None):
""" Get ETag, size, and last modified time for given objecct
"""
start_time = time.time()
session = self._app["session"]
self._renewToken()
async with session.create_client('s3', region_name=self._aws_region,
aws_secret_access_key=self._aws_secret_access_key,
aws_access_key_id=self._aws_access_key_id,
aws_session_token=self._aws_session_token,
endpoint_url=self._s3_gateway,
use_ssl=self._use_ssl,
config=self._aio_config) as _client:
try:
head_data = await _client.head_object(Bucket=bucket, Key=key)
finish_time = time.time()
log.info(f"head: {head_data}")
except ClientError:
# key does not exist?
msg = f"s3Client.get_key_stats: Key: {key} not found"
log.info(msg)
finish_time = time.time()
raise HTTPNotFound()
except CancelledError as cle:
self._s3_stats_increment("error_count")
msg = f"s3Client.get_key_stats: CancelledError getting head for s3 obj {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
self._s3_stats_increment("error_count")
msg = f"s3Client.get_key_stats: Unexpected Exception {type(e)} getting head for s3 obj {key}: {e}"
log.error(msg)
raise HTTPInternalServerError()
for head_key in ("ContentLength", "ETag", "LastModified"):
if head_key not in head_data:
msg = f"s3Client.get_key_stats, expected to find key: {head_key} in head_data"
log.error(msg)
raise HTTPInternalServerError()
last_modified_dt = head_data["LastModified"]
if not isinstance(last_modified_dt, datetime.datetime):
msg ="S3Client.get_key_stats, expected datetime object in head data"
log.error(msg)
raise HTTPInternalServerError()
key_stats = {}
key_stats["Size"] = head_data['ContentLength']
key_stats["ETag"] = head_data["ETag"]
key_stats["LastModified"] = datetime.datetime.timestamp(last_modified_dt)
log.info(f"s3Client.get_key_stats({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f}")
return key_stats
async def PUT_ACL(request):
""" Handler creating/update an ACL"""
log.request(request)
app = request.app
acl_username = request.match_info.get('username')
if not request.has_body:
msg = "Expected body in delete domain"
log.error(msg)
raise HTTPInternalServerError()
body_json = await request.json()
domain = get_domain(request, body=body_json)
log.info(f"put_acl - domain: {domain}, username: {acl_username}")
# raises exception if domain not found
domain_json = await get_metadata_obj(app, domain)
if "acls" not in domain_json:
log.error(f"unexpected domain data for domain: {domain}")
raise HTTPInternalServerError() # 500
acl_keys = getAclKeys()
acls = domain_json["acls"]
acl = {}
if acl_username in acls:
acl = acls[acl_username]
else:
# initialize acl with no perms
for k in acl_keys:
acl[k] = False
# replace any permissions given in the body
for k in body_json.keys():
acl[k] = body_json[k]
# replace/insert the updated/new acl
acls[acl_username] = acl
# update the timestamp
now = time.time()
domain_json["lastModified"] = now
# write back to S3
await save_metadata_obj(app, domain, domain_json, flush=True)
resp_json = {}
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
def __init__(self, app):
self._app = app
self._root_dir = config.get("root_dir")
if not self._root_dir:
log.error("FileClient init: root_dir config not set")
raise HTTPInternalServerError()
if not pp.isdir(self._root_dir):
log.error("FileClient init: root folder does not exist")
raise HTTPInternalServerError()
if not pp.isabs(self._root_dir):
log.error("FileClient init: root dir most have absolute path")
raise HTTPInternalServerError()
async def getPathForObjectId(app, parent_id, idpath_map, tgt_id=None):
""" Search the object starting with the given parent_id.
idpath should be a dict with at minimum the key: parent_id: <parent_path>.
If tgt_id is not None, returns first path that matches the tgt_id or None if not found.
If Tgt_id is no, returns the idpath_map.
"""
if not parent_id:
log.error("No parent_id passed to getPathForObjectId")
raise HTTPInternalServerError()
if parent_id not in idpath_map:
msg = "Obj {} expected to be found in idpath_map".format(parent_id)
log.error(msg)
raise HTTPInternalServerError()
parent_path = idpath_map[parent_id]
if parent_id == tgt_id:
return parent_path
req = getDataNodeUrl(app, parent_id)
req += "/groups/" + parent_id + "/links"
log.debug("getPathForObjectId LINKS: " + req)
links_json = await http_get(app, req)
log.debug(
"getPathForObjectId got links json from dn for parent_id: {}".format(
parent_id))
links = links_json["links"]
h5path = None
for link in links:
if link["class"] != "H5L_TYPE_HARD":
continue # ignore everything except hard links
link_id = link["id"]
if link_id in idpath_map:
continue # this node has already been visited
title = link["title"]
if tgt_id is not None and link_id == tgt_id:
# found it!
h5path = op.join(parent_path, title)
break
idpath_map[link_id] = op.join(parent_path, title)
if getCollectionForId(link_id) != "groups":
continue
h5path = await getPathForObjectId(app, link_id, idpath_map,
tgt_id) # recursive call
if tgt_id is not None and h5path:
break
return h5path
async def put_object(self, key, data, bucket=None):
""" Write data to given key.
Returns client specific dict on success
"""
if not bucket:
log.error("put_object - bucket not set")
raise HTTPInternalServerError()
start_time = time.time()
log.debug(f"azureBlobClient.put_object({bucket}/{key} start: {start_time}")
try:
async with self._client.get_blob_client(container=bucket, blob=key) as blob_client:
blob_rsp = await blob_client.upload_blob(data, blob_type='BlockBlob', overwrite=True)
finish_time = time.time()
ETag = blob_rsp["etag"]
lastModified = int(blob_rsp["last_modified"].timestamp())
data_size = len(data)
rsp = {"ETag": ETag, "size": data_size, "LastModified": lastModified }
log.debug(f"put_object {key} returning: {rsp}")
log.info(f"azureBlobClient.put_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}")
except CancelledError as cle:
self._azure_stats_increment("error_count")
msg = f"azureBlobClient.CancelledError for put_object {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
if isinstance(e, AzureError):
if e.status_code == 404:
msg = f"azureBlobClient.key: {key} not found "
log.warn(msg)
raise HTTPNotFound()
elif e.status_code in (401, 403):
msg = f"azureBlobClient.access denied for get key: {key}"
log.info(msg)
raise HTTPForbidden()
else:
self._azure_stats_increment("error_count")
log.error(f"azureBlobClient.got unexpected AzureError for get_object {key}: {e.message}")
raise HTTPInternalServerError()
else:
log.error(f"azureBlobClient.Unexpected exception for put_object {key}: {e}")
raise HTTPInternalServerError()
if data and len(data) > 0:
self._azure_stats_increment("bytes_out", inc=len(data))
log.debug(f"azureBlobClient.put_object {key} complete, rsp: {rsp}")
return rsp
async def getS3JSONObj(app, key):
""" Get S3 object identified by key and read as JSON
"""
client = getS3Client(app)
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
log.info("getS3JSONObj({})".format(key))
s3_stats_increment(app, "get_count")
try:
resp = await client.get_object(Bucket=bucket, Key=key)
data = await resp['Body'].read()
resp['Body'].close()
except ClientError as ce:
# key does not exist?
# check for not found status
# Note: Error.Code should always exist - cf https://github.com/boto/botocore/issues/885
log.info("ClientError on getS3JSONObj")
response_code = ce.response['Error']['Code']
# remove key from pending map if present
if "pending_s3_read" in app:
pending_s3_read = app["pending_s3_read"]
if key in pending_s3_read:
log.debug(f"remove {key} from pending_s3_read")
del pending_s3_read[key]
if response_code == "NoSuchKey":
msg = "s3_key: {} not found ".format(key,)
log.info(msg)
raise HTTPNotFound()
else:
s3_stats_increment(app, "error_count")
log.warn("got ClientError on s3 get: {}".format(str(ce)))
msg = "Error getting s3 obj: " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
s3_stats_increment(app, "bytes_in", inc=len(data))
try:
json_dict = json.loads(data.decode('utf8'))
except UnicodeDecodeError:
s3_stats_increment(app, "error_count")
log.error("Error loading JSON at key: {}".format(key))
msg = "Unexpected i/o error"
raise HTTPInternalServerError()
log.debug("s3 returned: {}".format(json_dict))
return json_dict
async def get_object(self, key, bucket=None, offset=0, length=-1):
""" Return data for object at given key.
If Range is set, return the given byte range.
"""
self._validateBucket(bucket)
self._validateKey(key)
if length > 0:
range = f"bytes={offset}-{offset+length-1}"
log.info(f"storage range request: {range}")
filepath = self._getFilePath(bucket, key)
log.info(f"get_object - filepath: {filepath}")
start_time = time.time()
log.debug(f"fileClient.get_object({bucket}/{key} start: {start_time}")
loop = asyncio.get_event_loop()
try:
async with aiofiles.open(filepath, loop=loop, mode='rb') as f:
if offset:
await f.seek(offset)
if length > 0:
data = await f.read(length)
else:
data = await f.read()
finish_time = time.time()
log.info(
f"fileClient.get_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}"
)
except FileNotFoundError:
msg = f"fileClient: {key} not found "
log.warn(msg)
raise HTTPNotFound()
except IOError as ioe:
msg = f"fileClient: IOError reading {bucket}/{key}: {ioe}"
log.warn(msg)
raise HTTPInternalServerError()
except CancelledError as cle:
self._file_stats_increment("error_count")
msg = f"CancelledError for get file obj {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
self._file_stats_increment("error_count")
msg = f"Unexpected Exception {type(e)} get get_object {key}: {e}"
log.error(msg)
raise HTTPInternalServerError()
return data
async def put_object(self, key, data, bucket=None):
""" Write data to given key.
Returns client specific dict on success
"""
if not bucket:
log.error("put_object - bucket not set")
raise HTTPInternalServerError()
start_time = time.time()
log.debug(f"s3Client.put_object({bucket}/{key} start: {start_time}")
session = self._app["session"]
self._renewToken()
async with session.create_client('s3', region_name=self._aws_region,
aws_secret_access_key=self._aws_secret_access_key,
aws_access_key_id=self._aws_access_key_id,
aws_session_token=self._aws_session_token,
endpoint_url=self._s3_gateway,
use_ssl=self._use_ssl,
config=self._aio_config) as _client:
try:
rsp = await _client.put_object(Bucket=bucket, Key=key, Body=data)
finish_time = time.time()
log.info(f"s3Client.put_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}")
s3_rsp = {"etag": rsp["ETag"], "size": len(data), "lastModified": int(finish_time)}
except ClientError as ce:
response_code = ce.response["Error"]["Code"]
if response_code == "NoSuchBucket":
msg = f"s3_bucket: {bucket} not found"
log.warn(msg)
raise HTTPNotFound()
else:
self._s3_stats_increment("error_count")
msg = f"Error putting s3 obj {key}: {ce}"
log.error(msg)
raise HTTPInternalServerError()
except CancelledError as cle:
#s3_stats_increment(app, "error_count")
msg = f"CancelledError for put s3 obj {key}: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
#s3_stats_increment(app, "error_count")
msg = f"Unexpected Exception {type(e)} putting s3 obj {key}: {e}"
log.error(msg)
raise HTTPInternalServerError()
if data and len(data) > 0:
self._s3_stats_increment("bytes_out", inc=len(data))
log.debug(f"s3Client.put_object {key} complete, s3_rsp: {s3_rsp}")
return s3_rsp
def aclCheck(app, obj_json, req_action, req_user):
log.info(f"aclCheck: {req_action} for user: {req_user}")
if isAdminUser(app, req_user):
return # allow admin user to do anything
if obj_json is None:
log.error("aclCheck: no obj json")
raise HTTPInternalServerError() # 500
if "acls" not in obj_json:
log.error("no acl key")
raise HTTPInternalServerError() # 500
acls = obj_json["acls"]
log.debug(f"acls: {acls}")
if req_action not in ("create", "read", "update", "delete", "readACL",
"updateACL"):
log.error(f"unexpected req_action: {req_action}")
if req_user in acls:
acl = acls[req_user]
log.debug(f"got acl: {acl} for user: {req_user}")
if req_action in acl and acl[req_action]:
log.debug("action permitted by user acl")
return
else:
# treat deny for username as authorative deny
log.warn(
f"Action: {req_action} not permitted for user: {req_user}")
raise HTTPForbidden() # 403
if "default" in acls:
acl = acls["default"]
log.debug(f"got default acl: {acl}")
if req_action in acl and acl[req_action]:
log.debug("action permitted by default acl")
return
user_group_db = app["user_group_db"]
if req_user in user_group_db:
user_groups = user_group_db[req_user]
for user_group in user_groups:
acl_name = GROUP_PREFIX + user_group
log.debug(f"checking group acl: {acl_name}")
if acl_name in acls:
acl = acls[acl_name]
if req_action in acl and acl[req_action]:
log.debug(f"action permitted by group acl: {acl_name}")
return
log.warn(f"Action: {req_action} not permitted for user: {req_user}")
raise HTTPForbidden() # 403
async def getS3Bytes(app, key, deflate_level=None, s3offset=0, s3size=None, bucket=None):
""" Get S3 object identified by key and read as bytes
"""
client = getS3Client(app)
if not bucket:
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
log.info("getS3Bytes({})".format(key))
s3_stats_increment(app, "get_count")
range=""
if s3size:
range = f"bytes={s3offset}-{s3offset+s3size-1}"
log.info(f"s3 range request: {range}")
try:
resp = await client.get_object(Bucket=bucket, Key=key, Range=range)
data = await resp['Body'].read()
resp['Body'].close()
except ClientError as ce:
# key does not exist?
# check for not found status
if ce.response["Error"]["Code"] == "NoSuchKey":
msg = "s3_key: {} not found ".format(key,)
log.warn(msg)
raise HTTPInternalServerError()
else:
s3_stats_increment(app, "error_count")
log.warn("got ClientError on s3 get: {}".format(str(ce)))
msg = "Error getting s3 obj: " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
if data and len(data) > 0:
s3_stats_increment(app, "bytes_in", inc=len(data))
log.info("read: {} bytes for S3 key: {}".format(len(data), key))
if deflate_level is not None:
try:
unzip_data = zlib.decompress(data)
log.info("uncompressed to {} bytes".format(len(unzip_data)))
data = unzip_data
except zlib.error as zlib_error:
log.info("zlib_err: {}".format(zlib_error))
log.warn("unable to uncompress s3 obj: {}, returning raw bytes".format(key))
return data
async def validate_token(self, token: str) -> dict:
"""
Request the JWT token validation to the associated UAA service
Args:
token: JWT token to validate
Returns: user authenticated if the token is valid
"""
async with ClientSession() as client:
validate_token_url = self.URL_BASE.format(protocol=self._protocol, host=self._host, port=self._port,
path=self.PATHS['validate_token'])
try:
async with client.post(validate_token_url, data=dict(token=token)) as response:
if response.status == 200:
response_json = await response.json()
elif response.status == 500:
raise HTTPUnauthorized(reason='Wrong authorization token')
else:
raise HTTPInternalServerError(reason='Error calling uaa service')
except Exception as ex:
if not isinstance(ex, HTTPUnauthorized) and not isinstance(ex, HTTPInternalServerError):
raise ConnectionError(f'Error calling uaa service {str(ex)}')
else:
raise ex
if 'id' in response_json:
return response_json
else:
raise HTTPUnauthorized(reason='Wrong authorization token')
async def deleteS3Obj(app, key):
""" Delete S3 object identfied by given key
"""
client = getS3Client(app)
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
log.info("deleteS3Obj({})".format(key))
s3_stats_increment(app, "delete_count")
try:
await client.delete_object(Bucket=bucket, Key=key)
except ClientError as ce:
# key does not exist?
key_found = await isS3Obj(app, key)
if not key_found:
log.warn(f"delete on s3key {key} but not found")
raise HTTPNotFound()
# else some other error
s3_stats_increment(app, "error_count")
msg = "Error deleting s3 obj: " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
log.debug("deleteS3Obj complete")
async def putS3JSONObj(app, key, json_obj):
""" Store JSON data as S3 object with given key
"""
client = getS3Client(app)
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
log.info("putS3JSONObj({})".format(key))
s3_stats_increment(app, "put_count")
data = json.dumps(json_obj)
data = data.encode('utf8')
try:
rsp = await client.put_object(Bucket=bucket, Key=key, Body=data)
now = int(time.time())
s3_rsp = {"etag": rsp["ETag"], "size": len(data), "lastModified": now}
except ClientError as ce:
s3_stats_increment(app, "error_count")
msg = "Error putting s3 obj: " + str(ce)
log.error(msg)
raise HTTPInternalServerError()
if data and len(data) > 0:
s3_stats_increment(app, "bytes_out", inc=len(data))
log.debug("putS3JSONObj complete, s3_rsp: {}".format(s3_rsp))
return s3_rsp
async def error_middleware(request, handler):
try:
http_exception = getattr(request.match_info, 'http_exception', None)
if http_exception:
raise http_exception
else:
r = await handler(request)
except HTTPException as e:
if request.method == METH_GET and e.status == 404 and request.rel_url.raw_path.endswith(
'/'):
possible_path = request.rel_url.raw_path[:-1]
for resource in request.app.router._resources:
match_dict = resource._match(possible_path)
if match_dict:
raise HTTPMovedPermanently(possible_path)
if e.status > 310:
await log_warning(request, e)
raise
except BaseException as e:
request_logger.exception(
'%s: %s',
e.__class__.__name__,
e,
extra={
'fingerprint': [e.__class__.__name__,
str(e)],
'data': await log_extra(request)
},
)
raise HTTPInternalServerError()
else:
if r.status > 310:
await log_warning(request, r)
return r
async def putStorBytes(app, key, data, filter_ops=None, bucket=None):
""" Store byte string as S3 object with given key
"""
client = _getStorageClient(app)
if not bucket:
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
shuffle = -1 # auto-shuffle
clevel = 5
cname = None # compressor name
if filter_ops:
if "compressor" in filter_ops:
cname = filter_ops["compressor"]
if "use_shuffle" in filter_ops and not filter_ops['use_shuffle']:
shuffle = 0 # client indicates to turn off shuffling
if "level" in filter_ops:
clevel = filter_ops["level"]
log.info(f"putStorBytes({bucket}/{key}), {len(data)} bytes shuffle: {shuffle} compressor: {cname} level: {clevel}")
if cname:
try:
blosc = codecs.Blosc(cname=cname, clevel=clevel, shuffle=shuffle)
cdata = blosc.encode(data)
# TBD: add cname in blosc constructor
log.info(f"compressed from {len(data)} bytes to {len(cdata)} bytes using filter: {blosc.cname} with level: {blosc.clevel}")
data = cdata
except Exception as e:
log.error(f"got exception using blosc encoding: {e}")
raise HTTPInternalServerError()
rsp = await client.put_object(key, data, bucket=bucket)
return rsp
async def getSNInstances(self):
if "is_dcos" not in self._app:
msg = "cannot use the MarathonClient in a non-DCOS context"
log.error(msg)
raise HTTPInternalServerError()
if "DCOS_PATH_SERVICE_NODE" in os.environ:
hsds_sn_node = os.environ["DCOS_PATH_SERVICE_NODE"]
else:
log.error(
"Must set DCOS_PATH_SERVICE_NODE environment variable to Marathon path to service node n order to query the correct marathon config"
)
return -1
req = "http://master.mesos/marathon/v2/apps/%s" % hsds_sn_node
try:
instancesJSON = await http_get(self._app, req)
except HTTPNotFound:
log.warn("Could not retrieve marathon app instance information.")
return -1
if instancesJSON is None or not isinstance(instancesJSON, dict):
log.warn("invalid marathon query response")
else:
if instancesJSON["app"] is not None and instancesJSON["app"][
"instances"] is not None:
log.debug("SN instances {}".format(
instancesJSON["app"]["instances"]))
return instancesJSON["app"]["instances"]
else:
log.warn("Incomplete or malformed JSON returned from SN node.")
return -1
async def DELETE_Dataset(request):
"""HTTP DELETE method for dataset
"""
log.request(request)
app = request.app
params = request.rel_url.query
dset_id = request.match_info.get('id')
log.info(f"DELETE dataset: {dset_id}")
if not isValidUuid(dset_id, obj_class="dataset"):
log.error(f"Unexpected dataset id: {dset_id}")
raise HTTPInternalServerError()
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
# verify the id exist
obj_found = await check_metadata_obj(app, dset_id, bucket=bucket)
if not obj_found:
raise HTTPNotFound()
log.debug(f"deleting dataset: {dset_id}")
notify=True
if "Notify" in params and not params["Notify"]:
notify=False
await delete_metadata_obj(app, dset_id, bucket=bucket, notify=notify)
resp_json = { }
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def GET_Group(request):
"""HTTP GET method to return JSON for /groups/
"""
log.request(request)
app = request.app
group_id = get_obj_id(request)
log.info("GET group: {}".format(group_id))
if not isValidUuid(group_id, obj_class="group"):
log.error("Unexpected group_id: {}".format(group_id))
raise HTTPInternalServerError()
group_json = await get_metadata_obj(app, group_id)
resp_json = {}
resp_json["id"] = group_json["id"]
resp_json["root"] = group_json["root"]
resp_json["created"] = group_json["created"]
resp_json["lastModified"] = group_json["lastModified"]
resp_json["linkCount"] = len(group_json["links"])
resp_json["attributeCount"] = len(group_json["attributes"])
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def getObjectIdByPath(app, obj_id, h5path, bucket=None, refresh=False):
""" Find the object at the provided h5path location.
If not found raise 404 error.
"""
log.info(
f"getObjectIdByPath obj_id: {obj_id} h5path: {h5path} refresh: {refresh}"
)
if h5path.startswith("./"):
h5path = h5path[2:] # treat as relative path
links = h5path.split('/')
for link in links:
if not link:
continue # skip empty link
log.debug(f"getObjectIdByPath for objid: {obj_id} got link: {link}")
if getCollectionForId(obj_id) != "groups":
# not a group, so won't have links
msg = f"h5path: {h5path} not found"
log.warn(msg)
raise HTTPNotFound()
req = getDataNodeUrl(app, obj_id)
req += "/groups/" + obj_id + "/links/" + link
log.debug("get LINK: " + req)
params = {}
if bucket:
params["bucket"] = bucket
link_json = await http_get(app, req, params=params)
log.debug("got link_json: " + str(link_json))
if link_json["class"] != 'H5L_TYPE_HARD':
# don't follow soft/external links
msg = f"h5path: {h5path} not found"
log.warn(msg)
raise HTTPInternalServerError()
obj_id = link_json["id"]
# if we get here, we've traveresed the entire path and found the object
return obj_id
async def GET_Datatype(request):
"""HTTP GET method to return JSON for /groups/
"""
log.request(request)
app = request.app
params = request.rel_url.query
ctype_id = get_obj_id(request)
if not isValidUuid(ctype_id, obj_class="type"):
log.error(f"Unexpected type_id: {ctype_id}")
raise HTTPInternalServerError()
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
ctype_json = await get_metadata_obj(app, ctype_id, bucket=bucket)
resp_json = { }
resp_json["id"] = ctype_json["id"]
resp_json["root"] = ctype_json["root"]
resp_json["created"] = ctype_json["created"]
resp_json["lastModified"] = ctype_json["lastModified"]
resp_json["type"] = ctype_json["type"]
resp_json["attributeCount"] = len(ctype_json["attributes"])
if "include_attrs" in params and params["include_attrs"]:
resp_json["attributes"] = ctype_json["attributes"]
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def GET_Attribute(request):
"""HTTP GET method to return JSON for /(obj)/<id>/attributes/<name>
"""
log.request(request)
app = request.app
params = request.rel_url.query
obj_id = get_obj_id(request)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
obj_json = await get_metadata_obj(app, obj_id, bucket=bucket)
log.info(f"GET attribute obj_id: {obj_id} name: {attr_name} bucket: {bucket}")
log.debug(f"got obj_json: {obj_json}")
if "attributes" not in obj_json:
log.error(f"unexpected obj data for id: {obj_id}")
raise HTTPInternalServerError()
attributes = obj_json["attributes"]
if attr_name not in attributes:
msg = f"Attribute '{attr_name}' not found for id: {obj_id}"
log.warn(msg)
raise HTTPNotFound()
attr_json = attributes[attr_name]
resp = json_response(attr_json)
log.response(request, resp=resp)
return resp
async def GET_Dataset(request):
"""HTTP GET method to return JSON for /groups/
"""
log.request(request)
app = request.app
dset_id = get_obj_id(request)
if not isValidUuid(dset_id, obj_class="dataset"):
log.error("Unexpected type_id: {}".format(dset_id))
raise HTTPInternalServerError()
dset_json = await get_metadata_obj(app, dset_id)
resp_json = {}
resp_json["id"] = dset_json["id"]
resp_json["root"] = dset_json["root"]
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["type"] = dset_json["type"]
resp_json["shape"] = dset_json["shape"]
resp_json["attributeCount"] = len(dset_json["attributes"])
if "creationProperties" in dset_json:
resp_json["creationProperties"] = dset_json["creationProperties"]
if "layout" in dset_json:
resp_json["layout"] = dset_json["layout"]
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def health_get(request: Request) -> Response:
try:
return Response(body=json.dumps({"status": "OK"}),
headers={'content-type': 'application/json'})
except Exception as ex:
log.warning(f"Endpoint: health, Method: get. Error:{str(ex)}")
return HTTPInternalServerError()
async def DELETE_Group(request):
"""HTTP DELETE method for /groups/
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
log.info("DELETE group: {}".format(group_id))
if not isValidUuid(group_id, obj_class="group"):
log.error("Unexpected group_id: {}".format(group_id))
raise HTTPInternalServerError()
# verify the id exist
obj_found = await check_metadata_obj(app, group_id)
if not obj_found:
log.debug(f"delete called on non-exsistet obj: {group_id}")
raise HTTPNotFound()
log.debug("deleting group: {}".format(group_id))
notify = True
if "Notify" in params and not params["Notify"]:
notify = False
await delete_metadata_obj(app, group_id, notify=notify)
resp_json = {}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def index(request: Request):
try:
return Response(body=json.dumps({"host": socket.gethostname()}),
headers={'content-type': 'application/json'})
except Exception as ex:
log.warning(f"Endpoint: /, Method: get. Error:{str(ex)}")
return HTTPInternalServerError()
async def check_metadata_obj(app, obj_id, bucket=None):
""" Return False is obj does not exist
"""
validateObjId(obj_id, bucket)
if isValidDomain(obj_id):
bucket = getBucketForDomain(obj_id)
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
msg = f"{obj_id} has been deleted"
log.info(msg)
return False
meta_cache = app['meta_cache']
if obj_id in meta_cache:
found = True
else:
# Not in chache, check s3 obj exists
s3_key = getS3Key(obj_id)
log.debug(f"check_metadata_obj({s3_key})")
# does key exist?
found = await isS3Obj(app, s3_key, bucket=bucket)
return found
async def DELETE_Domain(request):
"""HTTP DELETE method to delete a domain
"""
log.request(request)
app = request.app
domain = get_domain(request)
bucket = getBucketForDomain(domain)
log.info(f"delete domain: {domain}")
# raises exception if domain not found
if not bucket:
log.error(f"expected bucket to be used in domain: {domain}")
raise HTTPInternalServerError()
log.debug(f"using bucket: {bucket}")
domain_json = await get_metadata_obj(app, domain)
if domain_json:
log.debug("got domain json")
# delete domain
await delete_metadata_obj(app, domain, notify=True)
json_rsp = {"domain": domain}
resp = json_response(json_rsp)
log.response(request, resp=resp)
return resp
async def GET_Group(request):
"""HTTP GET method to return JSON for /groups/
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
log.info(f"GET group: {group_id} bucket: {bucket}")
if not isValidUuid(group_id, obj_class="group"):
log.error("Unexpected group_id: {}".format(group_id))
raise HTTPInternalServerError()
group_json = await get_metadata_obj(app, group_id, bucket=bucket)
resp_json = {}
resp_json["id"] = group_json["id"]
resp_json["root"] = group_json["root"]
resp_json["created"] = group_json["created"]
resp_json["lastModified"] = group_json["lastModified"]
resp_json["linkCount"] = len(group_json["links"])
resp_json["attributeCount"] = len(group_json["attributes"])
if "include_links" in params and params["include_links"]:
resp_json["links"] = group_json["links"]
if "include_attrs" in params and params["include_attrs"]:
resp_json["attributes"] = group_json["attributes"]
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
async def putS3JSONObj(app, key, json_obj, bucket=None):
""" Store JSON data as S3 object with given key
"""
client = getS3Client(app)
if not bucket:
bucket = app['bucket_name']
if key[0] == '/':
key = key[1:] # no leading slash
log.info(f"putS3JSONObj(s3://{bucket}/{key})")
s3_stats_increment(app, "put_count")
data = json.dumps(json_obj)
data = data.encode('utf8')
start_time = time.time()
try:
rsp = await client.put_object(Bucket=bucket, Key=key, Body=data)
finish_time = time.time()
log.info(
f"s3Util.putS3JSONObj({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}"
)
s3_rsp = {
"etag": rsp["ETag"],
"size": len(data),
"lastModified": int(finish_time)
}
except ClientError as ce:
s3_stats_increment(app, "error_count")
msg = f"Error putting s3 obj {key}: {ce}"
log.error(msg)
raise HTTPInternalServerError()
if data and len(data) > 0:
s3_stats_increment(app, "bytes_out", inc=len(data))
log.debug(f"putS3JSONObj {key} complete, s3_rsp: {s3_rsp}")
return s3_rsp
