async def get_key_stats(self, key, bucket=None): """ Get ETag, size, and last modified time for given objecct """ start_time = time.time() key_stats = {} try: async with self._client.get_blob_client(container=bucket, blob=key) as blob_client: blob_props = await blob_client.get_blob_properties() finish_time = time.time() except CancelledError as cle: self._azure_stats_increment("error_count") msg = f"azureBlobClient.CancelledError get_blob_properties {key}: {cle}" log.error(msg) raise HTTPInternalServerError() except Exception as e: if isinstance(e, AzureError): if e.status_code == 404: msg = f"storage key: {key} not found " log.warn(msg) raise HTTPNotFound() elif e.status_code in (401, 403): msg = f"azureBlobClient.access denied for get_blob_properties key: {key}" log.info(msg) raise HTTPForbidden() else: self._azure_stats_increment("error_count") log.error( f"azureBlobClient.got unexpected AzureError for get_blob_properties {key}: {e.message}" ) raise HTTPInternalServerError() else: log.error( f"azureBlobClient.Unexpected exception for get_blob_properties {key}: {e}" ) raise HTTPInternalServerError() last_modified_dt = blob_props.last_modified if not isinstance(last_modified_dt, datetime.datetime): msg = "azureBlobClient.get_key_stats, expected datetime object in head data" log.error(msg) raise HTTPInternalServerError() key_stats = {} key_stats["Size"] = blob_props.size key_stats["ETag"] = blob_props.etag key_stats["LastModified"] = datetime.datetime.timestamp( last_modified_dt) log.info( f"azureBlobClient.get_key_stats({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f}" ) return key_stats
async def get_key_stats(self, key, bucket=None): """ Get ETag, size, and last modified time for given objecct """ start_time = time.time() session = self._app["session"] self._renewToken() async with session.create_client('s3', region_name=self._aws_region, aws_secret_access_key=self._aws_secret_access_key, aws_access_key_id=self._aws_access_key_id, aws_session_token=self._aws_session_token, endpoint_url=self._s3_gateway, use_ssl=self._use_ssl, config=self._aio_config) as _client: try: head_data = await _client.head_object(Bucket=bucket, Key=key) finish_time = time.time() log.info(f"head: {head_data}") except ClientError: # key does not exist? msg = f"s3Client.get_key_stats: Key: {key} not found" log.info(msg) finish_time = time.time() raise HTTPNotFound() except CancelledError as cle: self._s3_stats_increment("error_count") msg = f"s3Client.get_key_stats: CancelledError getting head for s3 obj {key}: {cle}" log.error(msg) raise HTTPInternalServerError() except Exception as e: self._s3_stats_increment("error_count") msg = f"s3Client.get_key_stats: Unexpected Exception {type(e)} getting head for s3 obj {key}: {e}" log.error(msg) raise HTTPInternalServerError() for head_key in ("ContentLength", "ETag", "LastModified"): if head_key not in head_data: msg = f"s3Client.get_key_stats, expected to find key: {head_key} in head_data" log.error(msg) raise HTTPInternalServerError() last_modified_dt = head_data["LastModified"] if not isinstance(last_modified_dt, datetime.datetime): msg ="S3Client.get_key_stats, expected datetime object in head data" log.error(msg) raise HTTPInternalServerError() key_stats = {} key_stats["Size"] = head_data['ContentLength'] key_stats["ETag"] = head_data["ETag"] key_stats["LastModified"] = datetime.datetime.timestamp(last_modified_dt) log.info(f"s3Client.get_key_stats({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f}") return key_stats
async def PUT_ACL(request): """ Handler creating/update an ACL""" log.request(request) app = request.app acl_username = request.match_info.get('username') if not request.has_body: msg = "Expected body in delete domain" log.error(msg) raise HTTPInternalServerError() body_json = await request.json() domain = get_domain(request, body=body_json) log.info(f"put_acl - domain: {domain}, username: {acl_username}") # raises exception if domain not found domain_json = await get_metadata_obj(app, domain) if "acls" not in domain_json: log.error(f"unexpected domain data for domain: {domain}") raise HTTPInternalServerError() # 500 acl_keys = getAclKeys() acls = domain_json["acls"] acl = {} if acl_username in acls: acl = acls[acl_username] else: # initialize acl with no perms for k in acl_keys: acl[k] = False # replace any permissions given in the body for k in body_json.keys(): acl[k] = body_json[k] # replace/insert the updated/new acl acls[acl_username] = acl # update the timestamp now = time.time() domain_json["lastModified"] = now # write back to S3 await save_metadata_obj(app, domain, domain_json, flush=True) resp_json = {} resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
def __init__(self, app): self._app = app self._root_dir = config.get("root_dir") if not self._root_dir: log.error("FileClient init: root_dir config not set") raise HTTPInternalServerError() if not pp.isdir(self._root_dir): log.error("FileClient init: root folder does not exist") raise HTTPInternalServerError() if not pp.isabs(self._root_dir): log.error("FileClient init: root dir most have absolute path") raise HTTPInternalServerError()
async def getPathForObjectId(app, parent_id, idpath_map, tgt_id=None): """ Search the object starting with the given parent_id. idpath should be a dict with at minimum the key: parent_id: <parent_path>. If tgt_id is not None, returns first path that matches the tgt_id or None if not found. If Tgt_id is no, returns the idpath_map. """ if not parent_id: log.error("No parent_id passed to getPathForObjectId") raise HTTPInternalServerError() if parent_id not in idpath_map: msg = "Obj {} expected to be found in idpath_map".format(parent_id) log.error(msg) raise HTTPInternalServerError() parent_path = idpath_map[parent_id] if parent_id == tgt_id: return parent_path req = getDataNodeUrl(app, parent_id) req += "/groups/" + parent_id + "/links" log.debug("getPathForObjectId LINKS: " + req) links_json = await http_get(app, req) log.debug( "getPathForObjectId got links json from dn for parent_id: {}".format( parent_id)) links = links_json["links"] h5path = None for link in links: if link["class"] != "H5L_TYPE_HARD": continue # ignore everything except hard links link_id = link["id"] if link_id in idpath_map: continue # this node has already been visited title = link["title"] if tgt_id is not None and link_id == tgt_id: # found it! h5path = op.join(parent_path, title) break idpath_map[link_id] = op.join(parent_path, title) if getCollectionForId(link_id) != "groups": continue h5path = await getPathForObjectId(app, link_id, idpath_map, tgt_id) # recursive call if tgt_id is not None and h5path: break return h5path
async def put_object(self, key, data, bucket=None): """ Write data to given key. Returns client specific dict on success """ if not bucket: log.error("put_object - bucket not set") raise HTTPInternalServerError() start_time = time.time() log.debug(f"azureBlobClient.put_object({bucket}/{key} start: {start_time}") try: async with self._client.get_blob_client(container=bucket, blob=key) as blob_client: blob_rsp = await blob_client.upload_blob(data, blob_type='BlockBlob', overwrite=True) finish_time = time.time() ETag = blob_rsp["etag"] lastModified = int(blob_rsp["last_modified"].timestamp()) data_size = len(data) rsp = {"ETag": ETag, "size": data_size, "LastModified": lastModified } log.debug(f"put_object {key} returning: {rsp}") log.info(f"azureBlobClient.put_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}") except CancelledError as cle: self._azure_stats_increment("error_count") msg = f"azureBlobClient.CancelledError for put_object {key}: {cle}" log.error(msg) raise HTTPInternalServerError() except Exception as e: if isinstance(e, AzureError): if e.status_code == 404: msg = f"azureBlobClient.key: {key} not found " log.warn(msg) raise HTTPNotFound() elif e.status_code in (401, 403): msg = f"azureBlobClient.access denied for get key: {key}" log.info(msg) raise HTTPForbidden() else: self._azure_stats_increment("error_count") log.error(f"azureBlobClient.got unexpected AzureError for get_object {key}: {e.message}") raise HTTPInternalServerError() else: log.error(f"azureBlobClient.Unexpected exception for put_object {key}: {e}") raise HTTPInternalServerError() if data and len(data) > 0: self._azure_stats_increment("bytes_out", inc=len(data)) log.debug(f"azureBlobClient.put_object {key} complete, rsp: {rsp}") return rsp
async def getS3JSONObj(app, key): """ Get S3 object identified by key and read as JSON """ client = getS3Client(app) bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash log.info("getS3JSONObj({})".format(key)) s3_stats_increment(app, "get_count") try: resp = await client.get_object(Bucket=bucket, Key=key) data = await resp['Body'].read() resp['Body'].close() except ClientError as ce: # key does not exist? # check for not found status # Note: Error.Code should always exist - cf https://github.com/boto/botocore/issues/885 log.info("ClientError on getS3JSONObj") response_code = ce.response['Error']['Code'] # remove key from pending map if present if "pending_s3_read" in app: pending_s3_read = app["pending_s3_read"] if key in pending_s3_read: log.debug(f"remove {key} from pending_s3_read") del pending_s3_read[key] if response_code == "NoSuchKey": msg = "s3_key: {} not found ".format(key,) log.info(msg) raise HTTPNotFound() else: s3_stats_increment(app, "error_count") log.warn("got ClientError on s3 get: {}".format(str(ce))) msg = "Error getting s3 obj: " + str(ce) log.error(msg) raise HTTPInternalServerError() s3_stats_increment(app, "bytes_in", inc=len(data)) try: json_dict = json.loads(data.decode('utf8')) except UnicodeDecodeError: s3_stats_increment(app, "error_count") log.error("Error loading JSON at key: {}".format(key)) msg = "Unexpected i/o error" raise HTTPInternalServerError() log.debug("s3 returned: {}".format(json_dict)) return json_dict
async def get_object(self, key, bucket=None, offset=0, length=-1): """ Return data for object at given key. If Range is set, return the given byte range. """ self._validateBucket(bucket) self._validateKey(key) if length > 0: range = f"bytes={offset}-{offset+length-1}" log.info(f"storage range request: {range}") filepath = self._getFilePath(bucket, key) log.info(f"get_object - filepath: {filepath}") start_time = time.time() log.debug(f"fileClient.get_object({bucket}/{key} start: {start_time}") loop = asyncio.get_event_loop() try: async with aiofiles.open(filepath, loop=loop, mode='rb') as f: if offset: await f.seek(offset) if length > 0: data = await f.read(length) else: data = await f.read() finish_time = time.time() log.info( f"fileClient.get_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}" ) except FileNotFoundError: msg = f"fileClient: {key} not found " log.warn(msg) raise HTTPNotFound() except IOError as ioe: msg = f"fileClient: IOError reading {bucket}/{key}: {ioe}" log.warn(msg) raise HTTPInternalServerError() except CancelledError as cle: self._file_stats_increment("error_count") msg = f"CancelledError for get file obj {key}: {cle}" log.error(msg) raise HTTPInternalServerError() except Exception as e: self._file_stats_increment("error_count") msg = f"Unexpected Exception {type(e)} get get_object {key}: {e}" log.error(msg) raise HTTPInternalServerError() return data
async def put_object(self, key, data, bucket=None): """ Write data to given key. Returns client specific dict on success """ if not bucket: log.error("put_object - bucket not set") raise HTTPInternalServerError() start_time = time.time() log.debug(f"s3Client.put_object({bucket}/{key} start: {start_time}") session = self._app["session"] self._renewToken() async with session.create_client('s3', region_name=self._aws_region, aws_secret_access_key=self._aws_secret_access_key, aws_access_key_id=self._aws_access_key_id, aws_session_token=self._aws_session_token, endpoint_url=self._s3_gateway, use_ssl=self._use_ssl, config=self._aio_config) as _client: try: rsp = await _client.put_object(Bucket=bucket, Key=key, Body=data) finish_time = time.time() log.info(f"s3Client.put_object({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}") s3_rsp = {"etag": rsp["ETag"], "size": len(data), "lastModified": int(finish_time)} except ClientError as ce: response_code = ce.response["Error"]["Code"] if response_code == "NoSuchBucket": msg = f"s3_bucket: {bucket} not found" log.warn(msg) raise HTTPNotFound() else: self._s3_stats_increment("error_count") msg = f"Error putting s3 obj {key}: {ce}" log.error(msg) raise HTTPInternalServerError() except CancelledError as cle: #s3_stats_increment(app, "error_count") msg = f"CancelledError for put s3 obj {key}: {cle}" log.error(msg) raise HTTPInternalServerError() except Exception as e: #s3_stats_increment(app, "error_count") msg = f"Unexpected Exception {type(e)} putting s3 obj {key}: {e}" log.error(msg) raise HTTPInternalServerError() if data and len(data) > 0: self._s3_stats_increment("bytes_out", inc=len(data)) log.debug(f"s3Client.put_object {key} complete, s3_rsp: {s3_rsp}") return s3_rsp
def aclCheck(app, obj_json, req_action, req_user): log.info(f"aclCheck: {req_action} for user: {req_user}") if isAdminUser(app, req_user): return # allow admin user to do anything if obj_json is None: log.error("aclCheck: no obj json") raise HTTPInternalServerError() # 500 if "acls" not in obj_json: log.error("no acl key") raise HTTPInternalServerError() # 500 acls = obj_json["acls"] log.debug(f"acls: {acls}") if req_action not in ("create", "read", "update", "delete", "readACL", "updateACL"): log.error(f"unexpected req_action: {req_action}") if req_user in acls: acl = acls[req_user] log.debug(f"got acl: {acl} for user: {req_user}") if req_action in acl and acl[req_action]: log.debug("action permitted by user acl") return else: # treat deny for username as authorative deny log.warn( f"Action: {req_action} not permitted for user: {req_user}") raise HTTPForbidden() # 403 if "default" in acls: acl = acls["default"] log.debug(f"got default acl: {acl}") if req_action in acl and acl[req_action]: log.debug("action permitted by default acl") return user_group_db = app["user_group_db"] if req_user in user_group_db: user_groups = user_group_db[req_user] for user_group in user_groups: acl_name = GROUP_PREFIX + user_group log.debug(f"checking group acl: {acl_name}") if acl_name in acls: acl = acls[acl_name] if req_action in acl and acl[req_action]: log.debug(f"action permitted by group acl: {acl_name}") return log.warn(f"Action: {req_action} not permitted for user: {req_user}") raise HTTPForbidden() # 403
async def getS3Bytes(app, key, deflate_level=None, s3offset=0, s3size=None, bucket=None): """ Get S3 object identified by key and read as bytes """ client = getS3Client(app) if not bucket: bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash log.info("getS3Bytes({})".format(key)) s3_stats_increment(app, "get_count") range="" if s3size: range = f"bytes={s3offset}-{s3offset+s3size-1}" log.info(f"s3 range request: {range}") try: resp = await client.get_object(Bucket=bucket, Key=key, Range=range) data = await resp['Body'].read() resp['Body'].close() except ClientError as ce: # key does not exist? # check for not found status if ce.response["Error"]["Code"] == "NoSuchKey": msg = "s3_key: {} not found ".format(key,) log.warn(msg) raise HTTPInternalServerError() else: s3_stats_increment(app, "error_count") log.warn("got ClientError on s3 get: {}".format(str(ce))) msg = "Error getting s3 obj: " + str(ce) log.error(msg) raise HTTPInternalServerError() if data and len(data) > 0: s3_stats_increment(app, "bytes_in", inc=len(data)) log.info("read: {} bytes for S3 key: {}".format(len(data), key)) if deflate_level is not None: try: unzip_data = zlib.decompress(data) log.info("uncompressed to {} bytes".format(len(unzip_data))) data = unzip_data except zlib.error as zlib_error: log.info("zlib_err: {}".format(zlib_error)) log.warn("unable to uncompress s3 obj: {}, returning raw bytes".format(key)) return data
async def validate_token(self, token: str) -> dict: """ Request the JWT token validation to the associated UAA service Args: token: JWT token to validate Returns: user authenticated if the token is valid """ async with ClientSession() as client: validate_token_url = self.URL_BASE.format(protocol=self._protocol, host=self._host, port=self._port, path=self.PATHS['validate_token']) try: async with client.post(validate_token_url, data=dict(token=token)) as response: if response.status == 200: response_json = await response.json() elif response.status == 500: raise HTTPUnauthorized(reason='Wrong authorization token') else: raise HTTPInternalServerError(reason='Error calling uaa service') except Exception as ex: if not isinstance(ex, HTTPUnauthorized) and not isinstance(ex, HTTPInternalServerError): raise ConnectionError(f'Error calling uaa service {str(ex)}') else: raise ex if 'id' in response_json: return response_json else: raise HTTPUnauthorized(reason='Wrong authorization token')
async def deleteS3Obj(app, key): """ Delete S3 object identfied by given key """ client = getS3Client(app) bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash log.info("deleteS3Obj({})".format(key)) s3_stats_increment(app, "delete_count") try: await client.delete_object(Bucket=bucket, Key=key) except ClientError as ce: # key does not exist? key_found = await isS3Obj(app, key) if not key_found: log.warn(f"delete on s3key {key} but not found") raise HTTPNotFound() # else some other error s3_stats_increment(app, "error_count") msg = "Error deleting s3 obj: " + str(ce) log.error(msg) raise HTTPInternalServerError() log.debug("deleteS3Obj complete")
async def putS3JSONObj(app, key, json_obj): """ Store JSON data as S3 object with given key """ client = getS3Client(app) bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash log.info("putS3JSONObj({})".format(key)) s3_stats_increment(app, "put_count") data = json.dumps(json_obj) data = data.encode('utf8') try: rsp = await client.put_object(Bucket=bucket, Key=key, Body=data) now = int(time.time()) s3_rsp = {"etag": rsp["ETag"], "size": len(data), "lastModified": now} except ClientError as ce: s3_stats_increment(app, "error_count") msg = "Error putting s3 obj: " + str(ce) log.error(msg) raise HTTPInternalServerError() if data and len(data) > 0: s3_stats_increment(app, "bytes_out", inc=len(data)) log.debug("putS3JSONObj complete, s3_rsp: {}".format(s3_rsp)) return s3_rsp
async def error_middleware(request, handler): try: http_exception = getattr(request.match_info, 'http_exception', None) if http_exception: raise http_exception else: r = await handler(request) except HTTPException as e: if request.method == METH_GET and e.status == 404 and request.rel_url.raw_path.endswith( '/'): possible_path = request.rel_url.raw_path[:-1] for resource in request.app.router._resources: match_dict = resource._match(possible_path) if match_dict: raise HTTPMovedPermanently(possible_path) if e.status > 310: await log_warning(request, e) raise except BaseException as e: request_logger.exception( '%s: %s', e.__class__.__name__, e, extra={ 'fingerprint': [e.__class__.__name__, str(e)], 'data': await log_extra(request) }, ) raise HTTPInternalServerError() else: if r.status > 310: await log_warning(request, r) return r
async def putStorBytes(app, key, data, filter_ops=None, bucket=None): """ Store byte string as S3 object with given key """ client = _getStorageClient(app) if not bucket: bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash shuffle = -1 # auto-shuffle clevel = 5 cname = None # compressor name if filter_ops: if "compressor" in filter_ops: cname = filter_ops["compressor"] if "use_shuffle" in filter_ops and not filter_ops['use_shuffle']: shuffle = 0 # client indicates to turn off shuffling if "level" in filter_ops: clevel = filter_ops["level"] log.info(f"putStorBytes({bucket}/{key}), {len(data)} bytes shuffle: {shuffle} compressor: {cname} level: {clevel}") if cname: try: blosc = codecs.Blosc(cname=cname, clevel=clevel, shuffle=shuffle) cdata = blosc.encode(data) # TBD: add cname in blosc constructor log.info(f"compressed from {len(data)} bytes to {len(cdata)} bytes using filter: {blosc.cname} with level: {blosc.clevel}") data = cdata except Exception as e: log.error(f"got exception using blosc encoding: {e}") raise HTTPInternalServerError() rsp = await client.put_object(key, data, bucket=bucket) return rsp
async def getSNInstances(self): if "is_dcos" not in self._app: msg = "cannot use the MarathonClient in a non-DCOS context" log.error(msg) raise HTTPInternalServerError() if "DCOS_PATH_SERVICE_NODE" in os.environ: hsds_sn_node = os.environ["DCOS_PATH_SERVICE_NODE"] else: log.error( "Must set DCOS_PATH_SERVICE_NODE environment variable to Marathon path to service node n order to query the correct marathon config" ) return -1 req = "http://master.mesos/marathon/v2/apps/%s" % hsds_sn_node try: instancesJSON = await http_get(self._app, req) except HTTPNotFound: log.warn("Could not retrieve marathon app instance information.") return -1 if instancesJSON is None or not isinstance(instancesJSON, dict): log.warn("invalid marathon query response") else: if instancesJSON["app"] is not None and instancesJSON["app"][ "instances"] is not None: log.debug("SN instances {}".format( instancesJSON["app"]["instances"])) return instancesJSON["app"]["instances"] else: log.warn("Incomplete or malformed JSON returned from SN node.") return -1
async def DELETE_Dataset(request): """HTTP DELETE method for dataset """ log.request(request) app = request.app params = request.rel_url.query dset_id = request.match_info.get('id') log.info(f"DELETE dataset: {dset_id}") if not isValidUuid(dset_id, obj_class="dataset"): log.error(f"Unexpected dataset id: {dset_id}") raise HTTPInternalServerError() if "bucket" in params: bucket = params["bucket"] else: bucket = None # verify the id exist obj_found = await check_metadata_obj(app, dset_id, bucket=bucket) if not obj_found: raise HTTPNotFound() log.debug(f"deleting dataset: {dset_id}") notify=True if "Notify" in params and not params["Notify"]: notify=False await delete_metadata_obj(app, dset_id, bucket=bucket, notify=notify) resp_json = { } resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Group(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app group_id = get_obj_id(request) log.info("GET group: {}".format(group_id)) if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() group_json = await get_metadata_obj(app, group_id) resp_json = {} resp_json["id"] = group_json["id"] resp_json["root"] = group_json["root"] resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] resp_json["linkCount"] = len(group_json["links"]) resp_json["attributeCount"] = len(group_json["attributes"]) resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def getObjectIdByPath(app, obj_id, h5path, bucket=None, refresh=False): """ Find the object at the provided h5path location. If not found raise 404 error. """ log.info( f"getObjectIdByPath obj_id: {obj_id} h5path: {h5path} refresh: {refresh}" ) if h5path.startswith("./"): h5path = h5path[2:] # treat as relative path links = h5path.split('/') for link in links: if not link: continue # skip empty link log.debug(f"getObjectIdByPath for objid: {obj_id} got link: {link}") if getCollectionForId(obj_id) != "groups": # not a group, so won't have links msg = f"h5path: {h5path} not found" log.warn(msg) raise HTTPNotFound() req = getDataNodeUrl(app, obj_id) req += "/groups/" + obj_id + "/links/" + link log.debug("get LINK: " + req) params = {} if bucket: params["bucket"] = bucket link_json = await http_get(app, req, params=params) log.debug("got link_json: " + str(link_json)) if link_json["class"] != 'H5L_TYPE_HARD': # don't follow soft/external links msg = f"h5path: {h5path} not found" log.warn(msg) raise HTTPInternalServerError() obj_id = link_json["id"] # if we get here, we've traveresed the entire path and found the object return obj_id
async def GET_Datatype(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app params = request.rel_url.query ctype_id = get_obj_id(request) if not isValidUuid(ctype_id, obj_class="type"): log.error(f"Unexpected type_id: {ctype_id}") raise HTTPInternalServerError() if "bucket" in params: bucket = params["bucket"] else: bucket = None ctype_json = await get_metadata_obj(app, ctype_id, bucket=bucket) resp_json = { } resp_json["id"] = ctype_json["id"] resp_json["root"] = ctype_json["root"] resp_json["created"] = ctype_json["created"] resp_json["lastModified"] = ctype_json["lastModified"] resp_json["type"] = ctype_json["type"] resp_json["attributeCount"] = len(ctype_json["attributes"]) if "include_attrs" in params and params["include_attrs"]: resp_json["attributes"] = ctype_json["attributes"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Attribute(request): """HTTP GET method to return JSON for /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) attr_name = request.match_info.get('name') validateAttributeName(attr_name) if "bucket" in params: bucket = params["bucket"] else: bucket = None obj_json = await get_metadata_obj(app, obj_id, bucket=bucket) log.info(f"GET attribute obj_id: {obj_id} name: {attr_name} bucket: {bucket}") log.debug(f"got obj_json: {obj_json}") if "attributes" not in obj_json: log.error(f"unexpected obj data for id: {obj_id}") raise HTTPInternalServerError() attributes = obj_json["attributes"] if attr_name not in attributes: msg = f"Attribute '{attr_name}' not found for id: {obj_id}" log.warn(msg) raise HTTPNotFound() attr_json = attributes[attr_name] resp = json_response(attr_json) log.response(request, resp=resp) return resp
async def GET_Dataset(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app dset_id = get_obj_id(request) if not isValidUuid(dset_id, obj_class="dataset"): log.error("Unexpected type_id: {}".format(dset_id)) raise HTTPInternalServerError() dset_json = await get_metadata_obj(app, dset_id) resp_json = {} resp_json["id"] = dset_json["id"] resp_json["root"] = dset_json["root"] resp_json["created"] = dset_json["created"] resp_json["lastModified"] = dset_json["lastModified"] resp_json["type"] = dset_json["type"] resp_json["shape"] = dset_json["shape"] resp_json["attributeCount"] = len(dset_json["attributes"]) if "creationProperties" in dset_json: resp_json["creationProperties"] = dset_json["creationProperties"] if "layout" in dset_json: resp_json["layout"] = dset_json["layout"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def health_get(request: Request) -> Response: try: return Response(body=json.dumps({"status": "OK"}), headers={'content-type': 'application/json'}) except Exception as ex: log.warning(f"Endpoint: health, Method: get. Error:{str(ex)}") return HTTPInternalServerError()
async def DELETE_Group(request): """HTTP DELETE method for /groups/ """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info("DELETE group: {}".format(group_id)) if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() # verify the id exist obj_found = await check_metadata_obj(app, group_id) if not obj_found: log.debug(f"delete called on non-exsistet obj: {group_id}") raise HTTPNotFound() log.debug("deleting group: {}".format(group_id)) notify = True if "Notify" in params and not params["Notify"]: notify = False await delete_metadata_obj(app, group_id, notify=notify) resp_json = {} resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def index(request: Request): try: return Response(body=json.dumps({"host": socket.gethostname()}), headers={'content-type': 'application/json'}) except Exception as ex: log.warning(f"Endpoint: /, Method: get. Error:{str(ex)}") return HTTPInternalServerError()
async def check_metadata_obj(app, obj_id, bucket=None): """ Return False is obj does not exist """ validateObjId(obj_id, bucket) if isValidDomain(obj_id): bucket = getBucketForDomain(obj_id) try: validateInPartition(app, obj_id) except KeyError: log.error("Domain not in partition") raise HTTPInternalServerError() deleted_ids = app['deleted_ids'] if obj_id in deleted_ids: msg = f"{obj_id} has been deleted" log.info(msg) return False meta_cache = app['meta_cache'] if obj_id in meta_cache: found = True else: # Not in chache, check s3 obj exists s3_key = getS3Key(obj_id) log.debug(f"check_metadata_obj({s3_key})") # does key exist? found = await isS3Obj(app, s3_key, bucket=bucket) return found
async def DELETE_Domain(request): """HTTP DELETE method to delete a domain """ log.request(request) app = request.app domain = get_domain(request) bucket = getBucketForDomain(domain) log.info(f"delete domain: {domain}") # raises exception if domain not found if not bucket: log.error(f"expected bucket to be used in domain: {domain}") raise HTTPInternalServerError() log.debug(f"using bucket: {bucket}") domain_json = await get_metadata_obj(app, domain) if domain_json: log.debug("got domain json") # delete domain await delete_metadata_obj(app, domain, notify=True) json_rsp = {"domain": domain} resp = json_response(json_rsp) log.response(request, resp=resp) return resp
async def GET_Group(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) if "bucket" in params: bucket = params["bucket"] else: bucket = None log.info(f"GET group: {group_id} bucket: {bucket}") if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() group_json = await get_metadata_obj(app, group_id, bucket=bucket) resp_json = {} resp_json["id"] = group_json["id"] resp_json["root"] = group_json["root"] resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] resp_json["linkCount"] = len(group_json["links"]) resp_json["attributeCount"] = len(group_json["attributes"]) if "include_links" in params and params["include_links"]: resp_json["links"] = group_json["links"] if "include_attrs" in params and params["include_attrs"]: resp_json["attributes"] = group_json["attributes"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def putS3JSONObj(app, key, json_obj, bucket=None): """ Store JSON data as S3 object with given key """ client = getS3Client(app) if not bucket: bucket = app['bucket_name'] if key[0] == '/': key = key[1:] # no leading slash log.info(f"putS3JSONObj(s3://{bucket}/{key})") s3_stats_increment(app, "put_count") data = json.dumps(json_obj) data = data.encode('utf8') start_time = time.time() try: rsp = await client.put_object(Bucket=bucket, Key=key, Body=data) finish_time = time.time() log.info( f"s3Util.putS3JSONObj({key} bucket={bucket}) start={start_time:.4f} finish={finish_time:.4f} elapsed={finish_time-start_time:.4f} bytes={len(data)}" ) s3_rsp = { "etag": rsp["ETag"], "size": len(data), "lastModified": int(finish_time) } except ClientError as ce: s3_stats_increment(app, "error_count") msg = f"Error putting s3 obj {key}: {ce}" log.error(msg) raise HTTPInternalServerError() if data and len(data) > 0: s3_stats_increment(app, "bytes_out", inc=len(data)) log.debug(f"putS3JSONObj {key} complete, s3_rsp: {s3_rsp}") return s3_rsp