async def get(self, request: web.Request): params = request.query if request.query.get('embed'): link_id = request.query.get('url') if link_id not in LINKS: raise HTTPNotFound() link = LINKS[link_id] if link['ts'] and time.time() > link['ts']: LINKS.pop(link_id) raise HTTPGone() if link['limit']: link['limit'] -= 1 if link['limit'] == 0: LINKS.pop(link_id) params = link elif not request.get(KEY_AUTHENTICATED, False): # you shall not pass raise HTTPUnauthorized() ws_server = web.WebSocketResponse(autoclose=False, autoping=False) await ws_server.prepare(request) try: hass = request.app['hass'] url = await ws_connect(hass, params) async with async_get_clientsession(hass).ws_connect( url, autoclose=False, autoping=False) as ws_client: # Proxy requests await asyncio.wait([ _websocket_forward(ws_server, ws_client), _websocket_forward(ws_client, ws_server), ], return_when=asyncio.FIRST_COMPLETED) except Exception as e: await ws_server.send_json({'error': str(e)}) return ws_server
async def http_get(app, url, params=None, format="json"): log.info("http_get('{}')".format(url)) client = get_http_client(app) data = None status_code = None timeout = config.get("timeout") try: async with client.get(url, params=params, timeout=timeout) as rsp: log.info("http_get status: {}".format(rsp.status)) status_code = rsp.status if rsp.status != 200: log.warn(f"request to {url} failed with code: {status_code}") else: # 200, so read the response if format == "json": data = await rsp.json() else: data = await rsp.read() # read response as bytes except ClientError as ce: log.debug(f"ClientError: {ce}") status_code = 404 except CancelledError as cle: log.error("CancelledError for http_get({}): {}".format(url, str(cle))) raise HTTPInternalServerError() if status_code == 403: log.warn(f"Forbiden to access {url}") raise HTTPForbidden() elif status_code == 404: log.warn(f"Object: {url} not found") raise HTTPNotFound() elif status_code == 410: log.warn(f"Object: {url} removed") raise HTTPGone() elif status_code == 503: log.warn(f"503 error for http_get_Json {url}") raise HTTPServiceUnavailable() elif status_code != 200: log.error(f"Error for http_get_json({url}): {status_code}") raise HTTPInternalServerError() return data
async def GET_Datatype(request): """HTTP method to return JSON for committed datatype""" log.request(request) app = request.app params = request.rel_url.query include_attrs = False h5path = None getAlias = False ctype_id = request.match_info.get('id') if not ctype_id and "h5path" not in params: msg = "Missing type id" log.warn(msg) raise HTTPBadRequest(reason=msg) if "include_attrs" in params and params["include_attrs"]: include_attrs = True if ctype_id: if not isValidUuid(ctype_id, "Type"): msg = f"Invalid type id: {ctype_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) if "getalias" in params: if params["getalias"]: getAlias = True else: group_id = None if "grpid" in params: group_id = params["grpid"] if not isValidUuid(group_id, "Group"): msg = f"Invalid parent group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) if "h5path" not in params: msg = "Expecting either ctype id or h5path url param" log.warn(msg) raise HTTPBadRequest(reason=msg) h5path = params["h5path"] if h5path[0] != '/' and group_id is None: msg = "h5paths must be absolute" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info(f"GET_Datatype, h5path: {h5path}") username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) if h5path: domain_json = await getDomainJson(app, domain) verifyRoot(domain_json) if group_id is None: group_id = domain_json["root"] ctype_id = await getObjectIdByPath(app, group_id, h5path, bucket=bucket ) # throws 404 if not found if not isValidUuid(ctype_id, "Datatype"): msg = f"No datatype exist with the path: {h5path}" log.warn(msg) raise HTTPGone() log.info(f"got ctype_id: {ctype_id} from h5path: {h5path}") await validateAction(app, domain, ctype_id, username, "read") # get authoritative state for ctype from DN (even if it's in the meta_cache). type_json = await getObjectJson(app, ctype_id, bucket=bucket, refresh=True, include_attrs=include_attrs) type_json["domain"] = domain if getAlias: root_id = type_json["root"] alias = [] idpath_map = {root_id: '/'} h5path = await getPathForObjectId(app, root_id, idpath_map, tgt_id=ctype_id, bucket=bucket) if h5path: alias.append(h5path) type_json["alias"] = alias hrefs = [] ctype_uri = '/datatypes/' + ctype_id hrefs.append({'rel': 'self', 'href': getHref(request, ctype_uri)}) root_uri = '/groups/' + type_json["root"] hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)}) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({ 'rel': 'attributes', 'href': getHref(request, ctype_uri + '/attributes') }) type_json["hrefs"] = hrefs resp = await jsonResponse(request, type_json) log.response(request, resp=resp) return resp
async def get_metadata_obj(app, obj_id, bucket=None): """ Get object from metadata cache (if present). Otherwise fetch from S3 and add to cache """ log.info(f"get_metadata_obj: {obj_id} bucket: {bucket}") validateObjId(obj_id, bucket) # throws internal server error if invalid if isValidDomain(obj_id): bucket = getBucketForDomain(obj_id) """ try: validateInPartition(app, obj_id) except KeyError: log.error("Domain not in partition") raise HTTPInternalServerError() """ deleted_ids = app['deleted_ids'] if obj_id in deleted_ids: msg = f"{obj_id} has been deleted" log.warn(msg) raise HTTPGone() meta_cache = app['meta_cache'] obj_json = None if obj_id in meta_cache: log.debug(f"{obj_id} found in meta cache") obj_json = meta_cache[obj_id] else: s3_key = getS3Key(obj_id) pending_s3_read = app["pending_s3_read"] if obj_id in pending_s3_read: # already a read in progress, wait for it to complete read_start_time = pending_s3_read[obj_id] log.info( f"s3 read request for {s3_key} was requested at: {read_start_time}" ) while time.time() - read_start_time < 2.0: log.debug("waiting for pending s3 read, sleeping") await asyncio.sleep(1) # sleep for sub-second? if obj_id in meta_cache: log.info(f"object {obj_id} has arrived!") obj_json = meta_cache[obj_id] break if not obj_json: log.warn( f"s3 read for object {s3_key} timed-out, initiaiting a new read" ) # invoke S3 read unless the object has just come in from pending read if not obj_json: log.debug(f"getS3JSONObj({s3_key}, bucket={bucket})") if obj_id not in pending_s3_read: pending_s3_read[obj_id] = time.time() # read S3 object as JSON obj_json = await getS3JSONObj(app, s3_key, bucket=bucket) if obj_id in pending_s3_read: # read complete - remove from pending map elapsed_time = time.time() - pending_s3_read[obj_id] log.info(f"s3 read for {s3_key} took {elapsed_time}") del pending_s3_read[obj_id] meta_cache[obj_id] = obj_json # add to cache return obj_json
async def DELETE_Domain(request): """HTTP method to delete a domain resource""" log.request(request) app = request.app params = request.rel_url.query domain = None meta_only = False # if True, just delete the meta cache value keep_root = False if request.has_body: body = await request.json() if "domain" in body: domain = body["domain"] else: msg = "No domain in request body" log.warn(msg) raise HTTPBadRequest(reason=msg) if "meta_only" in body: meta_only = body["meta_only"] if "keep_root" in body: keep_root = body["keep_root"] else: # get domain from request uri try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) if "keep_root" in params: keep_root = params["keep_root"] log.info("meta_only domain delete: {}".format(meta_only)) if meta_only: # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: log.info("deleting {} from domain_cache".format(domain)) del domain_cache[domain] resp = await jsonResponse(request, {}) return resp username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) parent_domain = getParentDomain(domain) if (not parent_domain or parent_domain == '/') and username != "admin": msg = "Deletion of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code == 404: log.warn("domain not found") raise HTTPNotFound() elif ce.code == 410: log.warn("domain has been removed") raise HTTPGone() else: log.error(f"unexpected error: {ce.code}") raise HTTPInternalServerError() aclCheck(domain_json, "delete", username) # throws exception if not allowed # check for sub-objects if this is a folder if "root" not in domain_json: s3prefix = domain[1:] + '/' log.info(f"checking kets with prefix: {s3prefix} ") s3keys = await getS3Keys(app, include_stats=False, prefix=s3prefix, deliminator='/') for s3key in s3keys: if s3key.endswith("/"): log.warn(f"attempt to delete folder {domain} with sub-items") log.debug(f"got prefix: {s3keys[0]}") raise HTTPConflict(reason="folder has sub-items") req = getDataNodeUrl(app, domain) req += "/domains" body = {"domain": domain} rsp_json = await http_delete(app, req, data=body) if "root" in domain_json and not keep_root: # delete the root group root_id = domain_json["root"] req = getDataNodeUrl(app, root_id) req += "/groups/" + root_id await http_delete(app, req) # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: del domain_cache[domain] # delete domain cache from other sn_urls sn_urls = app["sn_urls"] body["meta_only"] = True for node_no in sn_urls: if node_no == app["node_number"]: continue # don't send to ourselves sn_url = sn_urls[node_no] req = sn_url + "/" log.info("sending sn request: {}".format(req)) try: sn_rsp = await http_delete(app, req, data=body) log.info("{} response: {}".format(req, sn_rsp)) except ClientResponseError as ce: log.warn("got error for sn_delete: {}".format(ce)) resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def PUT_Domain(request): """HTTP method to create a new domain""" log.request(request) app = request.app params = request.rel_url.query # verify username, password username, pswd = getUserPasswordFromRequest( request) # throws exception if user/password is not valid await validateUserPassword(app, username, pswd) # inital perms for owner and default owner_perm = { 'create': True, 'read': True, 'update': True, 'delete': True, 'readACL': True, 'updateACL': True } default_perm = { 'create': False, 'read': True, 'update': False, 'delete': False, 'readACL': False, 'updateACL': False } try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info("PUT domain: {}, username: {}".format(domain, username)) body = None if request.has_body: body = await request.json() log.debug("PUT domain with body: {}".format(body)) if ("flush" in params and params["flush"]) or (body and "flush" in body and body["flush"]): # flush domain - update existing domain rather than create a new resource domain_json = await getDomainJson(app, domain, reload=True) log.debug("got domain_json: {}".format(domain_json)) if domain_json is None: log.warn("domain: {} not found".format(domain)) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() aclCheck(domain_json, "update", username) # throws exception if not allowed if "root" in domain_json: # nothing to do for folder objects await doFlush(app, domain_json["root"]) # flush successful resp = await jsonResponse(request, None, status=204) log.response(request, resp=resp) return resp is_folder = False owner = username linked_domain = None root_id = None if body and "folder" in body: if body["folder"]: is_folder = True if body and "owner" in body: owner = body["owner"] if body and "linked_domain" in body: if is_folder: msg = "Folder domains can not be used for links" log.warn(msg) raise HTTPBadRequest(reason=msg) linked_domain = body["linked_domain"] log.info(f"linking to domain: {linked_domain}") if owner != username and username != "admin": log.warn("Only admin users are allowed to set owner for new domains") raise HTTPForbidden() parent_domain = getParentDomain(domain) log.debug("Parent domain: [{}]".format(parent_domain)) if (not parent_domain or parent_domain == '/') and not is_folder: msg = "Only folder domains can be created at the top-level" log.warn(msg) raise HTTPBadRequest(reason=msg) if (not parent_domain or parent_domain == '/') and username != "admin": msg = "creation of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() parent_json = None if parent_domain and parent_domain != '/': try: parent_json = await getDomainJson(app, parent_domain, reload=True) except ClientResponseError as ce: if ce.code == 404: msg = "Parent domain: {} not found".format(parent_domain) log.warn(msg) raise HTTPNotFound() elif ce.code == 410: msg = "Parent domain: {} removed".format(parent_domain) log.warn(msg) raise HTTPGone() else: log.error(f"Unexpected error: {ce.code}") raise HTTPInternalServerError() log.debug("parent_json {}: {}".format(parent_domain, parent_json)) if "root" in parent_json and parent_json["root"]: msg = "Parent domain must be a folder" log.warn(msg) raise HTTPBadRequest(reason=msg) if parent_json: aclCheck(parent_json, "create", username) # throws exception if not allowed if linked_domain: linked_json = await getDomainJson(app, linked_domain, reload=True) log.debug(f"got linked json: {linked_json}") if "root" not in linked_json: msg = "Folder domains cannot ber used as link target" log.warn(msg) raise HTTPBadRequest(reason=msg) root_id = linked_json["root"] aclCheck(linked_json, "read", username) aclCheck(linked_json, "delete", username) else: linked_json = None if not is_folder and not linked_json: # create a root group for the new domain root_id = createObjId("roots") log.debug("new root group id: {}".format(root_id)) group_json = {"id": root_id, "root": root_id, "domain": domain} log.debug("create group for domain, body: " + json.dumps(group_json)) # create root group req = getDataNodeUrl(app, root_id) + "/groups" try: group_json = await http_post(app, req, data=group_json) except ClientResponseError as ce: msg = "Error creating root group for domain -- " + str(ce) log.error(msg) raise HTTPInternalServerError() else: log.debug("no root group, creating folder") domain_json = {} domain_acls = {} # owner gets full control domain_acls[owner] = owner_perm if config.get("default_public") or is_folder: # this will make the domain public readable log.debug("adding default perm for domain: {}".format(domain)) domain_acls["default"] = default_perm # construct dn request to create new domain req = getDataNodeUrl(app, domain) req += "/domains" body = {"owner": owner, "domain": domain} body["acls"] = domain_acls if root_id: body["root"] = root_id log.debug("creating domain: {} with body: {}".format(domain, body)) try: domain_json = await http_put(app, req, data=body) except ClientResponseError as ce: msg = "Error creating domain state -- " + str(ce) log.error(msg) raise HTTPInternalServerError() # domain creation successful # maxin limits domain_json["limits"] = getLimits() domain_json["version"] = getVersion() resp = await jsonResponse(request, domain_json, status=201) log.response(request, resp=resp) return resp
async def GET_Datasets(request): """HTTP method to return dataset collection for given domain""" log.request(request) app = request.app params = request.rel_url.query (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) # verify the domain try: domain_json = await getDomainJson(app, domain) except ClientResponseError as ce: if ce.code == 404: msg = "Domain: {} not found".format(domain) log.warn(msg) raise HTTPNotFound() elif ce.code == 410: msg = "Domain: {} removed".format(domain) log.warn(msg) raise HTTPGone() else: log.error(f"Unexpected error: {ce.code}") raise HTTPInternalServerError() msg = "domain not found" log.warn(msg) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() log.debug("got domain_json: {}".format(domain_json)) # validate that the requesting user has permission to read this domain aclCheck(domain_json, "read", username) # throws exception if not authorized limit = None if "Limit" in params: try: limit = int(params["Limit"]) except ValueError: msg = "Bad Request: Expected int type for limit" log.warn(msg) raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] obj_ids = [] if "root" in domain_json or domain_json["root"]: # get the dataset collection list collections = await get_collections(app, domain_json["root"]) objs = collections["datasets"] obj_ids = getIdList(objs, marker=marker, limit=limit) log.debug("returning obj_ids: {}".format(obj_ids)) # create hrefs hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/datasets')}) if "root" in domain_json: root_uuid = domain_json["root"] hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + root_uuid) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) # return obj ids and hrefs rsp_json = {} rsp_json["datasets"] = obj_ids rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def DELETE_Domain(request): """HTTP method to delete a domain resource""" log.request(request) app = request.app params = request.rel_url.query meta_only = False # if True, just delete the meta cache value keep_root = False if request.has_body: body = await request.json() if "meta_only" in body: meta_only = body["meta_only"] if "keep_root" in body: keep_root = body["keep_root"] else: if "meta_only" in params: meta_only = params["meta_only"] if "keep_root" in params: keep_root = params["keep_root"] domain = None try: domain = getDomainFromRequest(request) except ValueError: log.warn(f"Invalid domain: {domain}") raise HTTPBadRequest(reason="Invalid domain name") bucket = getBucketForDomain(domain) log.debug(f"GET_Domain domain: {domain}") if not domain: msg = "No domain given" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info(f"meta_only domain delete: {meta_only}") if meta_only: # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: log.info(f"deleting {domain} from domain_cache") del domain_cache[domain] resp = await jsonResponse(request, {}) return resp username, pswd = getUserPasswordFromRequest(request) await validateUserPassword(app, username, pswd) parent_domain = getParentDomain(domain) if not parent_domain or getPathForDomain(parent_domain) == '/': is_toplevel = True else: is_toplevel = False if is_toplevel and username != "admin": msg = "Deletion of top-level domains is only supported by admin users" log.warn(msg) raise HTTPForbidden() try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code == 404: log.warn("domain not found") raise HTTPNotFound() elif ce.code == 410: log.warn("domain has been removed") raise HTTPGone() else: log.error(f"unexpected error: {ce.code}") raise HTTPInternalServerError() aclCheck(domain_json, "delete", username) # throws exception if not allowed # check for sub-objects if this is a folder if "root" not in domain_json: index = domain.find('/') s3prefix = domain[(index + 1):] + '/' log.info(f"checking s3key with prefix: {s3prefix} in bucket: {bucket}") s3keys = await getS3Keys(app, include_stats=False, prefix=s3prefix, deliminator='/', bucket=bucket) for s3key in s3keys: if s3key.endswith("/"): log.warn(f"attempt to delete folder {domain} with sub-items") log.debug(f"got prefix: {s3keys[0]}") raise HTTPConflict(reason="folder has sub-items") req = getDataNodeUrl(app, domain) req += "/domains" params = {} # for http_delete requests to DN nodes params["domain"] = domain rsp_json = await http_delete(app, req, params=params) if "root" in domain_json and not keep_root: # delete the root group root_id = domain_json["root"] req = getDataNodeUrl(app, root_id) req += "/groups/" + root_id await http_delete(app, req, params=params) # remove from domain cache if present domain_cache = app["domain_cache"] if domain in domain_cache: del domain_cache[domain] # delete domain cache from other sn_urls sn_urls = app["sn_urls"] params = {} params["domain"] = getPathForDomain(domain) params["bucket"] = getBucketForDomain(domain) params[ "meta_only"] = 1 # can't pass booleans as params, so use 1 instead of True for node_no in sn_urls: if node_no == app["node_number"]: continue # don't send to ourselves sn_url = sn_urls[node_no] req = sn_url + "/" log.info(f"sending sn request: {req}") try: sn_rsp = await http_delete(app, req, params=params) log.info(f"{req} response: {sn_rsp}") except ClientResponseError as ce: log.warn(f"got error for sn_delete: {ce}") resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def get_metadata_obj(app, obj_id, bucket=None): """ Get object from metadata cache (if present). Otherwise fetch from S3 and add to cache """ log.info(f"get_metadata_obj: {obj_id} bucket: {bucket}") if isValidDomain(obj_id): bucket = getBucketForDomain(obj_id) # don't call validateInPartition since this is used to pull in # immutable data from other nodes deleted_ids = app['deleted_ids'] # don't raise 410 for domains since a domain might have been # re-created outside the server if obj_id in deleted_ids and not isValidDomain(obj_id): msg = f"{obj_id} has been deleted" log.warn(msg) raise HTTPGone() meta_cache = app['meta_cache'] obj_json = None if obj_id in meta_cache: log.debug(f"{obj_id} found in meta cache") obj_json = meta_cache[obj_id] else: s3_key = getS3Key(obj_id) pending_s3_read = app["pending_s3_read"] if obj_id in pending_s3_read: # already a read in progress, wait for it to complete read_start_time = pending_s3_read[obj_id] log.info( f"s3 read request for {s3_key} was requested at: {read_start_time}" ) while time.time() - read_start_time < 2.0: log.debug("waiting for pending s3 read, sleeping") await asyncio.sleep(1) # sleep for sub-second? if obj_id in meta_cache: log.info(f"object {obj_id} has arrived!") obj_json = meta_cache[obj_id] break if not obj_json: log.warn( f"s3 read for object {obj_id} timed-out, initiaiting a new read" ) # invoke S3 read unless the object has just come in from pending read if not obj_json: log.debug(f"getS3JSONObj({obj_id}, bucket={bucket})") if obj_id not in pending_s3_read: pending_s3_read[obj_id] = time.time() # read S3 object as JSON try: obj_json = await getStorJSONObj(app, s3_key, bucket=bucket) # read complete - remove from pending map elapsed_time = time.time() - pending_s3_read[obj_id] log.info(f"s3 read for {obj_id} took {elapsed_time}") meta_cache[obj_id] = obj_json # add to cache except HTTPNotFound: log.warn( f"HTTPNotFound for {obj_id} bucket:{bucket} s3key: {s3_key}" ) if obj_id in deleted_ids and isValidDomain(obj_id): raise HTTPGone() raise except HTTPForbidden: log.warn( f"HTTPForbidden error for {obj_id} bucket:{bucket} s3key: {s3_key}" ) raise except HTTPInternalServerError: log.warn( f"HTTPInternalServerError error for {obj_id} bucket:{bucket} s3key: {s3_key}" ) raise finally: if obj_id in pending_s3_read: del pending_s3_read[obj_id] return obj_json
async def post(self): raise HTTPGone()
async def get(self): raise HTTPGone()