async def GET_Attribute(request): """HTTP GET method to return JSON for /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app obj_id = get_obj_id(request) attr_name = request.match_info.get('name') validateAttributeName(attr_name) obj_json = await get_metadata_obj(app, obj_id) log.info("GET attribute obj_id: {} name: {}".format(obj_id, attr_name)) log.debug(f"got obj_json: {obj_json}") if "attributes" not in obj_json: log.error("unexpected obj data for id: {}".format(obj_id)) raise HTTPInternalServerError() attributes = obj_json["attributes"] if attr_name not in attributes: msg = f"Attribute '{attr_name}' not found for id: {obj_id}" log.warn(msg) raise HTTPNotFound() attr_json = attributes[attr_name] resp = json_response(attr_json) log.response(request, resp=resp) return resp
async def GET_Group(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app group_id = get_obj_id(request) log.info("GET group: {}".format(group_id)) if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() group_json = await get_metadata_obj(app, group_id) resp_json = {} resp_json["id"] = group_json["id"] resp_json["root"] = group_json["root"] resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] resp_json["linkCount"] = len(group_json["links"]) resp_json["attributeCount"] = len(group_json["attributes"]) resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Attribute(request): """HTTP GET method to return JSON for /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) attr_name = request.match_info.get('name') validateAttributeName(attr_name) if "bucket" in params: bucket = params["bucket"] else: bucket = None obj_json = await get_metadata_obj(app, obj_id, bucket=bucket) log.info(f"GET attribute obj_id: {obj_id} name: {attr_name} bucket: {bucket}") log.debug(f"got obj_json: {obj_json}") if "attributes" not in obj_json: log.error(f"unexpected obj data for id: {obj_id}") raise HTTPInternalServerError() attributes = obj_json["attributes"] if attr_name not in attributes: msg = f"Attribute '{attr_name}' not found for id: {obj_id}" log.warn(msg) raise HTTPNotFound() attr_json = attributes[attr_name] resp = json_response(attr_json) log.response(request, resp=resp) return resp
async def DELETE_Datatype(request): """HTTP DELETE method for datatype """ log.request(request) app = request.app params = request.rel_url.query ctype_id = get_obj_id(request) log.info(f"DELETE ctype: {ctype_id}") if "bucket" in params: bucket = params["bucket"] else: bucket = None # verify the id exist obj_found = await check_metadata_obj(app, ctype_id) if not obj_found: log.warn(f"Delete on non-existent obj: {ctype_id}") raise HTTPNotFound log.info("deleting ctype: {}".format(ctype_id)) notify=True if "Notify" in params and not params["Notify"]: log.info("notify value: {}".format(params["Notify"])) notify=False log.info("notify: {}".format(notify)) await delete_metadata_obj(app, ctype_id, bucket=bucket, notify=notify) resp_json = { } resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def DELETE_Group(request): """HTTP DELETE method for /groups/ """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info("DELETE group: {}".format(group_id)) if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() # verify the id exist obj_found = await check_metadata_obj(app, group_id) if not obj_found: log.debug(f"delete called on non-exsistet obj: {group_id}") raise HTTPNotFound() log.debug("deleting group: {}".format(group_id)) notify = True if "Notify" in params and not params["Notify"]: notify = False await delete_metadata_obj(app, group_id, notify=notify) resp_json = {} resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Dataset(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app dset_id = get_obj_id(request) if not isValidUuid(dset_id, obj_class="dataset"): log.error("Unexpected type_id: {}".format(dset_id)) raise HTTPInternalServerError() dset_json = await get_metadata_obj(app, dset_id) resp_json = {} resp_json["id"] = dset_json["id"] resp_json["root"] = dset_json["root"] resp_json["created"] = dset_json["created"] resp_json["lastModified"] = dset_json["lastModified"] resp_json["type"] = dset_json["type"] resp_json["shape"] = dset_json["shape"] resp_json["attributeCount"] = len(dset_json["attributes"]) if "creationProperties" in dset_json: resp_json["creationProperties"] = dset_json["creationProperties"] if "layout" in dset_json: resp_json["layout"] = dset_json["layout"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Datatype(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app params = request.rel_url.query ctype_id = get_obj_id(request) if not isValidUuid(ctype_id, obj_class="type"): log.error(f"Unexpected type_id: {ctype_id}") raise HTTPInternalServerError() if "bucket" in params: bucket = params["bucket"] else: bucket = None ctype_json = await get_metadata_obj(app, ctype_id, bucket=bucket) resp_json = { } resp_json["id"] = ctype_json["id"] resp_json["root"] = ctype_json["root"] resp_json["created"] = ctype_json["created"] resp_json["lastModified"] = ctype_json["lastModified"] resp_json["type"] = ctype_json["type"] resp_json["attributeCount"] = len(ctype_json["attributes"]) if "include_attrs" in params and params["include_attrs"]: resp_json["attributes"] = ctype_json["attributes"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Group(request): """HTTP GET method to return JSON for /groups/ """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) if "bucket" in params: bucket = params["bucket"] else: bucket = None log.info(f"GET group: {group_id} bucket: {bucket}") if not isValidUuid(group_id, obj_class="group"): log.error("Unexpected group_id: {}".format(group_id)) raise HTTPInternalServerError() group_json = await get_metadata_obj(app, group_id, bucket=bucket) resp_json = {} resp_json["id"] = group_json["id"] resp_json["root"] = group_json["root"] resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] resp_json["linkCount"] = len(group_json["links"]) resp_json["attributeCount"] = len(group_json["attributes"]) if "include_links" in params and params["include_links"]: resp_json["links"] = group_json["links"] if "include_attrs" in params and params["include_attrs"]: resp_json["attributes"] = group_json["attributes"] resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def DELETE_Link(request): """HTTP DELETE method for group links """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info(f"DELETE link: {group_id}") if not isValidUuid(group_id, obj_class="group"): msg = f"Unexpected group_id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) link_title = request.match_info.get('title') validateLinkName(link_title) if "bucket" in params: bucket = params["bucket"] else: bucket = None group_json = await get_metadata_obj(app, group_id, bucket=bucket) # TBD: Possible race condition if "links" not in group_json: log.error(f"unexpected group data for id: {group_id}") raise HTTPInternalServerError() links = group_json["links"] if link_title not in links: msg = f"Link name {link_title} not found in group: {group_id}" log.warn(msg) raise HTTPNotFound() del links[link_title] # remove the link from dictionary # update the group lastModified now = time.time() group_json["lastModified"] = now # write back to S3 await save_metadata_obj(app, group_id, group_json, bucket=bucket) hrefs = [] # TBD resp_json = {"href": hrefs} resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def DELETE_Attribute(request): """HTTP DELETE method for /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) if "bucket" in params: bucket = params["bucket"] else: bucket = None attr_name = request.match_info.get('name') log.info(f"DELETE attribute {attr_name} in {obj_id} bucket: {bucket}") validateAttributeName(attr_name) obj_json = await get_metadata_obj(app, obj_id, bucket=bucket) log.debug(f"DELETE attribute obj_id: {obj_id} got json") if "attributes" not in obj_json: msg = f"unexpected data for obj id: {obj_id}" msg.error(msg) raise HTTPInternalServerError() # return a list of attributes based on sorted dictionary keys attributes = obj_json["attributes"] if attr_name not in attributes: msg = f"Attribute {attr_name} not found in objid: {obj_id}" log.warn(msg) raise HTTPNotFound() del attributes[attr_name] await save_metadata_obj(app, obj_id, obj_json, bucket=bucket) resp_json = { } resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def DELETE_Attribute(request): """HTTP DELETE method for /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app obj_id = get_obj_id(request) attr_name = request.match_info.get('name') log.info("DELETE attribute {} in {}".format(attr_name, obj_id)) validateAttributeName(attr_name) obj_json = await get_metadata_obj(app, obj_id) log.debug("DELETE attribute obj_id: {} got json".format(obj_id)) if "attributes" not in obj_json: msg = "unexpected data for obj id: {}".format(obj_id) msg.error(msg) raise HTTPInternalServerError() # return a list of attributes based on sorted dictionary keys attributes = obj_json["attributes"] if attr_name not in attributes: msg = "Attribute {} not found in id: {}".format(attr_name, obj_id) log.warn(msg) raise HTTPNotFound() del attributes[attr_name] await save_metadata_obj(app, obj_id, obj_json) resp_json = { } resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Link(request): """HTTP GET method to return JSON for a link """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info(f"GET link: {group_id}") if not isValidUuid(group_id, obj_class="group"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() link_title = request.match_info.get('title') validateLinkName(link_title) if "bucket" in params: bucket = params["bucket"] else: bucket = None group_json = await get_metadata_obj(app, group_id, bucket=bucket) log.info(f"for id: {group_id} got group json: {group_json}") if "links" not in group_json: log.error(f"unexpected group data for id: {group_id}") raise HTTPInternalServerError() links = group_json["links"] if link_title not in links: log.warn(f"Link name {link_title} not found in group: {group_id}") raise HTTPNotFound() link_json = links[link_title] resp = json_response(link_json) log.response(request, resp=resp) return resp
async def POST_Dataset(request): """ Handler for POST /datasets""" log.request(request) app = request.app if not request.has_body: msg = "POST_Dataset with no body" log.error(msg) raise HTTPBadRequest(reason=msg) body = await request.json() log.info("POST_Dataset, body: {}".format(body)) dset_id = get_obj_id(request, body=body) if not isValidUuid(dset_id, obj_class="dataset"): log.error("Unexpected dataset_id: {}".format(dset_id)) raise HTTPInternalServerError() # verify the id doesn't already exist obj_found = await check_metadata_obj(app, dset_id) if obj_found: log.error("Post with existing dset_id: {}".format(dset_id)) raise HTTPInternalServerError() if "root" not in body: msg = "POST_Dataset with no root" log.error(msg) raise HTTPInternalServerError() root_id = body["root"] try: validateUuid(root_id, "group") except ValueError: msg = "Invalid root_id: " + root_id log.error(msg) raise HTTPInternalServerError() if "type" not in body: msg = "POST_Dataset with no type" log.error(msg) raise HTTPInternalServerError() type_json = body["type"] if "shape" not in body: msg = "POST_Dataset with no shape" log.error(msg) raise HTTPInternalServerError() shape_json = body["shape"] layout = None if "layout" in body: layout = body["layout"] # client specified chunk layout # ok - all set, create committed type obj now = int(time.time()) log.debug("POST_dataset typejson: {}, shapejson: {}".format( type_json, shape_json)) dset_json = { "id": dset_id, "root": root_id, "created": now, "lastModified": now, "type": type_json, "shape": shape_json, "attributes": {} } if "creationProperties" in body: dset_json["creationProperties"] = body["creationProperties"] if layout is not None: dset_json["layout"] = layout await save_metadata_obj(app, dset_id, dset_json, notify=True, flush=True) resp_json = {} resp_json["id"] = dset_id resp_json["root"] = root_id resp_json["created"] = dset_json["created"] resp_json["type"] = type_json resp_json["shape"] = shape_json resp_json["lastModified"] = dset_json["lastModified"] resp_json["attributeCount"] = 0 resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
async def POST_Datatype(request): """ Handler for POST /datatypes""" log.info("Post_Datatype") log.request(request) app = request.app params = request.rel_url.query if not request.has_body: msg = "POST_Datatype with no body" log.error(msg) raise HTTPBadRequest(reason=msg) body = await request.json() if "bucket" in params: bucket = params["bucket"] elif "bucket" in body: bucket = params["bucket"] else: bucket = None ctype_id = get_obj_id(request, body=body) if not isValidUuid(ctype_id, obj_class="datatype"): log.error( "Unexpected type_id: {}".format(ctype_id)) raise HTTPInternalServerError() # verify the id doesn't already exist obj_found = await check_metadata_obj(app, ctype_id, bucket=bucket) if obj_found: log.error( "Post with existing type_id: {}".format(ctype_id)) raise HTTPInternalServerError() root_id = None if "root" not in body: msg = "POST_Datatype with no root" log.error(msg) raise HTTPInternalServerError() root_id = body["root"] try: validateUuid(root_id, "group") except ValueError: msg = "Invalid root_id: " + root_id log.error(msg) raise HTTPInternalServerError() if "type" not in body: msg = "POST_Datatype with no type" log.error(msg) raise HTTPInternalServerError() type_json = body["type"] # ok - all set, create committed type obj now = time.time() log.info("POST_datatype, typejson: {}". format(type_json)) ctype_json = {"id": ctype_id, "root": root_id, "created": now, "lastModified": now, "type": type_json, "attributes": {} } await save_metadata_obj(app, ctype_id, ctype_json, bucket=bucket, notify=True, flush=True) resp_json = {} resp_json["id"] = ctype_id resp_json["root"] = root_id resp_json["created"] = ctype_json["created"] resp_json["lastModified"] = ctype_json["lastModified"] resp_json["attributeCount"] = 0 resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
async def PUT_Link(request): """ Handler creating a new link""" log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info(f"PUT link: {group_id}") if not isValidUuid(group_id, obj_class="group"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() link_title = request.match_info.get('title') validateLinkName(link_title) log.info(f"link_title: {link_title}") if not request.has_body: msg = "PUT Link with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() if "class" not in body: msg = "PUT Link with no class key body" log.warn(msg) raise HTTPBadRequest(reason=msg) link_class = body["class"] link_json = {} link_json["class"] = link_class if "id" in body: link_json["id"] = body["id"] if "h5path" in body: link_json["h5path"] = body["h5path"] if "h5domain" in body: link_json["h5domain"] = body["h5domain"] if "bucket" in params: bucket = params["bucket"] elif "bucket" in body: bucket = params["bucket"] else: bucket = None group_json = await get_metadata_obj(app, group_id, bucket=bucket) if "links" not in group_json: log.error(f"unexpected group data for id: {group_id}") raise HTTPInternalServerError() links = group_json["links"] if link_title in links: msg = f"Link name {link_title} already found in group: {group_id}" log.warn(msg) raise HTTPConflict() now = time.time() link_json["created"] = now # add the link links[link_title] = link_json # update the group lastModified group_json["lastModified"] = now # write back to S3, save to metadata cache await save_metadata_obj(app, group_id, group_json, bucket=bucket) resp_json = { } resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
async def GET_Attributes(request): """ Return JSON for attribute collection """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) include_data = False if "IncludeData" in params and params["IncludeData"]: include_data = True limit = None if "Limit" in params: try: limit = int(params["Limit"]) log.info("GET_Links - using Limit: {}".format(limit)) except ValueError: msg = "Bad Request: Expected int type for limit" log.error(msg) # should be validated by SN raise HTTPInternalServerError() marker = None if "Marker" in params: marker = params["Marker"] log.info("GET_Links - using Marker: {}".format(marker)) obj_json = await get_metadata_obj(app, obj_id) log.debug("GET attributes obj_id: {} got json".format(obj_id)) if "attributes" not in obj_json: msg = "unexpected data for obj id: {}".format(obj_id) msg.error(msg) raise HTTPInternalServerError() # return a list of attributes based on sorted dictionary keys attr_dict = obj_json["attributes"] attr_names = list(attr_dict.keys()) attr_names.sort() # sort by key # TBD: provide an option to sort by create date start_index = 0 if marker is not None: start_index = index(attr_names, marker) + 1 if start_index == 0: # marker not found, return 404 msg = "attribute marker: {}, not found".format(marker) log.warn(msg) raise HTTPNotFound() end_index = len(attr_names) if limit is not None and (end_index - start_index) > limit: end_index = start_index + limit attr_list = [] for i in range(start_index, end_index): attr_name = attr_names[i] src_attr = attr_dict[attr_name] des_attr = {} des_attr["created"] = src_attr["created"] des_attr["type"] = src_attr["type"] des_attr["shape"] = src_attr["shape"] des_attr["name"] = attr_name if include_data: des_attr["value"] = src_attr["value"] attr_list.append(des_attr) resp_json = {"attributes": attr_list} resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def PUT_Attribute(request): """ Handler for PUT /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) attr_name = request.match_info.get('name') log.info("PUT attribute {} in {}".format(attr_name, obj_id)) validateAttributeName(attr_name) if not request.has_body: log.error( "PUT_Attribute with no body") raise HTTPBadRequest(message="body expected") body = await request.json() replace = False if "replace" in params and params["replace"]: replace = True log.info("replace attribute") datatype = None shape = None value = None if "type" not in body: log.error("PUT attribute with no type in body") raise HTTPInternalServerError() datatype = body["type"] if "shape" not in body: log.error("PUT attribute with no shape in body") raise HTTPInternalServerError() shape = body["shape"] if "value" in body: value = body["value"] obj_json = await get_metadata_obj(app, obj_id) log.debug("PUT attribute obj_id: {} got json".format(obj_id)) if "attributes" not in obj_json: log.error("unexpected obj data for id: {}".format(obj_id)) raise HTTPInternalServerError() attributes = obj_json["attributes"] if attr_name in attributes and not replace: # Attribute already exists, return a 409 log.warn("Attempt to overwrite attribute: {} in obj_id:".format(attr_name, obj_id)) raise HTTPConflict() if replace and attr_name not in attributes: # Replace requires attribute exists log.warn("Attempt to update missing attribute: {} in obj_id:".format(attr_name, obj_id)) raise HTTPNotFound() if replace: orig_attr = attributes[attr_name] create_time = orig_attr["created"] else: create_time = time.time() # ok - all set, create attribute obj attr_json = {"type": datatype, "shape": shape, "value": value, "created": create_time } attributes[attr_name] = attr_json # write back to S3, save to metadata cache await save_metadata_obj(app, obj_id, obj_json) resp_json = { } resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
async def POST_Group(request): """ Handler for POST /groups""" log.request(request) app = request.app params = request.rel_url.query if not request.has_body: msg = "POST_Group with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() if "bucket" in params: bucket = params["bucket"] elif "bucket" in body: bucket = params["bucket"] else: bucket = None group_id = get_obj_id(request, body=body) log.info(f"POST group: {group_id} bucket: {bucket}") if not isValidUuid(group_id, obj_class="group"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() if "root" not in body: msg = "POST_Group with no root" log.error(msg) raise HTTPInternalServerError() # verify the id doesn't already exist obj_found = await check_metadata_obj(app, group_id, bucket=bucket) if obj_found: log.error(f"Post with existing group_id: {group_id}") raise HTTPInternalServerError() root_id = body["root"] if not isValidUuid(root_id, obj_class="group"): msg = "Invalid root_id: " + root_id log.error(msg) raise HTTPInternalServerError() # ok - all set, create group obj now = time.time() group_json = { "id": group_id, "root": root_id, "created": now, "lastModified": now, "links": {}, "attributes": {} } await save_metadata_obj(app, group_id, group_json, bucket=bucket, notify=True, flush=True) # formulate response resp_json = {} resp_json["id"] = group_id resp_json["root"] = root_id resp_json["created"] = group_json["created"] resp_json["lastModified"] = group_json["lastModified"] resp_json["linkCount"] = 0 resp_json["attributeCount"] = 0 resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp
async def GET_Links(request): """HTTP GET method to return JSON for a link collection """ log.request(request) app = request.app params = request.rel_url.query group_id = get_obj_id(request) log.info(f"GET links: {group_id}") if not isValidUuid(group_id, obj_class="group"): log.error(f"Unexpected group_id: {group_id}") raise HTTPInternalServerError() limit = None if "Limit" in params: try: limit = int(params["Limit"]) log.info(f"GET_Links - using Limit: {limit}") except ValueError: msg = "Bad Request: Expected int type for limit" log.error(msg) # should be validated by SN raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] log.info(f"GET_Links - using Marker: {marker}") if "bucket" in params: bucket = params["bucket"] else: bucket = None group_json = await get_metadata_obj(app, group_id, bucket=bucket) log.info(f"for id: {group_id} got group json: {group_json}") if "links" not in group_json: msg.error(f"unexpected group data for id: {group_id}") raise HTTPInternalServerError() # return a list of links based on sorted dictionary keys link_dict = group_json["links"] titles = list(link_dict.keys()) titles.sort() # sort by key # TBD: provide an option to sort by create date start_index = 0 if marker is not None: start_index = index(titles, marker) + 1 if start_index == 0: # marker not found, return 404 msg = f"Link marker: {marker}, not found" log.warn(msg) raise HTTPNotFound() end_index = len(titles) if limit is not None and (end_index - start_index) > limit: end_index = start_index + limit link_list = [] for i in range(start_index, end_index): title = titles[i] link = copy(link_dict[title]) link["title"] = title link_list.append(link) resp_json = {"links": link_list} resp = json_response(resp_json) log.response(request, resp=resp) return resp