async def PUT_Link(request): """HTTP method to create a new link""" log.request(request) app = request.app group_id = request.match_info.get('id') if not group_id: msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(group_id, obj_class="Group"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) link_title = request.match_info.get('title') log.info(f"PUT Link_title: [{link_title}]") validateLinkName(link_title) username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "PUT Link with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() link_json = {} if "id" in body: if not isValidUuid(body["id"]): msg = "PUT Link with invalid id in body" log.warn(msg) raise HTTPBadRequest(reason=msg) link_json["id"] = body["id"] link_json["class"] = "H5L_TYPE_HARD" elif "h5path" in body: link_json["h5path"] = body["h5path"] # could be hard or soft link if "h5domain" in body: link_json["h5domain"] = body["h5domain"] link_json["class"] = "H5L_TYPE_EXTERNAL" else: # soft link link_json["class"] = "H5L_TYPE_SOFT" else: msg = "PUT Link with no id or h5path keys" log.warn(msg) raise HTTPBadRequest(reason=msg) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) await validateAction(app, domain, group_id, username, "create") # for hard links, verify that the referenced id exists and is in this domain if "id" in body: ref_id = body["id"] ref_json = await getObjectJson(app, ref_id, bucket=bucket) group_json = await getObjectJson(app, group_id, bucket=bucket) if ref_json["root"] != group_json["root"]: msg = "Hard link must reference an object in the same domain" log.warn(msg) raise HTTPBadRequest(reason=msg) # ready to add link now req = getDataNodeUrl(app, group_id) req += "/groups/" + group_id + "/links/" + link_title log.debug("PUT link - getting group: " + req) params = {} if bucket: params["bucket"] = bucket put_rsp = await http_put(app, req, data=link_json, params=params) log.debug("PUT Link resp: " + str(put_rsp)) hrefs = [] # TBD req_rsp = {"hrefs": hrefs} # link creation successful resp = await jsonResponse(request, req_rsp, status=201) log.response(request, resp=resp) return resp
async def GET_Links(request): """HTTP method to return JSON for link collection""" log.request(request) app = request.app params = request.rel_url.query group_id = request.match_info.get('id') if not group_id: msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(group_id, obj_class="Group"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) limit = None if "Limit" in params: try: limit = int(params["Limit"]) except ValueError: msg = "Bad Request: Expected int type for limit" log.warn(msg) raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) await validateAction(app, domain, group_id, username, "read") req = getDataNodeUrl(app, group_id) req += "/groups/" + group_id + "/links" query_sep = '?' if limit is not None: req += query_sep + "Limit=" + str(limit) query_sep = '&' if marker is not None: req += query_sep + "Marker=" + marker log.debug("get LINKS: " + req) params = {} if bucket: params["bucket"] = bucket links_json = await http_get(app, req, params=params) log.debug(f"got links json from dn for group_id: {group_id}") links = links_json["links"] # mix in collection key, target and hrefs for link in links: if link["class"] == "H5L_TYPE_HARD": collection_name = getCollectionForId(link["id"]) link["collection"] = collection_name target_uri = '/' + collection_name + '/' + link["id"] link["target"] = getHref(request, target_uri) link_uri = '/groups/' + group_id + '/links/' + link['title'] link["href"] = getHref(request, link_uri) resp_json = {} resp_json["links"] = links hrefs = [] group_uri = '/groups/' + group_id hrefs.append({ 'rel': 'self', 'href': getHref(request, group_uri + '/links') }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)}) resp_json["hrefs"] = hrefs resp = await jsonResponse(request, resp_json) log.response(request, resp=resp) return resp
async def PUT_Chunk(request): log.request(request) app = request.app params = request.rel_url.query chunk_id = request.match_info.get('id') if not chunk_id: msg = "Missing chunk id" log.error(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(chunk_id, "Chunk"): msg = "Invalid chunk id: {}".format(chunk_id) log.warn(msg) raise HTTPBadRequest(reason=msg) if not request.has_body: msg = "PUT Value with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) content_type = "application/octet-stream" if "Content-Type" in request.headers: # client should use "application/octet-stream" for binary transfer content_type = request.headers["Content-Type"] if content_type != "application/octet-stream": msg = "Unexpected content_type: {}".format(content_type) log.error(msg) raise HTTPBadRequest(reason=msg) validateInPartition(app, chunk_id) log.debug("request params: {}".format(list(params.keys()))) if "dset" not in params: msg = "Missing dset in GET request" log.error(msg) raise HTTPBadRequest(reason=msg) dset_json = json.loads(params["dset"]) log.debug("dset_json: {}".format(dset_json)) dims = getChunkLayout(dset_json) if "root" not in dset_json: msg = "expected root key in dset_json" log.error(msg) raise KeyError(msg) rank = len(dims) # get chunk selection from query params selection = [] for i in range(rank): dim_slice = getSliceQueryParam(request, i, dims[i]) selection.append(dim_slice) selection = tuple(selection) log.debug("got selection: {}".format(selection)) type_json = dset_json["type"] itemsize = 'H5T_VARIABLE' if "size" in type_json: itemsize = type_json["size"] dt = createDataType(type_json) log.debug("dtype: {}".format(dt)) if rank == 0: msg = "No dimension passed to PUT chunk request" log.error(msg) raise HTTPBadRequest(reason=msg) if len(selection) != rank: msg = "Selection rank does not match shape rank" log.error(msg) raise HTTPBadRequest(reason=msg) for i in range(rank): s = selection[i] log.debug("selection[{}]: {}".format(i, s)) mshape = getSelectionShape(selection) log.debug(f"mshape: {mshape}") num_elements = 1 for extent in mshape: num_elements *= extent # check that the content_length is what we expect if itemsize != 'H5T_VARIABLE': log.debug("expect content_length: {}".format(num_elements * itemsize)) log.debug("actual content_length: {}".format(request.content_length)) if itemsize != 'H5T_VARIABLE' and (num_elements * itemsize) != request.content_length: msg = "Expected content_length of: {}, but got: {}".format( num_elements * itemsize, request.content_length) log.error(msg) raise HTTPBadRequest(reason=msg) # create a numpy array for incoming data input_bytes = await request_read( request ) # TBD - will it cause problems when failures are raised before reading data? if len(input_bytes) != request.content_length: msg = "Read {} bytes, expecting: {}".format(len(input_bytes), request.content_length) log.error(msg) raise HTTPInternalServerError() input_arr = bytesToArray(input_bytes, dt, mshape) # TBD: Skipp read if the input shape is the entire chunk? chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=True) # update chunk array chunk_arr[selection] = input_arr chunk_cache = app["chunk_cache"] chunk_cache.setDirty(chunk_id) log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}") # async write to S3 dirty_ids = app["dirty_ids"] now = int(time.time()) dirty_ids[chunk_id] = now # chunk update successful resp = json_response({}, status=201) log.response(request, resp=resp) return resp
async def GET_Link(request): """HTTP method to return JSON for a group link""" log.request(request) app = request.app group_id = request.match_info.get('id') if not group_id: msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(group_id, obj_class="Group"): msg = f"Invalid group id: {group_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) link_title = request.match_info.get('title') validateLinkName(link_title) username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) await validateAction(app, domain, group_id, username, "read") req = getDataNodeUrl(app, group_id) req += "/groups/" + group_id + "/links/" + link_title log.debug("get LINK: " + req) params = {} if bucket: params["bucket"] = bucket link_json = await http_get(app, req, params=params) log.debug("got link_json: " + str(link_json)) resp_link = {} resp_link["title"] = link_title link_class = link_json["class"] resp_link["class"] = link_class if link_class == "H5L_TYPE_HARD": resp_link["id"] = link_json["id"] resp_link["collection"] = getCollectionForId(link_json["id"]) elif link_class == "H5L_TYPE_SOFT": resp_link["h5path"] = link_json["h5path"] elif link_class == "H5L_TYPE_EXTERNAL": resp_link["h5path"] = link_json["h5path"] resp_link["h5domain"] = link_json["h5domain"] else: log.warn(f"Unexpected link class: {link_class}") resp_json = {} resp_json["link"] = resp_link resp_json["created"] = link_json["created"] # links don't get modified, so use created timestamp as lastModified resp_json["lastModified"] = link_json["created"] hrefs = [] group_uri = '/groups/' + group_id hrefs.append({ 'rel': 'self', 'href': getHref(request, group_uri + '/links/' + link_title) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)}) if link_json["class"] == "H5L_TYPE_HARD": target = '/' + resp_link["collection"] + '/' + resp_link["id"] hrefs.append({'rel': 'target', 'href': getHref(request, target)}) resp_json["hrefs"] = hrefs resp = await jsonResponse(request, resp_json) log.response(request, resp=resp) return resp
async def GET_Datatypes(request): """HTTP method to return datatype collection for given domain""" log.request(request) app = request.app params = request.rel_url.query (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(reason=msg) # use reload to get authoritative domain json try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError as ce: if ce.code in (404, 410): msg = "domain not found" log.warn(msg) raise HTTPNotFound() else: log.error(f"Unexpected Error: {ce.code})") raise HTTPInternalServerError() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() log.debug(f"got domain_json: {domain_json}") # validate that the requesting user has permission to read this domain aclCheck(domain_json, "read", username) # throws exception if not authorized limit = None if "Limit" in params: try: limit = int(params["Limit"]) except ValueError: msg = "Bad Request: Expected int type for limit" log.warn(msg) raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] # get the datatype collection list obj_ids = [] if "root" in domain_json or domain_json["root"]: # get the groups collection list collections = await get_collections(app, domain_json["root"]) objs = collections["datatypes"] obj_ids = getIdList(objs, marker=marker, limit=limit) # create hrefs hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/datatypes')}) if "root" in domain_json: root_uuid = domain_json["root"] hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + root_uuid) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) # return obj ids and hrefs rsp_json = {} rsp_json["datatypes"] = obj_ids rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def GET_Attributes(request): """ Return JSON for attribute collection """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) include_data = False if "IncludeData" in params and params["IncludeData"]: include_data = True limit = None if "Limit" in params: try: limit = int(params["Limit"]) log.info("GET_Links - using Limit: {}".format(limit)) except ValueError: msg = "Bad Request: Expected int type for limit" log.error(msg) # should be validated by SN raise HTTPInternalServerError() marker = None if "Marker" in params: marker = params["Marker"] log.info("GET_Links - using Marker: {}".format(marker)) obj_json = await get_metadata_obj(app, obj_id) log.debug("GET attributes obj_id: {} got json".format(obj_id)) if "attributes" not in obj_json: msg = "unexpected data for obj id: {}".format(obj_id) msg.error(msg) raise HTTPInternalServerError() # return a list of attributes based on sorted dictionary keys attr_dict = obj_json["attributes"] attr_names = list(attr_dict.keys()) attr_names.sort() # sort by key # TBD: provide an option to sort by create date start_index = 0 if marker is not None: start_index = index(attr_names, marker) + 1 if start_index == 0: # marker not found, return 404 msg = "attribute marker: {}, not found".format(marker) log.warn(msg) raise HTTPNotFound() end_index = len(attr_names) if limit is not None and (end_index - start_index) > limit: end_index = start_index + limit attr_list = [] for i in range(start_index, end_index): attr_name = attr_names[i] src_attr = attr_dict[attr_name] des_attr = {} des_attr["created"] = src_attr["created"] des_attr["type"] = src_attr["type"] des_attr["shape"] = src_attr["shape"] des_attr["name"] = attr_name if include_data: des_attr["value"] = src_attr["value"] attr_list.append(des_attr) resp_json = {"attributes": attr_list} resp = json_response(resp_json) log.response(request, resp=resp) return resp
async def GET_Attributes(request): """HTTP method to return JSON for attribute collection""" log.request(request) app = request.app params = request.rel_url.query collection = getRequestCollectionName( request) # returns datasets|groups|datatypes obj_id = request.match_info.get('id') if not obj_id: msg = "Missing object id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(obj_id, obj_class=collection): msg = "Invalid obj id: {}".format(obj_id) log.warn(msg) raise HTTPBadRequest(reason=msg) include_data = False if "IncludeData" in params and params["IncludeData"]: include_data = True limit = None if "Limit" in params: try: limit = int(params["Limit"]) except ValueError: msg = "Bad Request: Expected int type for limit" log.warn(msg) raise HTTPBadRequest(reason=msg) marker = None if "Marker" in params: marker = params["Marker"] username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) # TBD - verify that the obj_id belongs to the given domain await validateAction(app, domain, obj_id, username, "read") req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes" params = {} if limit is not None: params["Limit"] = str(limit) if marker is not None: params["Marker"] = marker if include_data: params["IncludeData"] = '1' if bucket: params["bucket"] = bucket log.debug(f"get attributes: {req}") dn_json = await http_get(app, req, params=params) log.debug(f"got attributes json from dn for obj_id: {obj_id}") attributes = dn_json["attributes"] # mixin hrefs for attribute in attributes: attr_name = attribute["name"] attr_href = '/' + collection + '/' + obj_id + '/attributes/' + attr_name attribute["href"] = getHref(request, attr_href) resp_json = {} resp_json["attributes"] = attributes hrefs = [] obj_uri = '/' + collection + '/' + obj_id hrefs.append({ 'rel': 'self', 'href': getHref(request, obj_uri + '/attributes') }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)}) resp_json["hrefs"] = hrefs resp = await jsonResponse(request, resp_json) log.response(request, resp=resp) return resp
async def GET_Datatype(request): """HTTP method to return JSON for committed datatype""" log.request(request) app = request.app params = request.rel_url.query include_attrs = False h5path = None getAlias = False ctype_id = request.match_info.get('id') if not ctype_id and "h5path" not in params: msg = "Missing type id" log.warn(msg) raise HTTPBadRequest(reason=msg) if "include_attrs" in params and params["include_attrs"]: include_attrs = True if ctype_id: if not isValidUuid(ctype_id, "Type"): msg = "Invalid type id: {}".format(ctype_id) log.warn(msg) raise HTTPBadRequest(reason=msg) if "getalias" in params: if params["getalias"]: getAlias = True else: group_id = None if "grpid" in params: group_id = params["grpid"] if not isValidUuid(group_id, "Group"): msg = "Invalid parent group id: {}".format(group_id) log.warn(msg) raise HTTPBadRequest(reason=msg) if "h5path" not in params: msg = "Expecting either ctype id or h5path url param" log.warn(msg) raise HTTPBadRequest(reason=msg) h5path = params["h5path"] if h5path[0] != '/' and group_id is None: msg = "h5paths must be absolute" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info("GET_Datatype, h5path: {}".format(h5path)) username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = "Invalid host value: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) if h5path: domain_json = await getDomainJson(app, domain) if "root" not in domain_json: msg = "Expected root key for domain: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) if group_id is None: group_id = domain_json["root"] ctype_id = await getObjectIdByPath(app, group_id, h5path) # throws 404 if not found if not isValidUuid(ctype_id, "Datatype"): msg = "No datatype exist with the path: {}".format(h5path) log.warn(msg) raise HTTPGone() log.info("got ctype_id: {} from h5path: {}".format(ctype_id, h5path)) await validateAction(app, domain, ctype_id, username, "read") # get authoritative state for group from DN (even if it's in the meta_cache). type_json = await getObjectJson(app, ctype_id, refresh=True, include_attrs=include_attrs) type_json["domain"] = domain if getAlias: root_id = type_json["root"] alias = [] idpath_map = {root_id: '/'} h5path = await getPathForObjectId(app, root_id, idpath_map, tgt_id=ctype_id) if h5path: alias.append(h5path) type_json["alias"] = alias hrefs = [] ctype_uri = '/datatypes/'+ctype_id hrefs.append({'rel': 'self', 'href': getHref(request, ctype_uri)}) root_uri = '/groups/' + type_json["root"] hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)}) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'attributes', 'href': getHref(request, ctype_uri+'/attributes')}) type_json["hrefs"] = hrefs resp = await jsonResponse(request, type_json) log.response(request, resp=resp) return resp
async def GET_Attribute(request): """HTTP method to return JSON for an attribute""" log.request(request) app = request.app collection = getRequestCollectionName( request) # returns datasets|groups|datatypes obj_id = request.match_info.get('id') if not obj_id: msg = "Missing object id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(obj_id, obj_class=collection): msg = f"Invalid object id: {obj_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) attr_name = request.match_info.get('name') validateAttributeName(attr_name) username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) # TBD - verify that the obj_id belongs to the given domain await validateAction(app, domain, obj_id, username, "read") req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name log.debug("get Attribute: " + req) params = {} if bucket: params["bucket"] = bucket dn_json = await http_get(app, req, params=params) log.debug("got attributes json from dn for obj_id: " + str(obj_id)) resp_json = {} resp_json["name"] = attr_name resp_json["type"] = dn_json["type"] resp_json["shape"] = dn_json["shape"] if "value" in dn_json: resp_json["value"] = dn_json["value"] resp_json["created"] = dn_json["created"] # attributes don't get modified, so use created timestamp as lastModified resp_json["lastModified"] = dn_json["created"] hrefs = [] obj_uri = '/' + collection + '/' + obj_id attr_uri = obj_uri + '/attributes/' + attr_name hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)}) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)}) resp_json["hrefs"] = hrefs resp = await jsonResponse(request, resp_json) log.response(request, resp=resp) return resp
async def PUT_Attribute(request): """HTTP method to create a new attribute""" log.request(request) app = request.app collection = getRequestCollectionName( request) # returns datasets|groups|datatypes obj_id = request.match_info.get('id') if not obj_id: msg = "Missing object id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(obj_id, obj_class=collection): msg = f"Invalid object id: {obj_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) attr_name = request.match_info.get('name') log.debug(f"Attribute name: [{attr_name}]") validateAttributeName(attr_name) log.info(f"PUT Attribute id: {obj_id} name: {attr_name}") username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "PUT Attribute with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) # get domain JSON domain_json = await getDomainJson(app, domain) if "root" not in domain_json: log.error(f"Expected root key for domain: {domain}") raise HTTPBadRequest(reason="Unexpected Error") root_id = domain_json["root"] # TBD - verify that the obj_id belongs to the given domain await validateAction(app, domain, obj_id, username, "create") if "type" not in body: msg = "PUT attribute with no type in body" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = body["type"] if isinstance(datatype, str) and datatype.startswith("t-"): # Committed type - fetch type json from DN ctype_id = datatype log.debug(f"got ctypeid: {ctype_id}") ctype_json = await getObjectJson(app, ctype_id, bucket=bucket) log.debug(f"ctype {ctype_id}: {ctype_json}") if ctype_json["root"] != root_id: msg = "Referenced committed datatype must belong in same domain" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = ctype_json["type"] # add the ctype_id to type type datatype["id"] = ctype_id elif isinstance(datatype, str): try: # convert predefined type string (e.g. "H5T_STD_I32LE") to # corresponding json representation datatype = getBaseTypeJson(datatype) log.debug(f"got datatype: {datatype}") except TypeError: msg = "PUT attribute with invalid predefined type" log.warn(msg) raise HTTPBadRequest(reason=msg) validateTypeItem(datatype) dims = None shape_json = {} if "shape" in body: shape_body = body["shape"] shape_class = None if isinstance(shape_body, dict) and "class" in shape_body: shape_class = shape_body["class"] elif isinstance(shape_body, str): shape_class = shape_body if shape_class: if shape_class == "H5S_NULL": shape_json["class"] = "H5S_NULL" if isinstance(shape_body, dict) and "dims" in shape_body: msg = "can't include dims with null shape" log.warn(msg) raise HTTPBadRequest(reason=msg) if isinstance(shape_body, dict) and "value" in body: msg = "can't have H5S_NULL shape with value" log.warn(msg) raise HTTPBadRequest(reason=msg) elif shape_class == "H5S_SCALAR": shape_json["class"] = "H5S_SCALAR" dims = getShapeDims(shape_body) if len(dims) != 1 or dims[0] != 1: msg = "dimensions aren't valid for scalar attribute" log.warn(msg) raise HTTPBadRequest(reason=msg) elif shape_class == "H5S_SIMPLE": shape_json["class"] = "H5S_SIMPLE" dims = getShapeDims(shape_body) shape_json["dims"] = dims else: msg = f"Unknown shape class: {shape_class}" log.warn(msg) raise HTTPBadRequest(reason=msg) else: # no class, interpet shape value as dimensions and # use H5S_SIMPLE as class if isinstance(shape_body, list) and len(shape_body) == 0: shape_json["class"] = "H5S_SCALAR" dims = [ 1, ] else: shape_json["class"] = "H5S_SIMPLE" dims = getShapeDims(shape_body) shape_json["dims"] = dims else: shape_json["class"] = "H5S_SCALAR" dims = [ 1, ] if "value" in body: if dims is None: msg = "Bad Request: data can not be included with H5S_NULL space" log.warn(msg) raise HTTPBadRequest(reason=msg) value = body["value"] # validate that the value agrees with type/shape arr_dtype = createDataType(datatype) # np datatype if len(dims) == 0: np_dims = [ 1, ] else: np_dims = dims log.debug(f"attribute dims: {np_dims}") log.debug(f"attribute value: {value}") try: arr = jsonToArray(np_dims, arr_dtype, value) except ValueError: msg = "Bad Request: input data doesn't match selection" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info(f"Got: {arr.size} array elements") else: value = None # ready to add attribute now req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name log.info("PUT Attribute: " + req) attr_json = {} attr_json["type"] = datatype attr_json["shape"] = shape_json if value is not None: attr_json["value"] = value params = {} if bucket: params["bucket"] = bucket put_rsp = await http_put(app, req, params=params, data=attr_json) log.info(f"PUT Attribute resp: {put_rsp}") hrefs = [] # TBD req_rsp = {"hrefs": hrefs} # attribute creation successful resp = await jsonResponse(request, req_rsp, status=201) log.response(request, resp=resp) return resp
async def GET_Group(request): """HTTP method to return JSON for group""" log.request(request) app = request.app params = request.rel_url.query h5path = None getAlias = False include_links = False include_attrs = False group_id = request.match_info.get('id') if not group_id and "h5path" not in params: # no id, or path provided, so bad request msg = "Missing group id" log.warn(msg) raise HTTPBadRequest(reason=msg) if group_id: log.info("GET_Group, id: {}".format(group_id)) # is the id a group id and not something else? if not isValidUuid(group_id, "Group"): msg = "Invalid group id: {}".format(group_id) log.warn(msg) raise HTTPBadRequest(reason=msg) if "getalias" in params: if params["getalias"]: getAlias = True if "h5path" in params: h5path = params["h5path"] if not group_id and h5path[0] != '/': msg = "h5paths must be absolute if no parent id is provided" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info("GET_Group, h5path: {}".format(h5path)) if "include_links" in params and params["include_links"]: include_links = True if "include_attrs" in params and params["include_attrs"]: include_attrs = True username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = "Invalid host value: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) if h5path and h5path[0] == '/': # ignore the request path id (if given) and start # from root group for absolute paths domain_json = await getDomainJson(app, domain) if "root" not in domain_json: msg = "Expected root key for domain: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) group_id = domain_json["root"] if h5path: group_id = await getObjectIdByPath(app, group_id, h5path) # throws 404 if not found if not isValidUuid(group_id, "Group"): msg = "No group exist with the path: {}".format(h5path) log.warn(msg) raise HTTPNotFound() log.info("get group_id: {} from h5path: {}".format(group_id, h5path)) # verify authorization to read the group await validateAction(app, domain, group_id, username, "read") # get authoritative state for group from DN (even if it's in the meta_cache). group_json = await getObjectJson(app, group_id, refresh=True, include_links=include_links, include_attrs=include_attrs) group_json["domain"] = domain if getAlias: root_id = group_json["root"] alias = [] if group_id == root_id: alias.append('/') else: idpath_map = {root_id: '/'} h5path = await getPathForObjectId(app, root_id, idpath_map, tgt_id=group_id) if h5path: alias.append(h5path) group_json["alias"] = alias hrefs = [] group_uri = '/groups/' + group_id hrefs.append({'rel': 'self', 'href': getHref(request, group_uri)}) hrefs.append({ 'rel': 'links', 'href': getHref(request, group_uri + '/links') }) root_uri = '/groups/' + group_json["root"] hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)}) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({ 'rel': 'attributes', 'href': getHref(request, group_uri + '/attributes') }) group_json["hrefs"] = hrefs resp = await jsonResponse(request, group_json) log.response(request, resp=resp) return resp
async def POST_Group(request): """HTTP method to create new Group object""" log.request(request) app = request.app username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = "Invalid host value: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) domain_json = await getDomainJson(app, domain, reload=True) aclCheck(domain_json, "create", username) # throws exception if not allowed if "root" not in domain_json: msg = "Expected root key for domain: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) link_id = None link_title = None if request.has_body: body = await request.json() log.info("POST Group body: {}".format(body)) if body: if "link" in body: link_body = body["link"] log.debug("link_body: {}".format(link_body)) if "id" in link_body: link_id = link_body["id"] if "name" in link_body: link_title = link_body["name"] if link_id and link_title: log.debug("link id: {}".format(link_id)) # verify that the referenced id exists and is in this domain # and that the requestor has permissions to create a link await validateAction(app, domain, link_id, username, "create") if not link_id or not link_title: log.warn("POST Group body with no link: {}".format(body)) domain_json = await getDomainJson( app, domain) # get again in case cache was invalidated root_id = domain_json["root"] group_id = createObjId("groups", rootid=root_id) log.info("new group id: {}".format(group_id)) group_json = {"id": group_id, "root": root_id} log.debug("create group, body: " + json.dumps(group_json)) req = getDataNodeUrl(app, group_id) + "/groups" group_json = await http_post(app, req, data=group_json) # create link if requested if link_id and link_title: link_json = {} link_json["id"] = group_id link_json["class"] = "H5L_TYPE_HARD" link_req = getDataNodeUrl(app, link_id) link_req += "/groups/" + link_id + "/links/" + link_title log.debug("PUT link - : " + link_req) put_json_rsp = await http_put(app, link_req, data=link_json) log.debug("PUT Link resp: {}".format(put_json_rsp)) log.debug("returning resp") # group creation successful resp = await jsonResponse(request, group_json, status=201) log.response(request, resp=resp) return resp
async def register(request): """ HTTP method for nodes to register with head node""" log.request(request) app = request.app text = await request.text() # body = await request.json() body = json.loads(text) log.info("body: {}".format(body)) if 'id' not in body: msg = "Missing 'id'" log.response(request, code=400, message=msg) raise HTTPBadRequest(reason=msg) if 'port' not in body: msg = "missing key 'port'" log.response(request, code=400, message=msg) raise HTTPBadRequest(reason=msg) if 'node_type' not in body: raise HTTPBadRequest(reason="missing key 'node_type'") if body['node_type'] not in ('sn', 'dn'): msg = "invalid node_type" log.response(request, code=400, message=msg) raise HTTPBadRequest(reason=msg) peername = request.transport.get_extra_info('peername') if peername is None: raise HTTPBadRequest(reason="Can not determine caller IP") host, req_port = peername log.info("register host: {}, port: {}".format(host, req_port)) nodes = None ret_node = None node_ids = app['node_ids'] if body['id'] in node_ids: # already registered? ret_node = node_ids[body['id']] else: nodes = app['nodes'] for node in nodes: if node['host'] is None and node['node_type'] == body['node_type']: # found a free node log.info("got free node: {}".format(node)) node['host'] = host node['port'] = body["port"] node['id'] = body["id"] node["connected"] = unixTimeToUTC(int(time.time())) node['failcount'] = 0 ret_node = node node_ids[body["id"]] = ret_node break if ret_node is None: log.info("no free node to assign") inactive_node_count = getInactiveNodeCount(app) log.info("inactive_node_count: {}".format(inactive_node_count)) if inactive_node_count == 0: # all the nodes have checked in log.info("setting cluster state to ready") app['cluster_state'] = "READY" resp = StreamResponse() resp.headers['Content-Type'] = 'application/json' answer = {} if ret_node is not None: answer["node_number"] = ret_node["node_number"] else: # all nodes allocated, let caller know it's in the reserve pool answer["node_number"] = -1 answer["node_count"] = app["target_dn_count"] resp = json_response(answer) log.response(request, resp=resp) return resp
async def PUT_Chunk(request): log.request(request) app = request.app params = request.rel_url.query query = None if "query" in params: query = params["query"] log.info(f"PUT_Chunk query: {query}") chunk_id = request.match_info.get('id') if not chunk_id: msg = "Missing chunk id" log.error(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(chunk_id, "Chunk"): msg = f"Invalid chunk id: {chunk_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) if not request.has_body: msg = "PUT Value with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) if "bucket" in params: bucket = params["bucket"] log.debug(f"PUT_Chunk using bucket: {bucket}") else: bucket = None if query: expected_content_type = "text/plain; charset=utf-8" else: expected_content_type = "application/octet-stream" if "Content-Type" in request.headers: # client should use "application/octet-stream" for binary transfer content_type = request.headers["Content-Type"] if content_type != expected_content_type: msg = f"Unexpected content_type: {content_type}" log.error(msg) raise HTTPBadRequest(reason=msg) validateInPartition(app, chunk_id) if "dset" in params: msg = "Unexpected param dset in GET request" log.error(msg) raise HTTPBadRequest(reason=msg) log.debug(f"PUT_Chunk - id: {chunk_id}") dset_id = getDatasetId(chunk_id) dset_json = await get_metadata_obj(app, dset_id, bucket=bucket) log.debug(f"dset_json: {dset_json}") dims = getChunkLayout(dset_json) if "root" not in dset_json: msg = "expected root key in dset_json" log.error(msg) raise KeyError(msg) rank = len(dims) # get chunk selection from query params selection = [] for i in range(rank): dim_slice = getSliceQueryParam(request, i, dims[i]) selection.append(dim_slice) selection = tuple(selection) log.debug(f"got selection: {selection}") type_json = dset_json["type"] itemsize = 'H5T_VARIABLE' if "size" in type_json: itemsize = type_json["size"] dt = createDataType(type_json) log.debug(f"dtype: {dt}") if rank == 0: msg = "No dimension passed to PUT chunk request" log.error(msg) raise HTTPBadRequest(reason=msg) if len(selection) != rank: msg = "Selection rank does not match shape rank" log.error(msg) raise HTTPBadRequest(reason=msg) for i in range(rank): s = selection[i] log.debug(f"selection[{i}]: {s}") mshape = getSelectionShape(selection) log.debug(f"mshape: {mshape}") num_elements = 1 for extent in mshape: num_elements *= extent resp = {} query_update = None limit = 0 chunk_init = True input_arr = None if query: if not dt.fields: log.error("expected compound dtype for PUT query") raise HTTPInternalServerError() if rank != 1: log.error("expected one-dimensional array for PUT query") raise HTTPInternalServerError() query_update = await request.json() log.debug(f"query_update: {query_update}") if "Limit" in params: limit = int(params["Limit"]) chunk_init = False else: # regular chunk update # check that the content_length is what we expect if itemsize != 'H5T_VARIABLE': log.debug(f"expect content_length: {num_elements*itemsize}") log.debug(f"actual content_length: {request.content_length}") if itemsize != 'H5T_VARIABLE' and (num_elements * itemsize) != request.content_length: msg = f"Expected content_length of: {num_elements*itemsize}, but got: {request.content_length}" log.error(msg) raise HTTPBadRequest(reason=msg) # create a numpy array for incoming data input_bytes = await request_read( request ) # TBD - will it cause problems when failures are raised before reading data? if len(input_bytes) != request.content_length: msg = f"Read {len(input_bytes)} bytes, expecting: {request.content_length}" log.error(msg) raise HTTPInternalServerError() input_arr = bytesToArray(input_bytes, dt, mshape) # TBD: Skip read if the input shape is the entire chunk? chunk_arr = await getChunk(app, chunk_id, dset_json, chunk_init=chunk_init, bucket=bucket) is_dirty = False if query: values = [] indices = [] if chunk_arr is not None: # do query selection limit = 0 if "Limit" in params: limit = int(params["Limit"]) field_names = list(dt.fields.keys()) replace_mask = [ None, ] * len(field_names) for i in range(len(field_names)): field_name = field_names[i] if field_name in query_update: replace_mask[i] = query_update[field_name] log.debug(f"replace_mask: {replace_mask}") x = chunk_arr[selection] log.debug(f"put_query - x: {x}") eval_str = getEvalStr(query, "x", field_names) log.debug(f"put_query - eval_str: {eval_str}") where_result = np.where(eval(eval_str)) log.debug(f"put_query - where_result: {where_result}") where_result_index = where_result[0] log.debug(f"put_query - whare_result index: {where_result_index}") log.debug( f"put_query - boolean selection: {x[where_result_index]}") s = selection[0] count = 0 for index in where_result_index: log.debug(f"put_query - index: {index}") value = x[index].copy() log.debug(f"put_query - original value: {value}") for i in range(len(field_names)): if replace_mask[i] is not None: value[i] = replace_mask[i] log.debug(f"put_query - modified value: {value}") try: chunk_arr[index] = value except ValueError as ve: log.error(f"Numpy Value updating array: {ve}") raise HTTPInternalServerError() json_val = bytesArrayToList(value) log.debug(f"put_query - json_value: {json_val}") json_index = index.tolist( ) * s.step + s.start # adjust for selection indices.append(json_index) values.append(json_val) count += 1 is_dirty = True if limit > 0 and count >= limit: log.info("put_query - got limit items") break query_result = {} query_result["index"] = indices query_result["value"] = values log.info(f"query_result returning: {len(indices)} rows") log.debug(f"query_result: {query_result}") resp = json_response(query_result) else: # update chunk array try: chunk_arr[selection] = input_arr except ValueError as ve: log.error(f"Numpy Value updating array: {ve}") raise HTTPInternalServerError() is_dirty = True resp = json_response({}, status=201) if is_dirty: chunk_cache = app["chunk_cache"] chunk_cache.setDirty(chunk_id) log.info(f"PUT_Chunk dirty cache count: {chunk_cache.dirtyCount}") # async write to S3 dirty_ids = app["dirty_ids"] now = int(time.time()) dirty_ids[chunk_id] = (now, bucket) # chunk update successful log.response(request, resp=resp) return resp
async def GET_AttributeValue(request): """HTTP method to return an attribute value""" log.request(request) app = request.app log.info("GET_AttributeValue") collection = getRequestCollectionName( request) # returns datasets|groups|datatypes obj_id = request.match_info.get('id') if not obj_id: msg = "Missing object id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(obj_id, obj_class=collection): msg = f"Invalid object id: {obj_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) attr_name = request.match_info.get('name') validateAttributeName(attr_name) username, pswd = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain value: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) # get domain JSON domain_json = await getDomainJson(app, domain) if "root" not in domain_json: log.error(f"Expected root key for domain: {domain}") raise HTTPBadRequest(reason="Unexpected Error") # TBD - verify that the obj_id belongs to the given domain await validateAction(app, domain, obj_id, username, "read") req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name log.debug("get Attribute: " + req) params = {} if bucket: params["bucket"] = bucket dn_json = await http_get(app, req, params=params) log.debug("got attributes json from dn for obj_id: " + str(dn_json)) attr_shape = dn_json["shape"] log.debug(f"attribute shape: {attr_shape}") if attr_shape["class"] == 'H5S_NULL': msg = "Null space attributes can not be read" log.warn(msg) raise HTTPBadRequest(reason=msg) accept_type = getAcceptType(request) response_type = accept_type # will adjust later if binary not possible type_json = dn_json["type"] shape_json = dn_json["shape"] item_size = getItemSize(type_json) if item_size == 'H5T_VARIABLE' and accept_type != "json": msg = "Client requested binary, but only JSON is supported for variable length data types" log.info(msg) response_type = "json" if response_type == "binary": arr_dtype = createDataType(type_json) # np datatype np_shape = getShapeDims(shape_json) try: arr = jsonToArray(np_shape, arr_dtype, dn_json["value"]) except ValueError: msg = "Bad Request: input data doesn't match selection" log.warn(msg) raise HTTPBadRequest(reason=msg) output_data = arr.tobytes() log.debug( f"GET AttributeValue - returning {len(output_data)} bytes binary data" ) # write response try: resp = StreamResponse() resp.content_type = "application/octet-stream" resp.content_length = len(output_data) # allow CORS resp.headers['Access-Control-Allow-Origin'] = '*' resp.headers[ 'Access-Control-Allow-Methods'] = "GET, POST, DELETE, PUT, OPTIONS" resp.headers[ 'Access-Control-Allow-Headers'] = "Content-Type, api_key, Authorization" await resp.prepare(request) await resp.write(output_data) except Exception as e: log.error(f"Got exception: {e}") raise HTTPInternalServerError() finally: await resp.write_eof() else: resp_json = {} if "value" in dn_json: resp_json["value"] = dn_json["value"] hrefs = [] obj_uri = '/' + collection + '/' + obj_id attr_uri = obj_uri + '/attributes/' + attr_name hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)}) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)}) resp_json["hrefs"] = hrefs resp = await jsonResponse(request, resp_json) log.response(request, resp=resp) return resp
async def POST_Datatype(request): """HTTP method to create new committed datatype object""" log.request(request) app = request.app username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "POST Datatype with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) body = await request.json() if "type" not in body: msg = "POST Datatype has no type key in body" log.warn(msg) raise HTTPBadRequest(reason=msg) datatype = body["type"] if isinstance(datatype, str): try: # convert predefined type string (e.g. "H5T_STD_I32LE") to # corresponding json representation datatype = getBaseTypeJson(datatype) log.debug("got datatype: {}".format(datatype)) except TypeError: msg = "POST Dataset with invalid predefined type" log.warn(msg) raise HTTPBadRequest(reason=msg) validateTypeItem(datatype) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = "Invalid host value: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) domain_json = await getDomainJson(app, domain, reload=True) aclCheck(domain_json, "create", username) # throws exception if not allowed if "root" not in domain_json: msg = "Expected root key for domain: {}".format(domain) log.warn(msg) raise HTTPBadRequest(reason=msg) link_id = None link_title = None if "link" in body: link_body = body["link"] if "id" in link_body: link_id = link_body["id"] if "name" in link_body: link_title = link_body["name"] if link_id and link_title: log.debug("link id: {}".format(link_id)) # verify that the referenced id exists and is in this domain # and that the requestor has permissions to create a link await validateAction(app, domain, link_id, username, "create") root_id = domain_json["root"] ctype_id = createObjId("datatypes", rootid=root_id) log.debug("new type id: {}".format(ctype_id)) ctype_json = {"id": ctype_id, "root": root_id, "type": datatype } log.debug("create named type, body: " + json.dumps(ctype_json)) req = getDataNodeUrl(app, ctype_id) + "/datatypes" type_json = await http_post(app, req, data=ctype_json) # create link if requested if link_id and link_title: link_json={} link_json["id"] = ctype_id link_json["class"] = "H5L_TYPE_HARD" link_req = getDataNodeUrl(app, link_id) link_req += "/groups/" + link_id + "/links/" + link_title log.debug("PUT link - : " + link_req) put_rsp = await http_put(app, link_req, data=link_json) log.debug("PUT Link resp: {}".format(put_rsp)) # datatype creation successful resp = await jsonResponse(request, type_json, status=201) log.response(request, resp=resp) return resp
async def PUT_AttributeValue(request): """HTTP method to update an attributes data""" log.request(request) log.info("PUT_AttributeValue") app = request.app collection = getRequestCollectionName( request) # returns datasets|groups|datatypes obj_id = request.match_info.get('id') if not obj_id: msg = "Missing object id" log.warn(msg) raise HTTPBadRequest(reason=msg) if not isValidUuid(obj_id, obj_class=collection): msg = f"Invalid object id: {obj_id}" log.warn(msg) raise HTTPBadRequest(reason=msg) attr_name = request.match_info.get('name') log.debug(f"Attribute name: [{attr_name}]") validateAttributeName(attr_name) log.info(f"PUT Attribute Value id: {obj_id} name: {attr_name}") username, pswd = getUserPasswordFromRequest(request) # write actions need auth await validateUserPassword(app, username, pswd) if not request.has_body: msg = "PUT AttributeValue with no body" log.warn(msg) raise HTTPBadRequest(reason=msg) domain = getDomainFromRequest(request) if not isValidDomain(domain): msg = f"Invalid domain: {domain}" log.warn(msg) raise HTTPBadRequest(reason=msg) bucket = getBucketForDomain(domain) # get domain JSON domain_json = await getDomainJson(app, domain) if "root" not in domain_json: log.error(f"Expected root key for domain: {domain}") raise HTTPInternalServerError() # TBD - verify that the obj_id belongs to the given domain await validateAction(app, domain, obj_id, username, "update") req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name log.debug("get Attribute: " + req) params = {} if bucket: params["bucket"] = bucket dn_json = await http_get(app, req, params=params) log.debug("got attributes json from dn for obj_id: " + str(obj_id)) log.debug(f"got dn_json: {dn_json}") attr_shape = dn_json["shape"] if attr_shape["class"] == 'H5S_NULL': msg = "Null space attributes can not be updated" log.warn(msg) raise HTTPBadRequest(reason=msg) np_shape = getShapeDims(attr_shape) type_json = dn_json["type"] np_dtype = createDataType(type_json) # np datatype request_type = "json" if "Content-Type" in request.headers: # client should use "application/octet-stream" for binary transfer content_type = request.headers["Content-Type"] if content_type not in ("application/json", "application/octet-stream"): msg = f"Unknown content_type: {content_type}" log.warn(msg) raise HTTPBadRequest(reason=msg) if content_type == "application/octet-stream": log.debug("PUT AttributeValue - request_type is binary") request_type = "binary" else: log.debug("PUT AttribueValue - request type is json") binary_data = None if request_type == "binary": item_size = getItemSize(type_json) if item_size == 'H5T_VARIABLE': msg = "Only JSON is supported for variable length data types" log.warn(msg) raise HTTPBadRequest(reason=msg) # read binary data binary_data = await request.read() if len(binary_data) != request.content_length: msg = f"Read {len(binary_data)} bytes, expecting: {request.content_length}" log.error(msg) raise HTTPInternalServerError() arr = None # np array to hold request data if binary_data: npoints = getNumElements(np_shape) if npoints * item_size != len(binary_data): msg = "Expected: " + str( npoints * item_size) + " bytes, but got: " + str( len(binary_data)) log.warn(msg) raise HTTPBadRequest(reason=msg) arr = np.fromstring(binary_data, dtype=np_dtype) arr = arr.reshape(np_shape) # conform to selection shape # convert to JSON for transmission to DN data = arr.tolist() value = bytesArrayToList(data) else: body = await request.json() if "value" not in body: msg = "PUT attribute value with no value in body" log.warn(msg) raise HTTPBadRequest(reason=msg) value = body["value"] # validate that the value agrees with type/shape try: arr = jsonToArray(np_shape, np_dtype, value) except ValueError: msg = "Bad Request: input data doesn't match selection" log.warn(msg) raise HTTPBadRequest(reason=msg) log.info(f"Got: {arr.size} array elements") # ready to add attribute now attr_json = {} attr_json["type"] = type_json attr_json["shape"] = attr_shape attr_json["value"] = value req = getDataNodeUrl(app, obj_id) req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name log.info(f"PUT Attribute Value: {req}") dn_json["value"] = value params = {} params = {"replace": 1} # let the DN know we can overwrite the attribute if bucket: params["bucket"] = bucket put_rsp = await http_put(app, req, params=params, data=attr_json) log.info(f"PUT Attribute Value resp: {put_rsp}") hrefs = [] # TBD req_rsp = {"hrefs": hrefs} # attribute creation successful resp = await jsonResponse(request, req_rsp) log.response(request, resp=resp) return resp
async def GET_ACLs(request): """HTTP method to return JSON for domain/ACLs""" log.request(request) app = request.app (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) try: domain = getDomainFromRequest(request) except ValueError: msg = "Invalid domain" log.warn(msg) raise HTTPBadRequest(message=msg) # use reload to get authoritative domain json try: domain_json = await getDomainJson(app, domain, reload=True) except ClientResponseError: log.warn("domain not found") log.warn(msg) raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() acls = domain_json["acls"] log.debug(f"got domain_json: {domain_json}") # validate that the requesting user has permission to read this domain aclCheck(domain_json, "readACL", username) # throws exception if not authorized acl_list = [] acl_usernames = list(acls.keys()) acl_usernames.sort() for acl_username in acl_usernames: entry = {"userName": acl_username} acl = acls[acl_username] for k in acl.keys(): entry[k] = acl[k] acl_list.append(entry) # return just the keys as per the REST API rsp_json = {} rsp_json["acls"] = acl_list hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/acls')}) if "root" in domain_json: hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + domain_json["root"]) }) hrefs.append({'rel': 'home', 'href': getHref(request, '/')}) hrefs.append({'rel': 'owner', 'href': getHref(request, '/')}) rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def info(request): """HTTP Method to retun node state to caller""" log.debug("info request") app = request.app answer = {} # copy relevant entries from state dictionary to response node = {} node['id'] = request.app['id'] node['type'] = request.app['node_type'] node['start_time'] = app["start_time"] #unixTimeToUTC(app['start_time']) node['state'] = app['node_state'] node['node_number'] = app['node_number'] node['node_count'] = app['node_count'] answer["node"] = node # psutil info # see: http://pythonhosted.org/psutil/ for description of different fields cpu = {} cpu["percent"] = psutil.cpu_percent() cpu["cores"] = psutil.cpu_count() answer["cpu"] = cpu diskio = psutil.disk_io_counters() disk_stats = {} disk_stats["read_count"] = diskio.read_count disk_stats["read_time"] = diskio.read_time disk_stats["read_bytes"] = diskio.read_bytes disk_stats["write_count"] = diskio.write_count disk_stats["write_time"] = diskio.write_time disk_stats["write_bytes"] = diskio.write_bytes answer["diskio"] = disk_stats netio = psutil.net_io_counters() net_stats = {} net_stats["bytes_sent"] = netio.bytes_sent net_stats["bytes_sent"] = netio.bytes_recv net_stats["packets_sent"] = netio.packets_sent net_stats["packets_recv"] = netio.packets_recv net_stats["errin"] = netio.errin net_stats["errout"] = netio.errout net_stats["dropin"] = netio.dropin net_stats["dropout"] = netio.dropout answer["netio"] = net_stats mem_stats = {} svmem = psutil.virtual_memory() mem_stats["phys_total"] = svmem.total mem_stats["phys_available"] = svmem.available sswap = psutil.swap_memory() mem_stats["swap_total"] = sswap.total mem_stats["swap_used"] = sswap.used mem_stats["swap_free"] = sswap.free mem_stats["percent"] = sswap.percent answer["memory"] = mem_stats disk_stats = {} sdiskusage = psutil.disk_usage('/') disk_stats["total"] = sdiskusage.total disk_stats["used"] = sdiskusage.used disk_stats["free"] = sdiskusage.free disk_stats["percent"] = sdiskusage.percent answer["disk"] = disk_stats answer["log_stats"] = app["log_count"] answer["req_count"] = app["req_count"] answer["s3_stats"] = app["s3_stats"] mc_stats = {} if "meta_cache" in app: mc = app["meta_cache"] # only DN nodes have this mc_stats["count"] = len(mc) mc_stats["dirty_count"] = mc.dirtyCount mc_stats["utililization_per"] = mc.cacheUtilizationPercent mc_stats["mem_used"] = mc.memUsed mc_stats["mem_target"] = mc.memTarget answer["meta_cache_stats"] = mc_stats cc_stats = {} if "chunk_cache" in app: cc = app["chunk_cache"] # only DN nodes have this cc_stats["count"] = len(cc) cc_stats["dirty_count"] = cc.dirtyCount cc_stats["utililization_per"] = cc.cacheUtilizationPercent cc_stats["mem_used"] = cc.memUsed cc_stats["mem_target"] = cc.memTarget answer["chunk_cache_stats"] = cc_stats dc_stats = {} if "domain_cache" in app: dc = app["domain_cache"] # only DN nodes have this dc_stats["count"] = len(dc) dc_stats["dirty_count"] = dc.dirtyCount dc_stats["utililization_per"] = dc.cacheUtilizationPercent dc_stats["mem_used"] = dc.memUsed dc_stats["mem_target"] = dc.memTarget answer["domain_cache_stats"] = dc_stats resp = await jsonResponse(request, answer) log.response(request, resp=resp) return resp
async def GET_Domain(request): """HTTP method to return JSON for given domain""" log.request(request) app = request.app params = request.rel_url.query (username, pswd) = getUserPasswordFromRequest(request) if username is None and app['allow_noauth']: username = "******" else: await validateUserPassword(app, username, pswd) domain = None try: domain = getDomainFromRequest(request) except ValueError: log.warn(f"Invalid domain: {domain}") raise HTTPBadRequest(reason="Invalid domain name") bucket = getBucketForDomain(domain) log.debug(f"GET_Domain domain: {domain} bucket: {bucket}") if not bucket and not config.get("bucket_name"): # no bucket defined, raise 400 msg = "Bucket not provided" log.warn(msg) raise HTTPBadRequest(reason=msg) verbose = False if "verbose" in params and params["verbose"]: verbose = True if not domain: log.info("no domain passed in, returning all top-level domains") # no domain passed in, return top-level domains for this request domains = await get_domains(request) rsp_json = {"domains": domains} rsp_json["hrefs"] = [] resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp log.info(f"got domain: {domain}") domain_json = await getDomainJson(app, domain, reload=True) if domain_json is None: log.warn(f"domain: {domain} not found") raise HTTPNotFound() if 'owner' not in domain_json: log.error("No owner key found in domain") raise HTTPInternalServerError() if 'acls' not in domain_json: log.error("No acls key found in domain") raise HTTPInternalServerError() log.debug(f"got domain_json: {domain_json}") # validate that the requesting user has permission to read this domain aclCheck(domain_json, "read", username) # throws exception if not authorized if "h5path" in params: # if h5path is passed in, return object info for that path # (if exists) h5path = params["h5path"] root_id = domain_json["root"] obj_id = await getObjectIdByPath(app, root_id, h5path, bucket=bucket ) # throws 404 if not found log.info(f"get obj_id: {obj_id} from h5path: {h5path}") # get authoritative state for object from DN (even if it's in the meta_cache). obj_json = await getObjectJson(app, obj_id, refresh=True, bucket=bucket) obj_json["domain"] = domain # Not bothering with hrefs for h5path lookups... resp = await jsonResponse(request, obj_json) log.response(request, resp=resp) return resp # return just the keys as per the REST API rsp_json = await get_domain_response(app, domain_json, bucket=bucket, verbose=verbose) # include domain objects if requested if "getobjs" in params and params["getobjs"] and "root" in domain_json: root_id = domain_json["root"] include_attrs = False if "include_attrs" in params and params["include_attrs"]: include_attrs = True domain_objs = await getDomainObjects(app, root_id, include_attrs=include_attrs, bucket=bucket) rsp_json["domain_objs"] = domain_objs hrefs = [] hrefs.append({'rel': 'self', 'href': getHref(request, '/')}) if "root" in domain_json: root_uuid = domain_json["root"] hrefs.append({ 'rel': 'database', 'href': getHref(request, '/datasets') }) hrefs.append({'rel': 'groupbase', 'href': getHref(request, '/groups')}) hrefs.append({ 'rel': 'typebase', 'href': getHref(request, '/datatypes') }) hrefs.append({ 'rel': 'root', 'href': getHref(request, '/groups/' + root_uuid) }) hrefs.append({'rel': 'acls', 'href': getHref(request, '/acls')}) parent_domain = getParentDomain(domain) if not parent_domain or getPathForDomain(parent_domain) == '/': is_toplevel = True else: is_toplevel = False log.debug(f"href parent domain: {parent_domain}") if not is_toplevel: hrefs.append({ 'rel': 'parent', 'href': getHref(request, '/', domain=parent_domain) }) rsp_json["hrefs"] = hrefs resp = await jsonResponse(request, rsp_json) log.response(request, resp=resp) return resp
async def PUT_Attribute(request): """ Handler for PUT /(obj)/<id>/attributes/<name> """ log.request(request) app = request.app params = request.rel_url.query obj_id = get_obj_id(request) attr_name = request.match_info.get('name') log.info("PUT attribute {} in {}".format(attr_name, obj_id)) validateAttributeName(attr_name) if not request.has_body: log.error( "PUT_Attribute with no body") raise HTTPBadRequest(message="body expected") body = await request.json() replace = False if "replace" in params and params["replace"]: replace = True log.info("replace attribute") datatype = None shape = None value = None if "type" not in body: log.error("PUT attribute with no type in body") raise HTTPInternalServerError() datatype = body["type"] if "shape" not in body: log.error("PUT attribute with no shape in body") raise HTTPInternalServerError() shape = body["shape"] if "value" in body: value = body["value"] obj_json = await get_metadata_obj(app, obj_id) log.debug("PUT attribute obj_id: {} got json".format(obj_id)) if "attributes" not in obj_json: log.error("unexpected obj data for id: {}".format(obj_id)) raise HTTPInternalServerError() attributes = obj_json["attributes"] if attr_name in attributes and not replace: # Attribute already exists, return a 409 log.warn("Attempt to overwrite attribute: {} in obj_id:".format(attr_name, obj_id)) raise HTTPConflict() if replace and attr_name not in attributes: # Replace requires attribute exists log.warn("Attempt to update missing attribute: {} in obj_id:".format(attr_name, obj_id)) raise HTTPNotFound() if replace: orig_attr = attributes[attr_name] create_time = orig_attr["created"] else: create_time = time.time() # ok - all set, create attribute obj attr_json = {"type": datatype, "shape": shape, "value": value, "created": create_time } attributes[attr_name] = attr_json # write back to S3, save to metadata cache await save_metadata_obj(app, obj_id, obj_json) resp_json = { } resp = json_response(resp_json, status=201) log.response(request, resp=resp) return resp