async def sentence(request):
sentence_id = int(request.match_info['sentence_id'])
pool = request.app['db_pool']
async with pool.acquire() as connection:
async with connection.transaction():
sql = """
SELECT
s.id, s.text, s.indexed
FROM sentence s
WHERE s.id = $1
"""
res = await connection.fetchrow(sql, sentence_id)
if not res:
raise HTTPNotFound(reason="Sentence not found")
if not res['indexed']:
raise HTTPBadRequest(
reason="Sentence is too common or not indexed yet.")
index = request.app['data'].get('index')
model = request.app['data'].get('model')
if not index or not model:
raise HTTPBadRequest(
reason="Word2vec model or index is not loaded yet.")
word_embeddings = []
sentence_text = res['text']
words = filter(lambda x: x not in config.IGNORED_WORDS,
nltk.word_tokenize(sentence_text))
for w in words:
if w in model:
word_embeddings.append(model[w])
if not word_embeddings:
raise HTTPBadRequest(reason="Sentence embedding is empty.")
sentence_embedding = np.mean(word_embeddings, axis=0)
labels, distances = index.knn_query([sentence_embedding],
k=config.SEARCH_RESULT_COUNT)
sentence_ids = labels[0].tolist()
similarities = list(map(lambda x: 1 - abs(x), distances[0].tolist()))
async with connection.transaction():
sql = """
SELECT
r.similarity, s.document_id, s.id, s.text
FROM
sentence s JOIN (
SELECT unnest($1::int[]) as sentence_id, unnest($2::float[]) as similarity
) r
ON s.id = r.sentence_id
ORDER BY r.similarity DESC;
"""
res = await connection.fetch(sql, sentence_ids, similarities)
return web.json_response({
"success": True,
"sentence_id": sentence_id,
"sentence_text": sentence_text,
"search_results": [dict(item) for item in res]
})
async def authenticate(self, params: MultiDict):
request = self.request
try:
domain = self.get_domain_from_acr(params['acr_values'])
except ValueError:
logger.warning('Invalid acr_values parameter')
raise HTTPBadRequest()
try:
provider = await self.get_provider(domain)
logger.debug('Using provider %s', provider)
authorization_endpoint = provider.get_authorization_endpoint()
if not authorization_endpoint:
raise ConfigurationError('Authorization point is not defined')
client_id = provider.get_client_id()
if not client_id:
raise ConfigurationError('Client does not registered')
if 'code' not in provider.get_response_types_supported():
raise ConfigurationError(
'Responce type "code" is not supported by provider')
scopes = provider.get_scopes_supported()
if 'openid' not in scopes:
raise ConfigurationError(
'Scope "openid" is not supported by provider')
scopes = ' '.join(
list({'openid'} | {'email', 'profile'} & set(scopes)))
nonce = sha256(request['sid'].encode('utf-8')).hexdigest()
except ProviderError as e:
logger.warning('Error getting provider configuration: %s', e)
reason = str(e)
raise HTTPSeeOther('{}?{}'.format(
request.app['config'].get('http_server.endpoints.login.path'),
urlencode({
'error':
'bigur_oidc_provider_error',
'error_description':
reason,
'next':
('{}?{}'.format(request.path,
urlencode(query=params, doseq=True))),
})))
else:
raise HTTPSeeOther('{}?{}'.format(
authorization_endpoint,
urlencode({
'redirect_uri':
self.endpoint_uri,
'client_id':
client_id,
'state':
urlsafe_b64encode(
crypt(
request.app['cookie_key'],
dumps({
'n': nonce,
'u': request.path,
'p': dict(params)
}))).decode('utf-8'),
'scope':
scopes,
'response_type':
'code',
'nonce':
nonce
})))
async def POST_Group(request):
"""HTTP method to create new Group object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_id = None
link_title = None
if request.has_body:
body = await request.json()
log.info(f"POST Group body: {body}")
if body:
if "link" in body:
link_body = body["link"]
log.debug(f"link_body: {link_body}")
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username,
"create")
if not link_id or not link_title:
log.warn(f"POST Group body with no link: {body}")
domain_json = await getDomainJson(
app, domain) # get again in case cache was invalidated
root_id = domain_json["root"]
group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
log.debug("create group, body: " + json.dumps(group_json))
req = getDataNodeUrl(app, group_id) + "/groups"
params = {}
if bucket:
params["bucket"] = bucket
group_json = await http_post(app, req, data=group_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = group_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.debug("PUT link - : " + link_req)
put_json_rsp = await http_put(app,
link_req,
data=link_json,
params=params)
log.debug(f"PUT Link resp: {put_json_rsp}")
log.debug("returning resp")
# group creation successful
resp = await jsonResponse(request, group_json, status=201)
log.response(request, resp=resp)
return resp
async def GET_Dataset(request):
"""HTTP method to return JSON description of a dataset"""
log.request(request)
app = request.app
params = request.rel_url.query
include_attrs = False
h5path = None
getAlias = False
dset_id = request.match_info.get('id')
if not dset_id and "h5path" not in params:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
if dset_id:
if not isValidUuid(dset_id, "Dataset"):
msg = f"Invalid dataset id: {dset_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
else:
group_id = None
if "grpid" in params:
group_id = params["grpid"]
if not isValidUuid(group_id, "Group"):
msg = f"Invalid parent group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "h5path" not in params:
msg = "Expecting either ctype id or h5path url param"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Dataset, h5path: {h5path}")
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
verbose = False
if "verbose" in params and params["verbose"]:
verbose = True
if h5path:
if group_id is None:
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
dset_id = await getObjectIdByPath(app, group_id, h5path, bucket=bucket
) # throws 404 if not found
if not isValidUuid(dset_id, "Dataset"):
msg = f"No dataset exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get dataset_id: {dset_id} from h5path: {h5path}")
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app,
dset_id,
refresh=True,
include_attrs=include_attrs,
bucket=bucket)
# check that we have permissions to read the object
await validateAction(app, domain, dset_id, username, "read")
log.debug(f"got dset_json: {dset_json}")
resp_json = {}
resp_json["id"] = dset_json["id"]
resp_json["root"] = dset_json["root"]
resp_json["shape"] = dset_json["shape"]
resp_json["type"] = dset_json["type"]
if "creationProperties" in dset_json:
resp_json["creationProperties"] = dset_json["creationProperties"]
else:
resp_json["creationProperties"] = {}
if "layout" in dset_json:
resp_json["layout"] = dset_json["layout"]
resp_json["attributeCount"] = dset_json["attributeCount"]
resp_json["created"] = dset_json["created"]
resp_json["lastModified"] = dset_json["lastModified"]
resp_json["domain"] = getPathForDomain(domain)
if getAlias:
root_id = dset_json["root"]
alias = []
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=dset_id,
bucket=bucket)
if h5path:
alias.append(h5path)
resp_json["alias"] = alias
if include_attrs:
resp_json["attributes"] = dset_json["attributes"]
hrefs = []
dset_uri = '/datasets/' + dset_id
hrefs.append({'rel': 'self', 'href': getHref(request, dset_uri)})
root_uri = '/groups/' + dset_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, dset_uri + '/attributes')
})
# provide a value link if the dataset is relatively small,
# otherwise create a preview link that shows a limited number of data values
dset_shape = dset_json["shape"]
if dset_shape["class"] != 'H5S_NULL':
count = 1
if dset_shape["class"] == 'H5S_SIMPLE':
dims = dset_shape["dims"]
count = getNumElements(dims)
if count <= 100:
# small number of values, provide link to entire dataset
hrefs.append({
'rel': 'data',
'href': getHref(request, dset_uri + '/value')
})
else:
# large number of values, create preview link
previewQuery = getPreviewQuery(dset_shape["dims"])
hrefs.append({
'rel':
'preview',
'href':
getHref(request, dset_uri + '/value', query=previewQuery)
})
resp_json["hrefs"] = hrefs
if verbose:
# get allocated size and num_chunks for the dataset if available
dset_detail = await getDatasetDetails(app,
dset_id,
dset_json["root"],
bucket=bucket)
if dset_detail is not None:
if "num_chunks" in dset_detail:
resp_json["num_chunks"] = dset_detail["num_chunks"]
if "allocated_bytes" in dset_detail:
resp_json["allocated_size"] = dset_detail["allocated_bytes"]
if "lastModified" in dset_detail:
resp_json["lastModified"] = dset_detail["lastModified"]
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def PUT_DatasetShape(request):
"""HTTP method to update dataset's shape"""
log.request(request)
app = request.app
shape_update = None
extend = 0
extend_dim = 0
dset_id = request.match_info.get('id')
if not dset_id:
msg = "Missing dataset id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(dset_id, "Dataset"):
msg = f"Invalid dataset id: {dset_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
username, pswd = getUserPasswordFromRequest(request)
await validateUserPassword(app, username, pswd)
# validate request
if not request.has_body:
msg = "PUT shape with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
data = await request.json()
if "shape" not in data and "extend" not in data:
msg = "PUT shape has no shape or extend key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "shape" in data:
shape_update = data["shape"]
if isinstance(shape_update, int):
# convert to a list
shape_update = [
shape_update,
]
log.debug(f"shape_update: {shape_update}")
if "extend" in data:
try:
extend = int(data["extend"])
except ValueError:
msg = "extend value must be integer"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extend <= 0:
msg = "extend value must be positive"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "extend_dim" in data:
try:
extend_dim = int(data["extend_dim"])
except ValueError:
msg = "extend_dim value must be integer"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extend_dim < 0:
msg = "extend_dim value must be non-negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# verify the user has permission to update shape
await validateAction(app, domain, dset_id, username, "update")
# get authoritative state for dataset from DN (even if it's in the meta_cache).
dset_json = await getObjectJson(app, dset_id, refresh=True, bucket=bucket)
shape_orig = dset_json["shape"]
log.debug(f"shape_orig: {shape_orig}")
# verify that the extend request is valid
if shape_orig["class"] != "H5S_SIMPLE":
msg = "Unable to extend shape of datasets who are not H5S_SIMPLE"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "maxdims" not in shape_orig:
msg = "Dataset is not extensible"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dims = shape_orig["dims"]
rank = len(dims)
maxdims = shape_orig["maxdims"]
if shape_update and len(shape_update) != rank:
msg = "Extent of update shape request does not match dataset sahpe"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
if shape_update and shape_update[i] < dims[i]:
msg = "Dataspace can not be made smaller"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if shape_update and maxdims[i] != 0 and shape_update[i] > maxdims[i]:
msg = "Database can not be extended past max extent"
log.warn(msg)
raise HTTPConflict()
if extend_dim < 0 or extend_dim >= rank:
msg = "Extension dimension must be less than rank and non-negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# send request onto DN
req = getDataNodeUrl(app, dset_id) + "/datasets/" + dset_id + "/shape"
json_resp = {"hrefs": []}
params = {}
if bucket:
params["bucket"] = bucket
if extend:
data = {"extend": extend, "extend_dim": extend_dim}
else:
data = {"shape": shape_update}
try:
put_rsp = await http_put(app, req, data=data, params=params)
log.info(f"got shape put rsp: {put_rsp}")
if "selection" in put_rsp:
json_resp["selection"] = put_rsp["selection"]
except HTTPConflict:
log.warn("got 409 extending dataspace")
raise
resp = await jsonResponse(request, json_resp, status=201)
log.response(request, resp=resp)
return resp
async def GET_Link(request):
"""HTTP method to return JSON for a group link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("get LINK: " + req)
params = {}
if bucket:
params["bucket"] = bucket
link_json = await http_get(app, req, params=params)
log.debug("got link_json: " + str(link_json))
resp_link = {}
resp_link["title"] = link_title
link_class = link_json["class"]
resp_link["class"] = link_class
if link_class == "H5L_TYPE_HARD":
resp_link["id"] = link_json["id"]
resp_link["collection"] = getCollectionForId(link_json["id"])
elif link_class == "H5L_TYPE_SOFT":
resp_link["h5path"] = link_json["h5path"]
elif link_class == "H5L_TYPE_EXTERNAL":
resp_link["h5path"] = link_json["h5path"]
resp_link["h5domain"] = link_json["h5domain"]
else:
log.warn(f"Unexpected link class: {link_class}")
resp_json = {}
resp_json["link"] = resp_link
resp_json["created"] = link_json["created"]
# links don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = link_json["created"]
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links/' + link_title)
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
if link_json["class"] == "H5L_TYPE_HARD":
target = '/' + resp_link["collection"] + '/' + resp_link["id"]
hrefs.append({'rel': 'target', 'href': getHref(request, target)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Links(request):
"""HTTP method to return JSON for link collection"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "read")
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links"
query_sep = '?'
if limit is not None:
req += query_sep + "Limit=" + str(limit)
query_sep = '&'
if marker is not None:
req += query_sep + "Marker=" + marker
log.debug("get LINKS: " + req)
params = {}
if bucket:
params["bucket"] = bucket
links_json = await http_get(app, req, params=params)
log.debug(f"got links json from dn for group_id: {group_id}")
links = links_json["links"]
# mix in collection key, target and hrefs
for link in links:
if link["class"] == "H5L_TYPE_HARD":
collection_name = getCollectionForId(link["id"])
link["collection"] = collection_name
target_uri = '/' + collection_name + '/' + link["id"]
link["target"] = getHref(request, target_uri)
link_uri = '/groups/' + group_id + '/links/' + link['title']
link["href"] = getHref(request, link_uri)
resp_json = {}
resp_json["links"] = links
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({
'rel': 'self',
'href': getHref(request, group_uri + '/links')
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, group_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def save_metadata_obj(app, obj_id, obj_json, bucket=None, notify=False, flush=False):
""" Persist the given object """
log.info(f"save_metadata_obj {obj_id} bucket={bucket} notify={notify} flush={flush}")
if notify and not flush:
log.error("notify not valid when flush is false")
raise HTTPInternalServerError()
if not isinstance(obj_json, dict):
log.error("Passed non-dict obj to save_metadata_obj")
raise HTTPInternalServerError()
try:
validateInPartition(app, obj_id)
except KeyError:
log.error("Domain not in partition")
raise HTTPInternalServerError()
dirty_ids = app["dirty_ids"]
deleted_ids = app['deleted_ids']
if obj_id in deleted_ids:
if isValidUuid(obj_id):
# domain objects may be re-created, but shouldn't see repeats of
# deleted uuids
log.warn(f"{obj_id} has been deleted")
raise HTTPInternalServerError()
elif obj_id in deleted_ids:
deleted_ids.remove(obj_id) # un-gone the domain id
# update meta cache
meta_cache = app['meta_cache']
log.debug(f"save: {obj_id} to cache")
meta_cache[obj_id] = obj_json
meta_cache.setDirty(obj_id)
now = int(time.time())
if flush:
# write to S3 immediately
if isValidChunkId(obj_id):
log.warn("flush not supported for save_metadata_obj with chunks")
raise HTTPBadRequest()
try:
await write_s3_obj(app, obj_id, bucket=bucket)
except KeyError as ke:
log.error(f"s3 sync got key error: {ke}")
raise HTTPInternalServerError()
except HTTPInternalServerError:
log.warn(f" failed to write {obj_id}")
raise # re-throw
if obj_id in dirty_ids:
log.warn(f"save_metadata_obj flush - object {obj_id} is still dirty")
# message AN immediately if notify flag is set
# otherwise AN will be notified at next S3 sync
if notify:
if isValidUuid(obj_id) and isSchema2Id(obj_id):
root_id = getRootObjId(obj_id)
await notify_root(app, root_id, bucket=bucket)
else:
log.debug(f"setting dirty_ids[{obj_id}] = ({now}, {bucket})")
if isValidUuid(obj_id) and not bucket:
log.warn(f"bucket is not defined for save_metadata_obj: {obj_id}")
dirty_ids[obj_id] = (now, bucket)
async def wrapper(request):
try:
return await fun(request)
except MicroCurrencyConverterException as e:
Logger.debug('Exception handled')
raise HTTPBadRequest(reason=e.args and e.args[0] or str(e))
def _validateBucket(self, bucket):
if not bucket or pp.isabs(bucket) or pp.dirname(bucket):
msg = "invalid bucket name"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
def _validateKey(self, key):
if not key or pp.isabs(key):
msg = "invalid key name"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
async def list_keys(self,
prefix='',
deliminator='',
suffix='',
include_stats=False,
callback=None,
bucket=None,
limit=None):
""" return keys matching the arguments
"""
self._validateBucket(bucket)
if deliminator and deliminator != '/':
msg = "Only '/' is supported as deliminator"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(
f"list_keys('{prefix}','{deliminator}','{suffix}', include_stats={include_stats}"
)
await asyncio.sleep(0) # for async compat
basedir = pp.join(self._root_dir, bucket)
if prefix:
basedir = pp.join(basedir, prefix)
if not pp.isdir(basedir):
msg = f"listkeys - {basedir} not found"
log.warn(msg)
raise HTTPNotFound()
# return all files (but not directories) under basedir
files = []
for root, dirs, filelist in walk(basedir):
if deliminator:
dirs.sort()
for dirname in dirs:
if suffix and not dirname.endswith(suffix):
continue
log.debug(f"got dirname: {dirname}")
filename = pp.join(root[len(basedir):], dirname)
filename += '/'
files.append(filename)
if limit and len(files) >= limit:
break
break # don't recurse into subdirs
else:
filelist.sort()
for filename in filelist:
if suffix and not filename.endswith(suffix):
continue
files.append(pp.join(root[len(basedir):], filename))
if limit and len(files) >= limit:
break
if include_stats:
key_names = {}
else:
key_names = []
for filename in files:
if suffix and not filename.endswith(suffix):
continue
if include_stats:
filepath = pp.join(basedir, filename)
key_stats = self._getFileStats(filepath)
key_name = pp.join(prefix, filename)
key_names[key_name] = key_stats
else:
key_names.append(pp.join(prefix, filename))
if limit and len(key_names) == limit:
break
if callback:
if iscoroutinefunction(callback):
await callback(self._app, key_names)
else:
callback(self._app, key_names)
log.info(f"listKeys done, got {len(key_names)} keys")
return key_names
async def validateChunkLayout(app, shape_json, item_size, layout, bucket=None):
rank = 0
space_dims = None
chunk_dims = None
max_dims = None
if "dims" in shape_json:
space_dims = shape_json["dims"]
rank = len(space_dims)
if "maxdims" in shape_json:
max_dims = shape_json["maxdims"]
if "dims" in layout:
chunk_dims = layout["dims"]
if chunk_dims:
# validate that the chunk_dims are valid and correlates with the dataset shape
if isinstance(chunk_dims, int):
chunk_dims = [
chunk_dims,
] # promote to array
if len(chunk_dims) != rank:
msg = "Layout rank does not match shape rank"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
dim_extent = space_dims[i]
chunk_extent = chunk_dims[i]
if not isinstance(chunk_extent, int):
msg = "Layout dims must be integer or integer array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if chunk_extent <= 0:
msg = "Invalid layout value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if max_dims is None:
if chunk_extent > dim_extent:
msg = "Invalid layout value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif max_dims[i] != 0:
if chunk_extent > max_dims[i]:
msg = "Invalid layout value for extensible dimension"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
pass # allow any positive value for unlimited dimensions
if "class" not in layout:
msg = "class key not found in layout for creation property list"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
layout_class = layout["class"]
if layout_class == 'H5D_CONTIGUOUS_REF':
# reference to a dataset in a traditional HDF5 files with contigious storage
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "file_uri" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'file_uri' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "offset" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'offset' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "size" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'size' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" in layout:
# used defined chunk layout not allowed for H5D_CONTIGUOUS_REF
msg = "'dims' key can not be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout_class == 'H5D_CHUNKED_REF':
# reference to a dataset in a traditional HDF5 files with chunked storage
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "file_uri" not in layout:
# needed for H5D_CHUNKED_REF
msg = "'file_uri' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" not in layout:
# needed for H5D_CHUNKED_REF
msg = "'dimns' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "chunks" not in layout:
msg = "'chunks' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout_class == 'H5D_CHUNKED_REF_INDIRECT':
# reference to a dataset in a traditional HDF5 files with chunked storage using an auxillary dataset
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" not in layout:
# needed for H5D_CHUNKED_REF_INDIRECT
msg = "'dimns' key must be provided for H5D_CHUNKED_REF_INDIRECT layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "chunk_table" not in layout:
msg = "'chunk_table' key must be provided for H5D_CHUNKED_REF_INDIRECT layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
chunktable_id = layout["chunk_table"]
if not isValidUuid(chunktable_id, "Dataset"):
msg = f"Invalid chunk table id: {chunktable_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# verify the chunk table exists and is of reasonable shape
try:
chunktable_json = await getObjectJson(app,
chunktable_id,
bucket=bucket,
refresh=False)
except HTTPNotFound:
msg = f"chunk table id: {chunktable_id} not found"
log.warn(msg)
raise
chunktable_shape = chunktable_json["shape"]
if chunktable_shape["class"] == 'H5S_NULL':
msg = "Null space datasets can not be used as chunk tables"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
chunktable_dims = getShapeDims(chunktable_shape)
if len(chunktable_dims) != len(space_dims):
msg = "Chunk table rank must be same as dataspace rank"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout_class == 'H5D_CHUNKED':
if "dims" not in layout:
msg = "dims key not found in layout for creation property list"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if shape_json["class"] != 'H5S_SIMPLE':
msg = f"Bad Request: chunked layout not valid with shape class: {shape_json['class']}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
msg = f"Unexpected layout: {layout_class}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
def _get_object_id(self):
try:
object_id = ObjectId(self.request.match_info['product_id'])
except InvalidId:
raise HTTPBadRequest(reason='Invalid id')
return object_id
async def GET_AttributeValue(request):
"""HTTP method to return an attribute value"""
log.request(request)
app = request.app
log.info("GET_AttributeValue")
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain value: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(dn_json))
attr_shape = dn_json["shape"]
log.debug(f"attribute shape: {attr_shape}")
if attr_shape["class"] == 'H5S_NULL':
msg = "Null space attributes can not be read"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
accept_type = getAcceptType(request)
response_type = accept_type # will adjust later if binary not possible
type_json = dn_json["type"]
shape_json = dn_json["shape"]
item_size = getItemSize(type_json)
if item_size == 'H5T_VARIABLE' and accept_type != "json":
msg = "Client requested binary, but only JSON is supported for variable length data types"
log.info(msg)
response_type = "json"
if response_type == "binary":
arr_dtype = createDataType(type_json) # np datatype
np_shape = getShapeDims(shape_json)
try:
arr = jsonToArray(np_shape, arr_dtype, dn_json["value"])
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
output_data = arr.tobytes()
log.debug(
f"GET AttributeValue - returning {len(output_data)} bytes binary data"
)
cors_domain = config.get("cors_domain")
# write response
try:
resp = StreamResponse()
resp.content_type = "application/octet-stream"
resp.content_length = len(output_data)
# allow CORS
if cors_domain:
resp.headers['Access-Control-Allow-Origin'] = cors_domain
resp.headers[
'Access-Control-Allow-Methods'] = "GET, POST, DELETE, PUT, OPTIONS"
resp.headers[
'Access-Control-Allow-Headers'] = "Content-Type, api_key, Authorization"
await resp.prepare(request)
await resp.write(output_data)
except Exception as e:
log.error(f"Got exception: {e}")
raise HTTPInternalServerError()
finally:
await resp.write_eof()
else:
resp_json = {}
if "value" in dn_json:
resp_json["value"] = dn_json["value"]
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
attr_uri = obj_uri + '/attributes/' + attr_name
hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
def validate_required_fields(self, data):
for f in self.get_required_fields():
if f not in data.keys():
raise HTTPBadRequest(body=self.error_response(
'Field "{field_name}" is required'.format(field_name=f)))
async def PUT_AttributeValue(request):
"""HTTP method to update an attributes data"""
log.request(request)
log.info("PUT_AttributeValue")
app = request.app
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
log.debug(f"Attribute name: [{attr_name}]")
validateAttributeName(attr_name)
log.info(f"PUT Attribute Value id: {obj_id} name: {attr_name}")
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT AttributeValue with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPInternalServerError()
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "update")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(obj_id))
log.debug(f"got dn_json: {dn_json}")
attr_shape = dn_json["shape"]
if attr_shape["class"] == 'H5S_NULL':
msg = "Null space attributes can not be updated"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
np_shape = getShapeDims(attr_shape)
type_json = dn_json["type"]
np_dtype = createDataType(type_json) # np datatype
request_type = "json"
if "Content-Type" in request.headers:
# client should use "application/octet-stream" for binary transfer
content_type = request.headers["Content-Type"]
if content_type not in ("application/json",
"application/octet-stream"):
msg = f"Unknown content_type: {content_type}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if content_type == "application/octet-stream":
log.debug("PUT AttributeValue - request_type is binary")
request_type = "binary"
else:
log.debug("PUT AttribueValue - request type is json")
binary_data = None
if request_type == "binary":
item_size = getItemSize(type_json)
if item_size == 'H5T_VARIABLE':
msg = "Only JSON is supported for variable length data types"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# read binary data
binary_data = await request.read()
if len(binary_data) != request.content_length:
msg = f"Read {len(binary_data)} bytes, expecting: {request.content_length}"
log.error(msg)
raise HTTPInternalServerError()
arr = None # np array to hold request data
if binary_data:
npoints = getNumElements(np_shape)
if npoints * item_size != len(binary_data):
msg = "Expected: " + str(
npoints * item_size) + " bytes, but got: " + str(
len(binary_data))
log.warn(msg)
raise HTTPBadRequest(reason=msg)
arr = np.fromstring(binary_data, dtype=np_dtype)
arr = arr.reshape(np_shape) # conform to selection shape
# convert to JSON for transmission to DN
data = arr.tolist()
value = bytesArrayToList(data)
else:
body = await request.json()
if "value" not in body:
msg = "PUT attribute value with no value in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
value = body["value"]
# validate that the value agrees with type/shape
try:
arr = jsonToArray(np_shape, np_dtype, value)
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"Got: {arr.size} array elements")
# ready to add attribute now
attr_json = {}
attr_json["type"] = type_json
attr_json["shape"] = attr_shape
attr_json["value"] = value
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.info(f"PUT Attribute Value: {req}")
dn_json["value"] = value
params = {}
params = {"replace": 1} # let the DN know we can overwrite the attribute
if bucket:
params["bucket"] = bucket
put_rsp = await http_put(app, req, params=params, data=attr_json)
log.info(f"PUT Attribute Value resp: {put_rsp}")
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
# attribute creation successful
resp = await jsonResponse(request, req_rsp)
log.response(request, resp=resp)
return resp
async def POST_Group(request):
""" Handler for POST /groups"""
log.request(request)
app = request.app
if not request.has_body:
msg = "POST_Group with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
group_id = get_obj_id(request, body=body)
log.info("POST group: {}".format(group_id))
if not isValidUuid(group_id, obj_class="group"):
log.error("Unexpected group_id: {}".format(group_id))
raise HTTPInternalServerError()
# verify the id doesn't already exist
obj_found = await check_metadata_obj(app, group_id)
if obj_found:
log.error("Post with existing group_id: {}".format(group_id))
raise HTTPInternalServerError()
root_id = None
if "root" not in body:
msg = "POST_Group with no root"
log.error(msg)
raise HTTPInternalServerError()
root_id = body["root"]
if not isValidUuid(root_id, obj_class="group"):
msg = "Invalid root_id: " + root_id
log.error(msg)
raise HTTPInternalServerError()
# ok - all set, create group obj
now = time.time()
group_json = {
"id": group_id,
"root": root_id,
"created": now,
"lastModified": now,
"links": {},
"attributes": {}
}
await save_metadata_obj(app, group_id, group_json, notify=True, flush=True)
# formulate response
resp_json = {}
resp_json["id"] = group_id
resp_json["root"] = root_id
resp_json["created"] = group_json["created"]
resp_json["lastModified"] = group_json["lastModified"]
resp_json["linkCount"] = 0
resp_json["attributeCount"] = 0
resp = json_response(resp_json, status=201)
log.response(request, resp=resp)
return resp
async def PUT_Link(request):
"""HTTP method to create a new link"""
log.request(request)
app = request.app
group_id = request.match_info.get('id')
if not group_id:
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(group_id, obj_class="Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_title = request.match_info.get('title')
log.info(f"PUT Link_title: [{link_title}]")
validateLinkName(link_title)
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Link with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
link_json = {}
if "id" in body:
if not isValidUuid(body["id"]):
msg = "PUT Link with invalid id in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_json["id"] = body["id"]
link_json["class"] = "H5L_TYPE_HARD"
elif "h5path" in body:
link_json["h5path"] = body["h5path"]
# could be hard or soft link
if "h5domain" in body:
link_json["h5domain"] = body["h5domain"]
link_json["class"] = "H5L_TYPE_EXTERNAL"
else:
# soft link
link_json["class"] = "H5L_TYPE_SOFT"
else:
msg = "PUT Link with no id or h5path keys"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
await validateAction(app, domain, group_id, username, "create")
# for hard links, verify that the referenced id exists and is in this domain
if "id" in body:
ref_id = body["id"]
ref_json = await getObjectJson(app, ref_id, bucket=bucket)
group_json = await getObjectJson(app, group_id, bucket=bucket)
if ref_json["root"] != group_json["root"]:
msg = "Hard link must reference an object in the same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
# ready to add link now
req = getDataNodeUrl(app, group_id)
req += "/groups/" + group_id + "/links/" + link_title
log.debug("PUT link - getting group: " + req)
params = {}
if bucket:
params["bucket"] = bucket
try:
put_rsp = await http_put(app, req, data=link_json, params=params)
log.debug("PUT Link resp: " + str(put_rsp))
dn_status = 201
except HTTPConflict:
# check to see if this is just a duplicate put of an existing link
dn_status = 409
log.warn(f"PUT Link: got conflict error for link_json: {link_json}")
existing_link = await http_get(app, req, params=params)
log.warn(f"PUT Link: fetched existing link: {existing_link}")
for prop in ("class", "id", "h5path", "h5domain"):
if prop in link_json:
if prop not in existing_link:
log.warn(
f"PUT Link - prop {prop} not found in existing link, returning 409"
)
break
if link_json[prop] != existing_link[prop]:
log.warn(
f"PUT Link - prop {prop} value is different, old: {existing_link[prop]}, new: {link_json[prop]}, returning 409"
)
break
else:
log.info("PUT link is identical to existing value returning OK")
dn_status = 200 # return 200 since we didn't actually create a resource
if dn_status == 409:
raise # return 409 to client
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
# link creation successful
# returns 201 if new link was created, 200 if this is a duplicate
# of an existing link
resp = await jsonResponse(request, req_rsp, status=dn_status)
log.response(request, resp=resp)
return resp
async def real_resolve(self, request: IRequest) -> MatchInfo:
"""Main function to resolve a request."""
security = get_adapter(request, IInteraction)
if request.method not in app_settings['http_methods']:
raise HTTPMethodNotAllowed()
method = app_settings['http_methods'][request.method]
language = language_negotiation(request)
language_object = language(request)
try:
resource, tail = await self.traverse(request)
except ConflictError:
# can also happen from connection errors so we bubble this...
raise
except Exception as _exc:
logger.error('Unhandled exception occurred', exc_info=True)
request.resource = request.tail = None
request.exc = _exc
data = {
'success': False,
'exception_message': str(_exc),
'exception_type': getattr(type(_exc), '__name__',
str(type(_exc))), # noqa
}
if app_settings.get('debug'):
data['traceback'] = traceback.format_exc()
raise HTTPBadRequest(text=ujson.dumps(data))
request.record('traversed')
await notify(ObjectLoadedEvent(resource))
request.resource = resource
request.tail = tail
if request.resource is None:
raise HTTPBadRequest(text='Resource not found')
traverse_to = None
if tail and len(tail) == 1:
view_name = tail[0]
elif tail is None or len(tail) == 0:
view_name = ''
else:
view_name = tail[0]
traverse_to = tail[1:]
request.record('beforeauthentication')
await self.apply_authorization(request)
request.record('authentication')
translator = query_adapter(language_object,
ITranslated,
args=[resource, request])
if translator is not None:
resource = translator.translate()
# Add anonymous participation
if len(security.participations) == 0:
security.add(AnonymousParticipation(request))
# container registry lookup
try:
view = query_multi_adapter((resource, request),
method,
name=view_name)
except AttributeError:
view = None
request.found_view = view
request.view_name = view_name
# Traverse view if its needed
if traverse_to is not None and view is not None:
if not ITraversableView.providedBy(view):
return None
else:
try:
view = await view.publish_traverse(traverse_to)
except KeyError:
return None # not found, it's okay.
except Exception as e:
logger.error("Exception on view execution",
exc_info=e,
request=request)
return None
request.record('viewfound')
permission = get_utility(IPermission, name='guillotina.AccessContent')
if not security.check_permission(permission.id, resource):
# Check if its a CORS call:
if IOPTIONS != method:
# Check if the view has permissions explicit
if view is None or not view.__allow_access__:
logger.info(
"No access content {content} with {auths}".format(
content=resource,
auths=str([
x.principal.id for x in security.participations
])),
request=request)
raise HTTPUnauthorized()
if view is None and method == IOPTIONS:
view = DefaultOPTIONS(resource, request)
if view:
ViewClass = view.__class__
view_permission = get_view_permission(ViewClass)
if not security.check_permission(view_permission, view):
logger.info("No access for view {content} with {auths}".format(
content=resource,
auths=str(
[x.principal.id for x in security.participations])),
request=request)
raise HTTPUnauthorized()
request.record('authorization')
renderer = content_type_negotiation(request, resource, view)
renderer_object = renderer(request)
rendered = query_multi_adapter((renderer_object, view, request),
IRendered)
request.record('renderer')
if rendered is not None:
return MatchInfo(resource, request, view, rendered)
else:
return None
async def list_keys(self, prefix='', deliminator='', suffix='', include_stats=False, callback=None, bucket=None, limit=None):
""" return keys matching the arguments
"""
if not bucket:
log.error("list_keys - bucket not set")
raise HTTPInternalServerError()
log.info(f"list_keys('{prefix}','{deliminator}','{suffix}', include_stats={include_stats}")
if deliminator and deliminator != '/':
msg = "Only '/' is supported as deliminator"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if include_stats:
# use a dictionary to hold return values
key_names = {}
else:
# just use a list
key_names = []
continuation_token = None
page_result_count = 1000 # compatible with what S3 uses by default
if prefix == '':
prefix = None # azure sdk expects None for no prefix
try:
async with self._client.get_container_client(container=bucket) as container_client:
while True:
log.info(f"list_blobs: {prefix} continuation_token: {continuation_token}")
keyList = container_client.walk_blobs(name_starts_with=prefix, delimiter=deliminator, results_per_page=page_result_count).by_page(continuation_token)
async for key in await keyList.__anext__():
key_name = key["name"]
if include_stats:
ETag = key["etag"]
lastModified = int(key["last_modified"].timestamp())
data_size = key["size"]
key_names[key_name] = {"ETag": ETag, "Size": data_size, "LastModified": lastModified }
else:
if suffix and not key_name.endswith(suffix):
continue
if deliminator and key_name[-1] != '/':
# only return folders
continue
if limit and len(key_names) >= limit:
break
key_names.append(key_name)
if callback:
if iscoroutinefunction(callback):
await callback(self._app, key_names)
else:
callback(self._app, key_names)
if not keyList.continuation_token or (limit and len(key_names) >= limit):
# got all the keys (or as many as requested)
break
else:
# keep going
continuation_token = keyList.continuation_token
except CancelledError as cle:
self._azure_stats_increment("error_count")
msg = f"azureBlobClient.CancelledError for list_keys: {cle}"
log.error(msg)
raise HTTPInternalServerError()
except Exception as e:
if isinstance(e, AzureError):
if e.status_code == 404:
msg = "azureBlobClient not found error for list_keys"
log.warn(msg)
raise HTTPNotFound()
elif e.status_code in (401, 403):
msg = "azureBlobClient.access denied for list_keys"
log.info(msg)
raise HTTPForbidden()
else:
self._azure_stats_increment("error_count")
log.error(f"azureBlobClient.got unexpected AzureError for list_keys: {e.message}")
raise HTTPInternalServerError()
else:
log.error(f"azureBlobClient.Unexpected exception for list_keys: {e}")
raise HTTPInternalServerError()
log.info(f"list_keys done, got {len(key_names)} keys")
if limit and len(key_names) > limit:
# return requested number of keys
if include_stats:
keys = list(key_names.keys())
keys.sort()
for k in keys[limit:]:
del key_names[k]
else:
key_names = key_names[:limit]
return key_names
def fake_request(_):
raise HTTPBadRequest()
def validateChunkLayout(shape_json, item_size, layout):
rank = 0
space_dims = None
chunk_dims = None
max_dims = None
if "dims" in shape_json:
space_dims = shape_json["dims"]
rank = len(space_dims)
if "maxdims" in shape_json:
max_dims = shape_json["maxdims"]
if "dims" in layout:
chunk_dims = layout["dims"]
if chunk_dims:
# validate that the chunk_dims are valid and correlates with the dataset shape
if isinstance(chunk_dims, int):
chunk_dims = [
chunk_dims,
] # promote to array
if len(chunk_dims) != rank:
msg = "Layout rank does not match shape rank"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
for i in range(rank):
dim_extent = space_dims[i]
chunk_extent = chunk_dims[i]
if not isinstance(chunk_extent, int):
msg = "Layout dims must be integer or integer array"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if chunk_extent <= 0:
msg = "Invalid layout value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if max_dims is None:
if chunk_extent > dim_extent:
msg = "Invalid layout value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif max_dims[i] != 0:
if chunk_extent > max_dims[i]:
msg = "Invalid layout value for extensible dimension"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
pass # allow any positive value for unlimited dimensions
if "class" not in layout:
msg = "class key not found in layout for creation property list"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if layout["class"] == 'H5D_CONTIGUOUS_REF':
# reference to a dataset in a traditional HDF5 files with contigious storage
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "file_uri" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'file_uri' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "offset" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'offset' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "size" not in layout:
# needed for H5D_CONTIGUOUS_REF
msg = "'size' key must be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" in layout:
# used defined chunk layout not allowed for H5D_CONTIGUOUS_REF
msg = "'dims' key can not be provided for H5D_CONTIGUOUS_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout["class"] == 'H5D_CHUNKED_REF':
# reference to a dataset in a traditional HDF5 files with chunked storage
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "file_uri" not in layout:
# needed for H5D_CHUNKED_REF
msg = "'file_uri' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" not in layout:
# needed for H5D_CHUNKED_REF
msg = "'dimns' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "chunks" not in layout:
msg = "'chunks' key must be provided for H5D_CHUNKED_REF layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout["class"] == 'H5D_CHUNKED_REF_INDIRECT':
# reference to a dataset in a traditional HDF5 files with chunked storage using an auxillary dataset
if item_size == 'H5T_VARIABLE':
# can't be used with variable types..
msg = "Datsets with variable types cannot be used with reference layouts"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "dims" not in layout:
# needed for H5D_CHUNKED_REF_INDIRECT
msg = "'dimns' key must be provided for H5D_CHUNKED_REF_INDIRECT layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "chunk_table" not in layout:
msg = "'chunk_table' key must be provided for H5D_CHUNKED_REF_INDIRECT layout"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif layout["class"] == 'H5D_CHUNKED':
if "dims" not in layout:
msg = "dims key not found in layout for creation property list"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if shape_json["class"] != 'H5S_SIMPLE':
msg = f"Bad Request: chunked layout not valid with shape class: {shape_json['class']}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
msg = f"Unexpected layout: {layout['class']}"
async def GET_Attribute(request):
"""HTTP method to return JSON for an attribute"""
log.request(request)
app = request.app
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
validateAttributeName(attr_name)
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.debug("get Attribute: " + req)
params = {}
if bucket:
params["bucket"] = bucket
dn_json = await http_get(app, req, params=params)
log.debug("got attributes json from dn for obj_id: " + str(obj_id))
resp_json = {}
resp_json["name"] = attr_name
resp_json["type"] = dn_json["type"]
resp_json["shape"] = dn_json["shape"]
if "value" in dn_json:
resp_json["value"] = dn_json["value"]
resp_json["created"] = dn_json["created"]
# attributes don't get modified, so use created timestamp as lastModified
resp_json["lastModified"] = dn_json["created"]
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
attr_uri = obj_uri + '/attributes/' + attr_name
hrefs.append({'rel': 'self', 'href': getHref(request, attr_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def POST_Dataset(request):
"""HTTP method to create a new dataset object"""
log.request(request)
app = request.app
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "POST Datasets with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
# get domain, check authorization
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
domain_json = await getDomainJson(app, domain, reload=True)
root_id = domain_json["root"]
aclCheck(domain_json, "create",
username) # throws exception if not allowed
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
#
# validate type input
#
if "type" not in body:
msg = "POST Dataset has no type key in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str) and datatype.startswith("t-"):
# Committed type - fetch type json from DN
ctype_id = datatype
log.debug(f"got ctypeid: {ctype_id}")
ctype_json = await getObjectJson(app, ctype_id, bucket=bucket)
log.debug(f"ctype: {ctype_json}")
if ctype_json["root"] != root_id:
msg = "Referenced committed datatype must belong in same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = ctype_json["type"]
# add the ctype_id to type type
datatype["id"] = ctype_id
elif isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug(f"got datatype: {datatype}")
except TypeError:
msg = "POST Dataset with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
validateTypeItem(datatype)
item_size = getItemSize(datatype)
#
# Validate shape input
#
dims = None
shape_json = {}
rank = 0
if "shape" not in body:
shape_json["class"] = "H5S_SCALAR"
else:
shape = body["shape"]
if isinstance(shape, int):
shape_json["class"] = "H5S_SIMPLE"
dims = [
shape,
]
shape_json["dims"] = dims
rank = 1
elif isinstance(shape, str):
# only valid string value is H5S_NULL
if shape != "H5S_NULL":
msg = "POST Datset with invalid shape value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["class"] = "H5S_NULL"
elif isinstance(shape, list):
if len(shape) == 0:
shape_json["class"] = "H5S_SCALAR"
else:
shape_json["class"] = "H5S_SIMPLE"
shape_json["dims"] = shape
dims = shape
rank = len(dims)
else:
msg = "Bad Request: shape is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if dims is not None:
for i in range(rank):
extent = dims[i]
if not isinstance(extent, int):
msg = "Invalid shape type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "shape dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = None
if "maxdims" in body:
if dims is None:
msg = "Maxdims cannot be supplied if space is NULL"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
maxdims = body["maxdims"]
if isinstance(maxdims, int):
dim1 = maxdims
maxdims = [dim1]
elif isinstance(maxdims, list):
pass # can use as is
else:
msg = "Bad Request: maxdims is invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(dims) != len(maxdims):
msg = "Maxdims rank doesn't match Shape"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if maxdims is not None:
for extent in maxdims:
if not isinstance(extent, int):
msg = "Invalid maxdims type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if extent < 0:
msg = "maxdims dimension is negative"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if len(maxdims) != len(dims):
msg = "Bad Request: maxdims array length must equal shape array length"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
shape_json["maxdims"] = []
for i in range(rank):
maxextent = maxdims[i]
if not isinstance(maxextent, int):
msg = "Bad Request: maxdims must be integer type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif maxextent == 0:
# unlimited dimension
shape_json["maxdims"].append(0)
elif maxextent < dims[i]:
msg = "Bad Request: maxdims extent can't be smaller than shape extent"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
shape_json["maxdims"].append(maxextent)
layout = None
min_chunk_size = int(config.get("min_chunk_size"))
max_chunk_size = int(config.get("max_chunk_size"))
if 'creationProperties' in body:
creationProperties = body["creationProperties"]
if 'layout' in creationProperties:
layout = creationProperties["layout"]
validateChunkLayout(shape_json, item_size, layout)
if layout is None and shape_json["class"] != "H5S_NULL":
# default to chunked layout
layout = {"class": "H5D_CHUNKED"}
if layout and layout["class"] == 'H5D_CONTIGUOUS_REF':
chunk_dims = getContiguousLayout(shape_json,
item_size,
chunk_min=min_chunk_size,
chunk_max=max_chunk_size)
layout["dims"] = chunk_dims
log.debug(f"autoContiguous layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED' and "dims" not in layout:
# do autochunking
chunk_dims = guessChunk(shape_json, item_size)
layout["dims"] = chunk_dims
log.debug(f"initial autochunk layout: {layout}")
if layout and layout["class"] == 'H5D_CHUNKED':
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug(
f"chunk_size: {chunk_size}, min: {min_chunk_size}, max: {max_chunk_size}"
)
# adjust the chunk shape if chunk size is too small or too big
adjusted_chunk_dims = None
if chunk_size < min_chunk_size:
log.debug(
f"chunk size: {chunk_size} less than min size: {min_chunk_size}, expanding"
)
adjusted_chunk_dims = expandChunk(chunk_dims,
item_size,
shape_json,
chunk_min=min_chunk_size,
layout_class=layout["class"])
elif chunk_size > max_chunk_size:
log.debug(
f"chunk size: {chunk_size} greater than max size: {max_chunk_size}, shrinking"
)
adjusted_chunk_dims = shrinkChunk(chunk_dims,
item_size,
chunk_max=max_chunk_size)
if adjusted_chunk_dims:
log.debug(
f"requested chunk_dimensions: {chunk_dims} modified dimensions: {adjusted_chunk_dims}"
)
layout["dims"] = adjusted_chunk_dims
# set partition_count if needed:
max_chunks_per_folder = int(config.get("max_chunks_per_folder"))
if max_chunks_per_folder > 0 and "dims" in shape_json and "dims" in layout:
chunk_dims = layout["dims"]
shape_dims = shape_json["dims"]
if "maxdims" in shape_json:
max_dims = shape_json["maxdims"]
else:
max_dims = None
num_chunks = 1
rank = len(shape_dims)
unlimited_count = 0
if max_dims:
for i in range(rank):
if max_dims[i] == 0:
unlimited_count += 1
log.debug(f"number of unlimited dimensions: {unlimited_count}")
for i in range(rank):
max_dim = 1
if max_dims:
max_dim = max_dims[i]
if max_dim == 0:
# don't really know what the ultimate extent could be, but assume
# 10^6 for total number of elements and square-shaped array...
MAX_ELEMENT_GUESS = 10.0**6
max_dim = int(
math.pow(MAX_ELEMENT_GUESS, 1 / unlimited_count))
else:
max_dim = shape_dims[i]
num_chunks *= math.ceil(max_dim / chunk_dims[i])
if num_chunks > max_chunks_per_folder:
partition_count = math.ceil(num_chunks / max_chunks_per_folder)
log.info(
f"set partition count to: {partition_count}, num_chunks: {num_chunks}"
)
layout["partition_count"] = partition_count
else:
log.info(
f"do not need chunk partitions, num_chunks: {num_chunks} max_chunks_per_folder: {max_chunks_per_folder}"
)
if layout and layout["class"] in ('H5D_CHUNKED_REF',
'H5D_CHUNKED_REF_INDIRECT'):
chunk_dims = layout["dims"]
chunk_size = getChunkSize(chunk_dims, item_size)
log.debug(
f"chunk_size: {chunk_size}, min: {min_chunk_size}, max: {max_chunk_size}"
)
# adjust the chunk shape if chunk size is too small or too big
if chunk_size < min_chunk_size:
log.warn(
f"chunk size: {chunk_size} less than min size: {min_chunk_size} for H5D_CHUNKED_REF dataset"
)
elif chunk_size > max_chunk_size:
log.warn(
f"chunk size: {chunk_size} greater than max size: {max_chunk_size}, for H5D_CHUNKED_REF dataset"
)
link_id = None
link_title = None
if "link" in body:
link_body = body["link"]
if "id" in link_body:
link_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.info(f"link id: {link_id}")
# verify that the referenced id exists and is in this domain
# and that the requestor has permissions to create a link
await validateAction(app, domain, link_id, username, "create")
dset_id = createObjId("datasets", rootid=root_id)
log.info(f"new dataset id: {dset_id}")
dataset_json = {
"id": dset_id,
"root": root_id,
"type": datatype,
"shape": shape_json
}
if "creationProperties" in body:
# TBD - validate all creationProperties
creationProperties = body["creationProperties"]
if "fillValue" in creationProperties:
# validate fill value compatible with type
dt = createDataType(datatype)
fill_value = creationProperties["fillValue"]
if isinstance(fill_value, list):
fill_value = tuple(fill_value)
try:
np.asarray(fill_value, dtype=dt)
except (TypeError, ValueError):
msg = f"Fill value {fill_value} not compatible with dataset type: {datatype}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dataset_json["creationProperties"] = creationProperties
if layout is not None:
dataset_json["layout"] = layout
log.debug("create dataset: " + json.dumps(dataset_json))
req = getDataNodeUrl(app, dset_id) + "/datasets"
params = {}
if bucket:
params["bucket"] = bucket
post_json = await http_post(app, req, data=dataset_json, params=params)
# create link if requested
if link_id and link_title:
link_json = {}
link_json["id"] = dset_id
link_json["class"] = "H5L_TYPE_HARD"
link_req = getDataNodeUrl(app, link_id)
link_req += "/groups/" + link_id + "/links/" + link_title
log.info("PUT link - : " + link_req)
put_rsp = await http_put(app, link_req, data=link_json, params=params)
log.debug(f"PUT Link resp: {put_rsp}")
# dataset creation successful
resp = await jsonResponse(request, post_json, status=201)
log.response(request, resp=resp)
return resp
async def PUT_Attribute(request):
"""HTTP method to create a new attribute"""
log.request(request)
app = request.app
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = f"Invalid object id: {obj_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
attr_name = request.match_info.get('name')
log.debug(f"Attribute name: [{attr_name}]")
validateAttributeName(attr_name)
log.info(f"PUT Attribute id: {obj_id} name: {attr_name}")
username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
await validateUserPassword(app, username, pswd)
if not request.has_body:
msg = "PUT Attribute with no body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
body = await request.json()
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# get domain JSON
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
log.error(f"Expected root key for domain: {domain}")
raise HTTPBadRequest(reason="Unexpected Error")
root_id = domain_json["root"]
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "create")
if "type" not in body:
msg = "PUT attribute with no type in body"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = body["type"]
if isinstance(datatype, str) and datatype.startswith("t-"):
# Committed type - fetch type json from DN
ctype_id = datatype
log.debug(f"got ctypeid: {ctype_id}")
ctype_json = await getObjectJson(app, ctype_id, bucket=bucket)
log.debug(f"ctype {ctype_id}: {ctype_json}")
if ctype_json["root"] != root_id:
msg = "Referenced committed datatype must belong in same domain"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
datatype = ctype_json["type"]
# add the ctype_id to type type
datatype["id"] = ctype_id
elif isinstance(datatype, str):
try:
# convert predefined type string (e.g. "H5T_STD_I32LE") to
# corresponding json representation
datatype = getBaseTypeJson(datatype)
log.debug(f"got datatype: {datatype}")
except TypeError:
msg = "PUT attribute with invalid predefined type"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
try:
validateTypeItem(datatype)
except KeyError as ke:
msg = f"KeyError creating type: {ke}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
except TypeError as te:
msg = f"TypeError creating type: {te}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
except ValueError as ve:
msg = f"ValueError creating type: {ve}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
dims = None
shape_json = {}
if "shape" in body:
shape_body = body["shape"]
shape_class = None
if isinstance(shape_body, dict) and "class" in shape_body:
shape_class = shape_body["class"]
elif isinstance(shape_body, str):
shape_class = shape_body
if shape_class:
if shape_class == "H5S_NULL":
shape_json["class"] = "H5S_NULL"
if isinstance(shape_body, dict) and "dims" in shape_body:
msg = "can't include dims with null shape"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if isinstance(shape_body, dict) and "value" in body:
msg = "can't have H5S_NULL shape with value"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif shape_class == "H5S_SCALAR":
shape_json["class"] = "H5S_SCALAR"
dims = getShapeDims(shape_body)
if len(dims) != 1 or dims[0] != 1:
msg = "dimensions aren't valid for scalar attribute"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
elif shape_class == "H5S_SIMPLE":
shape_json["class"] = "H5S_SIMPLE"
dims = getShapeDims(shape_body)
shape_json["dims"] = dims
else:
msg = f"Unknown shape class: {shape_class}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
# no class, interpet shape value as dimensions and
# use H5S_SIMPLE as class
if isinstance(shape_body, list) and len(shape_body) == 0:
shape_json["class"] = "H5S_SCALAR"
dims = [
1,
]
else:
shape_json["class"] = "H5S_SIMPLE"
dims = getShapeDims(shape_body)
shape_json["dims"] = dims
else:
shape_json["class"] = "H5S_SCALAR"
dims = [
1,
]
if "value" in body:
if dims is None:
msg = "Bad Request: data can not be included with H5S_NULL space"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
value = body["value"]
# validate that the value agrees with type/shape
arr_dtype = createDataType(datatype) # np datatype
if len(dims) == 0:
np_dims = [
1,
]
else:
np_dims = dims
log.debug(f"attribute dims: {np_dims}")
log.debug(f"attribute value: {value}")
try:
arr = jsonToArray(np_dims, arr_dtype, value)
except ValueError:
msg = "Bad Request: input data doesn't match selection"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"Got: {arr.size} array elements")
else:
value = None
# ready to add attribute now
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes/" + attr_name
log.info("PUT Attribute: " + req)
attr_json = {}
attr_json["type"] = datatype
attr_json["shape"] = shape_json
if value is not None:
attr_json["value"] = value
params = {}
if bucket:
params["bucket"] = bucket
put_rsp = await http_put(app, req, params=params, data=attr_json)
log.info(f"PUT Attribute resp: {put_rsp}")
hrefs = [] # TBD
req_rsp = {"hrefs": hrefs}
# attribute creation successful
resp = await jsonResponse(request, req_rsp, status=201)
log.response(request, resp=resp)
return resp
async def get(self) -> Response:
request = self.request
config = self.request.app['config']
# Decode state parameter
if 'state' not in request.query:
raise HTTPBadRequest(reason='No state parameter in request')
try:
state = loads(
decrypt(request.app['cookie_key'],
urlsafe_b64decode(request.query['state'])))
except ValueError:
raise HTTPBadRequest(reason='Can\'t decode state')
logger.debug('Request state: %s', state)
domain = self.get_domain_from_acr(state['p']['acr_values'])
provider = await self.get_provider(domain)
return_uri = ('{}?{}'.format(state['u'],
urlencode(query=state['p'], doseq=True)))
# There is id token in state and user authenticated, so just make
# relationship existing user with oidc account.
if 't' in state:
logger.debug('Id token provided in state, checking authn')
# Check cookie
cookie = request.cookies.get(config.get('authn.cookie.id_name'))
if cookie:
logger.debug('Found authn cookie %s', cookie)
key = request.app['cookie_key']
userid: str = decrypt(key, urlsafe_b64decode(cookie))
await self.link_user_with_oidc(userid, provider.id,
state['t']['sub'])
return Response(status=303,
reason='See Other',
charset='utf-8',
headers={'Location': return_uri})
else:
logger.warning(
'Id token recieved, but authn cookie does not set')
# Overwise it is callback with code from oidc provider
def error_redirect(reason, source: Exception = None):
logger.warning('Redirecting with error: %s', reason)
exc = HTTPSeeOther('{}?{}'.format(
request.app['config'].get('http_server.endpoints.login.path'),
urlencode({
'error':
'bigur_oidc_provider_error',
'error_description':
reason,
'next':
('{}?{}'.format(request.path,
urlencode(query=state['p'], doseq=True))),
})))
if source is None:
raise exc
else:
raise exc from source
try:
code = request.query['code']
except KeyError as e:
error_redirect('No code provided', e)
try:
token_endpoint = provider.get_token_endpoint()
except AttributeError:
token_endpoint = None
if not token_endpoint:
error_redirect('No token endpoint')
response_types = provider.get_response_types_supported()
if 'id_token' not in response_types:
error_redirect('ID token not supported')
try:
client_id = provider.get_client_id()
client_secret = provider.get_client_secret()
except AttributeError as e:
error_redirect('Invalid client credentials', e)
logger.debug('Getting id_token from %s', token_endpoint)
async with ClientSession() as session:
try:
data = {
'redirect_uri': self.endpoint_uri,
'code': code,
'client_id': client_id,
'client_secret': client_secret,
'grant_type': 'authorization_code'
}
async with session.post(token_endpoint, data=data) as resp:
if resp.status != 200:
body = await resp.text()
logger.error('Invalid response from provider: %s',
body)
error_redirect('Can\'t obtain token from provider')
token_obj = await resp.json()
except ClientError as e:
error_redirect('Can\'t obtain token', e)
if not isinstance(token_obj, dict):
error_redirect('Invalid response from provider')
if 'id_token' not in token_obj:
error_redirect('Invalid response from provider')
if str(token_obj.get('token_type', '')).lower() != 'bearer':
error_redirect('Invalid token type returned by provider')
try:
header = get_unverified_header(token_obj['id_token'])
logger.debug('Key header: %s', header)
except DecodeError as e:
error_redirect('Can\'t decode token', e)
try:
alg = get_default_algorithms()[header['alg']]
except KeyError as e:
logger.error('Algorythm %s is not supported', header['alg'])
error_redirect('Key algorytm not supported', e)
try:
key = provider.get_key(header['kid'])
except KeyError:
# Key not found, update provider's keys
try:
await provider.update_keys()
key = provider.get_key(header['kid'])
except (ClientError, KeyError, TypeError) as e:
logger.warning(str(e))
error_redirect(
'Can\'t get key with kid'
' {}'.format(header['kid']), e)
try:
key = alg.from_jwk(dumps(key))
except InvalidKeyError as e:
error_redirect('Can\'t decode key', e)
try:
# XXX: hardcoded token encryption algorithm
payload = decode(token_obj['id_token'],
key,
audience=client_id,
algorithms=['RS256'])
except DecodeError as e:
error_redirect('Can\'t decode token', e)
logger.debug('Token id payload: %s', payload)
if payload['nonce'] != state['n']:
error_redirect('Can\'t verify nonce')
if 'sub' not in payload:
error_redirect('Invalid token')
try:
user = await self.request.app['store'].users.get_by_oidp(
provider.get_id(), payload['sub'])
except KeyError:
logger.warning('User %s:%s not found', domain, payload['sub'])
state['t'] = payload
template = 'oidc_user_not_exists.j2'
context = {
'login_endpoint':
config.get('http_server.endpoints.login.path'),
'registration_endpoint':
config.get('http_server.endpoints.registration.path'),
'next':
config.get('http_server.endpoints.oidc.path'),
'state':
urlsafe_b64encode(
crypt(request.app['cookie_key'],
dumps(state))).decode('utf-8')
}
return render_template(template, request, context)
response = Response(status=303,
reason='See Other',
charset='utf-8',
headers={'Location': return_uri})
self.set_cookie(request, response, user.get_id())
return response
async def GET_Attributes(request):
"""HTTP method to return JSON for attribute collection"""
log.request(request)
app = request.app
params = request.rel_url.query
collection = getRequestCollectionName(
request) # returns datasets|groups|datatypes
obj_id = request.match_info.get('id')
if not obj_id:
msg = "Missing object id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not isValidUuid(obj_id, obj_class=collection):
msg = "Invalid obj id: {}".format(obj_id)
log.warn(msg)
raise HTTPBadRequest(reason=msg)
include_data = False
if "IncludeData" in params and params["IncludeData"]:
include_data = True
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
# TBD - verify that the obj_id belongs to the given domain
await validateAction(app, domain, obj_id, username, "read")
req = getDataNodeUrl(app, obj_id)
req += '/' + collection + '/' + obj_id + "/attributes"
params = {}
if limit is not None:
params["Limit"] = str(limit)
if marker is not None:
params["Marker"] = marker
if include_data:
params["IncludeData"] = '1'
if bucket:
params["bucket"] = bucket
log.debug(f"get attributes: {req}")
dn_json = await http_get(app, req, params=params)
log.debug(f"got attributes json from dn for obj_id: {obj_id}")
attributes = dn_json["attributes"]
# mixin hrefs
for attribute in attributes:
attr_name = attribute["name"]
attr_href = '/' + collection + '/' + obj_id + '/attributes/' + attr_name
attribute["href"] = getHref(request, attr_href)
resp_json = {}
resp_json["attributes"] = attributes
hrefs = []
obj_uri = '/' + collection + '/' + obj_id
hrefs.append({
'rel': 'self',
'href': getHref(request, obj_uri + '/attributes')
})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({'rel': 'owner', 'href': getHref(request, obj_uri)})
resp_json["hrefs"] = hrefs
resp = await jsonResponse(request, resp_json)
log.response(request, resp=resp)
return resp
async def GET_Group(request):
"""HTTP method to return JSON for group"""
log.request(request)
app = request.app
params = request.rel_url.query
h5path = None
getAlias = False
include_links = False
include_attrs = False
group_id = request.match_info.get('id')
if not group_id and "h5path" not in params:
# no id, or path provided, so bad request
msg = "Missing group id"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if group_id:
log.info(f"GET_Group, id: {group_id}")
# is the id a group id and not something else?
if not isValidUuid(group_id, "Group"):
msg = f"Invalid group id: {group_id}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if "getalias" in params:
if params["getalias"]:
getAlias = True
if "h5path" in params:
h5path = params["h5path"]
if not group_id and h5path[0] != '/':
msg = "h5paths must be absolute if no parent id is provided"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.info(f"GET_Group, h5path: {h5path}")
if "include_links" in params and params["include_links"]:
include_links = True
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True
username, pswd = getUserPasswordFromRequest(request)
if username is None and app['allow_noauth']:
username = "******"
else:
await validateUserPassword(app, username, pswd)
domain = getDomainFromRequest(request)
if not isValidDomain(domain):
msg = f"Invalid domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
bucket = getBucketForDomain(domain)
if h5path and h5path[0] == '/':
# ignore the request path id (if given) and start
# from root group for absolute paths
domain_json = await getDomainJson(app, domain)
if "root" not in domain_json:
msg = f"Expected root key for domain: {domain}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
group_id = domain_json["root"]
if h5path:
group_id = await getObjectIdByPath(app,
group_id,
h5path,
bucket=bucket
) # throws 404 if not found
if not isValidUuid(group_id, "Group"):
msg = f"No group exist with the path: {h5path}"
log.warn(msg)
raise HTTPNotFound()
log.info(f"get group_id: {group_id} from h5path: {h5path}")
# verify authorization to read the group
await validateAction(app, domain, group_id, username, "read")
# get authoritative state for group from DN (even if it's in the meta_cache).
group_json = await getObjectJson(app,
group_id,
refresh=True,
include_links=include_links,
include_attrs=include_attrs,
bucket=bucket)
log.debug(f"domain from request: {domain}")
group_json["domain"] = getPathForDomain(domain)
if bucket:
group_json["bucket"] = bucket
if getAlias:
root_id = group_json["root"]
alias = []
if group_id == root_id:
alias.append('/')
else:
idpath_map = {root_id: '/'}
h5path = await getPathForObjectId(app,
root_id,
idpath_map,
tgt_id=group_id,
bucket=bucket)
if h5path:
alias.append(h5path)
group_json["alias"] = alias
hrefs = []
group_uri = '/groups/' + group_id
hrefs.append({'rel': 'self', 'href': getHref(request, group_uri)})
hrefs.append({
'rel': 'links',
'href': getHref(request, group_uri + '/links')
})
root_uri = '/groups/' + group_json["root"]
hrefs.append({'rel': 'root', 'href': getHref(request, root_uri)})
hrefs.append({'rel': 'home', 'href': getHref(request, '/')})
hrefs.append({
'rel': 'attributes',
'href': getHref(request, group_uri + '/attributes')
})
group_json["hrefs"] = hrefs
resp = await jsonResponse(request, group_json)
log.response(request, resp=resp)
return resp
async def GET_Links(request):
"""HTTP GET method to return JSON for a link collection
"""
log.request(request)
app = request.app
params = request.rel_url.query
group_id = get_obj_id(request)
log.info(f"GET links: {group_id}")
if not isValidUuid(group_id, obj_class="group"):
log.error(f"Unexpected group_id: {group_id}")
raise HTTPInternalServerError()
limit = None
if "Limit" in params:
try:
limit = int(params["Limit"])
log.info(f"GET_Links - using Limit: {limit}")
except ValueError:
msg = "Bad Request: Expected int type for limit"
log.error(msg) # should be validated by SN
raise HTTPBadRequest(reason=msg)
marker = None
if "Marker" in params:
marker = params["Marker"]
log.info(f"GET_Links - using Marker: {marker}")
if "bucket" in params:
bucket = params["bucket"]
else:
bucket = None
group_json = await get_metadata_obj(app, group_id, bucket=bucket)
log.info(f"for id: {group_id} got group json: {group_json}")
if "links" not in group_json:
msg.error(f"unexpected group data for id: {group_id}")
raise HTTPInternalServerError()
# return a list of links based on sorted dictionary keys
link_dict = group_json["links"]
titles = list(link_dict.keys())
titles.sort() # sort by key
# TBD: provide an option to sort by create date
start_index = 0
if marker is not None:
start_index = index(titles, marker) + 1
if start_index == 0:
# marker not found, return 404
msg = f"Link marker: {marker}, not found"
log.warn(msg)
raise HTTPNotFound()
end_index = len(titles)
if limit is not None and (end_index - start_index) > limit:
end_index = start_index + limit
link_list = []
for i in range(start_index, end_index):
title = titles[i]
link = copy(link_dict[title])
link["title"] = title
link_list.append(link)
resp_json = {"links": link_list}
resp = json_response(resp_json)
log.response(request, resp=resp)
return resp
