def _get_globus_identity(self, globus_id_filename: str):
"""
Get the first identity for the username in the .globus_id file
"""
try:
return self._get_globus_identities(
globus_id_filename)["identities"][0]["id"]
except FileNotFoundError as error:
response = {
"success": False,
"error_type": "FileNotFoundError",
"strerror": error.strerror,
"filename": error.filename,
"error_code": error.errno,
}
logging.error(response)
raise HTTPInternalServerError(text=json.dumps(response),
content_type="application/json")
except globus_sdk.GlobusAPIError as error:
response = {
"success": False,
"error_type": "GlobusAPIError",
"message": error.message,
"code": error.code,
"http_status": error.http_status,
}
logging.error(response)
raise HTTPInternalServerError(text=json.dumps(response),
content_type="application/json")
def _remove_acl(self, user_identity_id: str):
"""
Get all ACLS and attempt to remove the correct ACL for the given user_identity
"""
try:
acls = self.globus_transfer_client.endpoint_acl_list(self.endpoint_id)['DATA']
for acl in acls:
if user_identity_id == acl['principal']:
if 'id' in acl and acl['id'] is not None:
resp = self.globus_transfer_client.delete_endpoint_acl_rule(self.endpoint_id,
acl['id'])
return {'message': str(resp), 'Success': True}
else:
return {'message': 'Couldn\'t find ACL for principal. Did you already delete your ACL?' , 'Success': False, 'principal' : acl['principal']}
response = {'success': False,
'error_type': 'Could Not Find or Delete User Identity Id (ACL)',
'user_identity_id': user_identity_id}
raise HTTPInternalServerError(text=json.dumps(response), content_type='application/json')
except globus_sdk.GlobusAPIError as error:
response = {'success': False,
'error_type': 'GlobusAPIError',
'user_identity_id': user_identity_id}
raise HTTPInternalServerError(text=json.dumps(response), content_type='application/json')
async def health_get(request: Request) -> Response:
try:
return Response(body=json.dumps({"status": "OK"}),
headers={'content-type': 'application/json'})
except Exception as ex:
log.warning(f"Endpoint: health, Method: get. Error:{str(ex)}")
return HTTPInternalServerError()
async def acs(request):
session = await get_session(request)
post = await request.post()
req = await prepare_saml_req(request)
auth = init_saml_auth(req, request.app['saml_settings'])
request_id = None
if 'AuthNRequestID' in session:
request_id = session['AuthNRequestID']
auth.process_response(request_id=request_id)
errors = auth.get_errors()
not_auth_warn = not auth.is_authenticated()
if len(errors) == 0:
if 'AuthNRequestID' in session:
del session['AuthNRequestID']
session['samlUserdata'] = auth.get_attributes()
session['samlNameIdFormat'] = auth.get_nameid_format()
session['samlNameIdNameQualifier'] = auth.get_nameid_nq()
session['samlNameIdSPNameQualifier'] = auth.get_nameid_spnq()
session['samlSessionIndex'] = auth.get_session_index()
self_url = OneLogin_Saml2_Utils.get_self_url(req)
if 'RelayState' in post and self_url != post['RelayState']:
raise HTTPFound(auth.redirect_to(post['RelayState']))
elif auth.get_settings().is_debug_active():
raise HTTPInternalServerError(text=auth.get_last_error_reason())
def index(request: Request):
try:
return Response(body=json.dumps({"host": socket.gethostname()}),
headers={'content-type': 'application/json'})
except Exception as ex:
log.warning(f"Endpoint: /, Method: get. Error:{str(ex)}")
return HTTPInternalServerError()
def _add_acl(self, user_identity_id: str, shared_directory_basename: str):
"""
Attempt to add acl for the given user id and directory
"""
try:
resp = self.globus_transfer_client.add_endpoint_acl_rule(
self.endpoint_id,
dict(DATA_TYPE="access", principal=user_identity_id,
principal_type='identity', path=shared_directory_basename, permissions='rw'),
)
response = {'success': True, 'principal': user_identity_id,
'path': shared_directory_basename,
'permissions': 'rw'}
logging.info(response)
logging.info('Shared %s with %s\n' % (shared_directory_basename, user_identity_id))
logging.info(response)
return response
except globus_sdk.TransferAPIError as error:
response = {'success': False, 'error_type': 'TransferAPIError', 'error': error.message,
'error_code': error.code,
'shared_directory_basename': shared_directory_basename}
logging.error(response)
if error.code == "Exists":
raise HTTPOk(text=json.dumps(response), content_type='application/json')
raise HTTPInternalServerError(text=json.dumps(response), content_type='application/json')
async def request_dispatcher(self, request):
from aiohttp.web import HTTPInternalServerError, Response
with get_tracer().async_span(
service_name=self.__class__.__name__,
span_name="[1]http request",
is_root=True,
standalone=True,
sample_rate=0.001,
):
api_route = request.match_info.get("path")
if api_route in self.batch_handlers:
req = HTTPRequest(
tuple((k.decode(), v.decode())
for k, v in request.raw_headers),
await request.read(),
)
try:
resp = await self.batch_handlers[api_route](req)
except RemoteException as e:
# known remote exception
logger.error(traceback.format_exc())
resp = Response(
status=e.payload.status,
headers=e.payload.headers,
body=e.payload.body,
)
except Exception: # pylint: disable=broad-except
logger.error(traceback.format_exc())
resp = HTTPInternalServerError()
else:
resp = await self.relay_handler(request)
return resp
async def run_command(*args):
"""Run command in subprocess
Example from:
http://asyncio.readthedocs.io/en/latest/subprocess.html
"""
# Create subprocess
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
# Status
# print('Started:', args, '(pid = ' + str(process.pid) + ')')
# Wait for the subprocess to finish
stdout, stderr = await process.communicate()
# Progress
if process.returncode == 0:
return stdout.decode().strip()
else:
error_msg = 'command {cmd} failed\nreturn code: {returncode}\nerror: {error}'.format(
cmd=' '.join(args),
returncode=process.returncode,
error=stderr.decode().strip())
raise HTTPInternalServerError(text=error_msg)
def extract_device(devices, type):
ds = [d for d in devices if d['Device Type'] == type]
if len(ds) > 1:
raise HTTPInternalServerError('Two devices of same type')
elif len(ds) == 1:
return ds[0]
else:
return None
def get_template(app, template_name):
"""Helper to fetch a template"""
env = get_env(app)
try:
return env.get_template(template_name)
except (KeyError, jinja2.TemplateNotFound) as e:
raise HTTPInternalServerError(
text="Template '{}' not found".format(template_name)) from e
def _get_globus_identity(self, globus_id_filename: str):
"""
Get the first identity for the username in the .globus_id file
"""
try:
return self._get_globus_identities(globus_id_filename)['identities'][0]['id']
except FileNotFoundError as error:
response = {'success': False, 'error_type': 'FileNotFoundError',
'strerror': error.strerror,
'filename': error.filename, 'error_code': error.errno}
logging.error(response)
raise HTTPInternalServerError(text=json.dumps(response), content_type='application/json')
except globus_sdk.GlobusAPIError as error:
response = {'success': False, 'error_type': 'GlobusAPIError', 'message': error.message,
'code': error.code, 'http_status': error.http_status}
logging.error(response)
raise HTTPInternalServerError(text=json.dumps(response), content_type='application/json')
async def metadata(request):
auth = init_saml_auth(await prepare_saml_req(request),
request.app['saml_settings'])
settings = auth.get_settings()
metadata = settings.get_sp_metadata()
errors = settings.validate_metadata(metadata)
if len(errors) == 0:
return Response(text=metadata, content_type='text/xml')
else:
raise HTTPInternalServerError(text=', '.join(errors))
async def error_middleware(request: web.Request, handler: Callable[[web.Request], Awaitable[web.StreamResponse]]) \
-> web.StreamResponse:
try:
return await handler(request)
except web.HTTPException as ex:
resp = web.Response(body=str(ex), status=ex.status)
return resp
except Exception as ex:
log.warning(f"Endpoint: {request.path}, Method: {request.method}. Error:{str(ex)}")
resp = HTTPInternalServerError(body=str(ex))
return resp
async def aiohttp_error_middleware(request, handler):
try:
response = await handler(request)
return response
except BotActionNotImplementedError:
raise HTTPNotImplemented()
except PermissionError:
raise HTTPUnauthorized()
except KeyError:
raise HTTPNotFound()
except Exception:
raise HTTPInternalServerError()
def _remove_acl(self, user_identity_id: str):
"""
Get all ACLS and attempt to remove the correct ACL for the given user_identity
"""
try:
acls = self.globus_transfer_client.endpoint_acl_list(
self.endpoint_id)["DATA"]
for acl in acls:
if user_identity_id == acl["principal"]:
if "id" in acl and acl["id"] is not None:
resp = self.globus_transfer_client.delete_endpoint_acl_rule(
self.endpoint_id, acl["id"])
return {"message": str(resp), "Success": True}
else:
return {
"message":
"Couldn't find ACL for principal. Did you already delete your ACL?",
"Success": False,
"principal": acl["principal"],
}
response = {
"success": False,
"error_type":
"Could Not Find or Delete User Identity Id (ACL)",
"user_identity_id": user_identity_id,
}
raise HTTPInternalServerError(text=json.dumps(response),
content_type="application/json")
except globus_sdk.GlobusAPIError as error:
response = {
"success": False,
"error_type": "GlobusAPIError",
"user_identity_id": user_identity_id,
}
raise HTTPInternalServerError(text=json.dumps(response),
content_type="application/json")
async def _handler(self, request: Request) -> StreamResponse:
if not self._started:
raise HTTPInternalServerError()
assert self._scheduler is not None
assert self._webhook_token is not None
if self._check_address and not self._address_is_allowed(request):
raise HTTPNotFound()
if not compare_digest(self._webhook_token,
request.match_info['token']):
raise HTTPNotFound()
update_data = json.loads(await request.read())
update = Update.from_dict(update_data)
await self._scheduler.spawn(self._handle_update(update))
return Response()
async def circle_full_json(request: Request) -> Response:
ra, dec, radius = ra_dec_radius_from_request(request, MAX_RADIUS)
lcs = await get_lcs_in_circle(request.app['ch_client'], ra, dec, radius)
oids = set(obs['oid'] for obs in lcs)
metas = await get_meta_for_oids(request.app['ch_client'], oids)
metas = {meta['oid']: meta for meta in metas}
if oids != set(metas):
raise HTTPInternalServerError(reason='dr2 and dr2_meta return different oids')
data = {}
for obs in lcs:
obj = data.setdefault(obs['oid'], dict(meta=prepare_meta(metas[obs['oid']])))
lc = obj.setdefault('lc', [])
lc.append({k: v for k, v in obs.items() if k in LC_FIELDS})
return json_response(data)
async def aiohttp_error_middleware(request, handler):
try:
response = await handler(request)
return response
except BotActionNotImplementedError:
raise HTTPNotImplemented()
except NotImplementedError:
raise HTTPNotImplemented()
except PermissionError:
raise HTTPUnauthorized()
except KeyError:
raise HTTPNotFound()
except HTTPError as error:
# In the case the integration adapter raises a specific HTTPError
raise error
except Exception:
raise HTTPInternalServerError()
def _add_acl(self, user_identity_id: str, shared_directory_basename: str):
"""
Attempt to add acl for the given user id and directory
"""
try:
resp = self.globus_transfer_client.add_endpoint_acl_rule(
self.endpoint_id,
dict(
DATA_TYPE="access",
principal=user_identity_id,
principal_type="identity",
path=shared_directory_basename,
permissions="rw",
),
)
response = {
"success": True,
"principal": user_identity_id,
"path": shared_directory_basename,
"permissions": "rw",
}
logging.info(response)
logging.info("Shared %s with %s\n" %
(shared_directory_basename, user_identity_id))
logging.info(response)
return response
except globus_sdk.TransferAPIError as error:
response = {
"success": False,
"error_type": "TransferAPIError",
"error": error.message,
"error_code": error.code,
"shared_directory_basename": shared_directory_basename,
}
logging.error(response)
if error.code == "Exists":
raise HTTPOk(text=json.dumps(response),
content_type="application/json")
raise HTTPInternalServerError(text=json.dumps(response),
content_type="application/json")
async def my_middleware(self, request, handler):
try:
resp = await handler(request)
except Exception as e:
if self.publish_errors:
Framework.publish(
Event(
Signal.ERROR, {
"exc": e,
"traceback": traceback.format_exc(),
"location": self.__class__.__name__,
"request": request.url
}))
raise HTTPInternalServerError(
text=
f"Error handling request {request.path_qs}\n{traceback.format_exc()}"
)
return resp
def __init__(self):
"""
The ACLManager is used to add and remove acl endpoints for KBase Users on our Globus Share
"""
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
config = configparser.ConfigParser()
config.read("/etc/globus.cfg")
cf = config['globus']
self.endpoint_id = cf['endpoint_id']
client = globus_sdk.NativeAppAuthClient(cf['client_id'])
try:
transfer_authorizer = globus_sdk.RefreshTokenAuthorizer(cf['transfer_token'], client)
self.globus_transfer_client = globus_sdk.TransferClient(authorizer=transfer_authorizer)
auth_authorizer = globus_sdk.RefreshTokenAuthorizer(cf['auth_token'], client)
self.globus_auth_client = globus_sdk.AuthClient(authorizer=auth_authorizer)
except globus_sdk.GlobusAPIError as error:
logging.error(str(error.code) + error.raw_text)
raise HTTPInternalServerError(text=str("Invalid Token Specified in globus.cfg file"))
async def data_dr_circle_full_json(request: Request) -> Response:
dr = request.match_info['dr']
ra, dec, radius = ra_dec_radius_from_request(request, MAX_RADIUS)
lcs = await get_lcs_in_circle(request.app['ch_client'], dr, ra, dec,
radius)
if not lcs:
return json_response({})
oids = set(obs['oid'] for obs in lcs)
metas = await get_meta_for_oids(request.app['ch_client'], meta_table(dr),
oids)
if oids != set(metas):
raise HTTPInternalServerError(
reason='observation and meta requests returned different oids')
if metas_short := meta_short_table(dr):
metas_short = await get_meta_for_oids(request.app['ch_client'],
metas_short, oids)
async def sls(request):
session = await get_session(request)
auth = init_saml_auth(await prepare_saml_req(request),
request.app['saml_settings'])
request_id = None
if 'LogoutRequestID' in session:
request_id = session['LogoutRequestID']
dscb = lambda: session.invalidate()
url = auth.process_slo(request_id=request_id, delete_session_cb=dscb)
errors = auth.get_errors()
if len(errors) == 0:
if url is not None:
return redirect(url)
else:
# TODO better return from this method
return Response()
elif auth.get_settings().is_debug_active():
raise HTTPInternalServerError(text=auth.get_last_error_reason())
async def add_task(request: Request):
"""Provide the task to the main_loop"""
schema = AddTaskSchema()
try:
req_data = await request.json()
field_error = [(k, v) for k, v in schema.validate(req_data).items()]
if field_error:
return HTTPBadRequest(text=f"Request validation error: {field_error}")
run = req_data.pop("run", None)
call_params = req_data.pop("params", dict())
task = await request.app["task_dispatcher"].create_task(**req_data, **call_params)
if run:
job = get_running_loop().create_task(request.app["task_runner"].run_task(task))
job.set_name(task.task_id)
except json.decoder.JSONDecodeError:
return HTTPBadRequest()
except ValueError as e:
return HTTPBadRequest(text=e.__str__())
except Exception as e:
logging.getLogger(__name__).error(e)
return HTTPInternalServerError()
return HTTPAccepted()
def invalid_role_id(role_id):
msg = 'Invalid role ID: {}'.format(role_id)
logger.warn(msg, role_id=role_id)
raise HTTPInternalServerError(reason=msg)
