def remove_proxy_route(app_logger, uuidcode, app_hub_url_proxy_route,
jhubtoken, username, server_name):
app_logger.debug(
"uuidcode={} - Remove proxys from server_name, because the original host is not accessable any longer"
.format(uuidcode))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"uuidcode": uuidcode,
"Intern-Authorization": get_jhubtoken()
}
try:
app_logger.info("uuidcode={} - Remove Proxys for {}".format(
uuidcode, server_name))
url = app_hub_url_proxy_route
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
app_logger.trace("uuidcode={} - Delete Proxy Route: {} {}".format(
uuidcode, url, hub_header))
for i in range(0, 10):
with closing(
requests.delete(url,
headers=hub_header,
verify=False,
timeout=1800)) as r:
if r.status_code == 200:
app_logger.info(
"uuidcode={} - Proxy route deletion successful".format(
uuidcode))
return True
elif r.status_code == 503:
app_logger.info(
"uuidcode={} - Proxy route deletion status_code 503. Try again (Try {}/10)"
.format(uuidcode, i + 1))
else:
raise Exception(
"uuidcode={} - Could not remove proxy route for server_name {}: {} {}"
.format(uuidcode, server_name, r.text, r.status_code))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout reached (1800). Could not remove routes from proxy via JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
except:
app_logger.exception(
"uuidcode={} - Could not remove routes from proxy via JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
def token(app_logger, uuidcode, app_hub_url_proxy_route, app_hub_url_token,
jhubtoken, accesstoken, expire, username, server_name):
app_logger.debug(
"uuidcode={} - Send new token to JupyterHub".format(uuidcode))
app_logger.trace("uuidcode={} - Access_token: {} , expire: {}".format(
uuidcode, accesstoken, expire))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"uuidcode": uuidcode,
"Intern-Authorization": get_jhubtoken()
}
hub_json = {"accesstoken": accesstoken, "expire": str(expire)}
try:
app_logger.info(
"uuidcode={} - Update JupyterHub Token".format(uuidcode))
url = app_hub_url_token
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
app_logger.trace(
"uuidcode={} - Update JupyterHub Token: {} {} {}".format(
uuidcode, url, hub_header, hub_json))
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r:
if r.status_code == 201:
app_logger.trace(
"uuidcode={} - Token Update successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
return
elif r.status_code == 503:
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 201:
app_logger.trace(
"uuidcode={} - Token Update successful: {} {} {}".
format(uuidcode, r2.text, r2.status_code,
r2.headers))
return
else:
app_logger.warning(
"uuidcode={} - Token Update sent wrong status_code: {} {} {}"
.format(uuidcode, r2.text, r2.status_code,
remove_secret(r2.headers)))
else:
app_logger.warning(
"uuidcode={} - Token Update sent wrong status_code: {} {} {}"
.format(uuidcode, r.text, r.status_code,
remove_secret(r.headers)))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout reached (1800). Could not send update token to JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
except:
app_logger.exception(
"uuidcode={} - Could not send update token to JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
def cancel(app_logger,
uuidcode,
app_hub_url_proxy_route,
app_hub_url_cancel,
jhubtoken,
errormsg,
username,
server_name=''):
app_logger.debug(
"uuidcode={} - Send cancel to JupyterHub".format(uuidcode))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"Intern-Authorization": get_jhubtoken(),
"uuidcode": uuidcode,
"Error": errormsg,
"Stopped": "True"
}
try:
url = app_hub_url_cancel
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
app_logger.trace("uuidcode={} - Cancel JupyterHub: {} {}".format(
uuidcode, url, hub_header))
with closing(
requests.delete(url,
headers=hub_header,
verify=False,
timeout=1800)) as r:
if r.status_code == 202:
app_logger.trace(
"uuidcode={} - Cancel successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
return
elif r.status_code == 503:
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.delete(url,
headers=hub_header,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 202:
app_logger.trace(
"uuidcode={} - Cancel successful: {} {} {}".format(
uuidcode, r2.text, r2.status_code, r2.headers))
return
else:
app_logger.warning(
"uuidcode={} - JupyterHub.cancel sent wrong status_code: {} {} {}"
.format(uuidcode, r2.text, r2.status_code,
remove_secret(r2.headers)))
else:
app_logger.warning(
"uuidcode={} - JupyterHub.cancel sent wrong status_code: {} {} {}"
.format(uuidcode, r.text, r.status_code,
remove_secret(r.headers)))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout reached (1800). Could not send cancel to JupyterHub. Url: {}, Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
except:
if errormsg != "":
cancel(app_logger, uuidcode, app_hub_url_proxy_route,
app_hub_url_cancel, jhubtoken, "", username, server_name)
else:
app_logger.exception(
"uuidcode={} - Could not send cancel to JupyterHub. Url: {}, Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
def status(app_logger,
uuidcode,
app_hub_url_proxy_route,
app_hub_url_status,
jhubtoken,
status,
username,
server_name=''):
app_logger.debug("uuidcode={} - Send status to JupyterHub: {}".format(
uuidcode, status))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"uuidcode": uuidcode,
"Intern-Authorization": get_jhubtoken()
}
hub_json = {"Status": status}
try:
url = app_hub_url_status
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
app_logger.trace(
"uuidcode={} - Update JupyterHub Status: {} {} {}".format(
uuidcode, url, hub_header, hub_json))
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r:
if r.status_code == 201:
app_logger.trace(
"uuidcode={} - Status Update successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
return
elif r.status_code == 503:
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 201:
app_logger.trace(
"uuidcode={} - Status Update successful: {} {} {}".
format(uuidcode, r2.text, r2.status_code,
r2.headers))
return
elif r2.status_code == 404:
app_logger.info(
"uuidcode={} - JupyterHub doesn't know the spawner."
.format(uuidcode))
elif r.status_code == 404:
app_logger.info(
"uuidcode={} - JupyterHub doesn't know the spawner.".
format(uuidcode))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout reached (1800). Could not send status update to JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
except:
app_logger.exception(
"uuidcode={} - Could not send status update to JupyterHub. Url: {} Headers: {}"
.format(uuidcode, url, remove_secret(hub_header)))
def get_tokens(app_logger, uuidcode, app_hub_url_proxy_route,
app_hub_token_url, jhubtoken, username, server_name):
app_logger.info(
"uuidcode={} - Get oauth tokens from JupyterHub".format(uuidcode))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"uuidcode": uuidcode,
"Intern-Authorization": get_jhubtoken()
}
try:
app_logger.trace("uuidcode={} - Header: {}".format(
uuidcode, hub_header))
url = app_hub_token_url
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
with closing(
requests.get(url,
headers=hub_header,
verify=False,
timeout=1800)) as r:
if r.status_code == 201:
app_logger.trace(
"uuidcode={} - Token Get successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
tokens_json = json.loads(r.text)
return tokens_json.get('accesstoken'), tokens_json.get(
'refreshtoken'), tokens_json.get('expire')
elif r.status_code == 503:
app_logger.info(
"uuidcode={} - Try to remove the proxys for the dead host".
format(uuidcode))
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.get(url,
headers=hub_header,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 201:
app_logger.trace(
"uuidcode={} - Token Get successful: {} {} {}".
format(uuidcode, r2.text, r2.status_code,
r2.headers))
tokens_json = json.loads(r2.text)
return tokens_json.get('accesstoken'), tokens_json.get(
'refreshtoken'), tokens_json.get('expire')
else:
app_logger.error(
"uuidcode={} - Token Get sent wrong status_code: {} {} {}"
.format(uuidcode, r2.text, r2.status_code,
r2.headers))
else:
app_logger.error(
"uuidcode={} - Token Get sent wrong status_code: {} {} {}".
format(uuidcode, r.text, r.status_code, r.headers))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout reached (1800). Could not get token from JupyterHub"
.format(uuidcode))
except:
app_logger.exception(
"uuidcode={} - Could not get token from JupyterHub".format(
uuidcode))
raise Exception(
"{} - Could not get token from JupyterHub".format(uuidcode))
def get_thread(self, app_logger, uuidcode, request_headers, app_urls,
app_database):
try:
machines = request_headers.get('machines', '').split(' ')
h = {
"Accept":
"application/json",
"User-Agent":
request_headers.get('User-Agent'),
"Authorization":
"Bearer {}".format(request_headers.get('accesstoken'))
}
unicore = utils_file_loads.get_unicore()
xlogins = {}
for machine in machines:
url = unicore.get(machine, {}).get(
'link', '<no_url_found_for_{}>'.format(machine))
cert = unicore.get(machine, {}).get('certificate', False)
try:
with closing(
requests.get(url,
headers=h,
verify=cert,
timeout=1800)) as r:
if r.status_code == 200:
xlogins[machine] = r.json().get('client', {}).get(
'xlogin', {})
app_logger.trace(
"uuidcode={} - {} returned {}".format(
uuidcode, machine, xlogins[machine]))
else:
app_logger.warning(
"uuidcode={} - Could not get user information from {}. {} {} {}"
.format(uuidcode, machine, r.status_code,
r.text, r.headers))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout (1800) reached".format(
uuidcode))
except:
app_logger.exception(
"uuidcode={} - Could not get user information from {}".
format(uuidcode, machine))
ret = {}
resources = utils_file_loads.get_resources()
for system, xlogin in xlogins.items():
if system not in ret.keys():
ret[system] = {}
for account in xlogin.get('availableUIDs', []):
if account == '!!DISCLAIMER!!':
continue
if account not in ret[system].keys():
ret[system][account] = {}
groups = xlogin.get('availableGroups', [])
if len(groups) == 0:
groups = ["default"]
for group in groups:
if group not in ret[system][account].keys():
ret[system][account][group] = {}
for partition in resources.get(machine, {}).keys():
if partition not in ret[system][account][
group].keys():
ret[system][account][group][partition] = {}
# send update to jhub
url = app_urls.get('hub', {}).get('url_useraccs', '<No_url_found>')
url = url.replace('<user>', request_headers.get('username'))
hub_header = {
'uuidcode': uuidcode,
'Intern-Authorization': utils_file_loads.get_jhubtoken()
}
hub_json = {'useraccs': ret}
app_logger.trace(
"uuidcode={} - Send useraccs to jupyterhub: {} {} {}".format(
uuidcode, url, hub_header, hub_json))
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r:
if r.status_code == 204:
app_logger.trace(
"uuidcode={} - User accs sent successfully:{} {} {}".
format(uuidcode, r.text, r.status_code, r.headers))
elif r.status_code == 503:
app_logger.info(
"uuidcode={} - Try to remove the proxys for the dead host"
.format(uuidcode))
remove_proxy_route(
app_logger, uuidcode,
app_urls.get('hub', {}).get('url_proxy_route',
'<no_url_found>'),
request_headers.get('jhubtoken'),
request_headers.get('username'), '')
# try again
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 204:
app_logger.trace(
"uuidcode={} - User accs sent successfully:{} {} {}"
.format(uuidcode, r2.text, r2.status_code,
r2.headers))
return
else:
app_logger.error(
"uuidcode={} - Useracc update sent wrong status_code: {} {} {}"
.format(uuidcode, r2.text, r2.status_code,
r2.headers))
else:
app_logger.error(
"uuidcode={} - Usercc update sent wrong status_code: {} {} {}"
.format(uuidcode, r.text, r.status_code, r.headers))
except:
app_logger.exception("UNICORE/X get failed. Bugfix required")
return '', 500
return ret, 200