def validate_auth(app_logger, uuidcode, intern_authorization):
if not intern_authorization == None:
token = get_j4j_tunnel_token()
if intern_authorization == token:
app_logger.info(
"uuidcode={} - Intern-Authorization validated".format(
uuidcode))
return
app_logger.warning("uuidcode={} - Could not validate Token:\n{}".format(
uuidcode, intern_authorization))
abort(401)
def get_remote_node(app_logger, uuidcode, tunnel_url_remote, nodelist):
app_logger.debug("uuidcode={} - Get remote node".format(uuidcode))
header = {
'Intern-Authorization': get_j4j_tunnel_token(),
'uuidcode': uuidcode
}
while len(nodelist) > 0:
i = randint(0, len(nodelist) - 1)
try:
app_logger.info("uuidcode={} - Get J4J_Tunnel {}".format(
uuidcode, tunnel_url_remote))
with closing(
requests.get(tunnel_url_remote,
params={'node': nodelist[i]},
headers=header,
timeout=1800)) as r:
if r.status_code == 217:
app_logger.trace(
"uuidcode={} - Use {} as remote node".format(
uuidcode, nodelist[i]))
return nodelist[i]
elif r.status_code == 218:
try:
j4j_start_remote_tunnel(app_logger, uuidcode,
tunnel_url_remote, nodelist[i],
header)
app_logger.debug(
"uuidcode={} - Start remote tunnel for {}".format(
uuidcode, nodelist[i]))
app_logger.trace(
"uuidcode={} - Use {} as remote node".format(
uuidcode, nodelist[i]))
return nodelist[i]
except Exception as e:
app_logger.warning(
"uuidcode={} - Could not start remote tunnel for {}: {}"
.format(uuidcode, nodelist[i], str(e)))
del nodelist[i]
else:
app_logger.warning(
"uuidcode={} - Unexpected status_code ({}) for node: {}. Try another node."
.format(uuidcode, r.status_code, nodelist[i]))
del nodelist[i]
except:
app_logger.exception(
"uuidcode={} - Could not get remote tunnelnode".format(
uuidcode))
del nodelist[i]
app_logger.warning(
"uuidcode={} - All nodes failed. Cannot start service".format(
uuidcode))
raise Exception('{} - No Remote Tunnel active'.format(uuidcode))
def close(app_logger, uuidcode, hub_tunnel_url, tunnel_info):
app_logger.debug("uuidcode={} - Try to close tunnel.".format(uuidcode))
tunnel_header = {
"Intern-Authorization": get_j4j_tunnel_token(),
"Content-Type": "application/json",
"uuidcode": uuidcode
}
app_logger.info("uuidcode={} - Delete J4J_Tunnel {}".format(
uuidcode, hub_tunnel_url))
with closing(
requests.delete(hub_tunnel_url,
params=tunnel_info,
headers=tunnel_header,
timeout=1800)) as r:
if r.status_code == 204 or r.status_code == 200:
return
raise Exception("{} - Could not stop tunnel: {} {}".format(
uuidcode, r.text, r.status_code))
def post_thread(self, app_logger, uuidcode, request_headers, app_urls,
app_database):
username = request_headers.get('username')
app_logger.debug(
"uuidcode={} - Restore tunnels from database for {}".format(
uuidcode, username))
try:
serverinfos = utils_db.get_serverinfos_for_user(
app_logger, uuidcode, username, app_database)
except:
app_logger.exception(
"uuidcode={} - Could not collect serverinfos for {}. Bugfix required."
.format(uuidcode, username))
tunnel_header = {
'Intern-Authorization': utils_file_loads.get_j4j_tunnel_token(),
'uuidcode': uuidcode
}
for servername, system, hostname, port, tunnelup in serverinfos:
if tunnelup.lower() == 'true':
app_logger.debug(
"uuidcode={} - Tunnel for {} is already running. Skip this one"
.format(uuidcode, servername))
continue
tunnel_data = {
'account': servername,
'system': system,
'hostname': hostname,
'port': port
}
try:
tunnel_communication.j4j_start_tunnel(
app_logger, uuidcode,
app_urls.get('tunnel', {}).get('url_tunnel'),
tunnel_header, tunnel_data)
utils_db.set_tunnelup(app_logger, uuidcode, servername,
app_database, "true")
except:
app_logger.exception(
"uuidcode={} - Could not rebuild tunnel for server {}. Bugfix required"
.format(uuidcode, servername))
def create(app_logger, uuidcode, app_hub_url_proxy_route, app_tunnel_url,
app_hub_url_cancel, kernelurl, filedir, unicore_header, servername,
system, port, cert, jhubtoken, username, servername_short,
app_orchestrator_url_hostname):
app_logger.trace("uuidcode={} - Try to create a tunnel".format(uuidcode))
accept = unicore_header.get('Accept', False)
unicore_header['Accept'] = 'application/octet-stream'
hostname = ""
try:
method = "GET"
method_args = {
"url": filedir + '/.host',
"headers": unicore_header,
"certificate": cert,
"return_content": True
}
content, status_code, response_header = unicore_communication.request(
app_logger, uuidcode, method, method_args)
if status_code != 200:
app_logger.warning(
"uuidcode={} - Could not get hostname. UNICORE/X Response: {} {} {}"
.format(uuidcode, content, status_code,
remove_secret(response_header)))
raise Exception(
"{} - Could not get hostname. Throw exception because of wrong status_code: {}"
.format(uuidcode, status_code))
else:
unicore_header['X-UNICORE-SecuritySession'] = response_header[
'X-UNICORE-SecuritySession']
hostname = content.strip()
except:
app_logger.exception(
"uuidcode={} - Could not get hostname. {} {}".format(
uuidcode, method, remove_secret(method_args)))
app_logger.warning(
"uuidcode={} - Send cancel to JupyterHub.".format(uuidcode))
hub_communication.cancel(
app_logger, uuidcode, app_hub_url_proxy_route, app_hub_url_cancel,
jhubtoken,
"A mandatory backend service had a problem. An administrator is informed.",
username, servername_short)
unicore_utils.abort_job(app_logger, uuidcode, kernelurl,
unicore_header, cert)
unicore_utils.destroy_job(app_logger, uuidcode, kernelurl,
unicore_header, cert)
raise Exception("{} - Could not get hostname".format(uuidcode))
app_logger.trace(
'uuidcode={} - Inform J4J_Orchestrator about the hostname'.format(
uuidcode))
try:
orchestrator_communication.set_hostname(app_logger, uuidcode,
app_orchestrator_url_hostname,
servername, hostname)
except:
app_logger.exception(
"uuidcode={} - Could not set hostname to {} in J4J_Orchestrator database for {}"
.format(uuidcode, hostname, servername))
tunnel_header = {
'Intern-Authorization': utils_file_loads.get_j4j_tunnel_token(),
'uuidcode': uuidcode
}
if system == 'JUWELS' and hostname[:3] == 'jwc':
hostname = hostname.split('.')[0]
if system == 'JURON' and hostname[:6] == 'juronc':
hostname = hostname.split('.')[0]
tunnel_data = {
'account': servername, # for internal tunnel database
'system': system,
'hostname': hostname,
'port': port
}
tunnel_communication.j4j_start_tunnel(app_logger, uuidcode, app_tunnel_url,
tunnel_header, tunnel_data)
try:
method = "PUT"
method_args = {
"url": filedir + '/.tunnel',
"headers": unicore_header,
"data": '{}'.format(port),
"certificate": cert
}
text, status_code, response_header = unicore_communication.request(
app_logger, uuidcode, method, method_args)
if status_code != 204:
app_logger.warning(
"uuidcode={} - Could not create .tunnel file. UNICORE/X Response: {} {} {}"
.format(uuidcode, text, status_code,
remove_secret(response_header)))
raise Exception(
"{} - Could not create .tunnel file. Throw Exception because of wrong status_code: {}"
.format(uuidcode, status_code))
else:
unicore_header['X-UNICORE-SecuritySession'] = response_header[
'X-UNICORE-SecuritySession']
except:
app_logger.exception(
"uuidcode={} - Could not create .tunnel file. {} {}".format(
uuidcode, method, remove_secret(method_args)))
app_logger.warning(
"uuidcode={} - Send cancel to JupyterHub.".format(uuidcode))
hub_communication.cancel(
app_logger, uuidcode, app_hub_url_proxy_route, app_hub_url_cancel,
jhubtoken,
"A mandatory backend service had a problem. An administrator is informed.",
username, servername_short)
unicore_utils.abort_job(app_logger, uuidcode, kernelurl,
unicore_header, cert)
unicore_utils.destroy_job(app_logger, uuidcode, kernelurl,
unicore_header, cert)
raise Exception("{} - Could not create .tunnel file.".format(uuidcode))
if accept:
unicore_header['Accept'] = accept
else:
del unicore_header['Accept']
def start_docker_new(app_logger, uuidcode, app_database, request_headers, port,
service, dashboard, environment, app_urls):
"""
Headers:
intern-authorization
uuidcode
accesstoken
expire
servicelevel
Body:
servername
email
environments
image
port
jupyterhub_api_url
"""
servername = request_headers.get('servername')
account = request_headers.get('account')
jhubtoken = request_headers.get('jhubtoken')
servicelevel = request_headers.get('servicelevel', 'default')
app_tunnel_url = app_urls.get('tunnel', {}).get('url_tunnel')
app_tunnel_url_remote = app_urls.get('tunnel', {}).get('url_remote')
servername_at = servername.replace('@', '_at_')
email = servername_at.split(':')[0]
servername_short = servername_at.split(':')[1]
# Users should be allowed to use their access tokens in the JupyterLab. To get a maximum lifespan of this token for the user, we renew it before we're starting the JupyterLab.
accesstoken, expire = renew_token(
app_logger, uuidcode, request_headers.get('tokenurl'),
request_headers.get('authorizeurl'),
request_headers.get('refreshtoken'),
request_headers.get('accesstoken'), request_headers.get('expire'),
jhubtoken,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_token'),
request_headers.get('escapedusername'), servername, app_database, True)
dashboards = {}
if service == "JupyterLab":
dockerimage = utils_file_loads.image_name_to_image(account)
elif service == "Dashboard":
dashboards = utils_file_loads.get_dashboards()
dockerimage = dashboards.get(dashboard, {}).get("image")
else:
dockerimage = utils_file_loads.image_name_to_image(account)
app_logger.debug("uuidcode={} - Add server to database: {}".format(
uuidcode, servername_at))
utils_db.create_entry_docker(app_logger, uuidcode, app_database,
servername, jhubtoken, port, dockerimage)
docker_master_token = utils_file_loads.get_docker_master_token()
environment["HPCACCOUNTS"] = get_hpc_accounts(
app_logger, uuidcode, environment.get('hpcaccounts', []))
urls = utils_file_loads.get_urls()
url = urls.get('dockermaster', {}).get('url_jlab', '<no_jlab_url_defined>')
headers = {
"Intern-Authorization": docker_master_token,
"uuidcode": uuidcode,
"accesstoken": accesstoken,
"expire": str(expire),
"servicelevel": servicelevel
}
body = {
"servername":
servername_short,
"service":
service,
"dashboard":
dashboard,
"email":
email,
"environments":
environment,
"image":
dockerimage,
"port":
port,
"jupyterhub_api_url":
environment.get('JUPYTERHUB_API_URL', 'http://j4j_proxy:8000/hub/api')
}
with closing(
requests.post(url,
headers=headers,
json=body,
verify=False,
timeout=30)) as r:
if r.status_code == 200:
app_logger.debug(
"uuidcode={} - DockerMaster response: Positive".format(
uuidcode))
tunnel_header = {
'Intern-Authorization':
utils_file_loads.get_j4j_tunnel_token(),
'uuidcode': uuidcode
}
tunnel_data = {
'account': servername,
'system': "hdfcloud",
'hostname': uuidcode,
'port': port
}
hdfcloud = get_hdfcloud()
nodes = hdfcloud.get('nodes', [])
node = tunnel_communication.get_remote_node(
app_logger, uuidcode, app_tunnel_url_remote, nodes)
app_logger.debug("uuidcode={} - Use {} as node for tunnel".format(
uuidcode, node))
for i in range(0, 10):
try:
tunnel_communication.j4j_start_tunnel(
app_logger, uuidcode, app_tunnel_url, tunnel_header,
tunnel_data)
except:
if i == 9:
app_logger.exception(
"uuidcode={} - Could not start Tunnel for HDF-Cloud JupyterLab: {}"
.format(uuidcode, uuidcode))
return False
sleep(3)
break
return True
elif r.status_code == 501:
app_logger.debug(
"uuidcode={} - DockerMaster response: Negative".format(
uuidcode))
return False
else:
app_logger.error(
"uuidcode={} - DockerMaster unknown response: {} {}".format(
uuidcode, r.status_code, r.text))
return False