def delete(app_logger, uuidcode, request_headers, app_urls, app_database):
app_logger.trace("uuidcode={} - Begin of delete thread.".format(uuidcode))
if not request_headers.get('servername'):
app_logger.warning("uuidcode={} - No servername. Headers: {}".format(
uuidcode, utils_common.remove_secret(request_headers)))
return
servers = utils_db.get_entry_servername(app_logger, uuidcode,
request_headers.get('servername'),
app_database)
# nothing to delete
if len(servers) == 0:
return {}
# remove entrys
utils_db.remove_entrys(app_logger, uuidcode,
request_headers.get('servername'), app_database)
headers = {}
delete_header = {}
for server in servers:
docker = False
try:
system, kernelurl, filedir, port, account, project = server
if system.lower() == "docker" or system == "HDF-Cloud":
jobs_threads_docker.delete_docker_new(
app_logger, uuidcode, request_headers.get('servername'),
app_urls)
continue
else:
headers, delete_header = jobs_threads_unicore.delete_job(
app_logger, uuidcode, request_headers, delete_header,
app_urls, system, kernelurl, filedir, port, account,
project)
except:
if docker:
app_logger.exception(
"uuidcode={} - Could not delete docker container".format(
uuidcode))
else:
app_logger.exception(
"uuidcode={} - J4J_UNICORE communication failed. {}".
format(uuidcode, server))
return headers
def start_unicore_job(app_logger, uuidcode, request_headers, request_json, app_urls, app_database):
# Check if it's the cron test job
try:
cron_info = utils_file_loads.get_cron_info()
user, servernameshort = request_json.get('servername', ':').split(':') # @UnusedVariable
if cron_info.get('systems', {}).get(request_json.get('system').upper(), {}).get('servername', '<undefined>') == servernameshort:
if cron_info.get('systems', {}).get(request_json.get('system').upper(), {}).get('account', '<undefined>') == request_headers.get('account'):
if cron_info.get('systems', {}).get(request_json.get('system').upper(), {}).get('project', '<undefined>') == request_headers.get('project'):
if cron_info.get('systems', {}).get(request_json.get('system').upper(), {}).get('partition', '<undefined>') == request_json.get('partition'):
# get refresh token
unity_file = utils_file_loads.get_unity()
refresh_token = unity_file.get(request_headers.get('tokenurl'), {}).get('immune_tokens', [''])[0]
# get access token
client_id = unity_file.get(request_headers.get('tokenurl'), {}).get('client_id', '')
client_secret = unity_file.get(request_headers.get('tokenurl'), {}).get('client_secret', '')
unity_cert = unity_file.get(request_headers.get('tokenurl'), {}).get('certificate', False)
scopes = unity_file.get(request_headers.get('authorizeurl'), {}).get('scope')
tokeninfo_url = unity_file.get(request_headers.get('tokenurl'), {}).get('links', {}).get('tokeninfo', '')
b64_key = base64.b64encode(bytes('{}:{}'.format(client_id, client_secret),'utf8')).decode('utf8')
headers = {
'Accept': 'application/json',
'Authorization': 'Basic {}'.format(b64_key)
}
data = {
'refresh_token': refresh_token,
'grant_type': 'refresh_token',
'scope': ' '.join(scopes)
}
with closing(requests.post(request_headers.get('tokenurl'), headers=headers, data=data, verify=unity_cert)) as r:
if r.status_code != 200:
raise Exception("uuidcode={} Could not receive access_token: {} {}".format(uuidcode, r.status_code, r.text))
access_token = r.json().get('access_token')
# get expire
with closing(requests.get(tokeninfo_url,
headers = { 'Authorization': 'Bearer {}'.format(access_token) },
verify = unity_cert,
timeout = 1800)) as r:
expire = r.json().get('exp')
request_headers['accesstoken'] = access_token
request_headers['expire'] = '{}'.format(expire)
request_headers['refreshtoken'] = refresh_token
except:
app_logger.exception("uuidcode={}, Could not check if it's a cron job server".format(uuidcode))
# Delete all server with this name (there should be none, but better safe than sorry)
j4j_unicore_response_header = jobs_threads.delete(app_logger,
uuidcode,
request_headers,
app_urls,
app_database)
# All duplicated servers are deleted from the database and stopped
app_logger.trace('uuidcode={} - J4J_UNICORE_response_header: {}'.format(uuidcode, j4j_unicore_response_header))
try:
j4j_unicore_header = jobs_threads_unicore_utils.create_header(app_logger,
uuidcode,
request_headers,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_token'),
request_headers.get('escapedusername'),
request_headers.get('servername'),
app_database)
except (SpawnException, Exception) as e:
if type(e).__name__ == "SpawnException":
error_msg = str(e)
else:
error_msg = "A mandatory backend service for {} had a problem. An administrator is informed".format(request_json.get('system'))
app_logger.exception("uuidcode={} - error_msg: {} - Could not create Header. Send Cancel to JupyterHub and stop function. {} {}".format(uuidcode, error_msg, utils_common.remove_secret(request_headers), app_urls.get('hub', {}).get('url_token')))
utils_hub_update.cancel(app_logger,
uuidcode,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_cancel'),
request_headers.get("jhubtoken"),
error_msg,
request_headers.get('escapedusername'),
request_headers.get('servername'))
return
# update with Infos of j4j_unicore delete
if 'accesstoken' in j4j_unicore_response_header:
j4j_unicore_header['accesstoken'] = j4j_unicore_response_header['accesstoken']
j4j_unicore_header['expire'] = j4j_unicore_response_header['expire']
utils_hub_update.token(app_logger,
uuidcode,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_token'),
request_headers['jhubtoken'],
j4j_unicore_response_header['accesstoken'],
j4j_unicore_response_header['expire'],
request_headers.get('escapedusername'),
request_headers.get('servername'))
if 'session' in j4j_unicore_response_header:
j4j_unicore_header['X-UNICORE-SecuritySession'] = j4j_unicore_response_header['X-UNICORE-SecuritySession']
app_logger.trace("uuidcode={} - J4J_UNICORE_Header: {}".format(uuidcode, j4j_unicore_header))
j4j_unicore_json = jobs_threads_unicore_utils.create_json(app_logger,
uuidcode,
request_json)
app_logger.trace("uuidcode={} - J4J_UNICORE_Json: {}".format(uuidcode, j4j_unicore_json))
# Call UNICORE post
try:
method = "POST"
if request_json.get('service', '').lower() == 'dashboard':
url = app_urls.get('unicore', {}).get('url_dashboards')
else:
url = app_urls.get('unicore', {}).get('url_jobs')
method_args = {"url": url,
"headers": j4j_unicore_header,
"json": j4j_unicore_json,
"certificate": False}
app_logger.info("uuidcode={} - Post J4J_UNICORE".format(uuidcode))
text, status_code, headers = jobs_threads_unicore_utils.communicate(app_logger,
uuidcode,
method,
method_args)
if status_code != 201:
app_logger.warning("uuidcode={} - J4J_UNICORE Post failed. J4J_UNICORE Response: {} {} {}".format(uuidcode, text, status_code, utils_common.remove_secret(headers)))
raise Exception("{} - J4J_UNICORE Post failed. Throw exception because of wrong status_code: {}".format(uuidcode, status_code))
else:
app_logger.debug("uuidcode={} - J4J_UNICORE communication successful: {} {}".format(uuidcode, text, status_code))
app_logger.trace("uuidcode={} - J4J_UNICORE communication successful: {}".format(uuidcode, headers))
j4j_unicore_header['kernelurl'] = headers['kernelurl']
j4j_unicore_header['filedir'] = headers['filedir']
j4j_unicore_header['X-UNICORE-SecuritySession'] = headers['X-UNICORE-SecuritySession']
except (SpawnException, Exception) as e:
if type(e).__name__ == "SpawnException":
error_msg = str(e)
else:
error_msg = "A mandatory backend service for {} had a problem. An administrator is informed".format(request_json.get('system'))
app_logger.exception("uuidcode={} - error_msg: {} - J4J_UNICORE communication failed. {} {}".format(uuidcode, error_msg, method, utils_common.remove_secret(method_args)))
utils_hub_update.cancel(app_logger,
uuidcode,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_cancel'),
request_headers.get('jhubtoken'),
error_msg,
request_headers.get('escapedusername'),
request_headers.get('servername'))
return
# update database
app_logger.debug("uuidcode={} - Add userserver={} to database".format(uuidcode, request_json.get('servername')))
utils_db.create_entry(app_logger,
uuidcode,
request_headers,
request_json,
app_database,
j4j_unicore_header['kernelurl'],
j4j_unicore_header['filedir'])
# call unicore get (fire and forget)
j4j_unicore_header['servername'] = request_json.get('servername')
j4j_unicore_header['system'] = request_json.get('system')
j4j_unicore_header['port'] = str(request_json.get('port'))
j4j_unicore_header['spawnget'] = "True"
try:
utils_hub_update.status(app_logger,
uuidcode,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_status'),
request_headers.get('jhubtoken'),
'waitforhostname',
request_headers.get('escapedusername'),
request_headers.get('servername'))
except:
app_logger.warning("uuidcode={} - Could not update status for JupyterHub".format(uuidcode))
try:
method = "GET"
method_args = {"url": app_urls.get('unicore', {}).get('url_jobs'),
"headers": j4j_unicore_header,
"certificate": False,
"fire_and_forget": True}
app_logger.info("uuidcode={} - Get J4J_UNICORE".format(uuidcode))
jobs_threads_unicore_utils.communicate(app_logger,
uuidcode,
method,
method_args)
except (SpawnException, Exception) as e:
if type(e).__name__ == "SpawnException":
error_msg = str(e)
else:
error_msg = "A mandatory backend service for {} had a problem. An administrator is informed".format(request_json.get('system'))
app_logger.exception("uuidcode={} - error-msg: {} - J4J_UNICORE communication failed. Send errorcode 526 to JupyterHub.cancel. {} {}".format(uuidcode, error_msg, method, utils_common.remove_secret(method_args)))
utils_hub_update.cancel(app_logger,
uuidcode,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_cancel'),
request_headers.get('jhubtoken'),
error_msg,
request_headers.get('escapedusername'),
request_headers.get('servername'))
try:
request_headers['kernelurl'] = j4j_unicore_header['kernelurl']
request_headers['filedir'] = j4j_unicore_header['filedir']
request_headers['servername'] = request_json.get('servername')
request_headers['system'] = request_json.get('system')
request_headers['port'] = str(request_json.get('port'))
jobs_threads.delete(app_logger,
uuidcode,
request_headers,
app_database,
app_urls.get('tunnel', {}).get('url_tunnel'),
app_urls.get('unicore', {}).get('url_jobs'),
app_urls.get('docker', {}).get('delete_folder'),
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_token'))
except:
app_logger.exception("uuidcode={} - Could not delete/destroy Job via J4J_UNICORE. {}".format(uuidcode, utils_common.remove_secret(request_headers)))
def delete_job(app_logger, uuidcode, request_headers, delete_header, app_urls, system, kernelurl, filedir, port, account, project):
# Create Header to communicate with J4J_UNICORE
if len(delete_header) == 0:
# It's the first server with this name we want to delete. So we have to load the basic stuff
for key, value in request_headers.items():
delete_header[key] = value
delete_header['Intern-Authorization'] = utils_file_loads.get_j4j_unicore_token()
delete_header['system'] = system
delete_header['kernelurl'] = kernelurl
delete_header['filedir'] = filedir
delete_header['port'] = port
delete_header['account'] = account
delete_header['project'] = project
delete_header["tokenurl"] = request_headers.get("tokenurl")
delete_header["authorizeurl"] = request_headers.get("authorizeurl")
# Send DELETE to J4J_UNICORE
method = "DELETE"
method_args = {"url": app_urls.get('unicore', {}).get('url_jobs'),
"headers": delete_header,
"certificate": False}
app_logger.info("uuidcode={} - Delete J4J_UNICORE".format(uuidcode))
text, status_code, headers = jobs_threads_unicore_utils.communicate(app_logger,
uuidcode,
method,
method_args)
if status_code == 200:
app_logger.debug("uuidcode={} - J4J_UNICORE communication successful: {} {}".format(uuidcode, text, status_code))
app_logger.trace("uuidcode={} - J4J_UNICORE communication successful: {}".format(uuidcode, headers))
delete_header['accesstoken'] = headers['accesstoken']
delete_header['expire'] = headers['expire']
delete_header['X-UNICORE-SecuritySession'] = headers['X-UNICORE-SecuritySession']
else:
app_logger.warning("uuidcode={} - J4J_UNICORE communication not successful: {} {} {}".format(uuidcode, text, status_code, utils_common.remove_secret(headers)))
raise Exception("{} - J4J_UNICORE communication not successful. Throw exception because of wrong status_code: {}".format(uuidcode, status_code))
return headers, delete_header
def check_unicore_job_status(app_logger, uuidcode, app_urls, app_database, request_headers, escapedusername, servername, server_info):
try:
app_logger.trace("uuidcode={} - Call Create_get_header with: {} {} {} {} {}".format(uuidcode, request_headers, app_urls.get('hub', {}).get('url_proxy_route'), app_urls.get('hub', {}).get('url_token'), escapedusername, servername))
unicore_header = jobs_threads_unicore_utils.create_get_header(app_logger,
uuidcode,
request_headers,
app_urls.get('hub', {}).get('url_proxy_route'),
app_urls.get('hub', {}).get('url_token'),
escapedusername,
servername,
app_database)
except:
app_logger.exception("uuidcode={} - Could not create Header. Do nothing and return. {} {}".format(uuidcode, utils_common.remove_secret(request_headers), app_urls.get('hub', {}).get('url_token')))
utils_db.set_skip(app_logger,
uuidcode,
request_headers.get('servername'),
app_database,
'False')
return
try:
# call unicore get (fire and forget)
unicore_header['kernelurl'] = server_info.get('kernelurl')
unicore_header['filedir'] = server_info.get('filedir')
unicore_header['system'] = server_info.get('system')
unicore_header['port'] = server_info.get('port')
unicore_header['account'] = server_info.get('account')
unicore_header['project'] = server_info.get('project')
unicore_header['jhubtoken'] = server_info.get('jhubtoken')
unicore_header['spawning'] = server_info.get('spawning')
unicore_header['pollspawner'] = request_headers.get('pollspawner', 'false')
method = "GET"
method_args = {"url": app_urls.get('unicore', {}).get('url_jobs'),
"headers":unicore_header,
"certificate": False,
"fire_and_forget": True}
app_logger.info("uuidcode={} - Get J4J_UNICORE. Kernel_url: {}".format(uuidcode, unicore_header['kernelurl']))
jobs_threads_unicore_utils.communicate(app_logger,
uuidcode,
method,
method_args)
except:
utils_db.set_skip(app_logger,
uuidcode,
request_headers.get('servername'),
app_database,
'False')
app_logger.exception("uuidcode={} - J4J_UNICORE communication failed. {} {}".format(uuidcode, method, utils_common.remove_secret(method_args)))
def cancel(app_logger, uuidcode, app_hub_url_proxy_route, app_hub_cancel_url,
jhubtoken, errormsg, username, server_name):
app_logger.debug(
"uuidcode={} - Send cancel to JupyterHub".format(uuidcode))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"Intern-Authorization": get_jhubtoken(),
"uuidcode": uuidcode,
"Error": errormsg,
"Stopped": "True"
}
try:
app_logger.info(
"uuidcode={} - Cancel JupyterHub Server".format(uuidcode))
url = app_hub_cancel_url
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
with closing(
requests.delete(url,
headers=hub_header,
verify=False,
timeout=1800)) as r:
if r.status_code == 202:
app_logger.trace(
"uuidcode={} - Cancel successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
return
elif r.status_code == 503:
app_logger.info(
"uuidcode={} - Try to remove the proxys for the dead host".
format(uuidcode))
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.delete(url,
headers=hub_header,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 202:
app_logger.trace(
"uuidcode={} - Cancel successful: {} {} {}".format(
uuidcode, r2.text, r2.status_code, r2.headers))
return
else:
app_logger.warning(
"uuidcode={} - JupyterHub.cancel sent wrong status_code: {} {} {}"
.format(uuidcode, r2.text, r2.status_code,
remove_secret(r2.headers)))
else:
app_logger.warning(
"uuidcode={} - JupyterHub.cancel sent wrong status_code: {} {} {}"
.format(uuidcode, r.text, r.status_code,
remove_secret(r.headers)))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout (1800) reached. Could not send cancel to JupyterHub"
.format(uuidcode))
except:
app_logger.exception(
"uuidcode={} - Could not send cancel to JupyterHub".format(
uuidcode))
def status(app_logger, uuidcode, app_hub_url_proxy_route, app_hub_update_url,
jhubtoken, status, username, server_name):
app_logger.debug(
"uuidcode={} - Send job status to JupyterHub for userserver={} : {}".
format(uuidcode, server_name, status))
hub_header = {
"Authorization": "token {}".format(jhubtoken),
"uuidcode": uuidcode,
"Intern-Authorization": get_jhubtoken()
}
hub_json = {"Status": status}
try:
app_logger.info("uuidcode={} - Update JupyterHub Status ({})".format(
uuidcode, status))
url = app_hub_update_url
if ':' in server_name:
server_name = server_name.split(':')[1]
url = url + '/' + username
if server_name != '':
url = url + '/' + server_name
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r:
if r.status_code == 201:
app_logger.trace(
"uuidcode={} - Status Update successful: {} {} {}".format(
uuidcode, r.text, r.status_code, r.headers))
return
elif r.status_code == 503:
app_logger.info(
"uuidcode={} - Try to remove the proxys for the dead host".
format(uuidcode))
remove_proxy_route(app_logger, uuidcode,
app_hub_url_proxy_route, jhubtoken,
username, server_name)
# try again
with closing(
requests.post(url,
headers=hub_header,
json=hub_json,
verify=False,
timeout=1800)) as r2:
if r2.status_code == 201:
app_logger.trace(
"uuidcode={} - Status Update successful: {} {} {}".
format(uuidcode, r2.text, r2.status_code,
r2.headers))
return
elif r2.status_code == 404:
app_logger.info(
"uuidcode={} - JupyterHub doesn't know the spawner."
.format(uuidcode))
elif r.status_code == 404:
app_logger.info(
"uuidcode={} - JupyterHub doesn't know the spawner.".
format(uuidcode))
app_logger.warning(
"uuidcode={} - JupyterHub.update sent wrong status_code: {} {} {}"
.format(uuidcode, r.text, r.status_code,
remove_secret(r.headers)))
except requests.exceptions.ConnectTimeout:
app_logger.exception(
"uuidcode={} - Timeout (1800) reached. Could not send Status Update to JupyterHub"
.format(uuidcode))
except:
app_logger.exception(
"uuidcode={} - Could not send Status Update to JupyterHub".format(
uuidcode))
def communicate(app_logger, uuidcode, method, method_args, success_code=200):
app_logger.trace("uuidcode={} - Start unity.communicate()".format(uuidcode))
app_logger.trace("uuidcode={} - Method: {} - Method_args: {}".format(uuidcode, method, method_args))
if method == "POST":
try:
app_logger.info("uuidcode={} - Post to {}".format(uuidcode, method_args.get('url', '<no_url>')))
with closing(requests.post(method_args['url'],
headers = method_args.get('headers', {}),
data = method_args.get('data', "{}"),
verify = method_args.get('certificate', False),
timeout = 1800)) as r:
if r.status_code != success_code:
app_logger.warning("uuidcode={} - Unity communication response: {} {}".format(uuidcode, r.text, r.status_code))
app_logger.warning("uuidcode={} - arguments: method_args: {}".format(uuidcode, remove_secret(method_args)))
else:
app_logger.trace("uuidcode={} - Unity call successful".format(uuidcode))
except:
app_logger.exception("uuidcode={} - Unity communication failed".format(uuidcode))
if method == "GET":
try:
app_logger.info("uuidcode={} - Get to {}".format(uuidcode, method_args.get('url', '<no_url>')))
with closing(requests.get(method_args['url'],
headers = method_args.get('headers', {}),
verify = method_args.get('certificate', False),
timeout = 1800)) as r:
if r.status_code != success_code:
app_logger.warning("uuidcode={} - Unity communication response: {} {}".format(uuidcode, r.text, r.status_code))
app_logger.warning("uuidcode={} - arguments: method_args: {}".format(uuidcode, remove_secret(method_args)))
raise Exception("uuidcode={} - Unity communication failed".format(uuidcode))
else:
app_logger.trace("uuidcode={} - Unity call successful".format(uuidcode))
return r.json()
except:
app_logger.exception("uuidcode={} - Unity communication failed".format(uuidcode))