def get(self):
"""
Headers
Intern-Authorization
uuidcode
email
"""
try:
# Track actions through different webservices.
uuidcode = request.headers.get('uuidcode', '<no uuidcode>')
app.log.info("uuidcode={} - Get Deletion information".format(uuidcode))
app.log.trace("uuidcode={} - Headers: {}".format(uuidcode, request.headers))
# Check for the J4J intern token
utils_common.validate_auth(app.log,
uuidcode,
request.headers.get('intern-authorization', None))
request_headers = {}
for key, value in request.headers.items():
if 'Token' in key: # refresh, jhub, access
key = key.replace('-', '_')
request_headers[key.lower()] = value
config = utils_file_loads.get_general_config()
email = request_headers.get('email', '<no_email_submitted>')
email = email.replace("@", "_at_")
basefolder = config.get('basefolder', '<no basefolder defined>')
userfolder = os.path.join(basefolder, email)
workfolder = os.path.join(userfolder, "work")
projectfolder = os.path.join(userfolder, "Projects", "MyProjects")
totalfiles = 0
totalsize = 0
for root, dirs, files in os.walk(workfolder): # @UnusedVariable
if root.endswith(".hpc_mount") or \
root.endswith(".hpc_mount/.upload") or \
root.endswith(".davfs2") or \
root.endswith(".davfs2/cache") or \
root.endswith(".davfs2/cache/b2drop.eudat.eu-remote.php-webdav+home-jovyan-B2DROP+jovyan") or \
root.endswith(".davfs2/certs") or \
root.endswith(".davfs2/certs/private") or \
root.endswith(".ipynb_checkpoints"):
continue
else:
totalfiles += len(files)
for file in files:
filepath = os.path.join(root, file)
totalsize += os.path.getsize(filepath)
for root, dirs, files in os.walk(projectfolder): # @UnusedVariable
totalfiles += len(files)
for file in files:
filepath = os.path.join(root, file)
totalsize += os.path.getsize(filepath)
totalsize = sizeof_fmt(totalsize)
except:
app.log.exception("Deletion.get failed. Bugfix required")
return "", 500
return '{}:{}'.format(totalfiles, totalsize), 200
def post(self):
try:
"""
Headers:
Intern-Authorization: spawner_token
uuidcode
Body:
email
environments
image
port
servername
jupyterhub_api_url
Config:
basefolder # /etc/j4j/j4j_hdfcloud
network
cap-add
memory
memory-swap
device
storage-opt
"""
# Track actions through different webservices.
uuidcode = request.headers.get('uuidcode', '<no uuidcode>')
app.log.info("uuidcode={} - Start JupyterLab".format(uuidcode))
app.log.trace("uuidcode={} - Headers: {}".format(
uuidcode, request.headers))
app.log.trace("uuidcode={} - Json: {}".format(
uuidcode, request.json))
# Check for the J4J intern token
utils_common.validate_auth(
app.log, uuidcode,
request.headers.get('intern-authorization', None))
request_headers = {}
for key, value in request.headers.items():
if 'Token' in key: # refresh, jhub, access
key = key.replace('-', '_')
request_headers[key.lower()] = value
request_json = {}
for key, value in request.json.items():
if 'Token' in key: # refresh, jhub, access
key = key.replace('-', '_')
request_json[key.lower()] = value
app.log.trace("uuidcode={} - New Headers: {}".format(
uuidcode, request_headers))
app.log.trace("uuidcode={} - New Json: {}".format(
uuidcode, request_json))
config = utils_file_loads.get_general_config()
basefolder = config.get('basefolder', '<no basefolder defined>')
userfolder = os.path.join(
basefolder,
request_json.get('email').replace("@", "_at_"))
serverfolder = Path(
os.path.join(userfolder, '.{}'.format(uuidcode)))
mounts = jlab_utils.get_mounts(app.log, uuidcode, serverfolder,
userfolder)
cmd = ["docker", "run"]
cmd.append("--network")
cmd.append(config.get("network"))
cmd.append("--cap-add")
cmd.append(config.get("cap-add"))
cmd.append("--memory")
cmd.append(config.get("memory"))
cmd.append("--memory-swap")
cmd.append(config.get("memory-swap"))
cmd.append("--device")
cmd.append(config.get("device"))
cmd.append("--restart")
cmd.append(config.get("restart"))
#cmd.append("--storage-opt")
#cmd.append(config.get("storage-opt"))
cmd.append("--name")
cmd.append(uuidcode)
cmd.append("-e")
cmd.append("{}={}".format(
"HPCACCOUNTS",
request_json.get("environments", {}).get("HPCACCOUNTS", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_API_URL",
request_json.get("environments",
{}).get("JUPYTERHUB_API_URL", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_CLIENT_ID",
request_json.get("environments",
{}).get("JUPYTERHUB_CLIENT_ID", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_API_TOKEN",
request_json.get("environments",
{}).get("JUPYTERHUB_API_TOKEN", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_USER",
request_json.get("environments",
{}).get("JUPYTERHUB_USER", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_SERVICE_PREFIX",
request_json.get("environments",
{}).get("JUPYTERHUB_SERVICE_PREFIX", "")))
cmd.append("-e")
cmd.append("{}={}".format(
"JUPYTERHUB_BASE_URL",
request_json.get("environments",
{}).get("JUPYTERHUB_BASE_URL", "")))
cmd.extend(mounts)
cmd.append(request_json.get("image"))
cmd.append("/home/jovyan/.start.sh")
cmd.append(str(request_json.get("port")))
cmd.append(request_json.get("servername"))
cmd.append(request_json.get("jupyterhub_api_url"))
cmd.append("&")
app.log.debug("uuidcode={} - Run Command: {}".format(
uuidcode, cmd))
subprocess.Popen(cmd)
except:
app.log.exception("JLab.post failed. Bugfix required")
return "", 500
return "", 202
def delete(self):
"""
Headers
Intern-Authorization
uuidcode
email
"""
try:
# Track actions through different webservices.
uuidcode = request.headers.get('uuidcode', '<no uuidcode>')
app.log.info("uuidcode={} - Delete Account".format(uuidcode))
app.log.trace("uuidcode={} - Headers: {}".format(uuidcode, request.headers))
# Check for the J4J intern token
utils_common.validate_auth(app.log,
uuidcode,
request.headers.get('intern-authorization', None))
request_headers = {}
for key, value in request.headers.items():
if 'Token' in key: # refresh, jhub, access
key = key.replace('-', '_')
request_headers[key.lower()] = value
config = utils_file_loads.get_general_config()
email = request_headers.get('email', '<no_email_submitted>')
email = email.replace("@", "_at_")
basefolder = config.get('basefolder', '<no basefolder defined>')
userfolder = os.path.join(basefolder, email)
user_id = utils_db.get_user_id(app.log,
uuidcode,
app.database,
email)
servernames = utils_db.get_all_user_container_names(app.log,
uuidcode,
app.database,
user_id)
for servername in servernames:
results = jlab_utils.get_slave_infos(app.log,
uuidcode,
app.database,
servername,
email)
if len(results) > 0:
user_id, slave_id, slave_hostname, containername, running_no = results
else:
app.log.warning("uuidcode={} - {} not in database".format(uuidcode, servername))
continue;
url = app.urls.get('dockerspawner', {}).get('url_jlab_hostname', '<no_url_found>').replace('<hostname>', slave_hostname)
headers = {"Intern-Authorization": utils_file_loads.get_j4j_dockerspawner_token(),
"uuidcode": uuidcode,
"containername": containername}
try:
with closing(requests.delete(url,
headers=headers,
verify=False)) as r:
if r.status_code != 202:
app.log.error("uuidcode={} - DockerSpawner delete failed: {} {}".format(uuidcode, r.text, r.status_code))
except:
app.log.exception("uuidcode={} - Could not call DockerSpawner {}".format(uuidcode, slave_hostname))
serverfolder = Path(os.path.join(userfolder, '.{}'.format(containername)))
utils_db.decrease_slave_running(app.log, uuidcode, app.database, slave_id)
utils_db.remove_container(app.log, uuidcode, app.database, user_id, servername)
log_dir = Path(os.path.join(config.get('jobs_path', '<no_jobs_path>'), "{}-{}".format(email, containername)))
try:
os.makedirs(log_dir, exist_ok=True)
shutil.copy2(os.path.join(serverfolder, ".jupyterlabhub.log"), os.path.join(log_dir, "jupyterlabhub.log"))
except:
app.log.exception("uuidcode={} - Could not copy log".format(uuidcode))
jlab_output = "{};{};{}".format(userfolder,
containername,
running_no == 1)
jlab_delete_path = config.get('jlab_delete', '<no_jlab_delete_defined>')
app.log.debug("uuidcode={} - Write {} to {}".format(uuidcode, jlab_output, os.path.join(jlab_delete_path, uuidcode)))
with open(os.path.join(jlab_delete_path, uuidcode), 'w') as f:
f.write(jlab_output)
# end for loop all containers are stopped
utils_db.delete_account(app.log,
uuidcode,
app.database,
user_id)
try:
#shutil.rmtree(userfolder)
app.log.error("uuidcode={} - Account deletion. Please remove {} from the disk.".format(uuidcode, userfolder))
# remove user quota from xfs_quota
jlab_output = "{}".format(uuidcode)
jlab_delete_path = config.get('jlab_delete', '<no_jlab_delete_defined>')
app.log.debug("uuidcode={} - Write {} to {}.deletion".format(uuidcode, jlab_output, os.path.join(jlab_delete_path, email)))
with open('{}.deletion'.format(os.path.join(jlab_delete_path, email)), 'w') as f:
f.write(jlab_output)
except:
app.log.exception("uuidcode={} - Could not delete the users directories".format(uuidcode))
except:
app.log.exception("Deletion.delete failed. Bugfix required")
return "", 500
return '', 204
def post(self):
try:
"""
Headers:
intern-authorization
uuidcode
Body:
servername
service
dashboard
email
environments
image
port
jupyterhub_api_url
"""
# Track actions through different webservices.
uuidcode = request.headers.get('uuidcode', '<no uuidcode>')
app.log.info("uuidcode={} - Start JupyterLab".format(uuidcode))
app.log.trace("uuidcode={} - Headers: {}".format(
uuidcode, request.headers))
app.log.trace("uuidcode={} - Json: {}".format(
uuidcode, request.json))
# Check for the J4J intern token
utils_common.validate_auth(
app.log, uuidcode,
request.headers.get('intern-authorization', None))
request_json = {}
for key, value in request.json.items():
if 'Token' in key: # refresh, jhub, access
key = key.replace('-', '_')
request_json[key.lower()] = value
app.log.trace("uuidcode={} - New Json: {}".format(
uuidcode, request_json))
servername = request_json.get('servername')
email = request_json.get('email')
email = email.replace("@", "_at_")
environments = request_json.get('environments')
service = request_json.get('service')
dashboard = request_json.get('dashboard')
image = request_json.get('image')
port = request_json.get('port')
config = utils_file_loads.get_general_config()
quota_config = utils_file_loads.get_quota_config()
jupyterhub_api_url = config.get('jupyterhub_api_url')
basefolder = config.get('basefolder', '<no basefolder defined>')
userfolder = os.path.join(basefolder, email)
serverfolder = Path(
os.path.join(userfolder, '.{}'.format(uuidcode)))
os.umask(0)
user_id, set_user_quota = jlab_utils.create_user(
app.log, uuidcode, app.database, quota_config, email,
basefolder, userfolder)
jlab_utils.create_server_dirs(app.log, uuidcode, app.urls,
app.database, service, dashboard,
user_id, email, servername,
serverfolder, basefolder)
#jlab_utils.setup_server_quota(app.log, uuidcode, quota_config, serverfolder)
try:
start = jlab_utils.call_slave_start(
app.log, uuidcode, app.database, app.urls, userfolder,
config.get('jlab', '<no_jlab_path_defined>'), quota_config,
set_user_quota, user_id, servername, email, environments,
image, port, jupyterhub_api_url)
except:
app.log.exception(
"uuidcode={} - Could not start JupyterLab".format(
uuidcode))
start = False
if start:
return 200
else:
return 501
except:
app.log.exception("JLab.post failed. Bugfix required")
return 500
def create_server_dirs(app_logger, uuidcode, app_urls, app_database, service,
dashboard, user_id, email, servername, serverfolder,
basefolder):
results = utils_db.get_container_info(app_logger, uuidcode, app_database,
user_id, servername)
app_logger.debug("uuidcode={} - Container Info: {}".format(
uuidcode, results))
if len(results) > 0:
app_logger.debug(
"uuidcode={} - Server with name {} already exists. Delete it.".
format(uuidcode, serverfolder))
config = utils_file_loads.get_general_config()
user_id, slave_id, slave_hostname, containername, running_no = jlab_utils.get_slave_infos(
app_logger, uuidcode, app_database, servername, email)
url = app_urls.get('dockerspawner', {}).get('url_jlab_hostname',
'<no_url_found>').replace(
'<hostname>',
slave_hostname)
headers = {
"Intern-Authorization":
utils_file_loads.get_j4j_dockerspawner_token(),
"uuidcode": uuidcode,
"containername": containername
}
try:
with closing(requests.delete(url, headers=headers,
verify=False)) as r:
if r.status_code != 202:
app_logger.error(
"uuidcode={} - DockerSpawner delete failed: {} {}".
format(uuidcode, r.text, r.status_code))
except:
app_logger.exception(
"uuidcode={} - Could not call DockerSpawner {}".format(
uuidcode, slave_hostname))
basefolder = config.get('basefolder', '<no basefolder defined>')
userfolder = os.path.join(basefolder, email)
serverfolder = Path(
os.path.join(userfolder, '.{}'.format(containername)))
utils_db.decrease_slave_running(app_logger, uuidcode, app_database,
slave_id)
utils_db.remove_container(app_logger, uuidcode, app_database, user_id,
servername)
log_dir = Path(
os.path.join(config.get('jobs_path', '<no_jobs_path>'),
"{}-{}".format(email, containername)))
try:
os.makedirs(log_dir, exist_ok=True)
shutil.copy2(os.path.join(serverfolder, ".jupyterlabhub.log"),
os.path.join(log_dir, "jupyterlabhub.log"))
except:
app_logger.exception(
"uuidcode={} - Could not copy log".format(uuidcode))
jlab_output = "{};{};{}".format(userfolder, servername,
running_no == 1)
jlab_delete_path = config.get('jlab_delete',
'<no_jlab_delete_defined>')
app_logger.debug("uuidcode={} - Write {} to {}".format(
uuidcode, jlab_output, os.path.join(jlab_delete_path, uuidcode)))
with open(os.path.join(jlab_delete_path, uuidcode), 'w') as f:
f.write(jlab_output)
# Create folder for this JupyterLab
if not serverfolder.exists():
b2drop = Path(os.path.join(serverfolder, "B2DROP"))
hpcmount = Path(os.path.join(serverfolder, "HPCMOUNT"))
projects = Path(os.path.join(serverfolder, "Projects"))
myprojects = Path(os.path.join(projects, "MyProjects"))
sharedprojects = Path(os.path.join(projects, "SharedProjects"))
serverfolder.mkdir()
os.chmod(serverfolder, 0o777)
b2drop.mkdir()
os.chown(b2drop, 1000, 100)
hpcmount.mkdir()
os.chown(hpcmount, 1000, 100)
projects.mkdir()
os.chown(projects, 1000, 100)
myprojects.mkdir()
os.chown(myprojects, 1000, 100)
sharedprojects.mkdir()
os.chown(sharedprojects, 1000, 100)
# Copy files to user home
if service == "Dashboard":
app_logger.debug(
"{} - Try to copy base_home/.config_{}.py and base_home/.start_{}.sh"
.format(uuidcode, dashboard.replace(" ", "_"),
dashboard.replace(" ", "_")))
base_start_sh = Path(
os.path.join(basefolder, "base_home",
".start_{}.sh".format(dashboard.replace(" ", "_"))))
base_config_py = Path(
os.path.join(basefolder, "base_home",
".config_{}.py".format(dashboard.replace(" ", "_"))))
else:
base_config_py = Path(
os.path.join(basefolder, "base_home", ".config.py"))
base_start_sh = Path(os.path.join(basefolder, "base_home",
".start.sh"))
user_start_sh = Path(os.path.join(serverfolder, ".start.sh"))
shutil.copy2(base_start_sh, user_start_sh)
os.chown(user_start_sh, 1000, 100)
user_config_py = Path(os.path.join(serverfolder, ".config.py"))
shutil.copy2(base_config_py, user_config_py)
os.chown(user_config_py, 1000, 100)
base_jovyan = Path(
os.path.join(basefolder, "base_home", ".who_is_jovyan.txt"))
user_jovyan = Path(os.path.join(serverfolder, ".who_is_jovyan.txt"))
shutil.copy2(base_jovyan, user_jovyan)
os.chown(user_jovyan, 1000, 100)
base_mounthpc_sh = Path(
os.path.join(basefolder, "base_home", "mount_hpc.sh"))
user_mounthpc_sh = Path(os.path.join(serverfolder, "mount_hpc.sh"))
shutil.copy2(base_mounthpc_sh, user_mounthpc_sh)
os.chown(user_mounthpc_sh, 1000, 100)
base_manageprojects_sh = Path(
os.path.join(basefolder, "base_home", "manage_projects.sh"))
user_manageprojects_sh = Path(
os.path.join(serverfolder, "manage_projects.sh"))
shutil.copy2(base_manageprojects_sh, user_manageprojects_sh)
os.chown(user_manageprojects_sh, 1000, 100)
base_faq_ipynb = Path(os.path.join(basefolder, "base_home", "FAQ.ipynb"))
user_faq_ipynb = Path(os.path.join(serverfolder, "FAQ.ipynb"))
shutil.copy2(base_faq_ipynb, user_faq_ipynb)
os.chown(user_faq_ipynb, 1000, 100)