def get_list_of_notebooks(prefix, pattern="{{.Name}}: {{.Created}}"):
ret, stdout, stderr = shell("docker ps -qf name=%s" % prefix)
if not ret == 0:
raise Exception(stderr)
if stdout.strip() == "":
return []
ret, stdout, stderr = shell(
"docker inspect -f \"%s\" $(docker ps -qf name=%s)" %
(pattern, prefix))
return map(lambda x: x.strip().split(": "), stdout.strip().split("\n"))
def uploads():
target_dir = request.args.get("target_dir", None)
if target_dir == None:
return jsonify({"error": "target_dir cannot be None"}), 403
logger.debug("target_dir: " + target_dir)
ret, stdout, stderr = shell("mkdir -p %s" % target_dir)
if ret is not 0:
raise Exception(stderr)
file = request.files['file']
file.save(os.path.join(target_dir, file.filename))
response = {file.filename: "uploaded"}
return jsonify(response), 201
def run_docker_image(
port,
image_name,
username,
command="jupyter notebook --config=chorus_notebook_config.py",
**kargs):
options = ""
for key, value in kargs.iteritems():
if key in docker_mutiple_options:
for item in value:
options += key + "=" + item + " "
else:
options += key + "=" + value + " "
command = "/bin/sh -c \"source ./pyenv_paths.sh && export CONTAINER_PORT=%d && %s\"" % (
port, command)
ret, stdout, stderr = shell("docker run --name %s -d -p %d:8888 %s %s %s" %
(username, port, options, image_name, command))
if not ret == 0:
raise Exception(stderr)
container_name = username
return (container_name, port)
def run_notebook_in_docker():
logger.debug(request.form.to_dict())
username = request.form.get("username")
if username == None:
return jsonify({"error": "username cannot be None"}), 403
logger.debug("username: "******"master_address", "http://localhost:8080")
if master_address == None:
return jsonify({"error": "master_address cannot be None"}), 403
logger.debug("master_address: " + master_address)
image_name = request.form.get("image_name", "zh331873541/notebook")
if image_name == None:
return jsonify({"error": "image_name cannot be None"}), 403
logger.debug("image_name: " + image_name)
command = request.form.get("command",
app.config.get("COMMAND", "jupyter notebook --config=notebook_config.py"))
logger.debug("command: " + command)
options = {}
options["-e"] = ["MASTER_ADDRESS=%s" % master_address, "PYTHONWARNINGS=\"ignore:Unverified HTTPS request\""]
options["-v"] = []
if master_address.startswith("https") and app.config.get("CERTFILE", "") is not "" and app.config.get("KEYFILE",
"") is not "":
with open(app.config.get("CERTFILE", "")) as f:
cert = ""
for line in f:
cert += line.strip() + "\\n"
with open(app.config.get("KEYFILE", "")) as f:
key = ""
for line in f:
key += line.strip() + "\\n"
options["-e"].append("CERTFILE=certfile")
options["-e"].append("KEYFILE=keyfile")
command = ("echo -e \'%s\' > certfile && echo -e \'%s\' > keyfile && " % (cert, key)) + command
if app.config.get("MEM_LIMIT_PER_CONTAINER", "512m") is not "":
options["--memory"] = app.config.get("MEM_LIMIT_PER_CONTAINER", "512m")
if app.config.get("CPUSET_CPUS", "") is not "":
options["--cpuset-cpus"] = app.config.get("CPUSET_CPUS", "")
if app.config.get("NOTEBOOK_DATA_DIR", "") is not "":
mounted_path = os.path.abspath(
os.path.join(app.config.get("NOTEBOOK_DATA_DIR", ""), docker_container_name_prefix + username))
shell("mkdir -p %s && chmod -R 777 %s" % (mounted_path, mounted_path))
docker_path = "/home/notebook/notebook/" + docker_container_name_prefix + username
options["-v"].append(mounted_path + ":" + docker_path)
options["-e"].append("NOTEBOOK_DIR=%s" % docker_path)
if app.config.get("ENV", "") is not "":
for env in app.config.get("ENV", "").split(","):
options["-e"].append("\"%s\"" % env)
for key, value in request.form.to_dict().iteritems():
if not key in ["username", "master_address", "image_name", "command", "session_id"]:
if key in cnr.docker_mutiple_options:
if options.has_key(key):
options[key].append(value)
else:
options[key] = [value]
else:
options[key] = value
logger.debug(options)
try:
max_concurrent_users = app.config.get("MAX_CONCURRENT_USERS", 100)
running_containers = cnr.get_list_of_notebook_names(docker_container_name_prefix)
logger.debug(running_containers)
if (docker_container_name_prefix + username) in running_containers:
container_name = docker_container_name_prefix + username
port = cnr.get_docker_exposed_port(container_name)
else:
if len(running_containers) + 1 > max_concurrent_users:
return jsonify({"error": "exceed the docker containers limitation %s" % max_concurrent_users}), 403
container_name, port = cnr.run_notebook_in_docker(image_name,
docker_container_name_prefix + username,
command=command,
**options)
except Exception as e:
logger.debug(e)
return jsonify({"error": "%s" % e}), 403
response = {
"container_name": container_name,
"port": port
}
return jsonify(response), 201
def get_docker_exposed_port(container_name):
ret, stdout, stderr = shell("docker port %s" % container_name)
if not ret == 0:
raise Exception(stderr)
return int(stdout.split(":")[1])
def stop_all_docker_container(prefix):
ret, stdout, stderr = shell("docker rm -f $(docker ps -qf name=%s)" %
prefix)
if not ret == 0:
raise Exception(stderr)
return stdout
def stop_docker_container(container_id):
ret, stdout, stderr = shell("docker rm -f %s" % container_id)
if not ret == 0:
raise Exception(stderr)
return stdout
def start_docker_container(container_name):
ret, stdout, stderr = shell("docker start %s" % container_name)
if not ret == 0:
raise Exception(stderr)
return stdout