def catch_api_proxy_experiments_post():
json_obj = request.json
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir": app.config["HOST_USER_DIR"],
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
),
}
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}",
get_project_directory(json_obj["project_uuid"]),
"noninteractive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/experiments/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions_post():
json_obj = request.json
json_obj["project_dir"] = get_project_directory(
json_obj["project_uuid"], host_path=True)
json_obj["pipeline_path"] = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
json_obj["host_userdir"] = app.config["HOST_USER_DIR"]
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=json_obj,
)
tel_props = {
"project_uuid": json_obj["project_uuid"],
"pipeline_uuid": json_obj["pipeline_uuid"],
}
analytics.send_event(app, "session start", tel_props)
return resp.content, resp.status_code, resp.headers.items()
def pipeline_cwd(project_uuid, pipeline_uuid):
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
project_dir = get_project_directory(project_uuid)
cwd = pipeline_dir.replace(project_dir, "")
return jsonify({"cwd": cwd})
def catch_api_proxy_sessions():
json_obj = request.json
json_obj["project_dir"] = get_project_directory(
json_obj["project_uuid"], host_path=True
)
json_obj["pipeline_path"] = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
json_obj["host_userdir"] = app.config["HOST_USER_DIR"]
pipeline_json = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
json_obj["settings"] = pipeline_json.get("settings", {})
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"],
json_obj["project_uuid"])
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"])
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(json_obj["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments])
pipeline_environment_uuids = get_environments_from_pipeline_json(
json_obj["pipeline_definition"])
missing_environment_uuids = (pipeline_environment_uuids -
project_environment_uuids)
if len(missing_environment_uuids) > 0:
missing_environment_uuids_str = ", ".join(
missing_environment_uuids)
return (
jsonify({
"message":
"The pipeline definition references environments "
f"that do not exist in the project. "
"The following environments do not exist:"
f" [{missing_environment_uuids_str}].\n\n Please make sure all"
" pipeline steps are assigned an environment that exists"
" in the project."
}),
500,
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(job_uuid, json_obj["pipeline_uuid"],
json_obj["project_uuid"])
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
analytics.send_job_create(app, json_obj)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_runs():
if request.method == "POST":
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
json_obj["run_config"] = {
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_uuid_to_path(
json_obj["pipeline_definition"]["uuid"], json_obj["project_uuid"]
),
}
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_definition']['uuid']}",
get_project_directory(json_obj["project_uuid"]),
"interactive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "GET":
resp = requests.get(
"http://"
+ app.config["ORCHEST_API_ADDRESS"]
+ "/api/runs/?"
+ request.query_string.decode(),
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir": app.config["HOST_USER_DIR"],
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(
job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}",
get_project_directory(json_obj["project_uuid"]),
"noninteractive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def get_file_picker_tree(project_uuid):
allowed_file_extensions = ["ipynb", "R", "py", "sh"]
project_dir = get_project_directory(project_uuid)
if not os.path.isdir(project_dir):
return jsonify(
{"message": "Project dir %s not found." % project_dir}), 404
tree = {"type": "directory", "root": True, "name": "/", "children": []}
dir_nodes = {}
dir_nodes[project_dir] = tree
for root, dirs, files in os.walk(project_dir):
# exclude directories that start with "." from file_picker
dirs[:] = [
dirname for dirname in dirs if not dirname.startswith(".")
]
for dirname in dirs:
dir_path = os.path.join(root, dirname)
dir_node = {
"type": "directory",
"name": dirname,
"children": [],
}
dir_nodes[dir_path] = dir_node
dir_nodes[root]["children"].append(dir_node)
for filename in files:
if filename.split(".")[-1] in allowed_file_extensions:
file_node = {
"type": "file",
"name": filename,
}
# this key should always exist
try:
dir_nodes[root]["children"].append(file_node)
except KeyError as e:
app.logger.error(
"Key %s does not exist in dir_nodes %s. Error: %s"
% (root, dir_nodes, e))
except Exception as e:
app.logger.error("Error: %e" % e)
return jsonify(tree)
def catch_api_proxy_sessions_post():
json_obj = request.json
project_uuid = json_obj["project_uuid"]
pipeline_uuid = json_obj["pipeline_uuid"]
# Lock the project and pipeline row to avoid race conditions
# with RenameProject and MovePipeline, which are locking for
# update themselves.
Project.query.with_for_update().filter(
Project.uuid == project_uuid, ).one()
Pipeline.query.with_for_update().filter(
Pipeline.project_uuid == project_uuid,
Pipeline.uuid == pipeline_uuid,
).one()
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
project_dir = get_project_directory(json_obj["project_uuid"])
services = get_pipeline_json(json_obj["pipeline_uuid"],
json_obj["project_uuid"]).get(
"services", {})
session_config = {
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"pipeline_path": pipeline_path,
"project_dir": project_dir,
"userdir_pvc": app.config["USERDIR_PVC"],
"services": services,
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=session_config,
)
analytics.send_event(
app,
analytics.Event.SESSION_START,
{
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"services": services,
},
)
return resp.content, resp.status_code, resp.headers.items()
def create_job_spec(config) -> dict:
"""Returns a job spec based on the provided configuration.
Args: Initial configuration with which the job spec should be built.
project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name,
name are required. Optional entries such as env_variables can be
used to further customize the initial state of the newly created
job.
Returns:
A job spec that can be POSTED to the orchest-api to create a job
that is a duplicate of the job identified by the provided
job_uuid.
"""
job_spec = copy.deepcopy(config)
pipeline_path = pipeline_uuid_to_path(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
job_spec["pipeline_run_spec"]["run_config"] = {
"userdir_pvc": current_app.config["USERDIR_PVC"],
"project_dir": get_project_directory(job_spec["project_uuid"]),
"pipeline_path": pipeline_path,
}
job_spec["pipeline_definition"] = get_pipeline_json(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(job_spec["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments]
)
pipeline_environment_uuids = get_environments_from_pipeline_json(
job_spec["pipeline_definition"]
)
missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids
if len(missing_environment_uuids) > 0:
raise error.EnvironmentsDoNotExist(missing_environment_uuids)
# Jobs should always have eviction enabled.
job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
job_spec["uuid"] = job_uuid
create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"])
return job_spec
def catch_api_proxy_runs():
if request.method == "POST":
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
# All the paths are container path
json_obj["run_config"] = {
"userdir_pvc":
app.config["USERDIR_PVC"],
"project_dir":
get_project_directory(json_obj["project_uuid"]),
"pipeline_path":
pipeline_uuid_to_path(json_obj["pipeline_definition"]["uuid"],
json_obj["project_uuid"]),
"pipeline_uuid":
json_obj["pipeline_definition"]["uuid"],
"project_uuid":
json_obj["project_uuid"],
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
)
analytics.send_event(
app,
analytics.Event.PIPELINE_RUN_START,
{
"run_uuid": resp.json().get("uuid"),
"run_type": "interactive",
"pipeline_definition": json_obj["pipeline_definition"],
"step_uuids_to_execute": json_obj["uuids"],
},
)
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "GET":
resp = requests.get(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/" +
request_args_to_string(request.args), )
return resp.content, resp.status_code, resp.headers.items()
def create_project_file(project_uuid):
"""Create project file in specified directory within project."""
project_dir = get_project_directory(project_uuid)
# Client sends absolute path relative to project root, hence the
# starting / character is removed.
file_path = os.path.join(project_dir, request.json["file_path"][1:])
if os.path.isfile(file_path):
return jsonify({"message": "File already exists."}), 409
try:
open(file_path, "a").close()
return jsonify({"message": "File created."})
except IOError as e:
app.logger.error("Could not create file at %s. Error: %s" %
(file_path, e))
def catch_api_proxy_runs():
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
json_obj["run_config"] = {
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_uuid_to_path(json_obj["pipeline_description"]["uuid"],
json_obj["project_uuid"]),
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
stream=True,
)
return resp.raw.read(), resp.status_code, resp.headers.items()
def catch_api_proxy_experiments_post():
json_obj = request.json
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
),
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/experiments/",
json=json_obj,
stream=True,
)
return resp.raw.read(), resp.status_code, resp.headers.items()
