def catch_api_proxy_jobs_post():
try:
job_spec = jobs.create_job_spec(request.json)
except error.EnvironmentsDoNotExist as e:
return (
jsonify({
"message":
environments_missing_msg.format(missing_environment_uuids=[
",".join(e.environment_uuids)
]),
}),
500,
)
resp = requests.post(
"http://" + current_app.config["ORCHEST_API_ADDRESS"] +
"/api/jobs/",
json=job_spec,
)
analytics.send_event(
app,
analytics.Event.JOB_CREATE,
{
"job_definition":
job_spec,
"snapshot_size":
get_project_snapshot_size(job_spec["project_uuid"]),
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_runs_single(run_uuid):
if request.method == "GET":
resp = requests.get(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/runs/%s" % run_uuid, )
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "DELETE":
resp = requests.delete(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/runs/%s" % run_uuid, )
analytics.send_event(
app,
analytics.Event.PIPELINE_RUN_CANCEL,
{
"run_uuid": run_uuid,
"run_type": "interactive"
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_environment_image_builds():
environment_image_build_requests = request.json[
"environment_image_build_requests"]
for environment_image_build_request in environment_image_build_requests:
environment_image_build_request[
"project_path"] = project_uuid_to_path(
environment_image_build_request["project_uuid"])
resp = api_proxy_environment_image_builds(
environment_image_build_requests,
app.config["ORCHEST_API_ADDRESS"])
for environment_image_build_request in environment_image_build_requests:
environment_uuid = environment_image_build_request[
"environment_uuid"]
project_uuid = environment_image_build_request["project_uuid"]
env = get_environment(environment_uuid, project_uuid)
analytics.send_event(
app,
analytics.Event.ENVIRONMENT_BUILD_START,
{
"environment_uuid": environment_uuid,
"project_uuid": project_uuid,
"language": env.language,
"gpu_support": env.gpu_support,
"base_image": env.base_image,
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions_post():
json_obj = request.json
json_obj["project_dir"] = get_project_directory(
json_obj["project_uuid"], host_path=True)
json_obj["pipeline_path"] = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
json_obj["host_userdir"] = app.config["HOST_USER_DIR"]
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=json_obj,
)
tel_props = {
"project_uuid": json_obj["project_uuid"],
"pipeline_uuid": json_obj["pipeline_uuid"],
}
analytics.send_event(app, "session start", tel_props)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_job_cronjobs_resume(job_uuid):
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/jobs/cronjobs/resume/%s" % (job_uuid), )
analytics.send_event(app, analytics.Event.CRONJOB_RESUME,
{"job_uuid": job_uuid})
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_job_delete(job_uuid):
resp = requests.delete(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/%s" %
(job_uuid), )
analytics.send_event(app, analytics.Event.JOB_CANCEL,
{"job_uuid": job_uuid})
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_duplicate():
json_obj = request.json
try:
job_spec = jobs.duplicate_job_spec(json_obj["job_uuid"])
except error.ProjectDoesNotExist:
msg = ("The job cannot be duplicated because its project does "
"not exist anymore.")
return (
jsonify({"message": msg}),
409,
)
except error.PipelineDoesNotExist:
msg = ("The job cannot be duplicated because its pipeline does "
"not exist anymore.")
return (
jsonify({"message": msg}),
409,
)
except error.JobDoesNotExist:
msg = "The job cannot be duplicated because it does not exist anymore."
return (
jsonify({"message": msg}),
409,
)
except error.EnvironmentsDoNotExist as e:
return (
jsonify({
"message":
environments_missing_msg.format(missing_environment_uuids=[
",".join(e.environment_uuids)
]),
}),
500,
)
resp = requests.post(
"http://" + current_app.config["ORCHEST_API_ADDRESS"] +
"/api/jobs/",
json=job_spec,
)
analytics.send_event(
app,
analytics.Event.JOB_DUPLICATE,
{
"job_definition":
job_spec,
"duplicate_from":
json_obj["job_uuid"],
"snapshot_size":
get_project_snapshot_size(job_spec["project_uuid"]),
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_session_put(project_uuid, pipeline_uuid):
# check whether session is running
try:
resp = requests.get("http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/runs/?project_uuid=%s&pipeline_uuid=%s" %
(project_uuid, pipeline_uuid))
runs = resp.json()["runs"]
active_runs = False
for run in runs:
if run["status"] in ["PENDING", "STARTED"]:
active_runs = True
if active_runs:
analytics.send_event(
app,
analytics.Event.SESSION_RESTART,
{
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"active_runs": True,
},
)
return (
jsonify({
"message": ("Cannot restart the memory "
"server while the pipeline is running.")
}),
423,
)
else:
resp = requests.put(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/sessions/%s/%s" % (project_uuid, pipeline_uuid), )
analytics.send_event(
app,
analytics.Event.SESSION_RESTART,
{
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"active_runs": False,
},
)
return resp.content, resp.status_code, resp.headers.items()
except Exception as e:
app.logger.error(
"Could not get session information from orchest-api. Error: %s (%s)"
% (e, type(e)))
return "", 500
def catch_api_proxy_sessions_delete(project_uuid, pipeline_uuid):
resp = requests.delete(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/sessions/%s/%s" % (project_uuid, pipeline_uuid), )
tel_props = {
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
}
analytics.send_event(app, "session stop", tel_props)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions_post():
json_obj = request.json
project_uuid = json_obj["project_uuid"]
pipeline_uuid = json_obj["pipeline_uuid"]
# Lock the project and pipeline row to avoid race conditions
# with RenameProject and MovePipeline, which are locking for
# update themselves.
Project.query.with_for_update().filter(
Project.uuid == project_uuid, ).one()
Pipeline.query.with_for_update().filter(
Pipeline.project_uuid == project_uuid,
Pipeline.uuid == pipeline_uuid,
).one()
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
project_dir = get_project_directory(json_obj["project_uuid"])
services = get_pipeline_json(json_obj["pipeline_uuid"],
json_obj["project_uuid"]).get(
"services", {})
session_config = {
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"pipeline_path": pipeline_path,
"project_dir": project_dir,
"userdir_pvc": app.config["USERDIR_PVC"],
"services": services,
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=session_config,
)
analytics.send_event(
app,
analytics.Event.SESSION_START,
{
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"services": services,
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_job_pipeline_run_cleanup(job_uuid, run_uuid):
resp = requests.delete(
f"http://{current_app.config['ORCHEST_API_ADDRESS']}/api/"
f"jobs/cleanup/{job_uuid}/{run_uuid}")
analytics.send_event(
app,
analytics.Event.JOB_PIPELINE_RUN_DELETE,
{
"job_uuid": job_uuid,
"run_uuid": run_uuid
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_runs():
if request.method == "POST":
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
# All the paths are container path
json_obj["run_config"] = {
"userdir_pvc":
app.config["USERDIR_PVC"],
"project_dir":
get_project_directory(json_obj["project_uuid"]),
"pipeline_path":
pipeline_uuid_to_path(json_obj["pipeline_definition"]["uuid"],
json_obj["project_uuid"]),
"pipeline_uuid":
json_obj["pipeline_definition"]["uuid"],
"project_uuid":
json_obj["project_uuid"],
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
)
analytics.send_event(
app,
analytics.Event.PIPELINE_RUN_START,
{
"run_uuid": resp.json().get("uuid"),
"run_type": "interactive",
"pipeline_definition": json_obj["pipeline_definition"],
"step_uuids_to_execute": json_obj["uuids"],
},
)
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "GET":
resp = requests.get(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/" +
request_args_to_string(request.args), )
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_job_put(job_uuid):
resp = requests.put(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/%s" %
(job_uuid),
json=request.json,
)
analytics.send_event(
app,
analytics.Event.JOB_UPDATE,
{
"job_uuid": job_uuid,
"job_definition": request.json
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_environment_image_build_delete(
project_uuid,
environment_uuid,
image_tag,
):
resp = requests.delete(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/environment-builds/%s/%s/%s" %
(project_uuid, environment_uuid, image_tag), )
analytics.send_event(
app,
analytics.Event.ENVIRONMENT_BUILD_CANCEL,
{
"project_uuid": project_uuid,
"environment_uuid": environment_uuid,
"image_tag": image_tag,
},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_cleanup(job_uuid):
try:
# Get data before issuing deletion to the orchest-api. This
# is needed to retrieve the job pipeline uuid and project
# uuid. TODO: if the caller of the job knows about those
# ids, we could avoid making a request to the orchest-api.
resp = requests.get(
(f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api'
f"/jobs/{job_uuid}"))
data = resp.json()
if resp.status_code == 200:
pipeline_uuid = data["pipeline_uuid"]
project_uuid = data["project_uuid"]
# Tell the orchest-api that the job does not exist
# anymore, will be stopped if necessary then cleaned up
# from the orchest-api db.
resp = requests.delete(
f"http://{current_app.config['ORCHEST_API_ADDRESS']}/api/"
f"jobs/cleanup/{job_uuid}")
remove_job_directory(job_uuid, pipeline_uuid, project_uuid)
analytics.send_event(
app,
analytics.Event.JOB_DELETE,
{"job_uuid": job_uuid},
)
return resp.content, resp.status_code, resp.headers.items()
elif resp.status_code == 404:
raise ValueError(f"Job {job_uuid} does not exist.")
else:
raise Exception(f"{data}, {resp.status_code}")
except Exception as e:
msg = f"Error during job deletion:{e}"
return {"message": msg}, 500
def analytics_send_event():
event_name = request.json["event"]
try:
analytics_event = analytics.Event(event_name)
except ValueError:
app.logger.error(
f"No analytics event is defined for the given name: '{event_name}'."
)
return "Invalid analytics event name.", 500
success = analytics.send_event(app, analytics_event,
request.json["properties"])
if success:
return ""
else:
return "", 500
def catch_api_proxy_jupyter_builds_delete(build_uuid):
resp = requests.delete(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/jupyter-builds/%s" % build_uuid, )
analytics.send_event(app, "jupyter-build cancel", {})
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jupyter_builds_post():
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/jupyter-builds/", )
analytics.send_event(app, "jupyter-build start", {})
return resp.content, resp.status_code, resp.headers.items()
def analytics_send_event():
if send_event(app, request.json["event"], request.json["properties"]):
return ""
else:
return "", 500
def pipelines_json(project_uuid, pipeline_uuid):
if request.method == "POST":
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
None,
request.args.get("pipeline_run_uuid"),
)
pipeline_directory = get_pipeline_directory(
pipeline_uuid,
project_uuid,
None,
request.args.get("pipeline_run_uuid"),
)
# Parse JSON.
pipeline_json = json.loads(request.form.get("pipeline_json"))
# Normalize relative paths.
for step in pipeline_json["steps"].values():
is_project_file = is_valid_pipeline_relative_path(
project_uuid, pipeline_uuid, step["file_path"])
is_data_file = is_valid_data_path(step["file_path"])
if not (is_project_file or is_data_file):
raise app_error.OutOfAllowedDirectoryError(
"File is neither in the project, nor in the data directory."
)
if not step["file_path"].startswith("/"):
step["file_path"] = normalize_project_relative_path(
step["file_path"])
errors = check_pipeline_correctness(pipeline_json)
if errors:
msg = {}
msg = {"success": False}
reason = ", ".join([key for key in errors])
reason = f"Invalid value: {reason}."
msg["reason"] = reason
return jsonify(msg), 400
# Side effect: for each Notebook in de pipeline.json set the
# correct kernel.
try:
pipeline_set_notebook_kernels(pipeline_json,
pipeline_directory, project_uuid)
except KeyError:
msg = {
"success": False,
"reason": "Invalid Notebook metadata structure.",
}
return jsonify(msg), 400
with open(pipeline_json_path, "r") as json_file:
old_pipeline_json = json.load(json_file)
# Save the pipeline JSON again to make sure its keys are
# sorted.
with open(pipeline_json_path, "w") as json_file:
json.dump(pipeline_json, json_file, indent=4, sort_keys=True)
if old_pipeline_json["name"] != pipeline_json["name"]:
resp = requests.put(
(f'http://{current_app.config["ORCHEST_API_ADDRESS"]}'
f"/api/pipelines/{project_uuid}/{pipeline_uuid}"),
json={"name": pipeline_json["name"]},
)
if resp.status_code != 200:
return (
jsonify(
{"message": "Failed to PUT name to orchest-api."}),
resp.status_code,
)
# Analytics call.
analytics.send_event(
app,
analytics.Event.PIPELINE_SAVE,
{"pipeline_definition": pipeline_json},
)
return jsonify({
"success": True,
"message": "Successfully saved pipeline."
})
elif request.method == "GET":
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
if not os.path.isfile(pipeline_json_path):
return (
jsonify({
"success":
False,
"reason":
".orchest file doesn't exist at location " +
pipeline_json_path,
}),
404,
)
else:
pipeline_json = get_pipeline_json(pipeline_uuid, project_uuid)
return jsonify({
"success": True,
"pipeline_json": json.dumps(pipeline_json)
})
