def catch_api_proxy_sessions_post():
json_obj = request.json
json_obj["project_dir"] = get_project_directory(
json_obj["project_uuid"], host_path=True)
json_obj["pipeline_path"] = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
json_obj["host_userdir"] = app.config["HOST_USER_DIR"]
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=json_obj,
)
tel_props = {
"project_uuid": json_obj["project_uuid"],
"pipeline_uuid": json_obj["pipeline_uuid"],
}
analytics.send_event(app, "session start", tel_props)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_experiments_post():
json_obj = request.json
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir": app.config["HOST_USER_DIR"],
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
),
}
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}",
get_project_directory(json_obj["project_uuid"]),
"noninteractive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/experiments/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions():
json_obj = request.json
json_obj["project_dir"] = get_project_directory(
json_obj["project_uuid"], host_path=True
)
json_obj["pipeline_path"] = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
json_obj["host_userdir"] = app.config["HOST_USER_DIR"]
pipeline_json = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
json_obj["settings"] = pipeline_json.get("settings", {})
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def _transaction(
self,
project_uuid: str,
pipeline_uuid: str,
pipeline_name: str,
job_name: str,
draft: bool,
) -> Job:
job_uuid = str(uuid.uuid4())
pipeline_path = pipeline_uuid_to_path(pipeline_uuid, project_uuid)
new_ex = Job(
uuid=job_uuid,
name=job_name,
pipeline_uuid=pipeline_uuid,
project_uuid=project_uuid,
pipeline_name=pipeline_name,
pipeline_path=pipeline_path,
strategy_json="{}",
draft=draft,
)
db.session.add(new_ex)
self.collateral_kwargs["job_uuid"] = job_uuid
self.collateral_kwargs["pipeline_uuid"] = pipeline_uuid
self.collateral_kwargs["project_uuid"] = project_uuid
return new_ex
def post(self, experiment_uuid):
experiment_uuid = str(uuid.uuid4())
pipeline_path = pipeline_uuid_to_path(
request.json["pipeline_uuid"],
request.json["project_uuid"])
new_ex = Experiment(
uuid=experiment_uuid,
name=request.json["name"],
pipeline_uuid=request.json["pipeline_uuid"],
project_uuid=request.json["project_uuid"],
pipeline_name=request.json["pipeline_name"],
pipeline_path=pipeline_path,
strategy_json="{}",
draft=request.json["draft"],
)
db.session.add(new_ex)
db.session.commit()
create_experiment_directory(
experiment_uuid,
request.json["pipeline_uuid"],
request.json["project_uuid"],
)
return experiment_schema.dump(new_ex)
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"],
json_obj["project_uuid"])
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"])
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(json_obj["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments])
pipeline_environment_uuids = get_environments_from_pipeline_json(
json_obj["pipeline_definition"])
missing_environment_uuids = (pipeline_environment_uuids -
project_environment_uuids)
if len(missing_environment_uuids) > 0:
missing_environment_uuids_str = ", ".join(
missing_environment_uuids)
return (
jsonify({
"message":
"The pipeline definition references environments "
f"that do not exist in the project. "
"The following environments do not exist:"
f" [{missing_environment_uuids_str}].\n\n Please make sure all"
" pipeline steps are assigned an environment that exists"
" in the project."
}),
500,
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(job_uuid, json_obj["pipeline_uuid"],
json_obj["project_uuid"])
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
analytics.send_job_create(app, json_obj)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions_post():
json_obj = request.json
project_uuid = json_obj["project_uuid"]
pipeline_uuid = json_obj["pipeline_uuid"]
# Lock the project and pipeline row to avoid race conditions
# with RenameProject and MovePipeline, which are locking for
# update themselves.
Project.query.with_for_update().filter(
Project.uuid == project_uuid, ).one()
Pipeline.query.with_for_update().filter(
Pipeline.project_uuid == project_uuid,
Pipeline.uuid == pipeline_uuid,
).one()
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
)
project_dir = get_project_directory(json_obj["project_uuid"])
services = get_pipeline_json(json_obj["pipeline_uuid"],
json_obj["project_uuid"]).get(
"services", {})
session_config = {
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"pipeline_path": pipeline_path,
"project_dir": project_dir,
"userdir_pvc": app.config["USERDIR_PVC"],
"services": services,
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/",
json=session_config,
)
analytics.send_event(
app,
analytics.Event.SESSION_START,
{
"project_uuid": project_uuid,
"pipeline_uuid": pipeline_uuid,
"services": services,
},
)
return resp.content, resp.status_code, resp.headers.items()
def create_job_spec(config) -> dict:
"""Returns a job spec based on the provided configuration.
Args: Initial configuration with which the job spec should be built.
project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name,
name are required. Optional entries such as env_variables can be
used to further customize the initial state of the newly created
job.
Returns:
A job spec that can be POSTED to the orchest-api to create a job
that is a duplicate of the job identified by the provided
job_uuid.
"""
job_spec = copy.deepcopy(config)
pipeline_path = pipeline_uuid_to_path(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
job_spec["pipeline_run_spec"]["run_config"] = {
"userdir_pvc": current_app.config["USERDIR_PVC"],
"project_dir": get_project_directory(job_spec["project_uuid"]),
"pipeline_path": pipeline_path,
}
job_spec["pipeline_definition"] = get_pipeline_json(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(job_spec["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments]
)
pipeline_environment_uuids = get_environments_from_pipeline_json(
job_spec["pipeline_definition"]
)
missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids
if len(missing_environment_uuids) > 0:
raise error.EnvironmentsDoNotExist(missing_environment_uuids)
# Jobs should always have eviction enabled.
job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
job_spec["uuid"] = job_uuid
create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"])
return job_spec
def catch_api_proxy_runs():
if request.method == "POST":
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
# All the paths are container path
json_obj["run_config"] = {
"userdir_pvc":
app.config["USERDIR_PVC"],
"project_dir":
get_project_directory(json_obj["project_uuid"]),
"pipeline_path":
pipeline_uuid_to_path(json_obj["pipeline_definition"]["uuid"],
json_obj["project_uuid"]),
"pipeline_uuid":
json_obj["pipeline_definition"]["uuid"],
"project_uuid":
json_obj["project_uuid"],
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
)
analytics.send_event(
app,
analytics.Event.PIPELINE_RUN_START,
{
"run_uuid": resp.json().get("uuid"),
"run_type": "interactive",
"pipeline_definition": json_obj["pipeline_definition"],
"step_uuids_to_execute": json_obj["uuids"],
},
)
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "GET":
resp = requests.get(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/" +
request_args_to_string(request.args), )
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_runs():
if request.method == "POST":
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
json_obj["run_config"] = {
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_uuid_to_path(
json_obj["pipeline_definition"]["uuid"], json_obj["project_uuid"]
),
}
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_definition']['uuid']}",
get_project_directory(json_obj["project_uuid"]),
"interactive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
elif request.method == "GET":
resp = requests.get(
"http://"
+ app.config["ORCHEST_API_ADDRESS"]
+ "/api/runs/?"
+ request.query_string.decode(),
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir": app.config["HOST_USER_DIR"],
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(
job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}",
get_project_directory(json_obj["project_uuid"]),
"noninteractive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def get_pipeline_path(
pipeline_uuid,
project_uuid,
experiment_uuid=None,
pipeline_run_uuid=None,
host_path=False,
pipeline_path=None,
):
USER_DIR = app.config["USER_DIR"]
if host_path == True:
USER_DIR = app.config["HOST_USER_DIR"]
if pipeline_path is None:
pipeline_path = pipeline_uuid_to_path(pipeline_uuid, project_uuid)
project_path = project_uuid_to_path(project_uuid)
if pipeline_run_uuid is None:
return os.path.join(USER_DIR, "projects", project_path,
pipeline_path)
elif pipeline_run_uuid is not None and experiment_uuid is not None:
return os.path.join(
USER_DIR,
"experiments",
project_uuid,
pipeline_uuid,
experiment_uuid,
pipeline_run_uuid,
pipeline_path,
)
elif experiment_uuid is not None:
return os.path.join(
USER_DIR,
"experiments",
project_uuid,
pipeline_uuid,
experiment_uuid,
"snapshot",
pipeline_path,
)
def catch_api_proxy_runs():
json_obj = request.json
# add image mapping
# TODO: replace with dynamic mapping instead of hardcoded
json_obj["run_config"] = {
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_uuid_to_path(json_obj["pipeline_description"]["uuid"],
json_obj["project_uuid"]),
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/",
json=json_obj,
stream=True,
)
return resp.raw.read(), resp.status_code, resp.headers.items()
def catch_api_proxy_experiments_post():
json_obj = request.json
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_uuid_to_path(
json_obj["pipeline_uuid"],
json_obj["project_uuid"],
),
}
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/experiments/",
json=json_obj,
stream=True,
)
return resp.raw.read(), resp.status_code, resp.headers.items()
