def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"],
json_obj["project_uuid"])
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"])
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(json_obj["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments])
pipeline_environment_uuids = get_environments_from_pipeline_json(
json_obj["pipeline_definition"])
missing_environment_uuids = (pipeline_environment_uuids -
project_environment_uuids)
if len(missing_environment_uuids) > 0:
missing_environment_uuids_str = ", ".join(
missing_environment_uuids)
return (
jsonify({
"message":
"The pipeline definition references environments "
f"that do not exist in the project. "
"The following environments do not exist:"
f" [{missing_environment_uuids_str}].\n\n Please make sure all"
" pipeline steps are assigned an environment that exists"
" in the project."
}),
500,
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(job_uuid, json_obj["pipeline_uuid"],
json_obj["project_uuid"])
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
analytics.send_job_create(app, json_obj)
return resp.content, resp.status_code, resp.headers.items()
def create_job_spec(config) -> dict:
"""Returns a job spec based on the provided configuration.
Args: Initial configuration with which the job spec should be built.
project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name,
name are required. Optional entries such as env_variables can be
used to further customize the initial state of the newly created
job.
Returns:
A job spec that can be POSTED to the orchest-api to create a job
that is a duplicate of the job identified by the provided
job_uuid.
"""
job_spec = copy.deepcopy(config)
pipeline_path = pipeline_uuid_to_path(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
job_spec["pipeline_run_spec"]["run_config"] = {
"userdir_pvc": current_app.config["USERDIR_PVC"],
"project_dir": get_project_directory(job_spec["project_uuid"]),
"pipeline_path": pipeline_path,
}
job_spec["pipeline_definition"] = get_pipeline_json(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(job_spec["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments]
)
pipeline_environment_uuids = get_environments_from_pipeline_json(
job_spec["pipeline_definition"]
)
missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids
if len(missing_environment_uuids) > 0:
raise error.EnvironmentsDoNotExist(missing_environment_uuids)
# Jobs should always have eviction enabled.
job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
job_spec["uuid"] = job_uuid
create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"])
return job_spec
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir": app.config["HOST_USER_DIR"],
"project_dir": get_project_directory(
json_obj["project_uuid"], host_path=True
),
"pipeline_path": pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(
job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"]
)
# Analytics call
send_pipeline_run(
app,
f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}",
get_project_directory(json_obj["project_uuid"]),
"noninteractive",
)
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
return resp.content, resp.status_code, resp.headers.items()
def _collateral(self, job_uuid: str, pipeline_uuid: str, project_uuid: str):
create_job_directory(job_uuid, pipeline_uuid, project_uuid)