def projects_get():
discoverFSDeletedProjects()
discoverFSCreatedProjects()
# Projects that are in a INITIALIZING or DELETING state won't
# be shown until ready.
projects = projects_schema.dump(
Project.query.filter_by(status="READY").all())
for project in projects:
# Discover both pipelines of newly initialized projects and
# manually initialized pipelines of existing projects. Use a
# a TwoPhaseExecutor for each project so that issues in one
# project do not hinder the pipeline synchronization of
# others.
try:
with TwoPhaseExecutor(db.session) as tpe:
SyncProjectPipelinesDBState(tpe).transaction(
project["uuid"])
except Exception as e:
current_app.logger.error(
("Error during project pipelines synchronization of "
f'{project["path"]}: {e}.'))
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["job_count"] = Job.query.filter(
Job.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
return jsonify(projects)
def catch_api_proxy_jobs_post():
json_obj = request.json
pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"],
json_obj["project_uuid"])
json_obj["pipeline_run_spec"]["run_config"] = {
"host_user_dir":
app.config["HOST_USER_DIR"],
"project_dir":
get_project_directory(json_obj["project_uuid"], host_path=True),
"pipeline_path":
pipeline_path,
}
json_obj["pipeline_definition"] = get_pipeline_json(
json_obj["pipeline_uuid"], json_obj["project_uuid"])
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(json_obj["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments])
pipeline_environment_uuids = get_environments_from_pipeline_json(
json_obj["pipeline_definition"])
missing_environment_uuids = (pipeline_environment_uuids -
project_environment_uuids)
if len(missing_environment_uuids) > 0:
missing_environment_uuids_str = ", ".join(
missing_environment_uuids)
return (
jsonify({
"message":
"The pipeline definition references environments "
f"that do not exist in the project. "
"The following environments do not exist:"
f" [{missing_environment_uuids_str}].\n\n Please make sure all"
" pipeline steps are assigned an environment that exists"
" in the project."
}),
500,
)
# Jobs should always have eviction enabled.
json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
json_obj["uuid"] = job_uuid
create_job_directory(job_uuid, json_obj["pipeline_uuid"],
json_obj["project_uuid"])
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/",
json=json_obj,
)
analytics.send_job_create(app, json_obj)
return resp.content, resp.status_code, resp.headers.items()
def get(self, project_uuid):
if project_exists(project_uuid):
return {"message": "Project could not be found."}, 404
return environments_schema.dump(
get_environments(project_uuid,
language=request.args.get("language")))
def build_environments_for_project(project_uuid):
environments = get_environments(project_uuid)
for env in environments:
url = (f'http://{current_app.config["ORCHEST_API_ADDRESS"]}'
f"/api/environments/{project_uuid}")
requests.post(url, json={"uuid": env.uuid})
return build_environments(
[environment.uuid for environment in environments], project_uuid)
def create_job_spec(config) -> dict:
"""Returns a job spec based on the provided configuration.
Args: Initial configuration with which the job spec should be built.
project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name,
name are required. Optional entries such as env_variables can be
used to further customize the initial state of the newly created
job.
Returns:
A job spec that can be POSTED to the orchest-api to create a job
that is a duplicate of the job identified by the provided
job_uuid.
"""
job_spec = copy.deepcopy(config)
pipeline_path = pipeline_uuid_to_path(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
job_spec["pipeline_run_spec"]["run_config"] = {
"userdir_pvc": current_app.config["USERDIR_PVC"],
"project_dir": get_project_directory(job_spec["project_uuid"]),
"pipeline_path": pipeline_path,
}
job_spec["pipeline_definition"] = get_pipeline_json(
job_spec["pipeline_uuid"], job_spec["project_uuid"]
)
# Validate whether the pipeline contains environments
# that do not exist in the project.
project_environments = get_environments(job_spec["project_uuid"])
project_environment_uuids = set(
[environment.uuid for environment in project_environments]
)
pipeline_environment_uuids = get_environments_from_pipeline_json(
job_spec["pipeline_definition"]
)
missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids
if len(missing_environment_uuids) > 0:
raise error.EnvironmentsDoNotExist(missing_environment_uuids)
# Jobs should always have eviction enabled.
job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True
job_uuid = str(uuid.uuid4())
job_spec["uuid"] = job_uuid
create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"])
return job_spec
def catch_api_proxy_checks_gate():
project_uuid = request.json["project_uuid"]
environment_uuids = [
environment.uuid for environment in get_environments(project_uuid)
]
resp = requests.post(
"http://"
+ app.config["ORCHEST_API_ADDRESS"]
+ "/api/validations/environments",
json={"project_uuid": project_uuid, "environment_uuids": environment_uuids},
)
return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_checks_gate(project_uuid):
environment_uuids = [
environment.uuid for environment in get_environments(project_uuid)
]
resp = requests.post(
"http://" + app.config["ORCHEST_API_ADDRESS"] +
"/api/checks/gate/%s" % project_uuid,
json={
"type": "shallow",
"environment_uuids": environment_uuids
},
stream=True,
)
return resp.raw.read(), resp.status_code, resp.headers.items()
def projects_get():
discoverFSDeletedProjects()
discoverFSCreatedProjects()
# Projects that are in a INITIALIZING or DELETING state won't
# be shown until ready.
projects = projects_schema.dump(
Project.query.filter_by(status="READY").all())
for project in projects:
# Discover both pipelines of newly initialized projects and
# manually initialized pipelines of existing projects. Use a
# a TwoPhaseExecutor for each project so that issues in one
# project do not hinder the pipeline synchronization of
# others.
try:
with TwoPhaseExecutor(db.session) as tpe:
SyncProjectPipelinesDBState(tpe).transaction(
project["uuid"])
except Exception as e:
current_app.logger.error(
("Error during project pipelines synchronization of "
f'{project["path"]}: {e}.'))
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
resp = requests.get(
f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api/jobs/',
params={"project_uuid": project["uuid"]},
)
data = resp.json()
if resp.status_code != 200:
job_count = 0
else:
job_count = len(data.get("jobs", []))
project["job_count"] = job_count
return jsonify(projects)
def get(self, project_uuid):
return environments_schema.dump(
get_environments(project_uuid,
language=request.args.get("language")))
def build_environments_for_project(project_uuid):
environments = get_environments(project_uuid)
return build_environments(
[environment.uuid for environment in environments], project_uuid)
def populate_kernels(app, db, project_uuid):
# cleanup old kernels
cleanup_kernel(app, project_uuid)
kernels_root_path = os.path.join(app.config["USER_DIR"], ".orchest",
"kernels")
if not os.path.exists(kernels_root_path):
os.makedirs(kernels_root_path, exist_ok=True)
kernels_dir_path = os.path.join(kernels_root_path, project_uuid)
if not os.path.exists(kernels_dir_path):
os.makedirs(kernels_dir_path, exist_ok=True)
# kernel.json template
kernel_template_dir = os.path.join(app.config["RESOURCE_DIR"], "kernels",
"docker")
kernel_json_template_path = os.path.join(kernel_template_dir,
"kernel.json")
try:
with open(kernel_json_template_path, "r") as f:
kernel_json_template = f.read()
except Exception as e:
app.logger.info("Error reading kernel.json at path %s. Error: %s" %
(kernel_json_template_path, e))
raise e
environments = get_environments(project_uuid)
for environment in environments:
kernel_name = _config.KERNEL_NAME.format(
environment_uuid=environment.uuid)
image_name = _config.ENVIRONMENT_IMAGE_NAME.format(
project_uuid=project_uuid, environment_uuid=environment.uuid)
kernel_dir_path = os.path.join(kernels_dir_path, kernel_name)
os.makedirs(kernel_dir_path, exist_ok=True)
# copy kernel logo resources
copy_tree(kernel_template_dir, kernel_dir_path)
# write filled template kernel.json
filled_kernel_json = (kernel_json_template.replace(
"{image_name}",
image_name).replace("{language}", environment.language).replace(
"{display_name}", environment.name))
kernel_json_path = os.path.join(kernel_dir_path, "kernel.json")
# override kernel.json template
try:
with open(kernel_json_path, "w") as f:
f.write(filled_kernel_json)
except Exception as e:
app.logger.info("Error writing kernel.json at path %s. Error: %s" %
(kernel_json_path, e))
raise e
# copy launch_docker.py
launch_docker_path = os.path.join(app.config["RESOURCE_DIR"], "kernels",
"launch_docker.py")
launch_docker_dest_path = os.path.join(kernels_dir_path,
"launch_docker.py")
os.system('cp "%s" "%s"' % (launch_docker_path, launch_docker_dest_path))
def projects():
projects_dir = os.path.join(app.config["USER_DIR"], "projects")
project_paths = [
name for name in os.listdir(projects_dir)
if os.path.isdir(os.path.join(projects_dir, name))
]
# look for projects that have been removed through the filesystem by the user, cleanup
# dangling resources
fs_removed_projects = Project.query.filter(
Project.path.notin_(project_paths)).all()
for fs_removed_project in fs_removed_projects:
cleanup_project_from_orchest(fs_removed_project)
if len(fs_removed_projects) > 0:
# refresh kernels after change in environments
populate_kernels(app, db)
# detect new projects by detecting directories that were not registered in the db as projects
existing_project_paths = [
project.path for project in Project.query.filter(
Project.path.in_(project_paths)).all()
]
new_project_paths = set(project_paths) - set(existing_project_paths)
for new_project_path in new_project_paths:
try:
init_project(new_project_path)
except Exception as e:
logging.error(
f"Error during project initialization of {new_project_path}: {e}"
)
if request.method == "GET":
projects = projects_schema.dump(Project.query.all())
# Get counts for: pipelines, experiments and environments
for project in projects:
# catch both pipelines of newly initialized projects
# and manually initialized pipelines of existing
# projects
sync_project_pipelines_db_state(project["uuid"])
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["experiment_count"] = Experiment.query.filter(
Experiment.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
return jsonify(projects)
elif request.method == "DELETE":
project_uuid = request.json["project_uuid"]
project = Project.query.filter(
Project.uuid == project_uuid).first()
if project != None:
project_path = project_uuid_to_path(project_uuid)
full_project_path = os.path.join(projects_dir, project_path)
shutil.rmtree(full_project_path)
cleanup_project_from_orchest(project)
# refresh kernels after change in environments
populate_kernels(app, db)
return jsonify({"message": "Project deleted."})
else:
return (
jsonify({
"message":
"Project not found for UUID %s." % project_uuid
}),
404,
)
elif request.method == "POST":
project_path = request.json["name"]
if project_path not in project_paths:
full_project_path = os.path.join(projects_dir, project_path)
if not os.path.isdir(full_project_path):
os.makedirs(full_project_path)
# note that given the current pattern we have in the
# GUI, where we POST and then GET projects,
# this line does not strictly need to be there,
# since the new directory will be picked up
# on the GET request and initialized, placing it
# here is more explicit and less relying
# on the POST->GET pattern from the GUI
try:
init_project(project_path)
except Exception as e:
return (
jsonify({
"message":
"Failed to create the project. Error: %s" % e
}),
500,
)
else:
return (
jsonify(
{"message": "Project directory already exists."}),
409,
)
else:
return (
jsonify({"message": "Project name already exists."}),
409,
)
return jsonify({"message": "Project created."})
def populate_kernels(app, db):
# check whether all kernels are available in the userdir/.orchest/kernels
# use database to figure out which kernel directories should exist
projects = Project.query.all()
project_uuids = set([project.uuid for project in projects])
kernels_root_path = os.path.join(app.config["USER_DIR"], ".orchest", "kernels")
if not os.path.exists(kernels_root_path):
os.makedirs(kernels_root_path)
else:
# clear all the kernel folders
# without removing the project kernel folders of existing projects as to not
# disturb the mounted paths
for filename in os.listdir(kernels_root_path):
project_folder_path = os.path.join(kernels_root_path, filename)
if os.path.isdir(project_folder_path):
# if project exists clear kernel contents
if filename in project_uuids:
clear_folder(project_folder_path)
else:
# if project doesn't exist clear the entire folder to keep
# .orchest/kernels tidy
shutil.rmtree(project_folder_path)
for project in projects:
environments = get_environments(project.uuid)
kernels_dir_path = os.path.join(
app.config["USER_DIR"], ".orchest", "kernels", project.uuid
)
# remove all kernels
if not os.path.exists(kernels_dir_path):
os.makedirs(kernels_dir_path)
# kernel.json template
kernel_template_dir = os.path.join(
app.config["RESOURCE_DIR"], "kernels", "docker"
)
kernel_json_template_path = os.path.join(kernel_template_dir, "kernel.json")
try:
with open(kernel_json_template_path, "r") as f:
kernel_json_template = f.read()
except Exception as e:
logging.info(
"Error reading kernel.json at path %s. Error: %s"
% (kernel_json_template_path, e)
)
raise e
# create kernel_dirs
for environment in environments:
kernel_name = _config.KERNEL_NAME.format(environment_uuid=environment.uuid)
image_name = _config.ENVIRONMENT_IMAGE_NAME.format(
project_uuid=project.uuid, environment_uuid=environment.uuid
)
kernel_dir_path = os.path.join(kernels_dir_path, kernel_name)
os.makedirs(kernel_dir_path)
# copy kernel logo resources
copy_tree(kernel_template_dir, kernel_dir_path)
# write filled template kernel.json
filled_kernel_json = (
kernel_json_template.replace("{image_name}", image_name)
.replace("{language}", environment.language)
.replace("{display_name}", environment.name)
)
kernel_json_path = os.path.join(kernel_dir_path, "kernel.json")
# override kernel.json template
try:
with open(kernel_json_path, "w") as f:
f.write(filled_kernel_json)
except Exception as e:
logging.info(
"Error writing kernel.json at path %s. Error: %s"
% (kernel_json_path, e)
)
raise e
# copy launch_docker.py
launch_docker_path = os.path.join(
app.config["RESOURCE_DIR"], "kernels", "launch_docker.py"
)
launch_docker_dest_path = os.path.join(kernels_dir_path, "launch_docker.py")
os.system('cp "%s" "%s"' % (launch_docker_path, launch_docker_dest_path))
def projects():
project_dir = os.path.join(app.config["USER_DIR"], "projects")
project_paths = [
name for name in os.listdir(project_dir)
if os.path.isdir(os.path.join(project_dir, name))
]
# create UUID entry for all projects that do not yet exist
existing_project_paths = [
project.path for project in Project.query.filter(
Project.path.in_(project_paths)).all()
]
new_project_paths = set(project_paths) - set(existing_project_paths)
for new_project_path in new_project_paths:
new_project = Project(
uuid=str(uuid.uuid4()),
path=new_project_path,
)
db.session.add(new_project)
db.session.commit()
# build environments on project detection
build_environments_for_project(new_project.uuid)
# end of UUID creation
if request.method == "GET":
projects = projects_schema.dump(Project.query.all())
# Get counts for: pipelines, experiments and environments
for project in projects:
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["experiment_count"] = Experiment.query.filter(
Experiment.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
return jsonify(projects)
elif request.method == "DELETE":
project_uuid = request.json["project_uuid"]
project = Project.query.filter(
Project.uuid == project_uuid).first()
if project != None:
project_path = project_uuid_to_path(project_uuid)
full_project_path = os.path.join(project_dir, project_path)
shutil.rmtree(full_project_path)
db.session.delete(project)
db.session.commit()
# refresh kernels after change in environments
populate_kernels(app, db)
return jsonify({"message": "Project deleted."})
else:
return (
jsonify({
"message":
"Project not found for UUID %s." % project_uuid
}),
404,
)
elif request.method == "POST":
project_path = request.json["name"]
if project_path not in project_paths:
full_project_path = os.path.join(project_dir, project_path)
if not os.path.isdir(full_project_path):
new_project = Project(
uuid=str(uuid.uuid4()),
path=project_path,
)
db.session.add(new_project)
db.session.commit()
os.makedirs(full_project_path)
# initialize with default environments
populate_default_environments(new_project.uuid)
# refresh kernels after change in environments
populate_kernels(app, db)
# build environments on project creation
build_environments_for_project(new_project.uuid)
else:
return (
jsonify(
{"message": "Project directory already exists."}),
409,
)
else:
return (
jsonify({"message": "Project name already exists."}),
409,
)
return jsonify({"message": "Project created."})
