def projects_get(): discoverFSDeletedProjects() discoverFSCreatedProjects() # Projects that are in a INITIALIZING or DELETING state won't # be shown until ready. projects = projects_schema.dump( Project.query.filter_by(status="READY").all()) for project in projects: # Discover both pipelines of newly initialized projects and # manually initialized pipelines of existing projects. Use a # a TwoPhaseExecutor for each project so that issues in one # project do not hinder the pipeline synchronization of # others. try: with TwoPhaseExecutor(db.session) as tpe: SyncProjectPipelinesDBState(tpe).transaction( project["uuid"]) except Exception as e: current_app.logger.error( ("Error during project pipelines synchronization of " f'{project["path"]}: {e}.')) project["pipeline_count"] = Pipeline.query.filter( Pipeline.project_uuid == project["uuid"]).count() project["job_count"] = Job.query.filter( Job.project_uuid == project["uuid"]).count() project["environment_count"] = len( get_environments(project["uuid"])) return jsonify(projects)
def catch_api_proxy_jobs_post(): json_obj = request.json pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"], json_obj["project_uuid"]) json_obj["pipeline_run_spec"]["run_config"] = { "host_user_dir": app.config["HOST_USER_DIR"], "project_dir": get_project_directory(json_obj["project_uuid"], host_path=True), "pipeline_path": pipeline_path, } json_obj["pipeline_definition"] = get_pipeline_json( json_obj["pipeline_uuid"], json_obj["project_uuid"]) # Validate whether the pipeline contains environments # that do not exist in the project. project_environments = get_environments(json_obj["project_uuid"]) project_environment_uuids = set( [environment.uuid for environment in project_environments]) pipeline_environment_uuids = get_environments_from_pipeline_json( json_obj["pipeline_definition"]) missing_environment_uuids = (pipeline_environment_uuids - project_environment_uuids) if len(missing_environment_uuids) > 0: missing_environment_uuids_str = ", ".join( missing_environment_uuids) return ( jsonify({ "message": "The pipeline definition references environments " f"that do not exist in the project. " "The following environments do not exist:" f" [{missing_environment_uuids_str}].\n\n Please make sure all" " pipeline steps are assigned an environment that exists" " in the project." }), 500, ) # Jobs should always have eviction enabled. json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True job_uuid = str(uuid.uuid4()) json_obj["uuid"] = job_uuid create_job_directory(job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"]) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/", json=json_obj, ) analytics.send_job_create(app, json_obj) return resp.content, resp.status_code, resp.headers.items()
def get(self, project_uuid): if project_exists(project_uuid): return {"message": "Project could not be found."}, 404 return environments_schema.dump( get_environments(project_uuid, language=request.args.get("language")))
def build_environments_for_project(project_uuid): environments = get_environments(project_uuid) for env in environments: url = (f'http://{current_app.config["ORCHEST_API_ADDRESS"]}' f"/api/environments/{project_uuid}") requests.post(url, json={"uuid": env.uuid}) return build_environments( [environment.uuid for environment in environments], project_uuid)
def create_job_spec(config) -> dict: """Returns a job spec based on the provided configuration. Args: Initial configuration with which the job spec should be built. project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name, name are required. Optional entries such as env_variables can be used to further customize the initial state of the newly created job. Returns: A job spec that can be POSTED to the orchest-api to create a job that is a duplicate of the job identified by the provided job_uuid. """ job_spec = copy.deepcopy(config) pipeline_path = pipeline_uuid_to_path( job_spec["pipeline_uuid"], job_spec["project_uuid"] ) job_spec["pipeline_run_spec"]["run_config"] = { "userdir_pvc": current_app.config["USERDIR_PVC"], "project_dir": get_project_directory(job_spec["project_uuid"]), "pipeline_path": pipeline_path, } job_spec["pipeline_definition"] = get_pipeline_json( job_spec["pipeline_uuid"], job_spec["project_uuid"] ) # Validate whether the pipeline contains environments # that do not exist in the project. project_environments = get_environments(job_spec["project_uuid"]) project_environment_uuids = set( [environment.uuid for environment in project_environments] ) pipeline_environment_uuids = get_environments_from_pipeline_json( job_spec["pipeline_definition"] ) missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids if len(missing_environment_uuids) > 0: raise error.EnvironmentsDoNotExist(missing_environment_uuids) # Jobs should always have eviction enabled. job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True job_uuid = str(uuid.uuid4()) job_spec["uuid"] = job_uuid create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"]) return job_spec
def catch_api_proxy_checks_gate(): project_uuid = request.json["project_uuid"] environment_uuids = [ environment.uuid for environment in get_environments(project_uuid) ] resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/validations/environments", json={"project_uuid": project_uuid, "environment_uuids": environment_uuids}, ) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_checks_gate(project_uuid): environment_uuids = [ environment.uuid for environment in get_environments(project_uuid) ] resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/checks/gate/%s" % project_uuid, json={ "type": "shallow", "environment_uuids": environment_uuids }, stream=True, ) return resp.raw.read(), resp.status_code, resp.headers.items()
def projects_get(): discoverFSDeletedProjects() discoverFSCreatedProjects() # Projects that are in a INITIALIZING or DELETING state won't # be shown until ready. projects = projects_schema.dump( Project.query.filter_by(status="READY").all()) for project in projects: # Discover both pipelines of newly initialized projects and # manually initialized pipelines of existing projects. Use a # a TwoPhaseExecutor for each project so that issues in one # project do not hinder the pipeline synchronization of # others. try: with TwoPhaseExecutor(db.session) as tpe: SyncProjectPipelinesDBState(tpe).transaction( project["uuid"]) except Exception as e: current_app.logger.error( ("Error during project pipelines synchronization of " f'{project["path"]}: {e}.')) project["pipeline_count"] = Pipeline.query.filter( Pipeline.project_uuid == project["uuid"]).count() project["environment_count"] = len( get_environments(project["uuid"])) resp = requests.get( f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api/jobs/', params={"project_uuid": project["uuid"]}, ) data = resp.json() if resp.status_code != 200: job_count = 0 else: job_count = len(data.get("jobs", [])) project["job_count"] = job_count return jsonify(projects)
def get(self, project_uuid): return environments_schema.dump( get_environments(project_uuid, language=request.args.get("language")))
def build_environments_for_project(project_uuid): environments = get_environments(project_uuid) return build_environments( [environment.uuid for environment in environments], project_uuid)
def populate_kernels(app, db, project_uuid): # cleanup old kernels cleanup_kernel(app, project_uuid) kernels_root_path = os.path.join(app.config["USER_DIR"], ".orchest", "kernels") if not os.path.exists(kernels_root_path): os.makedirs(kernels_root_path, exist_ok=True) kernels_dir_path = os.path.join(kernels_root_path, project_uuid) if not os.path.exists(kernels_dir_path): os.makedirs(kernels_dir_path, exist_ok=True) # kernel.json template kernel_template_dir = os.path.join(app.config["RESOURCE_DIR"], "kernels", "docker") kernel_json_template_path = os.path.join(kernel_template_dir, "kernel.json") try: with open(kernel_json_template_path, "r") as f: kernel_json_template = f.read() except Exception as e: app.logger.info("Error reading kernel.json at path %s. Error: %s" % (kernel_json_template_path, e)) raise e environments = get_environments(project_uuid) for environment in environments: kernel_name = _config.KERNEL_NAME.format( environment_uuid=environment.uuid) image_name = _config.ENVIRONMENT_IMAGE_NAME.format( project_uuid=project_uuid, environment_uuid=environment.uuid) kernel_dir_path = os.path.join(kernels_dir_path, kernel_name) os.makedirs(kernel_dir_path, exist_ok=True) # copy kernel logo resources copy_tree(kernel_template_dir, kernel_dir_path) # write filled template kernel.json filled_kernel_json = (kernel_json_template.replace( "{image_name}", image_name).replace("{language}", environment.language).replace( "{display_name}", environment.name)) kernel_json_path = os.path.join(kernel_dir_path, "kernel.json") # override kernel.json template try: with open(kernel_json_path, "w") as f: f.write(filled_kernel_json) except Exception as e: app.logger.info("Error writing kernel.json at path %s. Error: %s" % (kernel_json_path, e)) raise e # copy launch_docker.py launch_docker_path = os.path.join(app.config["RESOURCE_DIR"], "kernels", "launch_docker.py") launch_docker_dest_path = os.path.join(kernels_dir_path, "launch_docker.py") os.system('cp "%s" "%s"' % (launch_docker_path, launch_docker_dest_path))
def projects(): projects_dir = os.path.join(app.config["USER_DIR"], "projects") project_paths = [ name for name in os.listdir(projects_dir) if os.path.isdir(os.path.join(projects_dir, name)) ] # look for projects that have been removed through the filesystem by the user, cleanup # dangling resources fs_removed_projects = Project.query.filter( Project.path.notin_(project_paths)).all() for fs_removed_project in fs_removed_projects: cleanup_project_from_orchest(fs_removed_project) if len(fs_removed_projects) > 0: # refresh kernels after change in environments populate_kernels(app, db) # detect new projects by detecting directories that were not registered in the db as projects existing_project_paths = [ project.path for project in Project.query.filter( Project.path.in_(project_paths)).all() ] new_project_paths = set(project_paths) - set(existing_project_paths) for new_project_path in new_project_paths: try: init_project(new_project_path) except Exception as e: logging.error( f"Error during project initialization of {new_project_path}: {e}" ) if request.method == "GET": projects = projects_schema.dump(Project.query.all()) # Get counts for: pipelines, experiments and environments for project in projects: # catch both pipelines of newly initialized projects # and manually initialized pipelines of existing # projects sync_project_pipelines_db_state(project["uuid"]) project["pipeline_count"] = Pipeline.query.filter( Pipeline.project_uuid == project["uuid"]).count() project["experiment_count"] = Experiment.query.filter( Experiment.project_uuid == project["uuid"]).count() project["environment_count"] = len( get_environments(project["uuid"])) return jsonify(projects) elif request.method == "DELETE": project_uuid = request.json["project_uuid"] project = Project.query.filter( Project.uuid == project_uuid).first() if project != None: project_path = project_uuid_to_path(project_uuid) full_project_path = os.path.join(projects_dir, project_path) shutil.rmtree(full_project_path) cleanup_project_from_orchest(project) # refresh kernels after change in environments populate_kernels(app, db) return jsonify({"message": "Project deleted."}) else: return ( jsonify({ "message": "Project not found for UUID %s." % project_uuid }), 404, ) elif request.method == "POST": project_path = request.json["name"] if project_path not in project_paths: full_project_path = os.path.join(projects_dir, project_path) if not os.path.isdir(full_project_path): os.makedirs(full_project_path) # note that given the current pattern we have in the # GUI, where we POST and then GET projects, # this line does not strictly need to be there, # since the new directory will be picked up # on the GET request and initialized, placing it # here is more explicit and less relying # on the POST->GET pattern from the GUI try: init_project(project_path) except Exception as e: return ( jsonify({ "message": "Failed to create the project. Error: %s" % e }), 500, ) else: return ( jsonify( {"message": "Project directory already exists."}), 409, ) else: return ( jsonify({"message": "Project name already exists."}), 409, ) return jsonify({"message": "Project created."})
def populate_kernels(app, db): # check whether all kernels are available in the userdir/.orchest/kernels # use database to figure out which kernel directories should exist projects = Project.query.all() project_uuids = set([project.uuid for project in projects]) kernels_root_path = os.path.join(app.config["USER_DIR"], ".orchest", "kernels") if not os.path.exists(kernels_root_path): os.makedirs(kernels_root_path) else: # clear all the kernel folders # without removing the project kernel folders of existing projects as to not # disturb the mounted paths for filename in os.listdir(kernels_root_path): project_folder_path = os.path.join(kernels_root_path, filename) if os.path.isdir(project_folder_path): # if project exists clear kernel contents if filename in project_uuids: clear_folder(project_folder_path) else: # if project doesn't exist clear the entire folder to keep # .orchest/kernels tidy shutil.rmtree(project_folder_path) for project in projects: environments = get_environments(project.uuid) kernels_dir_path = os.path.join( app.config["USER_DIR"], ".orchest", "kernels", project.uuid ) # remove all kernels if not os.path.exists(kernels_dir_path): os.makedirs(kernels_dir_path) # kernel.json template kernel_template_dir = os.path.join( app.config["RESOURCE_DIR"], "kernels", "docker" ) kernel_json_template_path = os.path.join(kernel_template_dir, "kernel.json") try: with open(kernel_json_template_path, "r") as f: kernel_json_template = f.read() except Exception as e: logging.info( "Error reading kernel.json at path %s. Error: %s" % (kernel_json_template_path, e) ) raise e # create kernel_dirs for environment in environments: kernel_name = _config.KERNEL_NAME.format(environment_uuid=environment.uuid) image_name = _config.ENVIRONMENT_IMAGE_NAME.format( project_uuid=project.uuid, environment_uuid=environment.uuid ) kernel_dir_path = os.path.join(kernels_dir_path, kernel_name) os.makedirs(kernel_dir_path) # copy kernel logo resources copy_tree(kernel_template_dir, kernel_dir_path) # write filled template kernel.json filled_kernel_json = ( kernel_json_template.replace("{image_name}", image_name) .replace("{language}", environment.language) .replace("{display_name}", environment.name) ) kernel_json_path = os.path.join(kernel_dir_path, "kernel.json") # override kernel.json template try: with open(kernel_json_path, "w") as f: f.write(filled_kernel_json) except Exception as e: logging.info( "Error writing kernel.json at path %s. Error: %s" % (kernel_json_path, e) ) raise e # copy launch_docker.py launch_docker_path = os.path.join( app.config["RESOURCE_DIR"], "kernels", "launch_docker.py" ) launch_docker_dest_path = os.path.join(kernels_dir_path, "launch_docker.py") os.system('cp "%s" "%s"' % (launch_docker_path, launch_docker_dest_path))
def projects(): project_dir = os.path.join(app.config["USER_DIR"], "projects") project_paths = [ name for name in os.listdir(project_dir) if os.path.isdir(os.path.join(project_dir, name)) ] # create UUID entry for all projects that do not yet exist existing_project_paths = [ project.path for project in Project.query.filter( Project.path.in_(project_paths)).all() ] new_project_paths = set(project_paths) - set(existing_project_paths) for new_project_path in new_project_paths: new_project = Project( uuid=str(uuid.uuid4()), path=new_project_path, ) db.session.add(new_project) db.session.commit() # build environments on project detection build_environments_for_project(new_project.uuid) # end of UUID creation if request.method == "GET": projects = projects_schema.dump(Project.query.all()) # Get counts for: pipelines, experiments and environments for project in projects: project["pipeline_count"] = Pipeline.query.filter( Pipeline.project_uuid == project["uuid"]).count() project["experiment_count"] = Experiment.query.filter( Experiment.project_uuid == project["uuid"]).count() project["environment_count"] = len( get_environments(project["uuid"])) return jsonify(projects) elif request.method == "DELETE": project_uuid = request.json["project_uuid"] project = Project.query.filter( Project.uuid == project_uuid).first() if project != None: project_path = project_uuid_to_path(project_uuid) full_project_path = os.path.join(project_dir, project_path) shutil.rmtree(full_project_path) db.session.delete(project) db.session.commit() # refresh kernels after change in environments populate_kernels(app, db) return jsonify({"message": "Project deleted."}) else: return ( jsonify({ "message": "Project not found for UUID %s." % project_uuid }), 404, ) elif request.method == "POST": project_path = request.json["name"] if project_path not in project_paths: full_project_path = os.path.join(project_dir, project_path) if not os.path.isdir(full_project_path): new_project = Project( uuid=str(uuid.uuid4()), path=project_path, ) db.session.add(new_project) db.session.commit() os.makedirs(full_project_path) # initialize with default environments populate_default_environments(new_project.uuid) # refresh kernels after change in environments populate_kernels(app, db) # build environments on project creation build_environments_for_project(new_project.uuid) else: return ( jsonify( {"message": "Project directory already exists."}), 409, ) else: return ( jsonify({"message": "Project name already exists."}), 409, ) return jsonify({"message": "Project created."})