def catch_api_proxy_sessions_post(): json_obj = request.json json_obj["project_dir"] = get_project_directory( json_obj["project_uuid"], host_path=True) json_obj["pipeline_path"] = pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"], ) json_obj["host_userdir"] = app.config["HOST_USER_DIR"] resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/", json=json_obj, ) tel_props = { "project_uuid": json_obj["project_uuid"], "pipeline_uuid": json_obj["pipeline_uuid"], } analytics.send_event(app, "session start", tel_props) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_experiments_post(): json_obj = request.json json_obj["pipeline_run_spec"]["run_config"] = { "host_user_dir": app.config["HOST_USER_DIR"], "project_dir": get_project_directory( json_obj["project_uuid"], host_path=True ), "pipeline_path": pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"], ), } # Analytics call send_pipeline_run( app, f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}", get_project_directory(json_obj["project_uuid"]), "noninteractive", ) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/experiments/", json=json_obj, ) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions(): json_obj = request.json json_obj["project_dir"] = get_project_directory( json_obj["project_uuid"], host_path=True ) json_obj["pipeline_path"] = pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"], ) json_obj["host_userdir"] = app.config["HOST_USER_DIR"] pipeline_json = get_pipeline_json( json_obj["pipeline_uuid"], json_obj["project_uuid"] ) json_obj["settings"] = pipeline_json.get("settings", {}) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/", json=json_obj, ) return resp.content, resp.status_code, resp.headers.items()
def _transaction( self, project_uuid: str, pipeline_uuid: str, pipeline_name: str, job_name: str, draft: bool, ) -> Job: job_uuid = str(uuid.uuid4()) pipeline_path = pipeline_uuid_to_path(pipeline_uuid, project_uuid) new_ex = Job( uuid=job_uuid, name=job_name, pipeline_uuid=pipeline_uuid, project_uuid=project_uuid, pipeline_name=pipeline_name, pipeline_path=pipeline_path, strategy_json="{}", draft=draft, ) db.session.add(new_ex) self.collateral_kwargs["job_uuid"] = job_uuid self.collateral_kwargs["pipeline_uuid"] = pipeline_uuid self.collateral_kwargs["project_uuid"] = project_uuid return new_ex
def post(self, experiment_uuid): experiment_uuid = str(uuid.uuid4()) pipeline_path = pipeline_uuid_to_path( request.json["pipeline_uuid"], request.json["project_uuid"]) new_ex = Experiment( uuid=experiment_uuid, name=request.json["name"], pipeline_uuid=request.json["pipeline_uuid"], project_uuid=request.json["project_uuid"], pipeline_name=request.json["pipeline_name"], pipeline_path=pipeline_path, strategy_json="{}", draft=request.json["draft"], ) db.session.add(new_ex) db.session.commit() create_experiment_directory( experiment_uuid, request.json["pipeline_uuid"], request.json["project_uuid"], ) return experiment_schema.dump(new_ex)
def catch_api_proxy_jobs_post(): json_obj = request.json pipeline_path = pipeline_uuid_to_path(json_obj["pipeline_uuid"], json_obj["project_uuid"]) json_obj["pipeline_run_spec"]["run_config"] = { "host_user_dir": app.config["HOST_USER_DIR"], "project_dir": get_project_directory(json_obj["project_uuid"], host_path=True), "pipeline_path": pipeline_path, } json_obj["pipeline_definition"] = get_pipeline_json( json_obj["pipeline_uuid"], json_obj["project_uuid"]) # Validate whether the pipeline contains environments # that do not exist in the project. project_environments = get_environments(json_obj["project_uuid"]) project_environment_uuids = set( [environment.uuid for environment in project_environments]) pipeline_environment_uuids = get_environments_from_pipeline_json( json_obj["pipeline_definition"]) missing_environment_uuids = (pipeline_environment_uuids - project_environment_uuids) if len(missing_environment_uuids) > 0: missing_environment_uuids_str = ", ".join( missing_environment_uuids) return ( jsonify({ "message": "The pipeline definition references environments " f"that do not exist in the project. " "The following environments do not exist:" f" [{missing_environment_uuids_str}].\n\n Please make sure all" " pipeline steps are assigned an environment that exists" " in the project." }), 500, ) # Jobs should always have eviction enabled. json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True job_uuid = str(uuid.uuid4()) json_obj["uuid"] = job_uuid create_job_directory(job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"]) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/", json=json_obj, ) analytics.send_job_create(app, json_obj) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_sessions_post(): json_obj = request.json project_uuid = json_obj["project_uuid"] pipeline_uuid = json_obj["pipeline_uuid"] # Lock the project and pipeline row to avoid race conditions # with RenameProject and MovePipeline, which are locking for # update themselves. Project.query.with_for_update().filter( Project.uuid == project_uuid, ).one() Pipeline.query.with_for_update().filter( Pipeline.project_uuid == project_uuid, Pipeline.uuid == pipeline_uuid, ).one() pipeline_path = pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"], ) project_dir = get_project_directory(json_obj["project_uuid"]) services = get_pipeline_json(json_obj["pipeline_uuid"], json_obj["project_uuid"]).get( "services", {}) session_config = { "project_uuid": project_uuid, "pipeline_uuid": pipeline_uuid, "pipeline_path": pipeline_path, "project_dir": project_dir, "userdir_pvc": app.config["USERDIR_PVC"], "services": services, } resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/sessions/", json=session_config, ) analytics.send_event( app, analytics.Event.SESSION_START, { "project_uuid": project_uuid, "pipeline_uuid": pipeline_uuid, "services": services, }, ) return resp.content, resp.status_code, resp.headers.items()
def create_job_spec(config) -> dict: """Returns a job spec based on the provided configuration. Args: Initial configuration with which the job spec should be built. project_uuid, pipeline_uuid, pipeline_run_spec, pipeline_name, name are required. Optional entries such as env_variables can be used to further customize the initial state of the newly created job. Returns: A job spec that can be POSTED to the orchest-api to create a job that is a duplicate of the job identified by the provided job_uuid. """ job_spec = copy.deepcopy(config) pipeline_path = pipeline_uuid_to_path( job_spec["pipeline_uuid"], job_spec["project_uuid"] ) job_spec["pipeline_run_spec"]["run_config"] = { "userdir_pvc": current_app.config["USERDIR_PVC"], "project_dir": get_project_directory(job_spec["project_uuid"]), "pipeline_path": pipeline_path, } job_spec["pipeline_definition"] = get_pipeline_json( job_spec["pipeline_uuid"], job_spec["project_uuid"] ) # Validate whether the pipeline contains environments # that do not exist in the project. project_environments = get_environments(job_spec["project_uuid"]) project_environment_uuids = set( [environment.uuid for environment in project_environments] ) pipeline_environment_uuids = get_environments_from_pipeline_json( job_spec["pipeline_definition"] ) missing_environment_uuids = pipeline_environment_uuids - project_environment_uuids if len(missing_environment_uuids) > 0: raise error.EnvironmentsDoNotExist(missing_environment_uuids) # Jobs should always have eviction enabled. job_spec["pipeline_definition"]["settings"]["auto_eviction"] = True job_uuid = str(uuid.uuid4()) job_spec["uuid"] = job_uuid create_job_directory(job_uuid, job_spec["pipeline_uuid"], job_spec["project_uuid"]) return job_spec
def catch_api_proxy_runs(): if request.method == "POST": json_obj = request.json # add image mapping # TODO: replace with dynamic mapping instead of hardcoded # All the paths are container path json_obj["run_config"] = { "userdir_pvc": app.config["USERDIR_PVC"], "project_dir": get_project_directory(json_obj["project_uuid"]), "pipeline_path": pipeline_uuid_to_path(json_obj["pipeline_definition"]["uuid"], json_obj["project_uuid"]), "pipeline_uuid": json_obj["pipeline_definition"]["uuid"], "project_uuid": json_obj["project_uuid"], } resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/", json=json_obj, ) analytics.send_event( app, analytics.Event.PIPELINE_RUN_START, { "run_uuid": resp.json().get("uuid"), "run_type": "interactive", "pipeline_definition": json_obj["pipeline_definition"], "step_uuids_to_execute": json_obj["uuids"], }, ) return resp.content, resp.status_code, resp.headers.items() elif request.method == "GET": resp = requests.get( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/" + request_args_to_string(request.args), ) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_runs(): if request.method == "POST": json_obj = request.json # add image mapping # TODO: replace with dynamic mapping instead of hardcoded json_obj["run_config"] = { "project_dir": get_project_directory( json_obj["project_uuid"], host_path=True ), "pipeline_path": pipeline_uuid_to_path( json_obj["pipeline_definition"]["uuid"], json_obj["project_uuid"] ), } # Analytics call send_pipeline_run( app, f"{json_obj['project_uuid']}-{json_obj['pipeline_definition']['uuid']}", get_project_directory(json_obj["project_uuid"]), "interactive", ) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/", json=json_obj, ) return resp.content, resp.status_code, resp.headers.items() elif request.method == "GET": resp = requests.get( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/?" + request.query_string.decode(), ) return resp.content, resp.status_code, resp.headers.items()
def catch_api_proxy_jobs_post(): json_obj = request.json pipeline_path = pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"] ) json_obj["pipeline_run_spec"]["run_config"] = { "host_user_dir": app.config["HOST_USER_DIR"], "project_dir": get_project_directory( json_obj["project_uuid"], host_path=True ), "pipeline_path": pipeline_path, } json_obj["pipeline_definition"] = get_pipeline_json( json_obj["pipeline_uuid"], json_obj["project_uuid"] ) # Jobs should always have eviction enabled. json_obj["pipeline_definition"]["settings"]["auto_eviction"] = True job_uuid = str(uuid.uuid4()) json_obj["uuid"] = job_uuid create_job_directory( job_uuid, json_obj["pipeline_uuid"], json_obj["project_uuid"] ) # Analytics call send_pipeline_run( app, f"{json_obj['project_uuid']}-{json_obj['pipeline_uuid']}", get_project_directory(json_obj["project_uuid"]), "noninteractive", ) resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/jobs/", json=json_obj, ) return resp.content, resp.status_code, resp.headers.items()
def get_pipeline_path( pipeline_uuid, project_uuid, experiment_uuid=None, pipeline_run_uuid=None, host_path=False, pipeline_path=None, ): USER_DIR = app.config["USER_DIR"] if host_path == True: USER_DIR = app.config["HOST_USER_DIR"] if pipeline_path is None: pipeline_path = pipeline_uuid_to_path(pipeline_uuid, project_uuid) project_path = project_uuid_to_path(project_uuid) if pipeline_run_uuid is None: return os.path.join(USER_DIR, "projects", project_path, pipeline_path) elif pipeline_run_uuid is not None and experiment_uuid is not None: return os.path.join( USER_DIR, "experiments", project_uuid, pipeline_uuid, experiment_uuid, pipeline_run_uuid, pipeline_path, ) elif experiment_uuid is not None: return os.path.join( USER_DIR, "experiments", project_uuid, pipeline_uuid, experiment_uuid, "snapshot", pipeline_path, )
def catch_api_proxy_runs(): json_obj = request.json # add image mapping # TODO: replace with dynamic mapping instead of hardcoded json_obj["run_config"] = { "project_dir": get_project_directory(json_obj["project_uuid"], host_path=True), "pipeline_path": pipeline_uuid_to_path(json_obj["pipeline_description"]["uuid"], json_obj["project_uuid"]), } resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/runs/", json=json_obj, stream=True, ) return resp.raw.read(), resp.status_code, resp.headers.items()
def catch_api_proxy_experiments_post(): json_obj = request.json json_obj["pipeline_run_spec"]["run_config"] = { "host_user_dir": app.config["HOST_USER_DIR"], "project_dir": get_project_directory(json_obj["project_uuid"], host_path=True), "pipeline_path": pipeline_uuid_to_path( json_obj["pipeline_uuid"], json_obj["project_uuid"], ), } resp = requests.post( "http://" + app.config["ORCHEST_API_ADDRESS"] + "/api/experiments/", json=json_obj, stream=True, ) return resp.raw.read(), resp.status_code, resp.headers.items()