def pipeline_cwd(project_uuid, pipeline_uuid): pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) project_dir = get_project_directory(project_uuid) cwd = pipeline_dir.replace(project_dir, "") return jsonify({"cwd": cwd})
def _collateral(self, project_uuid: str, pipeline_uuid: str, pipeline_name: str, **kwargs): pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid) os.makedirs(pipeline_dir, exist_ok=True) # Generate clean pipeline.json. pipeline_json = { "name": pipeline_name, "version": "1.0.0", "uuid": pipeline_uuid, "settings": { "auto_eviction": False, "data_passing_memory_size": "1GB", }, "steps": {}, "parameters": {}, } with open(pipeline_json_path, "w") as pipeline_json_file: json.dump(pipeline_json, pipeline_json_file, indent=4, sort_keys=True)
def pipelines_json(project_uuid, pipeline_uuid): pipeline_json_path = get_pipeline_path( pipeline_uuid, project_uuid, request.args.get("experiment_uuid"), request.args.get("pipeline_run_uuid"), ) if request.method == "POST": pipeline_directory = get_pipeline_directory( pipeline_uuid, project_uuid, request.args.get("experiment_uuid"), request.args.get("pipeline_run_uuid"), ) # parse JSON pipeline_json = json.loads(request.form.get("pipeline_json")) # first create all files part of pipeline_json definition # TODO: consider removing other files (no way to do this reliably, # special case might be rename) create_pipeline_files(pipeline_json, pipeline_directory, project_uuid) # side effect: for each Notebook in de pipeline.json set the correct kernel pipeline_set_notebook_kernels(pipeline_json, pipeline_directory, project_uuid) with open(pipeline_json_path, "w") as json_file: json_file.write(json.dumps(pipeline_json, indent=4)) # Analytics call send_anonymized_pipeline_definition(app, pipeline_json) return jsonify({"message": "Successfully saved pipeline."}) elif request.method == "GET": if not os.path.isfile(pipeline_json_path): return ( jsonify({ "success": False, "reason": ".orchest file doesn't exist at location %s" % pipeline_json_path, }), 404, ) else: with open(pipeline_json_path) as json_file: return jsonify({ "success": True, "pipeline_json": json_file.read() }) return ""
def filemanager_exists(): """Check whether file exists.""" path = request.args.get("path") project_uuid = request.args.get("project_uuid") pipeline_uuid = request.args.get("pipeline_uuid") # currently this endpoint only handles "/data" # if path is absolute if path.startswith("/") and not path.startswith("/data"): return jsonify({"message": "Illegal file path prefix."}), 400 file_path = None if path.startswith("/"): file_path = resolve_absolute_path(path) if not is_valid_data_path(file_path, True): raise app_error.OutOfDataDirectoryError( "Path points outside of the data directory.") else: pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) file_path = normalize_project_relative_path(path) file_path = os.path.join(pipeline_dir, file_path) if file_path is None: return jsonify({"message": "Failed to process file_path."}), 500 if os.path.isfile(file_path): return jsonify({"message": "File exists."}) else: return jsonify({"message": "File does not exists."}), 404
def _collateral(self, project_uuid: str, pipeline_uuid: str, pipeline_name: str, **kwargs): pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid) os.makedirs(pipeline_dir, exist_ok=True) # Generate clean pipeline.json. pipeline_json = { "name": pipeline_name, "version": "1.0.0", "uuid": pipeline_uuid, "settings": { "auto_eviction": False, "data_passing_memory_size": "1GB", }, "steps": {}, "parameters": {}, } resp = requests.post( f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api/pipelines/', json={ "project_uuid": project_uuid, "uuid": pipeline_uuid }, ) if resp.status_code != 201: raise Exception("Orchest-api pipeline creation failed.") with open(pipeline_json_path, "w") as pipeline_json_file: json.dump(pipeline_json, pipeline_json_file, indent=4, sort_keys=True)
def pipelines_json(project_uuid, pipeline_uuid): pipeline_json_path = get_pipeline_path( pipeline_uuid, project_uuid, request.args.get("job_uuid"), request.args.get("pipeline_run_uuid"), ) if request.method == "POST": pipeline_directory = get_pipeline_directory( pipeline_uuid, project_uuid, request.args.get("job_uuid"), request.args.get("pipeline_run_uuid"), ) # Parse JSON. pipeline_json = json.loads(request.form.get("pipeline_json")) # Side effect: for each Notebook in de pipeline.json set the # correct kernel. pipeline_set_notebook_kernels( pipeline_json, pipeline_directory, project_uuid ) with open(pipeline_json_path, "w") as json_file: json.dump(pipeline_json, json_file, indent=4, sort_keys=True) # Analytics call. send_anonymized_pipeline_definition(app, pipeline_json) return jsonify({"message": "Successfully saved pipeline."}) elif request.method == "GET": if not os.path.isfile(pipeline_json_path): return ( jsonify( { "success": False, "reason": ".orchest file doesn't exist at location %s" % pipeline_json_path, } ), 404, ) else: with open(pipeline_json_path, "r") as json_file: pipeline_json = json.load(json_file) # json.dumps because the front end expects it as a # string. return jsonify( {"success": True, "pipeline_json": json.dumps(pipeline_json)} ) return ""
def file_viewer(project_uuid, pipeline_uuid, step_uuid): experiment_uuid = request.args.get("experiment_uuid") pipeline_run_uuid = request.args.get("pipeline_run_uuid") pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid, experiment_uuid, pipeline_run_uuid) pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid, experiment_uuid, pipeline_run_uuid) if os.path.isfile(pipeline_json_path): with open(pipeline_json_path, "r") as json_file: pipeline_json = json.load(json_file) try: file_path = os.path.join( pipeline_dir, pipeline_json["steps"][step_uuid]["file_path"]) filename = pipeline_json["steps"][step_uuid]["file_path"] step_title = pipeline_json["steps"][step_uuid]["title"] except Exception as e: logging.info(e) return return_404("Invalid JSON for pipeline %s error: %e" % (pipeline_json_path, e)) else: return return_404("Could not find pipeline.json for pipeline %s" % pipeline_json_path) file_ext = file_path.split(".")[-1] file_content = "" if file_ext == "ipynb": if os.path.isfile(file_path): try: html_exporter = HTMLExporter() (file_content, _) = html_exporter.from_filename(file_path) except IOError as error: logging.info("Error opening notebook file %s error: %s" % (file_path, error)) return return_404("Could not find notebook file %s" % file_path) else: try: with open(file_path) as file: file_content = file.read() except (IOError, Exception) as e: return jsonify({"message": "Could not read file."}), 500 return jsonify({ "ext": file_ext, "content": file_content, "step_title": step_title, "filename": filename, })
def project_file_exists(project_uuid, pipeline_uuid): """Check whether file exists""" pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) file_path = os.path.join(pipeline_dir, request.json["relative_path"]) if os.path.isfile(file_path): return jsonify({"message": "File exists."}) else: return jsonify({"message": "File does not exists."}), 404
def pipelines_create(project_uuid): pipeline_path = request.json["pipeline_path"] if (Pipeline.query.filter( Pipeline.project_uuid == project_uuid).filter( Pipeline.path == pipeline_path).count() == 0): pipeline_uuid = str(uuid.uuid4()) pipeline = Pipeline(path=pipeline_path, uuid=pipeline_uuid, project_uuid=project_uuid) db.session.add(pipeline) db.session.commit() pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid) pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid) os.makedirs(pipeline_dir, exist_ok=True) # generate clean pipeline.json pipeline_json = { "name": request.json["name"], "version": "1.0.0", "uuid": pipeline_uuid, "settings": { "auto_eviction": False, "data_passing_memory_size": "1GB", }, "steps": {}, } with open(pipeline_json_path, "w") as pipeline_json_file: pipeline_json_file.write(json.dumps(pipeline_json, indent=4)) return jsonify({"success": True}) else: return ( jsonify({ "message": "Pipeline already exists at path '%s'." % pipeline_path }), 409, )
def notebook_html_get(project_uuid, pipeline_uuid, step_uuid): experiment_uuid = request.args.get("experiment_uuid") pipeline_run_uuid = request.args.get("pipeline_run_uuid") pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid, experiment_uuid, pipeline_run_uuid) pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid, experiment_uuid, pipeline_run_uuid) if os.path.isfile(pipeline_json_path): with open(pipeline_json_path, "r") as json_file: pipeline_json = json.load(json_file) try: notebook_path = os.path.join( pipeline_dir, pipeline_json["steps"][step_uuid]["file_path"]) except Exception as e: logging.info(e) return return_404("Invalid JSON for pipeline %s error: %e" % (pipeline_json_path, e)) else: return return_404("Could not find pipeline.json for pipeline %s" % pipeline_json_path) if os.path.isfile(notebook_path): try: html_exporter = HTMLExporter() (body, _) = html_exporter.from_filename(notebook_path) return body except IOError as error: logging.info("Error opening notebook file %s error: %s" % (notebook_path, error)) return return_404("Could not find notebook file %s" % notebook_path)
def pipelines_json(project_uuid, pipeline_uuid): pipeline_json_path = get_pipeline_path( pipeline_uuid, project_uuid, request.args.get("job_uuid"), request.args.get("pipeline_run_uuid"), ) if request.method == "POST": pipeline_directory = get_pipeline_directory( pipeline_uuid, project_uuid, request.args.get("job_uuid"), request.args.get("pipeline_run_uuid"), ) # Parse JSON. pipeline_json = json.loads(request.form.get("pipeline_json")) # First create all files part of pipeline_json definition # TODO: consider removing other files (no way to do this # reliably, special case might be rename). create_pipeline_files(pipeline_json, pipeline_directory, project_uuid) # Side effect: for each Notebook in de pipeline.json set the # correct kernel. pipeline_set_notebook_kernels(pipeline_json, pipeline_directory, project_uuid) with open(pipeline_json_path, "w") as json_file: json.dump(pipeline_json, json_file, indent=4, sort_keys=True) # Analytics call. send_anonymized_pipeline_definition(app, pipeline_json) return jsonify({"message": "Successfully saved pipeline."}) elif request.method == "GET": if not os.path.isfile(pipeline_json_path): return ( jsonify({ "success": False, "reason": ".orchest file doesn't exist at location %s" % pipeline_json_path, }), 404, ) else: with open(pipeline_json_path) as json_file: pipeline_json = json.load(json_file) # Take care of old pipelines with no defined params. if "parameters" not in pipeline_json: pipeline_json["parameters"] = {} # json.dumps because the front end expects it as a # string. return jsonify({ "success": True, "pipeline_json": json.dumps(pipeline_json) }) return ""
def pipelines_json(project_uuid, pipeline_uuid): if request.method == "POST": pipeline_json_path = get_pipeline_path( pipeline_uuid, project_uuid, None, request.args.get("pipeline_run_uuid"), ) pipeline_directory = get_pipeline_directory( pipeline_uuid, project_uuid, None, request.args.get("pipeline_run_uuid"), ) # Parse JSON. pipeline_json = json.loads(request.form.get("pipeline_json")) # Normalize relative paths. for step in pipeline_json["steps"].values(): is_project_file = is_valid_pipeline_relative_path( project_uuid, pipeline_uuid, step["file_path"]) is_data_file = is_valid_data_path(step["file_path"]) if not (is_project_file or is_data_file): raise app_error.OutOfAllowedDirectoryError( "File is neither in the project, nor in the data directory." ) if not step["file_path"].startswith("/"): step["file_path"] = normalize_project_relative_path( step["file_path"]) errors = check_pipeline_correctness(pipeline_json) if errors: msg = {} msg = {"success": False} reason = ", ".join([key for key in errors]) reason = f"Invalid value: {reason}." msg["reason"] = reason return jsonify(msg), 400 # Side effect: for each Notebook in de pipeline.json set the # correct kernel. try: pipeline_set_notebook_kernels(pipeline_json, pipeline_directory, project_uuid) except KeyError: msg = { "success": False, "reason": "Invalid Notebook metadata structure.", } return jsonify(msg), 400 with open(pipeline_json_path, "r") as json_file: old_pipeline_json = json.load(json_file) # Save the pipeline JSON again to make sure its keys are # sorted. with open(pipeline_json_path, "w") as json_file: json.dump(pipeline_json, json_file, indent=4, sort_keys=True) if old_pipeline_json["name"] != pipeline_json["name"]: resp = requests.put( (f'http://{current_app.config["ORCHEST_API_ADDRESS"]}' f"/api/pipelines/{project_uuid}/{pipeline_uuid}"), json={"name": pipeline_json["name"]}, ) if resp.status_code != 200: return ( jsonify( {"message": "Failed to PUT name to orchest-api."}), resp.status_code, ) # Analytics call. analytics.send_event( app, analytics.Event.PIPELINE_SAVE, {"pipeline_definition": pipeline_json}, ) return jsonify({ "success": True, "message": "Successfully saved pipeline." }) elif request.method == "GET": pipeline_json_path = get_pipeline_path( pipeline_uuid, project_uuid, request.args.get("job_uuid"), request.args.get("pipeline_run_uuid"), ) if not os.path.isfile(pipeline_json_path): return ( jsonify({ "success": False, "reason": ".orchest file doesn't exist at location " + pipeline_json_path, }), 404, ) else: pipeline_json = get_pipeline_json(pipeline_uuid, project_uuid) return jsonify({ "success": True, "pipeline_json": json.dumps(pipeline_json) })
def file_viewer(project_uuid, pipeline_uuid, step_uuid): job_uuid = request.args.get("job_uuid") pipeline_run_uuid = request.args.get("pipeline_run_uuid") pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid, job_uuid, pipeline_run_uuid) pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid, job_uuid, pipeline_run_uuid) if os.path.isfile(pipeline_json_path): with open(pipeline_json_path, "r") as json_file: pipeline_json = json.load(json_file) try: step_file_path = pipeline_json["steps"][step_uuid]["file_path"] if not is_valid_pipeline_relative_path( project_uuid, pipeline_uuid, step_file_path): raise app_error.OutOfProjectError( "Step path points outside of the project directory.") if step_file_path.startswith("/"): file_path = resolve_absolute_path(step_file_path) else: file_path = safe_join(pipeline_dir, step_file_path) filename = pipeline_json["steps"][step_uuid]["file_path"] step_title = pipeline_json["steps"][step_uuid]["title"] except Exception as e: app.logger.info(e) return return_404("Invalid JSON for pipeline %s error: %e" % (pipeline_json_path, e)) else: return return_404("Could not find pipeline.json for pipeline %s" % pipeline_json_path) file_ext = file_path.split(".")[-1] file_content = "" if file_ext == "ipynb": if os.path.isfile(file_path): try: html_exporter = HTMLExporter() (file_content, _) = html_exporter.from_filename(file_path) # custom CSS custom_style = "<style>.CodeMirror pre {overflow: auto}</style>" file_content = file_content.replace( "</head>", custom_style + "</head>", 1) except IOError as error: app.logger.info( "Error opening notebook file %s error: %s" % (file_path, error)) return return_404( ("Could not find notebook file %s") % file_path) else: try: with open(file_path) as file: file_content = file.read() except (IOError, Exception): return jsonify({"message": "Could not read file."}), 500 return jsonify({ "ext": file_ext, "content": file_content, "step_title": step_title, "filename": filename, })