def pipeline_cwd(project_uuid, pipeline_uuid):
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
project_dir = get_project_directory(project_uuid)
cwd = pipeline_dir.replace(project_dir, "")
return jsonify({"cwd": cwd})
def _collateral(self, project_uuid: str, pipeline_uuid: str,
pipeline_name: str, **kwargs):
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid)
os.makedirs(pipeline_dir, exist_ok=True)
# Generate clean pipeline.json.
pipeline_json = {
"name": pipeline_name,
"version": "1.0.0",
"uuid": pipeline_uuid,
"settings": {
"auto_eviction": False,
"data_passing_memory_size": "1GB",
},
"steps": {},
"parameters": {},
}
with open(pipeline_json_path, "w") as pipeline_json_file:
json.dump(pipeline_json,
pipeline_json_file,
indent=4,
sort_keys=True)
def pipelines_json(project_uuid, pipeline_uuid):
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
request.args.get("experiment_uuid"),
request.args.get("pipeline_run_uuid"),
)
if request.method == "POST":
pipeline_directory = get_pipeline_directory(
pipeline_uuid,
project_uuid,
request.args.get("experiment_uuid"),
request.args.get("pipeline_run_uuid"),
)
# parse JSON
pipeline_json = json.loads(request.form.get("pipeline_json"))
# first create all files part of pipeline_json definition
# TODO: consider removing other files (no way to do this reliably,
# special case might be rename)
create_pipeline_files(pipeline_json, pipeline_directory,
project_uuid)
# side effect: for each Notebook in de pipeline.json set the correct kernel
pipeline_set_notebook_kernels(pipeline_json, pipeline_directory,
project_uuid)
with open(pipeline_json_path, "w") as json_file:
json_file.write(json.dumps(pipeline_json, indent=4))
# Analytics call
send_anonymized_pipeline_definition(app, pipeline_json)
return jsonify({"message": "Successfully saved pipeline."})
elif request.method == "GET":
if not os.path.isfile(pipeline_json_path):
return (
jsonify({
"success":
False,
"reason":
".orchest file doesn't exist at location %s" %
pipeline_json_path,
}),
404,
)
else:
with open(pipeline_json_path) as json_file:
return jsonify({
"success": True,
"pipeline_json": json_file.read()
})
return ""
def filemanager_exists():
"""Check whether file exists."""
path = request.args.get("path")
project_uuid = request.args.get("project_uuid")
pipeline_uuid = request.args.get("pipeline_uuid")
# currently this endpoint only handles "/data"
# if path is absolute
if path.startswith("/") and not path.startswith("/data"):
return jsonify({"message": "Illegal file path prefix."}), 400
file_path = None
if path.startswith("/"):
file_path = resolve_absolute_path(path)
if not is_valid_data_path(file_path, True):
raise app_error.OutOfDataDirectoryError(
"Path points outside of the data directory.")
else:
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
file_path = normalize_project_relative_path(path)
file_path = os.path.join(pipeline_dir, file_path)
if file_path is None:
return jsonify({"message": "Failed to process file_path."}), 500
if os.path.isfile(file_path):
return jsonify({"message": "File exists."})
else:
return jsonify({"message": "File does not exists."}), 404
def _collateral(self, project_uuid: str, pipeline_uuid: str,
pipeline_name: str, **kwargs):
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid)
os.makedirs(pipeline_dir, exist_ok=True)
# Generate clean pipeline.json.
pipeline_json = {
"name": pipeline_name,
"version": "1.0.0",
"uuid": pipeline_uuid,
"settings": {
"auto_eviction": False,
"data_passing_memory_size": "1GB",
},
"steps": {},
"parameters": {},
}
resp = requests.post(
f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api/pipelines/',
json={
"project_uuid": project_uuid,
"uuid": pipeline_uuid
},
)
if resp.status_code != 201:
raise Exception("Orchest-api pipeline creation failed.")
with open(pipeline_json_path, "w") as pipeline_json_file:
json.dump(pipeline_json,
pipeline_json_file,
indent=4,
sort_keys=True)
def pipelines_json(project_uuid, pipeline_uuid):
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
if request.method == "POST":
pipeline_directory = get_pipeline_directory(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
# Parse JSON.
pipeline_json = json.loads(request.form.get("pipeline_json"))
# Side effect: for each Notebook in de pipeline.json set the
# correct kernel.
pipeline_set_notebook_kernels(
pipeline_json, pipeline_directory, project_uuid
)
with open(pipeline_json_path, "w") as json_file:
json.dump(pipeline_json, json_file, indent=4, sort_keys=True)
# Analytics call.
send_anonymized_pipeline_definition(app, pipeline_json)
return jsonify({"message": "Successfully saved pipeline."})
elif request.method == "GET":
if not os.path.isfile(pipeline_json_path):
return (
jsonify(
{
"success": False,
"reason": ".orchest file doesn't exist at location %s"
% pipeline_json_path,
}
),
404,
)
else:
with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)
# json.dumps because the front end expects it as a
# string.
return jsonify(
{"success": True, "pipeline_json": json.dumps(pipeline_json)}
)
return ""
def file_viewer(project_uuid, pipeline_uuid, step_uuid):
experiment_uuid = request.args.get("experiment_uuid")
pipeline_run_uuid = request.args.get("pipeline_run_uuid")
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid,
experiment_uuid,
pipeline_run_uuid)
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid,
experiment_uuid,
pipeline_run_uuid)
if os.path.isfile(pipeline_json_path):
with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)
try:
file_path = os.path.join(
pipeline_dir,
pipeline_json["steps"][step_uuid]["file_path"])
filename = pipeline_json["steps"][step_uuid]["file_path"]
step_title = pipeline_json["steps"][step_uuid]["title"]
except Exception as e:
logging.info(e)
return return_404("Invalid JSON for pipeline %s error: %e" %
(pipeline_json_path, e))
else:
return return_404("Could not find pipeline.json for pipeline %s" %
pipeline_json_path)
file_ext = file_path.split(".")[-1]
file_content = ""
if file_ext == "ipynb":
if os.path.isfile(file_path):
try:
html_exporter = HTMLExporter()
(file_content, _) = html_exporter.from_filename(file_path)
except IOError as error:
logging.info("Error opening notebook file %s error: %s" %
(file_path, error))
return return_404("Could not find notebook file %s" %
file_path)
else:
try:
with open(file_path) as file:
file_content = file.read()
except (IOError, Exception) as e:
return jsonify({"message": "Could not read file."}), 500
return jsonify({
"ext": file_ext,
"content": file_content,
"step_title": step_title,
"filename": filename,
})
def project_file_exists(project_uuid, pipeline_uuid):
"""Check whether file exists"""
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
file_path = os.path.join(pipeline_dir, request.json["relative_path"])
if os.path.isfile(file_path):
return jsonify({"message": "File exists."})
else:
return jsonify({"message": "File does not exists."}), 404
def pipelines_create(project_uuid):
pipeline_path = request.json["pipeline_path"]
if (Pipeline.query.filter(
Pipeline.project_uuid == project_uuid).filter(
Pipeline.path == pipeline_path).count() == 0):
pipeline_uuid = str(uuid.uuid4())
pipeline = Pipeline(path=pipeline_path,
uuid=pipeline_uuid,
project_uuid=project_uuid)
db.session.add(pipeline)
db.session.commit()
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid)
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid)
os.makedirs(pipeline_dir, exist_ok=True)
# generate clean pipeline.json
pipeline_json = {
"name": request.json["name"],
"version": "1.0.0",
"uuid": pipeline_uuid,
"settings": {
"auto_eviction": False,
"data_passing_memory_size": "1GB",
},
"steps": {},
}
with open(pipeline_json_path, "w") as pipeline_json_file:
pipeline_json_file.write(json.dumps(pipeline_json, indent=4))
return jsonify({"success": True})
else:
return (
jsonify({
"message":
"Pipeline already exists at path '%s'." % pipeline_path
}),
409,
)
def notebook_html_get(project_uuid, pipeline_uuid, step_uuid):
experiment_uuid = request.args.get("experiment_uuid")
pipeline_run_uuid = request.args.get("pipeline_run_uuid")
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid,
experiment_uuid,
pipeline_run_uuid)
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid,
experiment_uuid,
pipeline_run_uuid)
if os.path.isfile(pipeline_json_path):
with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)
try:
notebook_path = os.path.join(
pipeline_dir,
pipeline_json["steps"][step_uuid]["file_path"])
except Exception as e:
logging.info(e)
return return_404("Invalid JSON for pipeline %s error: %e" %
(pipeline_json_path, e))
else:
return return_404("Could not find pipeline.json for pipeline %s" %
pipeline_json_path)
if os.path.isfile(notebook_path):
try:
html_exporter = HTMLExporter()
(body, _) = html_exporter.from_filename(notebook_path)
return body
except IOError as error:
logging.info("Error opening notebook file %s error: %s" %
(notebook_path, error))
return return_404("Could not find notebook file %s" %
notebook_path)
def pipelines_json(project_uuid, pipeline_uuid):
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
if request.method == "POST":
pipeline_directory = get_pipeline_directory(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
# Parse JSON.
pipeline_json = json.loads(request.form.get("pipeline_json"))
# First create all files part of pipeline_json definition
# TODO: consider removing other files (no way to do this
# reliably, special case might be rename).
create_pipeline_files(pipeline_json, pipeline_directory,
project_uuid)
# Side effect: for each Notebook in de pipeline.json set the
# correct kernel.
pipeline_set_notebook_kernels(pipeline_json, pipeline_directory,
project_uuid)
with open(pipeline_json_path, "w") as json_file:
json.dump(pipeline_json, json_file, indent=4, sort_keys=True)
# Analytics call.
send_anonymized_pipeline_definition(app, pipeline_json)
return jsonify({"message": "Successfully saved pipeline."})
elif request.method == "GET":
if not os.path.isfile(pipeline_json_path):
return (
jsonify({
"success":
False,
"reason":
".orchest file doesn't exist at location %s" %
pipeline_json_path,
}),
404,
)
else:
with open(pipeline_json_path) as json_file:
pipeline_json = json.load(json_file)
# Take care of old pipelines with no defined params.
if "parameters" not in pipeline_json:
pipeline_json["parameters"] = {}
# json.dumps because the front end expects it as a
# string.
return jsonify({
"success": True,
"pipeline_json": json.dumps(pipeline_json)
})
return ""
def pipelines_json(project_uuid, pipeline_uuid):
if request.method == "POST":
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
None,
request.args.get("pipeline_run_uuid"),
)
pipeline_directory = get_pipeline_directory(
pipeline_uuid,
project_uuid,
None,
request.args.get("pipeline_run_uuid"),
)
# Parse JSON.
pipeline_json = json.loads(request.form.get("pipeline_json"))
# Normalize relative paths.
for step in pipeline_json["steps"].values():
is_project_file = is_valid_pipeline_relative_path(
project_uuid, pipeline_uuid, step["file_path"])
is_data_file = is_valid_data_path(step["file_path"])
if not (is_project_file or is_data_file):
raise app_error.OutOfAllowedDirectoryError(
"File is neither in the project, nor in the data directory."
)
if not step["file_path"].startswith("/"):
step["file_path"] = normalize_project_relative_path(
step["file_path"])
errors = check_pipeline_correctness(pipeline_json)
if errors:
msg = {}
msg = {"success": False}
reason = ", ".join([key for key in errors])
reason = f"Invalid value: {reason}."
msg["reason"] = reason
return jsonify(msg), 400
# Side effect: for each Notebook in de pipeline.json set the
# correct kernel.
try:
pipeline_set_notebook_kernels(pipeline_json,
pipeline_directory, project_uuid)
except KeyError:
msg = {
"success": False,
"reason": "Invalid Notebook metadata structure.",
}
return jsonify(msg), 400
with open(pipeline_json_path, "r") as json_file:
old_pipeline_json = json.load(json_file)
# Save the pipeline JSON again to make sure its keys are
# sorted.
with open(pipeline_json_path, "w") as json_file:
json.dump(pipeline_json, json_file, indent=4, sort_keys=True)
if old_pipeline_json["name"] != pipeline_json["name"]:
resp = requests.put(
(f'http://{current_app.config["ORCHEST_API_ADDRESS"]}'
f"/api/pipelines/{project_uuid}/{pipeline_uuid}"),
json={"name": pipeline_json["name"]},
)
if resp.status_code != 200:
return (
jsonify(
{"message": "Failed to PUT name to orchest-api."}),
resp.status_code,
)
# Analytics call.
analytics.send_event(
app,
analytics.Event.PIPELINE_SAVE,
{"pipeline_definition": pipeline_json},
)
return jsonify({
"success": True,
"message": "Successfully saved pipeline."
})
elif request.method == "GET":
pipeline_json_path = get_pipeline_path(
pipeline_uuid,
project_uuid,
request.args.get("job_uuid"),
request.args.get("pipeline_run_uuid"),
)
if not os.path.isfile(pipeline_json_path):
return (
jsonify({
"success":
False,
"reason":
".orchest file doesn't exist at location " +
pipeline_json_path,
}),
404,
)
else:
pipeline_json = get_pipeline_json(pipeline_uuid, project_uuid)
return jsonify({
"success": True,
"pipeline_json": json.dumps(pipeline_json)
})
def file_viewer(project_uuid, pipeline_uuid, step_uuid):
job_uuid = request.args.get("job_uuid")
pipeline_run_uuid = request.args.get("pipeline_run_uuid")
pipeline_json_path = get_pipeline_path(pipeline_uuid, project_uuid,
job_uuid, pipeline_run_uuid)
pipeline_dir = get_pipeline_directory(pipeline_uuid, project_uuid,
job_uuid, pipeline_run_uuid)
if os.path.isfile(pipeline_json_path):
with open(pipeline_json_path, "r") as json_file:
pipeline_json = json.load(json_file)
try:
step_file_path = pipeline_json["steps"][step_uuid]["file_path"]
if not is_valid_pipeline_relative_path(
project_uuid, pipeline_uuid, step_file_path):
raise app_error.OutOfProjectError(
"Step path points outside of the project directory.")
if step_file_path.startswith("/"):
file_path = resolve_absolute_path(step_file_path)
else:
file_path = safe_join(pipeline_dir, step_file_path)
filename = pipeline_json["steps"][step_uuid]["file_path"]
step_title = pipeline_json["steps"][step_uuid]["title"]
except Exception as e:
app.logger.info(e)
return return_404("Invalid JSON for pipeline %s error: %e" %
(pipeline_json_path, e))
else:
return return_404("Could not find pipeline.json for pipeline %s" %
pipeline_json_path)
file_ext = file_path.split(".")[-1]
file_content = ""
if file_ext == "ipynb":
if os.path.isfile(file_path):
try:
html_exporter = HTMLExporter()
(file_content, _) = html_exporter.from_filename(file_path)
# custom CSS
custom_style = "<style>.CodeMirror pre {overflow: auto}</style>"
file_content = file_content.replace(
"</head>", custom_style + "</head>", 1)
except IOError as error:
app.logger.info(
"Error opening notebook file %s error: %s" %
(file_path, error))
return return_404(
("Could not find notebook file %s") % file_path)
else:
try:
with open(file_path) as file:
file_content = file.read()
except (IOError, Exception):
return jsonify({"message": "Could not read file."}), 500
return jsonify({
"ext": file_ext,
"content": file_content,
"step_title": step_title,
"filename": filename,
})
