def fork_run(run_id: str) -> None:
run_dir: Path = get_run_dir(run_id)
stdout: Path = get_path(run_id, "stdout")
stderr: Path = get_path(run_id, "stderr")
cmd: str = f"/bin/bash {current_app.config['RUN_SH']} {run_dir}"
with stdout.open(mode="w") as f_stdout, stderr.open(mode="w") as f_stderr:
process = Popen(shlex.split(cmd), stdout=f_stdout, stderr=f_stderr)
pid: Optional[int] = process.pid
if pid is not None:
write_file(run_id, "pid", str(pid))
def write_workflow_attachment(run_id: str, run_request: RunRequest,
files: Dict[str, FileStorage]) -> None:
exe_dir: Path = get_path(run_id, "exe_dir")
host = request.host_url.strip("/")
url_prefix = current_app.config['URL_PREFIX'].strip("/")
endpoint = f"{host}/{url_prefix}".strip("/")
for file in run_request["workflow_attachment"]:
if "file_name" in file and "file_url" in file:
file_name: str = file["file_name"]
file_url: str = file["file_url"]
parsed_url = parse.urlparse(file_url)
if parsed_url.scheme in ["http", "https"] and \
not file_url.startswith(endpoint):
file_path = \
exe_dir.joinpath(secure_filepath(file_name)).resolve()
file_path.parent.mkdir(parents=True, exist_ok=True)
response: Response = requests.get(file["file_url"])
with file_path.open(mode="wb") as f:
f.write(response.content)
if current_app.config["WORKFLOW_ATTACHMENT"]:
workflow_attachment = \
files.getlist("workflow_attachment[]") # type: ignore
for file in workflow_attachment:
file_name = secure_filepath(file.filename) # type: ignore
file_path = exe_dir.joinpath(file_name).resolve()
file_path.parent.mkdir(parents=True, exist_ok=True)
file.save(file_path) # type: ignore
def prepare_exe_dir(run_id: str, request_files: Dict[str,
FileStorage]) -> None:
exe_dir: Path = get_path(run_id, "exe_dir")
exe_dir.mkdir(parents=True, exist_ok=True)
run_request: RunRequest = read_file(run_id, "run_request")
if current_app.config["REGISTERED_ONLY_MODE"]:
for attached_file in run_request["workflow_attachment"]:
file_name: str = secure_filename(attached_file["file_name"])
file_path: Path = exe_dir.joinpath(file_name).resolve()
file_path.parent.mkdir(parents=True, exist_ok=True)
response: Response = requests.get(attached_file["file_url"])
with file_path.open(mode="wb") as f:
f.write(response.content)
if "workflow_attachment" not in run_request:
run_request["workflow_attachment"] = []
if current_app.config["WORKFLOW_ATTACHMENT"]:
for file in request_files.values():
if file.filename != "":
file_name = secure_filename(file.filename)
run_request["workflow_attachment"].append({
"file_name": file_name,
"file_url": ""
})
file_path = exe_dir.joinpath(file_name).resolve()
file_path.parent.mkdir(parents=True, exist_ok=True)
file.save(file_path) # type: ignore
write_file(run_id, "run_request", json.dumps(run_request, indent=2))
def fork_run(run_id: str) -> None:
run_dir: Path = get_run_dir(run_id)
stdout: Path = get_path(run_id, "stdout")
stderr: Path = get_path(run_id, "stderr")
cmd: str = f"/bin/bash {current_app.config['RUN_SH']} {run_dir}"
write_file(run_id, "state", State.QUEUED.name)
with stdout.open(mode="w", encoding="utf-8") as f_stdout, \
stderr.open(mode="w", encoding="utf-8") as f_stderr:
process = Popen(shlex.split(cmd),
cwd=str(run_dir),
env={"PATH": os.environ.get("PATH", "")},
encoding="utf-8",
stdout=f_stdout,
stderr=f_stderr)
pid: Optional[int] = process.pid
if pid is not None:
write_file(run_id, "pid", str(pid))
def prepare_run_dir(
run_id: str,
run_request: RunRequest, # type: ignore
files: Dict[str, FileStorage]) -> RunRequest:
run_dir: Path = get_run_dir(run_id)
run_dir.mkdir(parents=True, exist_ok=True)
exe_dir: Path = get_path(run_id, "exe_dir")
exe_dir.mkdir(parents=True, exist_ok=True)
exe_dir.chmod(0o777)
outputs_dir: Path = get_path(run_id, "outputs_dir")
outputs_dir.mkdir(parents=True, exist_ok=True)
outputs_dir.chmod(0o777)
write_file(run_id, "sapporo_config", dump_sapporo_config(run_id))
write_file(run_id, "run_request", json.dumps(run_request, indent=2))
write_file(run_id, "wf_params", run_request["workflow_params"])
write_file(run_id, "wf_engine_params",
generate_wf_engine_params_str(run_request))
write_workflow_attachment(run_id, run_request, files)
def validate_and_update_run_request(run_id: str,
run_request: RunRequest,
files: Dict[str, FileStorage]) \
-> RunRequest:
if current_app.config["REGISTERED_ONLY_MODE"] and \
"workflow_url" in run_request:
abort(
400, "Currently, sapporo-service is running with "
"registered_only_mode. "
"Therefore, you need to specify a workflow using "
"`workflow_name` field. A list of executable workflows can "
"be retrieved requesting `GET /service-info`")
if "workflow_name" in run_request:
wf: Workflow = get_workflow(run_request["workflow_name"])
run_request["workflow_url"] = wf["workflow_url"]
run_request["workflow_type"] = wf["workflow_type"]
run_request["workflow_type_version"] = wf["workflow_type_version"]
if "workflow_attachment" not in run_request:
run_request["workflow_attachment"] = wf["workflow_attachment"]
for field in [
"workflow_params", "workflow_type", "workflow_type_version",
"workflow_url", "workflow_engine_name"
]:
if field not in run_request:
abort(400,
f"{field} not included in the form data of the request.")
if "workflow_attachment" in run_request:
if type(run_request["workflow_attachment"]) is str: # type: ignore
run_request["workflow_attachment"] = \
json.loads(run_request["workflow_attachment"]) # type: ignore
else:
run_request["workflow_attachment"] = []
if "workflow_engine_parameters" not in run_request:
run_request["workflow_engine_parameters"] = "{}"
if "tags" not in run_request:
run_request["tags"] = "{}"
if "workflow_name" not in run_request:
tags = json.loads(run_request["tags"])
if "workflow_name" in tags:
run_request["workflow_name"] = tags["workflow_name"]
else:
run_request["workflow_name"] = \
parse.urlparse(run_request["workflow_url"]).path.split("/")[-1]
if current_app.config["WORKFLOW_ATTACHMENT"]:
workflow_attachment = \
files.getlist("workflow_attachment[]") # type: ignore
exe_dir: Path = get_path(run_id, "exe_dir")
host = request.host_url.strip("/")
url_prefix = current_app.config['URL_PREFIX'].strip("/")
endpoint = f"{host}/{url_prefix}".strip("/")
base_remote_url = f"{endpoint}/runs/{run_id}/data/"
for f in workflow_attachment:
file_name: Path = secure_filepath(f.filename)
file_path: Path = exe_dir.joinpath(file_name).resolve()
run_request["workflow_attachment"].append({
"file_name":
str(file_name),
"file_url":
base_remote_url + str(file_path.relative_to(exe_dir.parent))
})
validate_wf_type(run_request["workflow_type"],
run_request["workflow_type_version"])
validate_meta_charactors("workflow_url", run_request["workflow_url"])
validate_meta_charactors("workflow_engine_name",
run_request["workflow_engine_name"])
return run_request