def import_wf(
wf_path, wf_type=None, name=None,
translate_to_cwl=True, # only if wf_type == "janis"
translate_to_wdl=True, # only if wf_type == "janis"
wf_imports_zip_path=None, # only if wf_type == "WDL"
wf_name_in_script=None # only if wf_type == "janis"
):
if wf_type is None:
wf_type = get_workflow_type_from_file_ext(wf_path)
else:
assert wf_type in supported_workflow_types, f"Provided workflow type \"{wf_type}\" is not supported."
if name is None or name == "":
name = os.path.splitext(os.path.basename(wf_path))[0]
if os.path.splitext(name)[1] in supported_workflow_exts[wf_type]:
name = os.path.splitext(name)[0]
if wf_type == "CWL":
import_cwl(wf_path, name)
elif wf_type == "janis":
import_janis(wf_path, name, translate_to_cwl, translate_to_wdl, wf_name_in_script)
else:
import_wdl(wf_path, name, wf_imports_zip_path)
def create_job(job_name,
username,
job_param_sheet=None,
run_inputs=None,
wf_target=None,
validate_uris=True,
search_paths=False,
search_subdirs=False,
search_dir=None,
sheet_format="xlsx"):
assert not (
job_param_sheet is None and (run_inputs is None or wf_target is None)
), "You have to either provide a job_param_sheet or a list of run_inputs plus a wf_target document"
runs_yaml_dir = get_path("runs_yaml_dir", job_name=job_name)
if wf_target is None:
job_param_sheet_dest_path = get_path("job_param_sheet",
job_name=job_name,
param_sheet_format=sheet_format)
copyfile(job_param_sheet, job_param_sheet_dest_path)
wf_target = get_job_templ_info(
"metadata",
job_templ_path=job_param_sheet_dest_path)["workflow_name"]
wf_type = get_workflow_type_from_file_ext(wf_target)
# make directories:
make_job_dir_tree(job_name)
# make run yamls:
if not job_param_sheet is None:
assert not (
search_paths and search_dir is None
), "search_paths was set to True but no search dir has been defined."
make_runs(
sheet_file=job_param_sheet_dest_path,
wf_type=wf_type,
output_basename="",
output_dir=runs_yaml_dir,
validate_uris=validate_uris,
search_paths=search_paths,
search_subdirs=search_subdirs,
allow_remote_uri=app.config["INPUT_SOURCES"]["URL"],
allow_local_path=app.config["INPUT_SOURCES"]["local_file_system"],
input_dir=search_dir)
else:
[copy(run_input, runs_yaml_dir) for run_input in run_inputs]
# get run names from produced yamls_
runs_yaml_dir = get_path("runs_yaml_dir", job_name)
run_yamls = fetch_files_in_dir(dir_path=runs_yaml_dir,
file_exts=["yaml"],
ignore_subdirs=True)
run_names = [r["file_nameroot"] for r in run_yamls]
# check if wf_target is absolute path and exists, else search for it in the wf_target dir:
if os.path.exists(wf_target):
wf_target = os.path.abspath(wf_target)
allowed_dirs = get_allowed_base_dirs(job_name=job_name,
allow_input=True,
allow_upload=False,
allow_download=False)
assert not check_if_path_in_dirs(
wf_target, allowed_dirs
) is None, "The provided wf_target file does not exit or you have no permission to access it."
else:
wf_target = get_path("wf", wf_target=wf_target)
# copy wf_target document:
copyfile(wf_target, get_path("job_wf", job_name=job_name, wf_type=wf_type))
# make output directories:
for run_name in run_names:
run_out_dir = get_path("run_out_dir", job_name, run_name)
if not os.path.exists(run_out_dir):
os.mkdir(run_out_dir)
# add job to database:
_ = job_manager.create_job(job_name=job_name,
username=username,
wf_target=wf_target)
# add runs to database:
job_manager.create_runs(run_names=run_names, job_name=job_name)
def create_job(job_id,
job_param_sheet=None,
run_inputs=None,
wf_target=None,
validate_paths=True,
search_paths=False,
search_subdirs=False,
search_dir=None,
sheet_format="xlsx"):
assert not (
job_param_sheet is None and (run_inputs is None or wf_target is None)
), "You have to either provide a job_param_sheet or a list of run_inputs plus a wf_target document"
runs_yaml_dir = get_path("runs_yaml_dir", job_id=job_id)
if wf_target is None:
job_param_sheet_dest_path = get_path("job_param_sheet",
job_id=job_id,
param_sheet_format=sheet_format)
copyfile(job_param_sheet, job_param_sheet_dest_path)
wf_target = get_job_templ_info(
"metadata",
job_templ_path=job_param_sheet_dest_path)["workflow_name"]
wf_type = get_workflow_type_from_file_ext(wf_target)
# make directories:
make_job_dir_tree(job_id)
# make run yamls:
if not job_param_sheet is None:
assert not (
search_paths and search_dir is None
), "search_paths was set to True but no search dir has been defined."
make_runs(sheet_file=job_param_sheet_dest_path,
wf_type=wf_type,
output_basename="",
output_dir=runs_yaml_dir,
validate_paths=validate_paths,
search_paths=search_paths,
search_subdirs=search_subdirs,
input_dir=search_dir)
else:
[copy(run_input, runs_yaml_dir) for run_input in run_inputs]
# check if wf_target is absolute path and exists, else search for it in the wf_target dir:
if os.path.exists(wf_target):
wf_target = os.path.abspath(wf_target)
allowed_dirs = get_allowed_base_dirs(job_id=job_id,
allow_input=True,
allow_upload=False,
allow_download=False)
assert not check_if_path_in_dirs(
wf_target, allowed_dirs
) is None, "The provided wf_target file does not exit or you have no permission to access it."
else:
wf_target = get_path("wf", wf_target=wf_target)
# copy wf_target document:
copyfile(wf_target, get_path("job_wf", job_id=job_id, wf_type=wf_type))
# make output directories:
run_ids = get_run_ids(job_id)
for run_id in run_ids:
run_out_dir = get_path("run_out_dir", job_id, run_id)
if not os.path.exists(run_out_dir):
os.mkdir(run_out_dir)