def prepare_job_env(): # generate param form sheet with data sent
# by the client
messages = []
data = {}
try:
login_required()
request_json = request.get_json()
job_id = request_json["job_id"]
# prepare job directory
job_dir = get_path("job_dir", job_id)
if not os.path.exists(job_dir):
os.mkdir(job_dir)
runs_yaml_dir = get_path("runs_yaml_dir", job_id)
if not os.path.exists(runs_yaml_dir):
os.mkdir(runs_yaml_dir)
runs_out_dir = get_path("runs_out_dir", job_id)
if not os.path.exists(runs_out_dir):
os.mkdir(runs_out_dir)
runs_log_dir = get_path("runs_log_dir", job_id)
if not os.path.exists(runs_log_dir):
os.mkdir(runs_log_dir)
runs_input_dir = get_path("runs_input_dir", job_id)
if not os.path.exists(runs_input_dir):
os.mkdir(runs_input_dir)
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
return jsonify({"data": data, "messages": messages})
def terminate_runs(
job_id,
run_ids,
mode="terminate" # can be one of terminate, reset, or delete
):
could_not_be_terminated = []
could_not_be_cleaned = []
succeeded = []
run_info, db_request = get_run_info(job_id, run_ids, return_pid=True, return_db_request=True)
db_changed = False
for run_id in run_info.keys():
if isinstance(run_info[run_id]["time_started"], datetime) and \
not isinstance(run_info[run_id]["time_finished"], datetime):
if run_info[run_id]["pid"] != -1:
is_killed = kill_proc_tree(run_info[run_id]["pid"])
if not is_killed:
could_not_be_terminated.append(run_id)
continue
cleanup_zombie_process(run_info[run_id]["pid"])
db_run_entry = db_request.filter(Exec.id==run_info[run_id]["db_id"])
db_run_entry.time_finished = datetime.now()
db_run_entry.status = "terminated by user"
db_changed = True
if mode in ["reset", "delete"]:
try:
log_path = get_path("run_log", job_id, run_id)
if os.path.exists(log_path):
os.remove(log_path)
run_out_dir = get_path("run_out_dir", job_id, run_id)
if os.path.exists(run_out_dir):
rmtree(run_out_dir)
if isinstance(run_info[run_id]["time_started"], datetime):
db_request.filter(Exec.run_id==run_id).delete(synchronize_session=False)
db_changed = True
except:
could_not_be_cleaned.append(run_id)
continue
if mode == "delete":
try:
yaml_path = get_path("run_yaml", job_id, run_id)
if os.path.exists(yaml_path):
os.remove(yaml_path)
except:
could_not_be_cleaned.append(run_id)
continue
succeeded.append(run_id)
if db_changed:
db_commit()
return succeeded, could_not_be_terminated, could_not_be_cleaned
def send_filled_param_values():
messages = []
data = []
try:
login_required()
request_json = request.get_json()
param_values = request_json["param_values"]
param_configs = request_json["param_configs"]
cwl_target = request_json["cwl_target"]
job_id = request_json["job_id"]
import_filepath = get_path("job_param_sheet_temp",
job_id=job_id,
param_sheet_format="xlsx")
validate_paths = request_json["validate_paths"]
search_paths = request_json["search_paths"]
search_dir = os.path.abspath(
remove_non_printable_characters(request_json["search_dir"]))
include_subdirs_for_searching = request_json[
"include_subdirs_for_searching"]
if search_paths:
# test if search dir exists:
if not os.path.isdir(search_dir):
sys.exit("The specified search dir \"" + search_dir +
"\" does not exist or is not a directory.")
generate_xls_from_param_values(
param_values=param_values,
configs=param_configs,
output_file=import_filepath,
validate_paths=validate_paths,
search_paths=search_paths,
search_subdirs=include_subdirs_for_searching,
input_dir=search_dir,
config_attributes={"CWL": cwl_target})
except SystemExit as e:
messages.append({
"type":
"error",
"text":
"The provided form failed validation: " + str(e)
})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
if len(messages) == 0:
messages.append({
"type":
"success",
"text":
"The filled form was successfully imported and validated."
})
return jsonify({"data": data, "messages": messages})
def generate_param_form_sheet(): # generate param form sheet with data sent
# by the client
messages = []
data = {}
try:
login_required()
request_json = request.get_json()
sheet_format = request_json["sheet_format"]
job_id = request_json["job_id"]
cwl_target = request_json["cwl_target"]
param_modes = request_json["param_modes"]
run_names = request_json["run_names"]
run_mode = request_json["run_mode"]
try:
param_form_sheet = get_path("job_param_sheet_temp", job_id=job_id)
os.remove(param_form_sheet)
except:
pass
output_file_path = get_path("job_param_sheet_temp",
job_id=job_id,
param_sheet_format=sheet_format)
print(output_file_path)
gen_form_sheet(output_file_path=output_file_path,
template_config_file_path=get_path(
"job_templ", cwl_target=cwl_target),
has_multiple_runs=run_mode,
run_names=run_names,
param_is_run_specific=param_modes,
show_please_fill=True,
config_attributes={"CWL": cwl_target})
data["get_form_sheet_href"] = url_for("get_param_form_sheet",
job_id=job_id)
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
return jsonify({"data": data, "messages": messages})
def get_param_form_sheet(job_id):
messages = []
data = {}
try:
login_required()
sheet_path = get_path("job_param_sheet_temp", job_id=job_id)
return send_from_directory(os.path.dirname(sheet_path),
os.path.basename(sheet_path),
attachment_filename=job_id +
".input_params" +
os.path.splitext(sheet_path)[1],
as_attachment=True)
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
return jsonify({"data": data, "messages": messages})
def import_cwl():
messages = []
data = []
# try:
login_required()
if 'file' not in request.files:
sys.exit('No file received.')
import_file = request.files['file']
if import_file.filename == '':
sys.exit("No file specified.")
if not is_allowed_file(import_file.filename, type="CWL"):
sys.exit(
"Wrong file type. Only files with following extensions are allowed: "
+ ", ".join(allowed_extensions_by_type["CWL"]))
# save the file to the CWL directory:
import_filename = secure_filename(import_file.filename)
imported_filepath = os.path.join(app.config['CWL_DIR'], import_filename)
import_file.save(imported_filepath)
# generate job template config:
job_templ_filepath = get_path("job_templ", cwl_target=import_filename)
generate_job_template_from_cwl(cwl_file=imported_filepath,
output_file=job_templ_filepath,
show_please_fill=True)
messages.append({
"type": "success",
"text": import_file.filename + " successfully imported."
})
# except SystemExit as e:
# messages.append( { "type":"error", "text": str(e) } )
# except:
# messages.append( {
# "type":"error",
# "text":"An uknown error occured."
# } )
return jsonify({"data": data, "messages": messages})
def get_param_values():
messages = []
data = {}
try:
login_required()
request_json = request.get_json()
param_values, configs = gen_form_sheet(
output_file_path=None,
template_config_file_path=get_path(
"job_templ", cwl_target=request_json["cwl_target"]),
has_multiple_runs=request_json["run_mode"],
run_names=request_json["run_names"],
param_is_run_specific=request_json["param_modes"],
show_please_fill=True,
config_attributes={"CWL": request_json["cwl_target"]})
data = {"param_values": param_values, "configs": configs}
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An unkown error occured."})
return jsonify({"data": data, "messages": messages})
def delete_job(job_id):
run_ids = get_run_ids(job_id)
_, could_not_be_terminated, could_not_be_cleaned = terminate_runs(job_id, run_ids, mode="delete")
if len(could_not_be_terminated) > 0 or len(could_not_be_cleaned) > 0 :
return {
"status": "failed run termination",
"could_not_be_terminated": could_not_be_terminated,
"could_not_be_cleaned": could_not_be_cleaned
}
try:
job_dir = get_path("job_dir", job_id)
if os.path.exists(job_dir):
rmtree(job_dir)
return {
"status": "success"
}
except Exception as e:
return {
"status": "failed to remove job dir",
"errorMessage": str(e)
}
def create_job(): # generate param form sheet with data sent
# by the client
messages = []
data = {}
try:
login_required()
request_json = request.get_json()
job_id = request_json["job_id"]
sheet_format = request_json["sheet_format"]
sheet_form_temp = get_path("job_param_sheet_temp",
job_id=job_id,
param_sheet_format=sheet_format)
if not os.path.isfile(sheet_form_temp):
sys.exit("Could not find the filled parameter sheet \"" +
sheet_form_temp + "\".")
if not is_allowed_file(sheet_form_temp, type="spreadsheet"):
sys.exit("The filled parameter sheet \"" + sheet_form_temp +
"\" has the wrong file type. " +
"Only files with following extensions are allowed: " +
", ".join(allowed_extensions_by_type["spreadsheet"]))
validate_paths = request_json["validate_paths"]
search_paths = request_json["search_paths"]
search_dir = os.path.abspath(
remove_non_printable_characters(request_json["search_dir"]))
include_subdirs_for_searching = request_json[
"include_subdirs_for_searching"]
if search_paths:
# test if search dir exists:
if not os.path.isdir(search_dir):
sys.exit("The specified search dir \"" + search_dir +
"\" does not exist or is not a directory.")
# Move form sheet to job dir:
try:
sheet_form = get_path("job_param_sheet", job_id=job_id)
os.remove(sheet_form)
except:
pass
sheet_form_dest_path = get_path("job_param_sheet",
job_id=job_id,
param_sheet_format=sheet_format)
move(sheet_form_temp, sheet_form_dest_path)
# create yaml runs:
make_yaml_runs(sheet_file=sheet_form_dest_path,
output_basename="",
default_run_id=get_job_name_from_job_id(job_id),
always_include_run_in_output_name=True,
output_suffix=".yaml",
output_dir=get_path("runs_yaml_dir", job_id=job_id),
validate_paths=validate_paths,
search_paths=search_paths,
search_subdirs=include_subdirs_for_searching,
input_dir=search_dir)
# make output directories:
run_ids = get_run_ids(job_id)
for run_id in run_ids:
run_out_dir = get_path("run_out_dir", job_id, run_id)
if not os.path.exists(run_out_dir):
os.mkdir(run_out_dir)
messages.append({
"type": "success",
"text": "Successfully created job \"" + job_id + "\"."
})
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
return jsonify({"data": data, "messages": messages})
def send_filled_param_form_sheet():
messages = []
data = []
try:
login_required()
if 'file' not in request.files:
sys.exit('No file received.')
import_file = request.files['file']
if import_file.filename == '':
sys.exit("No file specified.")
if not is_allowed_file(import_file.filename, type="spreadsheet"):
sys.exit(
"Wrong file type. Only files with following extensions are allowed: "
+ ", ".join(allowed_extensions_by_type["spreadsheet"]))
import_fileext = os.path.splitext(
import_file.filename)[1].strip(".").lower()
# save the file to the CWL directory:
metadata = json_loads(request.form.get("meta"))
job_id = metadata["job_id"]
import_filepath = get_path("job_param_sheet_temp",
job_id=job_id,
param_sheet_format=import_fileext)
import_file.save(import_filepath)
validate_paths = metadata["validate_paths"]
search_paths = metadata["search_paths"]
search_dir = os.path.abspath(
remove_non_printable_characters(metadata["search_dir"]))
include_subdirs_for_searching = metadata[
"include_subdirs_for_searching"]
if search_paths:
# test if search dir exists:
if not os.path.isdir(search_dir):
sys.exit("The specified search dir \"" + search_dir +
"\" does not exist or is not a directory.")
except SystemExit as e:
messages.append({"type": "error", "text": str(e)})
except:
messages.append({"type": "error", "text": "An uknown error occured."})
if len(messages) == 0:
try:
# validate the uploaded form sheet:
validation_result = only_validate_xls(
sheet_file=import_filepath,
validate_paths=validate_paths,
search_paths=search_paths,
search_subdirs=include_subdirs_for_searching,
input_dir=search_dir)
if validation_result != "VALID":
os.remove(import_filepath)
sys.exit(validation_result)
except SystemExit as e:
messages.append({
"type":
"error",
"text":
"The provided form failed validation: " + str(e)
})
except:
messages.append({
"type": "error",
"text": "An uknown error occured."
})
if len(messages) == 0:
messages.append({
"type":
"success",
"text":
"The filled form was successfully imported and validated."
})
return jsonify({"data": data, "messages": messages})
def get_job_list():
messages = []
jobs = []
try:
login_required()
job_ids = get_job_ids()
# for each dir:
# - check if form sheet present
# - if yes:
# - read in form sheet attributes
# - get list of runs
for job_id in job_ids:
job_dir = get_path("job_dir", job_id=job_id)
try:
job_param_sheet = get_path("job_param_sheet", job_id=job_id)
except SystemExit as e:
continue
job_param_sheet_attributes = get_job_templ_info("attributes", job_templ_filepath=job_param_sheet)
if "CWL" not in job_param_sheet_attributes.keys() or job_param_sheet_attributes["CWL"] == "":
messages.append( {
"type":"warning",
"text":"No CWL target was specified in the job_param_sheet of job \"" +
job_id + "\". Ignoring."
} )
continue
cwl_target = job_param_sheet_attributes["CWL"]
jobs.append({
"job_id": job_id,
"job_abs_path": job_dir,
"cwl_target": cwl_target
})
except SystemExit as e:
messages.append( {
"type":"error",
"text": str(e)
} )
except:
messages.append( {
"type":"error",
"text":"An unkown error occured reading the execution directory."
} )
# get exec profiles names:
exec_profile_names = list(app.config["EXEC_PROFILES"].keys())
exec_profile_params = {}
for exec_profile_name in exec_profile_names:
exec_profile_params[exec_profile_name] = {
"max_retries": app.config["EXEC_PROFILES"][exec_profile_name]["max_retries"],
"max_parallel_exec": app.config["EXEC_PROFILES"][exec_profile_name]["max_parallel_exec"],
"allow_user_decrease_max_parallel_exec": app.config["EXEC_PROFILES"][exec_profile_name]["allow_user_decrease_max_parallel_exec"],
}
return jsonify({
"data": {
"exec_profiles": exec_profile_names,
"exec_profile_params": exec_profile_params,
"jobs": jobs
},
"messages": messages
}
)
def exec_runs(job_id, run_ids, exec_profile_name, cwl, user_id=None, max_parrallel_exec_user_def=None):
# check if runs are already running:
already_running_runs = []
db_job_id_request = query_info_from_db(job_id)
for run_id in run_ids:
db_run_id_request = db_job_id_request.filter(Exec.run_id==run_id).distinct()
if db_run_id_request.count() > 0:
# find latest:
run_info = db_run_id_request.filter(Exec.id==max([r.id for r in db_run_id_request])).first()
if run_info.time_finished is None or run_info.status == "finished":
already_running_runs.append(run_id)
run_ids = sorted(list(set(run_ids) - set(already_running_runs)))
# create new exec entry in database:
exec_profile = app.config["EXEC_PROFILES"][exec_profile_name]
if not max_parrallel_exec_user_def is None and \
exec_profile["allow_user_decrease_max_parallel_exec"] and \
max_parrallel_exec_user_def < exec_profile["max_parallel_exec"]:
exec_profile["max_parallel_exec"] = max_parrallel_exec_user_def
exec_db_entry = {}
for run_id in run_ids:
exec_db_entry[run_id] = Exec(
job_id=job_id,
run_id=run_id,
cwl=get_path("cwl", cwl_target=cwl),
yaml=get_path("run_yaml", job_id=job_id, run_id=run_id),
out_dir=get_path("run_out_dir", job_id=job_id, run_id=run_id),
global_temp_dir=app.config["TEMP_DIR"],
log=get_path("run_log", job_id=job_id, run_id=run_id),
status="queued",
err_message="",
retry_count=0,
time_started=datetime.now(),
time_finished=None, #*
timeout_limit=None, #*
pid=-1, #*
user_id=user_id if not user_id is None else None,
exec_profile=exec_profile,
exec_profile_name=exec_profile_name
)
#* will be set by the background process itself
db.session.add(exec_db_entry[run_id])
db_commit()
# start the background process:
# the child process will be detached from the parent
# and manages the its status in the database autonomously,
# even if the parent process is terminated / fails,
# the child process will continue
started_runs = []
for run_id in run_ids:
create_background_process(
[
python_interpreter,
os.path.join(basedir, "cwlab_bg_exec.py"),
app.config["SQLALCHEMY_DATABASE_URI"],
str(exec_db_entry[run_id].id),
str(app.config["DEBUG"])
],
get_path("debug_run_log", job_id=job_id, run_id=run_id)
)
started_runs.append(run_id)
return started_runs, already_running_runs
def read_run_yaml(job_id, run_id):
yaml_path = get_path("run_yaml", job_id, run_id)
content, _ = read_file_content(yaml_path)
return content
def read_run_log(job_id, run_id):
log_path = get_path("run_log", job_id, run_id)
if not os.path.isfile(log_path):
return "Run not started yet."
content, _ = read_file_content(log_path)
return content