def prepare_job_env(): # generate param form sheet with data sent # by the client messages = [] data = {} try: login_required() request_json = request.get_json() job_id = request_json["job_id"] # prepare job directory job_dir = get_path("job_dir", job_id) if not os.path.exists(job_dir): os.mkdir(job_dir) runs_yaml_dir = get_path("runs_yaml_dir", job_id) if not os.path.exists(runs_yaml_dir): os.mkdir(runs_yaml_dir) runs_out_dir = get_path("runs_out_dir", job_id) if not os.path.exists(runs_out_dir): os.mkdir(runs_out_dir) runs_log_dir = get_path("runs_log_dir", job_id) if not os.path.exists(runs_log_dir): os.mkdir(runs_log_dir) runs_input_dir = get_path("runs_input_dir", job_id) if not os.path.exists(runs_input_dir): os.mkdir(runs_input_dir) except SystemExit as e: messages.append({"type": "error", "text": str(e)}) except: messages.append({"type": "error", "text": "An uknown error occured."}) return jsonify({"data": data, "messages": messages})
def change_password(): messages = [] data = {"success": False} try: login_required() data_req = request.get_json() old_password = data_req["old_password"] new_password = data_req["new_password"] new_rep_password = data_req["new_rep_password"] change_password_(current_user.get_id(), old_password, new_password, new_rep_password) data = {"success": True} messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully changed password. You will be logged out." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def download(): messages = [] data = {} try: login_required() data_req = json_loads(request.form.get("meta")) job_id = data_req["job_id"] run_id = data_req["run_id"] path = data_req["path"] send_file = data_req["send_file"] assert path != "" and os.path.exists( path), "Path does not exist or you have no permission to enter it." path = normalize_path(path) allowed_dirs = get_allowed_base_dirs(job_id=job_id, run_id=run_id, allow_input=False, allow_upload=False, allow_download=True) base_dir = check_if_path_in_dirs(path, allowed_dirs) assert base_dir is not None, "Path does not exist or you have no permission to enter it." if os.path.isdir(path): data["zip_path"] = zip_dir(path) if send_file: return send_from_directory( os.path.dirname(path), os.path.basename(path), attachment_filename=os.path.basename(path), as_attachment=True) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def delete_account(): messages = [] data = {"success": False} try: login_required() data_req = request.get_json() username = data_req["username"] current_user_id = current_user.get_id() assert username == load_user( current_user_id ).username, "The entered username does not match your account." delete_user(current_user_id) data = {"success": True} messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully deleted your account. You will be logged out." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_param_form_sheet(): messages = [] data = {} try: data_req = request.args.to_dict() access_token = data_req["access_token"] login_required(access_token=access_token) job_name = data_req["job_name"] temp_dir_name = data_req["temp_dir_name"] temp_dir = os.path.join(app.config["TEMP_DIR"], temp_dir_name) hits = fetch_files_in_dir( dir_path=temp_dir, file_exts=allowed_extensions_by_type["spreadsheet"], search_string=job_name, return_abspaths=True) assert len(hits) == 1, \ "The requested file does not exist or you have no permission access it" sheet_path = hits[0]["file_abspath"] return send_from_directory(os.path.dirname(sheet_path), os.path.basename(sheet_path), attachment_filename=job_name + "_inputs" + os.path.splitext(sheet_path)[1], as_attachment=True) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def change_password(): messages = [] data = {"success": False} try: validate_local_login_enabled() data_req = request.get_json() access_token = data_req["access_token"] username = data_req["username"] old_password = data_req["old_password"] new_password = data_req["new_password"] new_rep_password = data_req["new_rep_password"] login_required(access_token=access_token, username=username) db_connector.user_manager.change_password(username, old_password, new_password, new_rep_password) data = {"success": True} messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully changed password. You will be logged out." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_run_details(): messages = [] data = {} try: login_required() req_data = request.get_json() job_id = req_data["job_id"] run_id = req_data["run_id"] log_content = read_run_log(job_id, run_id) yaml_content = read_run_yaml(job_id, run_id) data = { "log": log_content, "yaml": yaml_content } except SystemExit as e: messages.append( { "type":"error", "text": str(e) } ) except: messages.append( { "type":"error", "text":"An unkown error occured." } ) return jsonify({ "data":data, "messages":messages })
def get_run_list(): messages = [] data = {} try: login_required() data_req = request.get_json() job_id = data_req["job_id"] run_ids = get_run_ids(job_id) run_ids.sort() data["run_ids"] = run_ids except SystemExit as e: messages.append( { "type":"error", "text": str(e) } ) except: messages.append( { "type":"error", "text":"An unkown error occured reading the execution directory." } ) return jsonify({ "data": data, "messages": messages } )
def get_run_list(): messages = [] data = {} try: data_req = request.get_json() access_token = data_req["access_token"] username = data_req["username"] login_required(access_token=access_token, username=username) job_name = data_req["job_name"] run_names = job_manager.get_run_names(job_name) run_names.sort() data["run_names"] = run_names if len(run_names) == 0: messages.append({ "type": "info", "text": f"No runs available in this job." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append( handle_unknown_error( e, alt_err_message= "An unkown error occured reading the execution directory", return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def import_wf_by_trs_uri(): messages = [] data = [] try: data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token) trs_uri = data_req["trs_uri"] import_name = data_req["import_name"] import_worflow_by_trs(uri=trs_uri, name=import_name, access_token=access_token) messages.append({ "time": get_time_string(), "type": "success", "text": import_name + " successfully imported." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def send_filled_param_values(): messages = [] data = [] try: login_required() request_json = request.get_json() param_values = request_json["param_values"] param_configs = request_json["param_configs"] cwl_target = request_json["cwl_target"] job_id = request_json["job_id"] import_filepath = get_path("job_param_sheet_temp", job_id=job_id, param_sheet_format="xlsx") validate_paths = request_json["validate_paths"] search_paths = request_json["search_paths"] search_dir = os.path.abspath( remove_non_printable_characters(request_json["search_dir"])) include_subdirs_for_searching = request_json[ "include_subdirs_for_searching"] if search_paths: # test if search dir exists: if not os.path.isdir(search_dir): sys.exit("The specified search dir \"" + search_dir + "\" does not exist or is not a directory.") generate_xls_from_param_values( param_values=param_values, configs=param_configs, output_file=import_filepath, validate_paths=validate_paths, search_paths=search_paths, search_subdirs=include_subdirs_for_searching, input_dir=search_dir, config_attributes={"CWL": cwl_target}) except SystemExit as e: messages.append({ "type": "error", "text": "The provided form failed validation: " + str(e) }) except: messages.append({"type": "error", "text": "An uknown error occured."}) if len(messages) == 0: messages.append({ "type": "success", "text": "The filled form was successfully imported and validated." }) return jsonify({"data": data, "messages": messages})
def get_general_user_info(): messages = [] data = {} try: login_required() data = get_user_info(current_user.get_id()) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def modify_or_delete_users(): messages = [] data = [] try: validate_local_login_enabled() data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token, admin=True) action = data_req["action"] user_selection = data_req["user_selection"] value = data_req["value"] if action == "delete": for username in user_selection: user_manager.delete(username) messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully deleted users: \"" + ", ".join(user_selection) + "\"" }) if action == "set_status": for user in user_selection: change_user_status_or_level(get_user_by_username(user).id, new_status=value) messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully set status on users: \"" + ", ".join(user_selection) + "\"" }) if action == "set_level": for user in user_selection: change_user_status_or_level(get_user_by_username(user).id, new_level=value) messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully set level on users: \"" + ", ".join(user_selection) + "\"" }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_all_users_info(): messages = [] data = [] try: login_required(admin=True) data = get_all_users_info_() except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_job_templ_list(): # returns list of job templates # for already imported CWL documents messages = [] templates = [] try: login_required() templates = get_job_templates() except SystemExit as e: messages.append({"type": "error", "text": str(e)}) except: messages.append({"type": "error", "text": "An uknown error occured."}) return jsonify({"data": templates, "messages": messages})
def logout(): messages = [] data = {"success": False} try: login_required() logout_user() data["success"] = True except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def upload_cwl_zip(): messages = [] data = {} try: metadata = json_loads(request.form.get("meta")) access_token = metadata["access_token"] login_required(access_token=access_token) assert 'file' in request.files, 'No file received.' import_wf_file = request.files['file'] assert import_wf_file.filename != '', "No file specified." assert is_allowed_file(import_wf_file.filename, type="zip"), ( "Wrong file type. Only files with following extensions are allowed: " + ", ".join(allowed_extensions_by_type["CWL"])) # save the file to the CWL directory: import_wf_filename = secure_filename(import_wf_file.filename) temp_upload_dir = make_temp_dir() imported_filepath = os.path.join(temp_upload_dir, import_wf_filename) import_wf_file.save(imported_filepath) temp_extract_dir = make_temp_dir() unzip_dir(imported_filepath, temp_extract_dir) try: rmtree(temp_upload_dir) except Exception as e: pass data["temp_dir"] = temp_extract_dir messages.append({ "time": get_time_string(), "type": "success", "text": import_wf_file.filename + " was successfully uploaded and extracted." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def start_exec( ): # returns all parmeter and its default mode (global/job specific) # for a given xls config messages = [] try: data_req = request.get_json() access_token = data_req["access_token"] username = data_req["username"] login_required(access_token=access_token, username=username) access_token = data_req["access_token"] job_name = data_req["job_name"] run_names = sorted(data_req["run_names"]) exec_profile_name = data_req["exec_profile"] max_parrallel_exec_user_def = int( data_req["parallel_exec"]) if "parallel_exec" in data_req.keys( ) else None started_runs, already_running_runs = exec_runs( job_name, run_names, exec_profile_name, username=username, max_parrallel_exec_user_def=max_parrallel_exec_user_def, access_token=access_token) if len(started_runs) > 0: messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully started execution for runs: " + ", ".join(started_runs) }) if len(already_running_runs) > 0: messages.append({ "time": get_time_string(), "type": "warning", "text": "Following runs are already running or have already finished: " + ", ".join(already_running_runs) + ". To restart them, reset them first." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": {}, "messages": messages})
def get_job_templ_list(): # returns list of job templates # for already imported CWL documents messages = [] templates = [] try: data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token) templates = get_job_templates() except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": templates, "messages": messages})
def get_all_users_info(): messages = [] data = [] try: validate_local_login_enabled() data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token, admin=True) data = get_all_users_info_() except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_job_list(): messages = [] jobs = [] try: data_req = request.get_json() access_token = data_req["access_token"] username = data_req["username"] login_required(access_token=access_token, username=username) job_info = [] for job in job_manager.get_jobs_info_for_user( username ): #! should be changed once workflows are integrated into the database job["wf_name"] = os.path.basename(job["wf_target"]) job_info.append(job) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) messages.append( handle_unknown_error( e, alt_err_message= "An unkown error occured reading the execution directory", return_front_end_message=True)) # get exec profiles names: exec_profile_names = list(app.config["EXEC_PROFILES"].keys()) exec_profile_params = {} for exec_profile_name in exec_profile_names: exec_profile_params[exec_profile_name] = { "workflow_type": app.config["EXEC_PROFILES"][exec_profile_name]["workflow_type"], "max_retries": app.config["EXEC_PROFILES"][exec_profile_name]["max_retries"], "max_parallel_exec": app.config["EXEC_PROFILES"][exec_profile_name] ["max_parallel_exec"], "enable_queueing": app.config["EXEC_PROFILES"][exec_profile_name]["enable_queueing"], "allow_user_decrease_max_parallel_exec": app.config["EXEC_PROFILES"][exec_profile_name] ["allow_user_decrease_max_parallel_exec"], } return jsonify({ "data": { "exec_profiles": exec_profile_names, "exec_profile_params": exec_profile_params, "jobs": job_info }, "messages": messages })
def upload_file(): messages = [] data = {} try: login_required() if 'file' not in request.files: sys.exit('No file received.') import_file = request.files['file'] if import_file.filename == '': sys.exit("No file specified.") filename = secure_filename(import_file.filename) # save the file to the CWL directory: metadata = json_loads(request.form.get("meta")) dir_path = metadata["dir_path"] job_id = metadata["job_id"] if "job_id" in metadata.keys() else None # check if dir path allowed: allowed_dirs = get_allowed_base_dirs(job_id=job_id, allow_input=False, allow_upload=True, allow_download=False) if dir_path == "": sys.exit( "Path does not exist or you have no permission to enter it.") dir_path = os.path.realpath(dir_path) if not os.path.exists(dir_path) or \ not os.path.isdir(dir_path) or \ check_if_path_in_dirs(dir_path, allowed_dirs) is None: sys.exit( "Path does not exist or you have no permission to enter it.") import_filepath = os.path.join(dir_path, filename) import_file.save(import_filepath) data["file_path"] = import_filepath messages.append({ "type": "success", "text": "Successfully uploaded file." }) except SystemExit as e: messages.append({"type": "error", "text": str(e)}) except: messages.append({"type": "error", "text": "An unkown error occured."}) return jsonify({"data": data, "messages": messages})
def get_run_details(): messages = [] data = {} try: login_required() req_data = request.get_json() job_id = req_data["job_id"] run_id = req_data["run_id"] log_content = read_run_log(job_id, run_id) yaml_content = read_run_input(job_id, run_id) data = {"log": log_content, "yaml": yaml_content} except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def upload_file(): messages = [] data = {} try: # load metadata: metadata = json_loads(request.form.get("meta")) dir_path = metadata["dir_path"] job_name = metadata["job_name"] if "job_name" in metadata.keys( ) else None access_token = metadata["access_token"] login_required(access_token=access_token) assert 'file' in request.files, 'No file received.' import_file = request.files['file'] assert import_file.filename != '', "No file specified." filename = secure_filename(import_file.filename) # check if dir path allowed: allowed_dirs = get_allowed_base_dirs(job_name=job_name, allow_input=False, allow_upload=True, allow_download=False) assert dir_path != "", "Path does not exist or you have no permission to enter it." dir_path = normalize_path(dir_path) assert os.path.exists(dir_path) and \ os.path.isdir(dir_path) and \ check_if_path_in_dirs(dir_path, allowed_dirs) is not None, \ "Path does not exist or you have no permission to enter it." import_filepath = os.path.join(dir_path, filename) import_file.save(import_filepath) data["file_path"] = import_filepath messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully uploaded file." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def terminate_runs(): messages = [] data = {} try: data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token) job_name = data_req["job_name"] run_names = sorted(data_req["run_names"]) mode = data_req["mode"] # one of terminate, reset, delete succeeded, could_not_be_terminated, could_not_be_cleaned = terminate_runs_by_name( job_name, run_names, mode) if len(succeeded) > 0: messages.append({ "time": get_time_string(), "type": "success", "text": "Successfully terminated/reset/deleted runs: " + ", ".join(succeeded) }) if len(could_not_be_terminated) > 0: messages.append({ "time": get_time_string(), "type": "warning", "text": "Following runs could not be terminated: " + ", ".join(could_not_be_terminated) }) if len(could_not_be_cleaned) > 0: messages.append({ "time": get_time_string(), "type": "warning", "text": "Following runs could not be cleaned: " + ", ".join(could_not_be_cleaned) }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_run_status(): messages = [] data = {} try: login_required() data_req = request.get_json() data = get_run_info(data_req["job_id"], data_req["run_ids"]) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append( handle_unknown_error( e, alt_err_message= "An unkown error occured reading the execution directory", return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def delete_job(): messages = [] data = {} try: data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token) job_name = data_req["job_name"] results = delete_job_by_name(job_name) if results["status"] == "success": pass elif results["status"] == "failed run termination": if len(results["could_not_be_terminated"]) > 0: messages.append({ "time": get_time_string(), "type": "error", "text": "Following runs could not be terminated: " + ", ".join(results["could_not_be_terminated"]) }) if len(results["could_not_be_cleaned"]) > 0: messages.append({ "time": get_time_string(), "type": "error", "text": "Following runs could not be cleaned: " + ", ".join(results["could_not_be_cleaned"]) }) else: messages.append({ "time": get_time_string(), "type": "error", "text": "Could not delete job dir for \"" + job_name + "\"." }) except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})
def get_param_form_sheet(job_id): messages = [] data = {} try: login_required() sheet_path = get_path("job_param_sheet_temp", job_id=job_id) return send_from_directory(os.path.dirname(sheet_path), os.path.basename(sheet_path), attachment_filename=job_id + ".input_params" + os.path.splitext(sheet_path)[1], as_attachment=True) except SystemExit as e: messages.append({"type": "error", "text": str(e)}) except: messages.append({"type": "error", "text": "An uknown error occured."}) return jsonify({"data": data, "messages": messages})
def start_exec(): # returns all parmeter and its default mode (global/job specific) # for a given xls config messages = [] # try: login_required() user_id = current_user.get_id() if app.config["ENABLE_USERS"] else None data = request.get_json() cwl_target = data["cwl_target"] job_id = data["job_id"] run_ids = sorted(data["run_ids"]) exec_profile_name = data["exec_profile"] max_parrallel_exec_user_def = int(data["parallel_exec"]) if "parallel_exec" in data.keys() else None started_runs, already_running_runs = exec_runs( job_id, run_ids, exec_profile_name, cwl_target, user_id, max_parrallel_exec_user_def ) if len(started_runs) > 0: messages.append({ "type":"success", "text":"Successfully started execution for runs: " + ", ".join(started_runs) }) if len(already_running_runs) > 0: messages.append({ "type":"warning", "text":"Following runs are already running or have already finished: " + ", ".join(already_running_runs) + ". To restart them, reset them first." }) # except SystemExit as e: # messages.append( { # "type":"error", # "text": str(e) # } ) # except: # messages.append( { # "type":"error", # "text":"An unkown error occured." # } ) return jsonify({ "data":{}, "messages":messages })
def get_run_details(): messages = [] data = {} try: data_req = request.get_json() access_token = data_req["access_token"] login_required(access_token=access_token) data_req = request.get_json() job_name = data_req["job_name"] run_name = data_req["run_name"] log_content = read_run_log(job_name, run_name) yaml_content = read_run_input(job_name, run_name) data = {"log": log_content, "yaml": yaml_content} except AssertionError as e: messages.append(handle_known_error(e, return_front_end_message=True)) except Exception as e: messages.append(handle_unknown_error(e, return_front_end_message=True)) return jsonify({"data": data, "messages": messages})