def move_all_to_iRODS_permanent_coll_atomic(submission_id): result = {} files = db_model_operations.retrieve_all_files_for_submission( submission_id) for file_to_submit in files: if not file_to_submit.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE: return models.Result( False, message= "The metadata must be added before moving the file to the iRODS permanent coll." ) for file_to_submit in files: task_id = launch_move_to_permanent_coll_task(file_to_submit.id) if task_id: tasks_dict = { 'type': move_to_permanent_coll_task.name, 'status': constants.PENDING_ON_WORKER_STATUS } update_dict = { 'set__file_submission_status': constants.SUBMISSION_IN_PROGRESS_STATUS, 'set__tasks_dict__' + task_id: tasks_dict } db_model_operations.update_file_from_dict(file_to_submit.id, update_dict) result[str(file_to_submit.id)] = True else: result[str(file_to_submit.id)] = False return models.Result(result)
def resubmit_jobs_for_file(submission_id, file_id, file_to_resubmit=None): ''' Function called for resubmitting the jobs for a file, as a result of a POST request on a specific file. It checks for permission and resubmits the jobs in the corresponding queue, depending on permissions. Throws: InvalidId -- InvalidId -- if the submission_id is not corresponding to MongoDB rules - checking done offline (pymongo specific error) DoesNotExist -- if there is not submission with this id in the DB (Mongoengine specific error) #### -- NOT ANY MORE! -- ResourceNotFoundException -- my custom exception, thrown if a file with the file_id does not exist within this submission. ''' if file_to_resubmit == None: file_to_resubmit = data_access.FileDataAccess.retrieve_submitted_file( file_id) permissions = utils.get_file_permissions(file_to_resubmit.file_path_client) submission = data_access.SubmissionDataAccess.retrieve_only_submission_fields( submission_id, ['_is_uploaded_as_serapis', 'sanger_user_id']) upld_as_srp_flag = submission._is_uploaded_as_serapis if permissions == constants.NOACCESS: if upld_as_srp_flag == True: result = models.Result( False, error_dict={constants.PERMISSION_DENIED: [file_id]}) result.message = "ERROR: serapis attempting to upload files to iRODS but hasn't got read access. " result.message = result.message + "Please give access to serapis user or resubmit your request with '_is_uploaded_as_serapis' : False." result.message = result.message + "In the latter case you will also be required to run the following script ... on the cluster." return result file_logic = app_logic.FileBusinessLogicBuilder.build_from_type( file_to_resubmit.file_type) result = file_logic.resubmit_presubmission_failed_tasks( file_id, file_to_resubmit) if result: file_logic.status_checker.check_and_update_all_statuses( file_id, file_to_resubmit) #db_model_operations.check_and_update_all_file_statuses(file_id) return models.Result(result)
def check_file_ready_for_move_to_permanent_coll_task(cls, file_obj): error_dict = {} if not file_obj.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE: utils.append_to_errors_dict( file_obj.id, constants.FILE_NOT_READY_FOR_THIS_OPERATION, error_dict) return models.Result(False, error_dict) return models.Result(True)
def check_file_ready_for_add_meta_task(cls, file_obj): error_dict = {} if not file_obj.file_submission_status == constants.READY_FOR_IRODS_SUBMISSION_STATUS: utils.append_to_errors_dict( file_obj.id, constants.FILE_NOT_READY_FOR_THIS_OPERATION, error_dict) md5_check = cls._check_file_md5(file_obj) if not md5_check.result: utils.append_to_errors_dict(file_obj.id, constants.UNEQUAL_MD5, error_dict) if error_dict: return models.Result(False, error_dict) return models.Result(True)
def resubmit_tasks_by_type(cls, task_types_list, file_obj, submission): tasks_to_resubmit = cls._select_and_remove_tasks_by_type( file_obj.tasks, task_types_list) if tasks_to_resubmit: return cls.resubmit_presubmission_tasks(tasks_to_resubmit, file_obj, submission) return models.Result(False)
def submit_all_to_irods_nonatomic(submission_id): files = db_model_operations.retrieve_all_files_for_submission( submission_id) results = {} for file_to_submit in files: submission_result = submit_file_to_irods(file_to_submit.id) results[str(file_to_submit.id)] = submission_result.result return models.Result(results)
def resubmit_jobs_for_submission(submission_id): files = data_access.SubmissionDataAccess.retrieve_all_files_for_submission( submission_id) result = {} for f in files: f_resubm_result = resubmit_jobs_for_file(submission_id, str(f.id), f) result[str(f.id)] = f_resubm_result.result return models.Result(result)
def add_meta_to_all_staged_files_nonatomic(submission_id): results = {} files = db_model_operations.retrieve_all_files_for_submission( submission_id) for file_to_submit in files: add_meta_result = add_meta_to_staged_file(file_to_submit.id, file_to_submit) results[str(file_to_submit.id)] = add_meta_result.result if add_meta_result.error_dict: logging.error("ERRORs dict: %s", str(add_meta_result.error_dict)) return models.Result(results)
def add_meta_to_all_staged_files_atomic(submission_id): files = db_model_operations.retrieve_all_files_for_submission( submission_id) results = {} # Check all files are ok: ready_to_submit = True error_dict = {} for file_to_submit in files: file_check_result = check_file(file_to_submit.id, file_to_submit) if file_check_result.result == False: ready_to_submit = False error_dict.update(file_check_result.error_dict) results[str(file_to_submit.id)] = False else: results[str(file_to_submit.id)] = True if not ready_to_submit: return models.Result(results, error_dict) # Submit all files to iRODS if they are ok: results = {} for file_to_submit in files: task_id = launch_add_mdata2irods_task(file_to_submit.id, submission_id) if task_id: tasks_dict = { 'type': add_mdata_to_IRODS_file_task.name, 'status': constants.PENDING_ON_WORKER_STATUS } update_dict = { 'set__file_submission_status': constants.SUBMISSION_IN_PREPARATION_STATUS, 'set__tasks_dict__' + task_id: tasks_dict } db_model_operations.update_file_from_dict(file_to_submit.id, update_dict) results[str(file_to_submit.id)] = True else: results[str(file_to_submit.id)] = False return models.Result(results)
def move_all_to_iRODS_permanent_coll_nonatomic(submission_id): result = {} files = db_model_operations.retrieve_all_files_for_submission( submission_id) for file_to_submit in files: subm_result = move_file_to_iRODS_permanent_coll( file_to_submit.id, file_to_submit) result[str(file_to_submit.id)] = subm_result.result if subm_result.error_dict: logging.error( "MOVE file from staging area to permanent coll - FAILED: %s", str(subm_result.error_dict)) return models.Result(result)
def resubmit_presubmission_tasks(cls, tasks_to_resubmit, file_obj, submission): resubm_tasks_dict = cls.batch_tasks_launcher.submit_list_of_tasks( tasks_to_resubmit, file_obj, user_id=submission.sanger_user_id, as_serapis=submission._is_uploaded_as_serapis) file_obj.tasks.update(resubm_tasks_dict) file_status = cls._decide_file_presubmission_status( submission._is_uploaded_as_serapis) cls.after_tasks_submission(file_obj.id, tasks_to_resubmit, file_obj.tasks, file_status) return models.Result(True)
def _check_file_md5(cls, file_obj): error_dict = {} f_md5_correct = cls._check_file_md5_eq(file_obj.file_path_client, file_obj.md5) if not f_md5_correct: logging.error( "Unequal md5: calculated file's md5 is different than the contents of %s.md5", file_obj.file_path_client) utils.append_to_errors_dict(str(file_obj.id), constants.UNEQUAL_MD5, error_dict) if file_obj.index_file.file_path_client: index_md5_correct = cls._check_file_md5_eq( file_obj.index_file.file_path_client, file_obj.index_file.md5) if not index_md5_correct: logging.error( "Unequal md5: calculated index file's md5 is different than the contents of %s.md5", file_obj.index_file.file_path_client) utils.append_to_errors_dict("index - " + str(file_obj.id), constants.UNEQUAL_MD5, error_dict) if error_dict: return models.Result(False, error_dict=error_dict) return models.Result(True)
def move_file_to_iRODS_permanent_coll(file_id, file_obj=None): if not file_obj: file_obj = db_model_operations.retrieve_submitted_file(file_id) if not file_obj.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE: return models.Result( False, message= "The metadata must be added before moving the file to the iRODS permanent coll." ) task_id = launch_move_to_permanent_coll_task(file_id) if task_id: tasks_dict = { 'type': move_to_permanent_coll_task.name, 'status': constants.PENDING_ON_WORKER_STATUS } update_dict = { 'set__file_submission_status': constants.SUBMISSION_IN_PROGRESS_STATUS, 'set__tasks_dict__' + task_id: tasks_dict } db_model_operations.update_file_from_dict(file_obj.id, update_dict) return models.Result(True) return models.Result(False, message="No task id returned.")
def submit_submission_task(cls, task_name, file_id, file_obj=None): ''' This method submits a list of submission tasks to the queues in order to be executed on the workers.''' if not task_name: logging.error( "Submit_submission_tasks error -- No submission task submitted, because the list of tasks received as param is empty!" ) return models.Result(False) if not file_obj: file_obj = data_access.FileDataAccess.retrieve_submitted_file( file_id) is_ready = status_checker.FileStatusCheckerForSubmissionTasks.check_file_ready_for_task( task_name, file_obj) if not is_ready.result: return is_ready # Submitting a list of tasks: submitted_tasks_dict = cls.batch_tasks_launcher.submit_list_of_tasks( [task_name], file_obj=file_obj, user_id=None) file_obj.tasks.update(submitted_tasks_dict) file_submission_status = cls._infer_file_submission_status(task_name) cls.after_tasks_submission(file_id, [task_name], file_obj.tasks, file_submission_status) return models.Result(True)
def submit_file_to_irods(file_id): file_check_result = check_file(file_id, None) if file_check_result.result == True: task_id = launch_submit2irods_task(file_id) if task_id: tasks_dict = { 'type': submit_to_permanent_iRODS_coll_task.name, 'status': constants.PENDING_ON_WORKER_STATUS } update_dict = { 'set__file_submission_status': constants.SUBMISSION_IN_PROGRESS_STATUS, 'set__tasks_dict__' + task_id: tasks_dict } db_model_operations.update_file_from_dict(file_id, update_dict) return models.Result(True) return file_check_result
def add_meta_to_staged_file(file_id, file_to_submit=None): if not file_to_submit: file_to_submit = db_model_operations.retrieve_submitted_file(file_id) file_check_result = check_file(file_to_submit.id, file_to_submit) if file_check_result.result == True: task_id = launch_add_mdata2irods_task(file_to_submit.id, file_to_submit.submission_id) if task_id: tasks_dict = { 'type': add_mdata_to_IRODS_file_task.name, 'status': constants.PENDING_ON_WORKER_STATUS } update_dict = { 'set__file_submission_status': constants.SUBMISSION_IN_PREPARATION_STATUS, 'set__tasks_dict__' + task_id: tasks_dict } db_model_operations.update_file_from_dict(file_to_submit.id, update_dict) return models.Result(True) logging.error("File check for adding metadata FAILED: %s", str(file_check_result.error_dict)) return file_check_result
def create_submission(user_id, data): ''' Creates a submission - given a list of files: initializes a submission object and submits jobs for all the files in the list. Params: list of files that the new submission contains Returns: a dictionary containing: { submission_id : 123 , errors: {..dictionary of errors..} Throws: ValueError - when the data provided in data parameter is incorrect. ''' # Make a copy of the request data submission_data = copy.deepcopy(data) user_id = submission_data.pop('sanger_user_id') # Get files from the request data: file_paths_list = get_files_list_from_request(submission_data) if not file_paths_list: raise exceptions.NotEnoughInformationProvidedException( msg="Files list is empty.") verif_result = verify_file_paths(file_paths_list) if verif_result.error_dict: return verif_result #submission_data['files_list'] = file_paths_list try: submission_data.pop('files_list') except KeyError: pass try: submission_data.pop('dir_path') except KeyError: pass # Verify the upload permissions: upld_as_serapis = True # the default if '_is_uploaded_as_serapis' in submission_data: upld_as_serapis = submission_data['_is_uploaded_as_serapis'] if upld_as_serapis and constants.NOACCESS in verif_result.warning_dict: result = models.Result(False, verif_result.warning_dict, None) result.message = "ERROR: serapis attempting to upload files to iRODS but hasn't got read access. " result.message = result.message + "Please give access to serapis user or resubmit your request with '_is_uploaded_as_serapis' : False." result.message = result.message + "In the latter case you will also be required to run the following script ... on the cluster." return result # Should ref genome be smth mandatory????? if 'reference_genome' in submission_data: ref_gen = submission_data.pop('reference_genome') ref_gen = data_access.ReferenceGenomeDataAccess.get_or_insert_reference_genome( ref_gen) submission_data['file_reference_genome_id'] = ref_gen.id else: logging.warning("NO reference provided!") # raise exceptions.NotEnoughInformationProvidedException(msg="There was no information regarding the reference genome provided") # Split the files_list in files and indexes: file_et_index_map = associate_files_with_indexes(file_paths_list).result if hasattr(file_et_index_map, 'error_dict') and getattr( file_et_index_map, 'error_dict'): return models.Result(False, error_dict=file_et_index_map.error_dict, warning_dict=file_et_index_map.warning_dict) files_type = utils.check_all_files_same_type(file_et_index_map) if not files_type: return models.Result( False, message="All the files in a submission must be of the same type.") submission_data['file_type'] = files_type print("FILE TYPE -- taken from files: ", submission_data['file_type']) # Build the submission: submission_id = model_builder.SubmissionBuilder.build_and_save( submission_data, user_id) if not submission_id: return models.Result(False, message="Submission couldn't be created.") submission = data_access.SubmissionDataAccess.retrieve_submission( submission_id) print("SUBMISSION FILE TYPE ---- ", submission.file_type, vars(submission)) submission_logic_layer = app_logic.SubmissionBusinessLogic( submission.file_type) files_init = submission_logic_layer.init_and_submit_files( file_et_index_map, submission) if not files_init: return models.Result( False, message= 'Files could not be initialised, the submission was not created.') if not upld_as_serapis: return models.Result( str(submission.id), warning_dict="You have requested to upload the files as " + user_id + ", therefore you need to run the following...script on the cluster" ) return models.Result(str(submission.id))