def move_all_to_iRODS_permanent_coll_atomic(submission_id):
result = {}
files = db_model_operations.retrieve_all_files_for_submission(
submission_id)
for file_to_submit in files:
if not file_to_submit.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE:
return models.Result(
False,
message=
"The metadata must be added before moving the file to the iRODS permanent coll."
)
for file_to_submit in files:
task_id = launch_move_to_permanent_coll_task(file_to_submit.id)
if task_id:
tasks_dict = {
'type': move_to_permanent_coll_task.name,
'status': constants.PENDING_ON_WORKER_STATUS
}
update_dict = {
'set__file_submission_status':
constants.SUBMISSION_IN_PROGRESS_STATUS,
'set__tasks_dict__' + task_id: tasks_dict
}
db_model_operations.update_file_from_dict(file_to_submit.id,
update_dict)
result[str(file_to_submit.id)] = True
else:
result[str(file_to_submit.id)] = False
return models.Result(result)
def resubmit_jobs_for_file(submission_id, file_id, file_to_resubmit=None):
''' Function called for resubmitting the jobs for a file, as a result
of a POST request on a specific file. It checks for permission and
resubmits the jobs in the corresponding queue, depending on permissions.
Throws:
InvalidId -- InvalidId -- if the submission_id is not corresponding to MongoDB rules - checking done offline (pymongo specific error)
DoesNotExist -- if there is not submission with this id in the DB (Mongoengine specific error)
#### -- NOT ANY MORE! -- ResourceNotFoundException -- my custom exception, thrown if a file with the file_id does not exist within this submission.
'''
if file_to_resubmit == None:
file_to_resubmit = data_access.FileDataAccess.retrieve_submitted_file(
file_id)
permissions = utils.get_file_permissions(file_to_resubmit.file_path_client)
submission = data_access.SubmissionDataAccess.retrieve_only_submission_fields(
submission_id, ['_is_uploaded_as_serapis', 'sanger_user_id'])
upld_as_srp_flag = submission._is_uploaded_as_serapis
if permissions == constants.NOACCESS:
if upld_as_srp_flag == True:
result = models.Result(
False, error_dict={constants.PERMISSION_DENIED: [file_id]})
result.message = "ERROR: serapis attempting to upload files to iRODS but hasn't got read access. "
result.message = result.message + "Please give access to serapis user or resubmit your request with '_is_uploaded_as_serapis' : False."
result.message = result.message + "In the latter case you will also be required to run the following script ... on the cluster."
return result
file_logic = app_logic.FileBusinessLogicBuilder.build_from_type(
file_to_resubmit.file_type)
result = file_logic.resubmit_presubmission_failed_tasks(
file_id, file_to_resubmit)
if result:
file_logic.status_checker.check_and_update_all_statuses(
file_id, file_to_resubmit)
#db_model_operations.check_and_update_all_file_statuses(file_id)
return models.Result(result)
def check_file_ready_for_move_to_permanent_coll_task(cls, file_obj):
error_dict = {}
if not file_obj.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE:
utils.append_to_errors_dict(
file_obj.id, constants.FILE_NOT_READY_FOR_THIS_OPERATION,
error_dict)
return models.Result(False, error_dict)
return models.Result(True)
def check_file_ready_for_add_meta_task(cls, file_obj):
error_dict = {}
if not file_obj.file_submission_status == constants.READY_FOR_IRODS_SUBMISSION_STATUS:
utils.append_to_errors_dict(
file_obj.id, constants.FILE_NOT_READY_FOR_THIS_OPERATION,
error_dict)
md5_check = cls._check_file_md5(file_obj)
if not md5_check.result:
utils.append_to_errors_dict(file_obj.id, constants.UNEQUAL_MD5,
error_dict)
if error_dict:
return models.Result(False, error_dict)
return models.Result(True)
def resubmit_tasks_by_type(cls, task_types_list, file_obj, submission):
tasks_to_resubmit = cls._select_and_remove_tasks_by_type(
file_obj.tasks, task_types_list)
if tasks_to_resubmit:
return cls.resubmit_presubmission_tasks(tasks_to_resubmit,
file_obj, submission)
return models.Result(False)
def submit_all_to_irods_nonatomic(submission_id):
files = db_model_operations.retrieve_all_files_for_submission(
submission_id)
results = {}
for file_to_submit in files:
submission_result = submit_file_to_irods(file_to_submit.id)
results[str(file_to_submit.id)] = submission_result.result
return models.Result(results)
def resubmit_jobs_for_submission(submission_id):
files = data_access.SubmissionDataAccess.retrieve_all_files_for_submission(
submission_id)
result = {}
for f in files:
f_resubm_result = resubmit_jobs_for_file(submission_id, str(f.id), f)
result[str(f.id)] = f_resubm_result.result
return models.Result(result)
def add_meta_to_all_staged_files_nonatomic(submission_id):
results = {}
files = db_model_operations.retrieve_all_files_for_submission(
submission_id)
for file_to_submit in files:
add_meta_result = add_meta_to_staged_file(file_to_submit.id,
file_to_submit)
results[str(file_to_submit.id)] = add_meta_result.result
if add_meta_result.error_dict:
logging.error("ERRORs dict: %s", str(add_meta_result.error_dict))
return models.Result(results)
def add_meta_to_all_staged_files_atomic(submission_id):
files = db_model_operations.retrieve_all_files_for_submission(
submission_id)
results = {}
# Check all files are ok:
ready_to_submit = True
error_dict = {}
for file_to_submit in files:
file_check_result = check_file(file_to_submit.id, file_to_submit)
if file_check_result.result == False:
ready_to_submit = False
error_dict.update(file_check_result.error_dict)
results[str(file_to_submit.id)] = False
else:
results[str(file_to_submit.id)] = True
if not ready_to_submit:
return models.Result(results, error_dict)
# Submit all files to iRODS if they are ok:
results = {}
for file_to_submit in files:
task_id = launch_add_mdata2irods_task(file_to_submit.id, submission_id)
if task_id:
tasks_dict = {
'type': add_mdata_to_IRODS_file_task.name,
'status': constants.PENDING_ON_WORKER_STATUS
}
update_dict = {
'set__file_submission_status':
constants.SUBMISSION_IN_PREPARATION_STATUS,
'set__tasks_dict__' + task_id: tasks_dict
}
db_model_operations.update_file_from_dict(file_to_submit.id,
update_dict)
results[str(file_to_submit.id)] = True
else:
results[str(file_to_submit.id)] = False
return models.Result(results)
def move_all_to_iRODS_permanent_coll_nonatomic(submission_id):
result = {}
files = db_model_operations.retrieve_all_files_for_submission(
submission_id)
for file_to_submit in files:
subm_result = move_file_to_iRODS_permanent_coll(
file_to_submit.id, file_to_submit)
result[str(file_to_submit.id)] = subm_result.result
if subm_result.error_dict:
logging.error(
"MOVE file from staging area to permanent coll - FAILED: %s",
str(subm_result.error_dict))
return models.Result(result)
def resubmit_presubmission_tasks(cls, tasks_to_resubmit, file_obj,
submission):
resubm_tasks_dict = cls.batch_tasks_launcher.submit_list_of_tasks(
tasks_to_resubmit,
file_obj,
user_id=submission.sanger_user_id,
as_serapis=submission._is_uploaded_as_serapis)
file_obj.tasks.update(resubm_tasks_dict)
file_status = cls._decide_file_presubmission_status(
submission._is_uploaded_as_serapis)
cls.after_tasks_submission(file_obj.id, tasks_to_resubmit,
file_obj.tasks, file_status)
return models.Result(True)
def _check_file_md5(cls, file_obj):
error_dict = {}
f_md5_correct = cls._check_file_md5_eq(file_obj.file_path_client,
file_obj.md5)
if not f_md5_correct:
logging.error(
"Unequal md5: calculated file's md5 is different than the contents of %s.md5",
file_obj.file_path_client)
utils.append_to_errors_dict(str(file_obj.id),
constants.UNEQUAL_MD5, error_dict)
if file_obj.index_file.file_path_client:
index_md5_correct = cls._check_file_md5_eq(
file_obj.index_file.file_path_client, file_obj.index_file.md5)
if not index_md5_correct:
logging.error(
"Unequal md5: calculated index file's md5 is different than the contents of %s.md5",
file_obj.index_file.file_path_client)
utils.append_to_errors_dict("index - " + str(file_obj.id),
constants.UNEQUAL_MD5, error_dict)
if error_dict:
return models.Result(False, error_dict=error_dict)
return models.Result(True)
def move_file_to_iRODS_permanent_coll(file_id, file_obj=None):
if not file_obj:
file_obj = db_model_operations.retrieve_submitted_file(file_id)
if not file_obj.file_submission_status == constants.METADATA_ADDED_TO_STAGED_FILE:
return models.Result(
False,
message=
"The metadata must be added before moving the file to the iRODS permanent coll."
)
task_id = launch_move_to_permanent_coll_task(file_id)
if task_id:
tasks_dict = {
'type': move_to_permanent_coll_task.name,
'status': constants.PENDING_ON_WORKER_STATUS
}
update_dict = {
'set__file_submission_status':
constants.SUBMISSION_IN_PROGRESS_STATUS,
'set__tasks_dict__' + task_id: tasks_dict
}
db_model_operations.update_file_from_dict(file_obj.id, update_dict)
return models.Result(True)
return models.Result(False, message="No task id returned.")
def submit_submission_task(cls, task_name, file_id, file_obj=None):
''' This method submits a list of submission tasks to the queues in order to be executed on the workers.'''
if not task_name:
logging.error(
"Submit_submission_tasks error -- No submission task submitted, because the list of tasks received as param is empty!"
)
return models.Result(False)
if not file_obj:
file_obj = data_access.FileDataAccess.retrieve_submitted_file(
file_id)
is_ready = status_checker.FileStatusCheckerForSubmissionTasks.check_file_ready_for_task(
task_name, file_obj)
if not is_ready.result:
return is_ready
# Submitting a list of tasks:
submitted_tasks_dict = cls.batch_tasks_launcher.submit_list_of_tasks(
[task_name], file_obj=file_obj, user_id=None)
file_obj.tasks.update(submitted_tasks_dict)
file_submission_status = cls._infer_file_submission_status(task_name)
cls.after_tasks_submission(file_id, [task_name], file_obj.tasks,
file_submission_status)
return models.Result(True)
def submit_file_to_irods(file_id):
file_check_result = check_file(file_id, None)
if file_check_result.result == True:
task_id = launch_submit2irods_task(file_id)
if task_id:
tasks_dict = {
'type': submit_to_permanent_iRODS_coll_task.name,
'status': constants.PENDING_ON_WORKER_STATUS
}
update_dict = {
'set__file_submission_status':
constants.SUBMISSION_IN_PROGRESS_STATUS,
'set__tasks_dict__' + task_id: tasks_dict
}
db_model_operations.update_file_from_dict(file_id, update_dict)
return models.Result(True)
return file_check_result
def add_meta_to_staged_file(file_id, file_to_submit=None):
if not file_to_submit:
file_to_submit = db_model_operations.retrieve_submitted_file(file_id)
file_check_result = check_file(file_to_submit.id, file_to_submit)
if file_check_result.result == True:
task_id = launch_add_mdata2irods_task(file_to_submit.id,
file_to_submit.submission_id)
if task_id:
tasks_dict = {
'type': add_mdata_to_IRODS_file_task.name,
'status': constants.PENDING_ON_WORKER_STATUS
}
update_dict = {
'set__file_submission_status':
constants.SUBMISSION_IN_PREPARATION_STATUS,
'set__tasks_dict__' + task_id: tasks_dict
}
db_model_operations.update_file_from_dict(file_to_submit.id,
update_dict)
return models.Result(True)
logging.error("File check for adding metadata FAILED: %s",
str(file_check_result.error_dict))
return file_check_result
def create_submission(user_id, data):
''' Creates a submission - given a list of files: initializes
a submission object and submits jobs for all the files in the list.
Params:
list of files that the new submission contains
Returns:
a dictionary containing:
{ submission_id : 123 , errors: {..dictionary of errors..}
Throws:
ValueError - when the data provided in data parameter is incorrect.
'''
# Make a copy of the request data
submission_data = copy.deepcopy(data)
user_id = submission_data.pop('sanger_user_id')
# Get files from the request data:
file_paths_list = get_files_list_from_request(submission_data)
if not file_paths_list:
raise exceptions.NotEnoughInformationProvidedException(
msg="Files list is empty.")
verif_result = verify_file_paths(file_paths_list)
if verif_result.error_dict:
return verif_result
#submission_data['files_list'] = file_paths_list
try:
submission_data.pop('files_list')
except KeyError:
pass
try:
submission_data.pop('dir_path')
except KeyError:
pass
# Verify the upload permissions:
upld_as_serapis = True # the default
if '_is_uploaded_as_serapis' in submission_data:
upld_as_serapis = submission_data['_is_uploaded_as_serapis']
if upld_as_serapis and constants.NOACCESS in verif_result.warning_dict:
result = models.Result(False, verif_result.warning_dict, None)
result.message = "ERROR: serapis attempting to upload files to iRODS but hasn't got read access. "
result.message = result.message + "Please give access to serapis user or resubmit your request with '_is_uploaded_as_serapis' : False."
result.message = result.message + "In the latter case you will also be required to run the following script ... on the cluster."
return result
# Should ref genome be smth mandatory?????
if 'reference_genome' in submission_data:
ref_gen = submission_data.pop('reference_genome')
ref_gen = data_access.ReferenceGenomeDataAccess.get_or_insert_reference_genome(
ref_gen)
submission_data['file_reference_genome_id'] = ref_gen.id
else:
logging.warning("NO reference provided!")
# raise exceptions.NotEnoughInformationProvidedException(msg="There was no information regarding the reference genome provided")
# Split the files_list in files and indexes:
file_et_index_map = associate_files_with_indexes(file_paths_list).result
if hasattr(file_et_index_map, 'error_dict') and getattr(
file_et_index_map, 'error_dict'):
return models.Result(False,
error_dict=file_et_index_map.error_dict,
warning_dict=file_et_index_map.warning_dict)
files_type = utils.check_all_files_same_type(file_et_index_map)
if not files_type:
return models.Result(
False,
message="All the files in a submission must be of the same type.")
submission_data['file_type'] = files_type
print("FILE TYPE -- taken from files: ", submission_data['file_type'])
# Build the submission:
submission_id = model_builder.SubmissionBuilder.build_and_save(
submission_data, user_id)
if not submission_id:
return models.Result(False, message="Submission couldn't be created.")
submission = data_access.SubmissionDataAccess.retrieve_submission(
submission_id)
print("SUBMISSION FILE TYPE ---- ", submission.file_type, vars(submission))
submission_logic_layer = app_logic.SubmissionBusinessLogic(
submission.file_type)
files_init = submission_logic_layer.init_and_submit_files(
file_et_index_map, submission)
if not files_init:
return models.Result(
False,
message=
'Files could not be initialised, the submission was not created.')
if not upld_as_serapis:
return models.Result(
str(submission.id),
warning_dict="You have requested to upload the files as " +
user_id +
", therefore you need to run the following...script on the cluster"
)
return models.Result(str(submission.id))
