def _from_to(self, get_from, put_to):
module_logger.debug(debug_utils.get_name() + " get_from: " + get_from +
" put_to: " + put_to)
if self.copy:
module_logger.debug(debug_utils.get_name() + " copying")
os.makedirs(put_to, exist_ok=True)
if self.show_log:
rsync_cmd = 'rsync -auLv '
else:
rsync_cmd = 'rsync -auL '
rsync_cmd += get_from + os.sep + '*' + ' ' + put_to
module_logger.info("rsync_cmd: " + rsync_cmd)
completed_rsync_process = subprocess.run(rsync_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
module_logger.info(completed_rsync_process.stdout)
else:
module_logger.debug(debug_utils.get_name() + " linking")
os.makedirs(put_to, exist_ok=True)
os_utils.lndir(get_from,
put_to,
self.show_log,
ignore_existing_dst_files=True)
def get_apply_hand_reclassification_data(self, subject_info,
output_study_dir):
module_logger.debug(debug_utils.get_name())
for directory in self.archive.available_apply_handreclassification_dir_fullpaths(
subject_info):
get_from = directory
module_logger.debug(debug_utils.get_name() + " get_from: " +
get_from)
put_to = output_study_dir
self._from_to(get_from, put_to)
def _get_processed_data(self, directories, output_dir):
for directory in directories:
get_from = directory
module_logger.debug(debug_utils.get_name() + " get_from: " +
get_from)
put_to = output_dir
module_logger.debug(debug_utils.get_name() + " put_to: " +
put_to)
self._from_to(get_from, put_to)
def _get_unprocessed_data(self, directories, subject_info, output_dir):
for directory in directories:
get_from = directory
module_logger.debug(debug_utils.get_name() + " get_from: " +
get_from)
last_sep_loc = get_from.rfind(os.sep)
unproc_loc = get_from.rfind("_" + 'unproc')
sub_dir = get_from[last_sep_loc + 1:unproc_loc]
put_to = output_dir + os.sep + subject_info.subject_id + "_" + subject_info.classifier + os.sep + 'unprocessed' + os.sep + sub_dir
module_logger.debug(debug_utils.get_name() + " put_to: " +
put_to)
self._from_to(get_from, put_to)
def get_supplemental_structural_preproc_data(self, subject_info,
output_dir):
module_logger.debug(debug_utils.get_name())
self._get_preprocessed_data(
self.archive.
available_supplemental_structural_preproc_dir_full_paths(
subject_info), output_dir)
def submit_process_data_jobs(self, stage, prior_job=None):
module_logger.debug(debug_utils.get_name())
# go ahead and submit the standard process data job and then
# submit an additional freesurfer assessor job
standard_process_data_jobno, all_process_data_jobs = super().submit_process_data_jobs(stage, prior_job)
if OneSubjectJobSubmitter._SUPPRESS_FREESURFER_ASSESSOR_JOB:
module_logger.info("freesufer assessor job not submitted because freesurfer assessor creation has been suppressed")
return standard_process_data_jobno, all_process_data_jobs
if stage >= ccf_processing_stage.ProcessingStage.PROCESS_DATA:
if standard_process_data_jobno:
fs_submit_cmd = 'qsub -W depend=afterok:' + standard_process_data_jobno + ' ' + self.freesurfer_assessor_script_name
else:
fs_submit_cmd = 'qsub ' + self.freesurfer_assessor_script_name
completed_submit_process = subprocess.run(
fs_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)
fs_job_no = str_utils.remove_ending_new_lines(completed_submit_process.stdout)
all_process_data_jobs.append(fs_job_no)
return fs_job_no, all_process_data_jobs
else:
module_logger.info("freesurfer assessor job not submitted because of requested processing stage")
return standard_process_data_jobno, all_process_data_jobs
def starttime_file_name(self):
module_logger.debug(debug_utils.get_name())
starttime_file_name = self.working_directory_name
starttime_file_name += os.path.sep
starttime_file_name += self.PIPELINE_NAME
starttime_file_name += '.starttime'
return starttime_file_name
def mark_running_status(self, stage):
module_logger.debug(debug_utils.get_name())
if stage > ccf_processing_stage.ProcessingStage.PREPARE_SCRIPTS:
mark_cmd = self._xnat_pbs_jobs_home
mark_cmd += os.sep + self.PIPELINE_NAME
mark_cmd += os.sep + self.PIPELINE_NAME
mark_cmd += '.XNAT_MARK_RUNNING_STATUS'
mark_cmd += ' --user='******' --password='******' --server=' + str_utils.get_server_name(
self.put_server)
mark_cmd += ' --project=' + self.project
mark_cmd += ' --subject=' + self.subject
mark_cmd += ' --classifier=' + self.classifier
mark_cmd += ' --resource=RunningStatus'
mark_cmd += ' --queued'
completed_mark_cmd_process = subprocess.run(
mark_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
print(completed_mark_cmd_process.stdout)
return
def create_get_data_job_script(self):
"""Create the script to be submitted to perform the get data job"""
module_logger.debug(debug_utils.get_name())
script_name = self.get_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.get_data_program_path + ' \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --classifier=' + self.classifier + ' \\' + os.linesep)
script.write(' --working-dir=' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('## Need to convert some files to symlinks as they are added to or rewritten by MSMAll' + os.linesep)
script.write('if [ -d "' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '" ] ; then ' + os.linesep)
script.write(' pushd ' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + os.linesep)
script.write(' find . -type l | egrep "\.spec$|prefiltered_func_data.*clean*" | xargs -I \'{}\' sh -c \'cp --remove-destination $(readlink {}) {}\'' + os.linesep)
script.write(' popd' + os.linesep)
script.write('fi' + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_get_data_job_script(self):
"""Create the script to be submitted to perform the get data job"""
module_logger.debug(debug_utils.get_name())
script_name = self.get_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=4gb' +
os.linesep)
script.write('#PBS -q HCPput' + os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write(self.get_data_program_path + ' \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --ref-project=' + self.structural_reference_project +
' \\' + os.linesep)
script.write(' --working-dir=' + self.working_directory_name +
os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_get_data_script(self):
logger.debug(debug_utils.get_name())
script_name = self._get_data_script_name()
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=4gb' +
os.linesep)
script.write('#PBS -q HCPput' + os.linesep)
script.write('#PBS -o ' + self._working_directory_name + os.linesep)
script.write('#PBS -e ' + self._working_directory_name + os.linesep)
script.write(os.linesep)
script.write(self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME +
os.sep + self.PIPELINE_NAME + '.XNAT_GET.sh \\' +
os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --working-dir=' + self._working_directory_name +
os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_clean_data_script(self):
logger.debug(debug_utils.get_name())
script_name = self._clean_data_script_name()
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=4gb' + os.linesep)
script.write('#PBS -o ' + self._working_directory_name + os.linesep)
script.write('#PBS -e ' + self._working_directory_name + os.linesep)
script.write(os.linesep)
script.write('echo "Newly created or modified files:"' + os.linesep)
script.write('find ' + self._working_directory_name + os.path.sep + self.subject +
' -type f -newer ' + self._starttime_file_name() + os.linesep)
script.write(os.linesep)
script.write('echo "Removing NOT newly created or modified files."' + os.linesep)
script.write('find ' + self._working_directory_name + os.path.sep + self.subject +
' -not -newer ' + self._starttime_file_name() + ' -delete')
script.write(os.linesep)
script.write('echo "Removing any XNAT catalog files still around."' + os.linesep)
script.write('find ' + self._working_directory_name +
' -name "*_catalog.xml" -delete')
script.write(os.linesep)
script.write('echo "Remaining files:"' + os.linesep)
script.write('find ' + self._working_directory_name + os.path.sep + self.subject + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_get_data_job_script(self):
"""Create the script to be submitted to perform the get data job"""
module_logger.debug(debug_utils.get_name())
script_name = self.get_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.get_data_program_path + ' \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --classifier=' + self.classifier + ' \\' + os.linesep)
script.write(' --working-dir=' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_mark_no_longer_running_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.mark_no_longer_running_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.mark_running_status_program_path + ' \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' + str_utils.get_server_name(self.put_server) + '" \\' + os.linesep)
script.write(' --project="' + self.project + '" \\' + os.linesep)
script.write(' --subject="' + self.subject + '" \\' + os.linesep)
script.write(' --classifier="' + self.classifier + '" \\' + os.linesep)
if self.scan:
script.write(' --scan="' + self.scan + '" \\' + os.linesep)
script.write(' --resource="' + 'RunningStatus' + '" \\' + os.linesep)
script.write(' --done' + os.linesep)
script.write(os.linesep)
script.write("rm -rf " + self.mark_completion_directory_name)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _get_unprocessed_data(self, directories, subject_info, output_dir):
for directory in directories:
get_from = directory
module_logger.debug(debug_utils.get_name() + " get_from: " +
get_from)
last_sep_loc = get_from.rfind(os.sep)
unproc_loc = get_from.rfind(self.archive.NAME_DELIMITER +
self.archive.UNPROC_SUFFIX)
sub_dir = get_from[last_sep_loc + 1:unproc_loc]
put_to = output_dir + os.sep + subject_info.subject_id + os.sep + 'unprocessed'
put_to += os.sep + subject_info.classifier + os.sep + sub_dir
module_logger.debug(debug_utils.get_name() + " put_to: " +
put_to)
self._from_to(get_from, put_to)
def get_structural_unproc_data(self, subject_info, output_study_dir):
for directory in self.archive.available_structural_unproc_dir_fullpaths(
subject_info):
get_from = directory
module_logger.debug(debug_utils.get_name() + " get_from: " +
get_from)
last_sep_loc = get_from.rfind(os.sep)
unproc_loc = get_from.rfind('_' + self.archive.UNPROC_SUFFIX)
sub_dir = get_from[last_sep_loc + 1:unproc_loc]
put_to = output_study_dir + os.sep + subject_info.subject_id + os.sep + 'unprocessed' + \
os.sep + self.archive.TESLA_SPEC + os.sep + sub_dir
module_logger.debug(debug_utils.get_name() + " put_to: " + put_to)
self._from_to(get_from, put_to)
def create_process_data_job_script(self):
module_logger.debug(debug_utils.get_name())
xnat_pbs_jobs_control_folder = os_utils.getenv_required(
'XNAT_PBS_JOBS_CONTROL')
subject_info = ccf_subject.SubjectInfo(self.project, self.subject,
self.classifier, self.scan)
script_name = self.process_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
mem_limit_str = str(self.mem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN) + ':gpus=1:K20x'
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',mem=' + mem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
load_cuda = "module load cuda-9.1"
xnat_pbs_setup_singularity_load = 'module load ' + self._get_xnat_pbs_setup_script_singularity_version(
)
xnat_pbs_setup_singularity_process = 'singularity exec --nv -B ' + xnat_pbs_jobs_control_folder + ':/opt/xnat_pbs_jobs_control,' \
+ self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() \
+ ',' + self._get_xnat_pbs_setup_script_gradient_coefficient_path() + ':/export/HCP/gradient_coefficient_files' \
+ ' ' + self._get_xnat_pbs_setup_script_singularity_container_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_qunexrun_path()
parameter_line = ' --parameterfolder=' + self._get_xnat_pbs_setup_script_singularity_qunexparameter_path(
)
studyfolder_line = ' --studyfolder=' + self.working_directory_name + '/' + self.subject + '_' + self.classifier
subject_line = ' --subjects=' + self.subject + '_' + self.classifier
overwrite_line = ' --overwrite=yes'
hcppipelineprocess_line = ' --hcppipelineprocess=DiffusionPreprocessing'
with open(script_name, 'w') as script:
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(load_cuda + os.linesep)
script.write(xnat_pbs_setup_singularity_load + os.linesep)
script.write(os.linesep)
script.write(xnat_pbs_setup_singularity_process + ' \\' +
os.linesep)
script.write(parameter_line + ' \\' + os.linesep)
script.write(studyfolder_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(overwrite_line + ' \\' + os.linesep)
self._group_list = []
script.write(' --boldlist="' + self._expand(self.groups) +
'" \\' + os.linesep)
script.write(hcppipelineprocess_line + os.linesep)
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_get_data_job_script(self):
"""Create the script to be submitted to perform the get data job"""
module_logger.debug(debug_utils.get_name())
script_name = self.get_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' +
os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' +
self._get_db_name() + os.linesep)
script.write('module load ' +
self._get_xnat_pbs_setup_script_singularity_version() +
os.linesep)
script.write(os.linesep)
script.write(
'singularity exec -B ' +
self._get_xnat_pbs_setup_script_archive_root() + ',' +
self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' +
self._get_xnat_pbs_setup_script_singularity_container_xnat_path() +
' ' + self.get_data_program_path + ' \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --classifier=' + self.classifier + ' \\' + os.linesep)
if self.scan:
script.write(' --scan=' + self.scan + ' \\' + os.linesep)
script.write(' --working-dir=' + self.working_directory_name + ' \\' +
os.linesep)
script.write(os.linesep)
script.write('find ' + self.working_directory_name + os.sep +
self.subject + '_' + self.classifier +
'/unprocessed/ -maxdepth 1 -mindepth 1 ')
script.write('\( -type d -not -path ' + self.working_directory_name +
os.sep + self.subject + '_' + self.classifier +
'/unprocessed/' + self.scan)
script.write(' -path \'' + self.working_directory_name + os.sep +
self.subject + '_' + self.classifier +
'/unprocessed/[rt]fMRI_*_[AP][PA]\'')
script.write(' -o -path \'' + self.working_directory_name + os.sep +
self.subject + '_' + self.classifier +
'/unprocessed/T1w_*\'')
script.write(' -o -path \'' + self.working_directory_name + os.sep +
self.subject + '_' + self.classifier +
'/unprocessed/T2w_*\'')
script.write(' -o -path \'' + self.working_directory_name + os.sep +
self.subject + '_' + self.classifier +
'/unprocessed/Diffusion\' \) -exec rm -rf \'{}\' \;' +
os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_freesurfer_assessor_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT_CREATE_FREESURFER_ASSESSOR script to the working directory
freesurfer_assessor_source_path = self.xnat_pbs_jobs_home
freesurfer_assessor_source_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_source_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_source_path += '.XNAT_CREATE_FREESURFER_ASSESSOR'
freesurfer_assessor_dest_path = self.working_directory_name
freesurfer_assessor_dest_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_dest_path += '.XNAT_CREATE_FREESURFER_ASSESSOR'
shutil.copy(freesurfer_assessor_source_path,
freesurfer_assessor_dest_path)
os.chmod(freesurfer_assessor_dest_path, stat.S_IRWXU | stat.S_IRWXG)
# write the freesurfer assessor submission script (that calls the .XNAT_CREATE_FREESURFER_ASSESSOR script)
script_name = self.freesurfer_assessor_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' +
os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' +
self._get_db_name() + os.linesep)
script.write(os.linesep)
script_line = freesurfer_assessor_dest_path
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
session_classifier_line = ' --session-classifier=' + self.classifier
wdir_line = ' --working-dir=' + self.working_directory_name
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(session_classifier_line + ' \\' + os.linesep)
script.write(wdir_line + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_scripts(self, stage):
module_logger.debug(debug_utils.get_name())
super().create_scripts(stage)
if OneSubjectJobSubmitter._SUPPRESS_FREESURFER_ASSESSOR_JOB:
return
if stage >= ccf_processing_stage.ProcessingStage.PREPARE_SCRIPTS:
self.create_freesurfer_assessor_script()
def create_work_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.work_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',vmem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep
script_line += self.PIPELINE_NAME + os.sep + self.PIPELINE_NAME + '.XNAT.sh'
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.setup_script
script = open(script_name, 'w')
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
for group in self._group_list:
script.write(' --group=' + group + ' \\' + os.linesep)
for name in self._concat_name_list:
script.write(' --concat-name=' + name + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def mark_no_longer_running_script_name(self):
module_logger.debug(debug_utils.get_name())
name = self.mark_completion_directory_name
name += os.sep + self.subject
name += '.' + self.PIPELINE_NAME
if self.scan:
name += '_' + self.scan
name += '.' + self.project
name += '.' + 'MARK_RUNNING_STATUS_job.sh'
return name
def create_setup_file(self):
module_logger.debug(debug_utils.get_name())
setup_source_file_name = self.PIPELINE_NAME + '.SetUp.sh'
xnat_pbs_jobs_control = os.getenv('XNAT_PBS_JOBS_CONTROL')
if xnat_pbs_jobs_control:
setup_source_file_name = xnat_pbs_jobs_control + os.sep + setup_source_file_name
shutil.copyfile(setup_source_file_name, self.setup_file_name)
os.chmod(self.setup_file_name, stat.S_IRWXU | stat.S_IRWXG)
def submit_no_longer_running_jobs(self, stage, prior_job=None): module_logger.debug(debug_utils.get_name()) if prior_job: cmd = 'qsub -W depend=afterany:' + prior_job + ' ' + self.mark_no_longer_running_script_name else: cmd = 'qsub ' + self.mark_no_longer_running_script_name completed_submit_process = subprocess.run( cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True) job_no = str_utils.remove_ending_new_lines(completed_submit_process.stdout) return job_no, [job_no]
def create_get_data_job_script(self):
"""Create the script to be submitted to perform the get data job"""
module_logger.debug(debug_utils.get_name())
script_name = self.get_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ')
script.write(self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.get_data_program_path + ' \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --classifier=' + self.classifier + ' \\' + os.linesep)
if self.scan:
script.write(' --scan=' + self.scan + ' \\' + os.linesep)
script.write(' --working-dir=' + self.working_directory_name + ' \\' + os.linesep)
# if self.use_prescan_normalized:
# script.write(' --use-prescan-normalized' + ' \\' + os.linesep)
# script.write(' --delay-seconds=60' + os.linesep)
script.write(os.linesep)
script.write('rm -rf ' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '/unprocessed/T1w_MPR_vNav_4e_RMS' + os.linesep)
script.write('## Convert FreeSurfer output from links to files for rerun' + os.linesep)
#script.write('if [ -d "' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '/T1w/' + self.subject + '_' + self.classifier + '" ] ; then ' + os.linesep)
#script.write(' pushd ' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '/T1w/' + self.subject + '_' + self.classifier + os.linesep)
#script.write('if [ -d "' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '/T1w" ] ; then ' + os.linesep)
#script.write(' pushd ' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '/T1w' + os.linesep)
script.write('if [ -d "' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + '" ] ; then ' + os.linesep)
script.write(' pushd ' + self.working_directory_name + os.sep + self.subject + '_' + self.classifier + os.linesep)
script.write(' find . -type l | xargs -I \'{}\' sh -c \'cp --remove-destination $(readlink {}) {}\'' + os.linesep)
script.write(' popd' + os.linesep)
script.write('fi' + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_scripts(self, stage):
module_logger.debug(debug_utils.get_name())
if stage >= ccf_processing_stage.ProcessingStage.PREPARE_SCRIPTS:
self.create_get_data_job_script()
self.create_process_data_job_script()
self.create_clean_data_script()
self.create_put_data_script()
self.create_check_data_job_script()
self.create_mark_no_longer_running_script()
else:
module_logger.info("Scripts not created")
def check_data_job_script_name(self):
"""
Name of script to be submitted as a job to perform the check data functionality.
"""
module_logger.debug(debug_utils.get_name())
name = self.check_data_directory_name
name += os.sep + self.subject
name += '.' + self.PIPELINE_NAME
if self.scan:
name += '_' + self.scan
name += '.' + self.project
name += '.' + self.session
name += '.' + 'XNAT_CHECK_DATA_job.sh'
return name
def starttime_file_name(self): module_logger.debug(debug_utils.get_name()) starttime_file_name = self.working_directory_name starttime_file_name += os.path.sep starttime_file_name +=self.subject starttime_file_name += os.path.sep starttime_file_name +='ProcessingInfo' starttime_file_name += os.path.sep starttime_file_name += self.subject if self.scan: starttime_file_name += '_' + self.scan starttime_file_name += '.' + self.PIPELINE_NAME starttime_file_name += '.starttime' return starttime_file_name
def _create_work_script(self):
logger.debug(debug_utils.get_name())
script_name = self._work_script_name()
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit = str(self.walltime_limit_hours) + ':00:00'
vmem_limit = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=1:ppn=1,walltime=' + walltime_limit + ',vmem=' + vmem_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.PIPELINE_NAME + '.XNAT.sh'
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
scan_line = ' --scan=' + self.scan
wdir_line = ' --working-dir=' + self._working_directory_name
setup_line = ' --setup-script=' + self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.setup_script
reg_name_line = ' --reg-name=' + self.reg_name
work_script = open(self._work_script_name(), 'w')
futils.wl(work_script, resources_line)
futils.wl(work_script, stdout_line)
futils.wl(work_script, stderr_line)
futils.wl(work_script, '')
futils.wl(work_script, script_line + self._continue)
futils.wl(work_script, user_line + self._continue)
futils.wl(work_script, password_line + self._continue)
futils.wl(work_script, server_line + self._continue)
futils.wl(work_script, project_line + self._continue)
futils.wl(work_script, subject_line + self._continue)
futils.wl(work_script, session_line + self._continue)
futils.wl(work_script, scan_line + self._continue)
futils.wl(work_script, wdir_line + self._continue)
if self.reg_name != 'MSMSulc':
futils.wl(work_script, reg_name_line + self._continue)
futils.wl(work_script, setup_line)
work_script.close()
os.chmod(self._work_script_name(), stat.S_IRWXU | stat.S_IRWXG)
def create_put_data_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.put_data_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=12gb' +
os.linesep)
script.write('#PBS -q HCPput' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' +
self._get_db_name() + os.linesep)
script.write(os.linesep)
script.write(self.xnat_pbs_jobs_home + os.sep + 'WorkingDirPut' +
os.sep + 'XNAT_working_dir_put.sh \\' + os.linesep)
script.write(' --leave-subject-id-level \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' +
str_utils.get_server_name(self.put_server) + '" \\' +
os.linesep)
script.write(' --project="' + self.project + '" \\' + os.linesep)
script.write(' --subject="' + self.subject + '" \\' + os.linesep)
script.write(' --session="' + self.session + '" \\' + os.linesep)
script.write(' --working-dir="' + self.working_directory_name +
'" \\' + os.linesep)
script.write(' --use-http' + ' \\' + os.linesep)
if self.scan:
script.write(' --scan="' + self.scan + '" \\' + os.linesep)
script.write(' --resource-suffix="' +
self.output_resource_suffix + '" \\' + os.linesep)
else:
script.write(' --resource-suffix="' + self.output_resource_name +
'" \\' + os.linesep)
script.write(' --reason="' + self.PIPELINE_NAME + '"' + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)