def delete_resource(user, password, server, project, subject, session, resource, perform_delete=True):
# get XNAT session id
xnat_session_id = xnat_access.get_session_id(server=str_utils.get_server_name(server), username=user, password=password,
project=project, subject=subject, session=session)
resource_url = ''
resource_url += 'https://' + str_utils.get_server_name(server)
resource_url += '/REST/projects/' + project
resource_url += '/subjects/' + subject
resource_url += '/experiments/' + xnat_session_id
resource_url += '/resources/' + resource
variable_values = '?removeFiles=true'
resource_uri = resource_url + variable_values
delete_cmd = 'java -Xmx1024m -jar /home/HCPpipeline/pipeline/lib/xnat-data-client-1.6.4-SNAPSHOT-jar-with-dependencies.jar'
delete_cmd += ' -u ' + user
delete_cmd += ' -p ' + password
delete_cmd += ' -m DELETE'
delete_cmd += ' -r ' + resource_uri
if perform_delete:
_inform("Deleting")
_inform(" Server: " + server)
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
completed_delete_process = subprocess.run(delete_cmd, shell=True, check=True)
else:
_inform("delete_cmd: " + delete_cmd)
_inform("Deletion not attempted")
def delete_resource(user,
password,
server,
project,
subject,
session,
resource,
perform_delete=True):
# get XNAT session id
xnat_session_id = xnat_access.get_session_id(
server=str_utils.get_server_name(server),
username=user,
password=password,
project=project,
subject=subject,
session=session)
resource_url = ''
resource_url += 'https://' + str_utils.get_server_name(server)
resource_url += '/REST/projects/' + project
resource_url += '/subjects/' + subject
resource_url += '/experiments/' + xnat_session_id
resource_url += '/resources/' + resource
variable_values = '?removeFiles=true'
resource_uri = resource_url + variable_values
pipeline_engine = os_utils.getenv_required('XNAT_PBS_JOBS_PIPELINE_ENGINE')
delete_cmd = 'java -Xmx1024m -jar ' + pipeline_engine + os.sep + 'lib' + os.sep + 'xnat-data-client-1.6.4-SNAPSHOT-jar-with-dependencies.jar'
delete_cmd += ' -u ' + user
delete_cmd += ' -p ' + password
delete_cmd += ' -m DELETE'
delete_cmd += ' -r ' + resource_uri
if perform_delete:
_inform("Deleting")
_inform(" Server: " + server)
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
completed_delete_process = subprocess.run(delete_cmd,
shell=True,
check=True)
else:
_inform("delete_cmd: " + delete_cmd)
_inform("Deletion not attempted")
def mark_running_status(self, stage):
module_logger.debug(debug_utils.get_name())
if stage > ccf_processing_stage.ProcessingStage.PREPARE_SCRIPTS:
mark_cmd = self._xnat_pbs_jobs_home
mark_cmd += os.sep + self.PIPELINE_NAME
mark_cmd += os.sep + self.PIPELINE_NAME
mark_cmd += '.XNAT_MARK_RUNNING_STATUS'
mark_cmd += ' --user='******' --password='******' --server=' + str_utils.get_server_name(
self.put_server)
mark_cmd += ' --project=' + self.project
mark_cmd += ' --subject=' + self.subject
mark_cmd += ' --classifier=' + self.classifier
mark_cmd += ' --resource=RunningStatus'
mark_cmd += ' --queued'
completed_mark_cmd_process = subprocess.run(
mark_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
print(completed_mark_cmd_process.stdout)
return
def create_mark_no_longer_running_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.mark_no_longer_running_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.mark_running_status_program_path + ' \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' + str_utils.get_server_name(self.put_server) + '" \\' + os.linesep)
script.write(' --project="' + self.project + '" \\' + os.linesep)
script.write(' --subject="' + self.subject + '" \\' + os.linesep)
script.write(' --classifier="' + self.classifier + '" \\' + os.linesep)
if self.scan:
script.write(' --scan="' + self.scan + '" \\' + os.linesep)
script.write(' --resource="' + 'RunningStatus' + '" \\' + os.linesep)
script.write(' --done' + os.linesep)
script.write(os.linesep)
script.write("rm -rf " + self.mark_completion_directory_name)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_check_data_job_script(self):
"""
Create the script to be submitted as a job to perform the check data functionality.
"""
module_logger.debug(debug_utils.get_name())
script_name = self.check_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.check_data_program_path + ' \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' + str_utils.get_server_name(self.put_server) + '" \\' + os.linesep)
script.write(' --project=' + self.project + ' \\' + os.linesep)
script.write(' --subject=' + self.subject + ' \\' + os.linesep)
script.write(' --classifier=' + self.classifier + ' \\' + os.linesep)
if self.scan:
script.write(' --scan=' + self.scan + ' \\' + os.linesep)
script.write(' --working-dir=' + self.check_data_directory_name + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_put_script(self, put_script_name, username, password, put_server, project, subject, session,
working_directory_name, output_resource_name, reason):
"""Create a script to put the working directory in the DB"""
with contextlib.suppress(FileNotFoundError):
os.remove(put_script_name)
put_script = open(put_script_name, 'w')
put_script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=12gb' + os.linesep)
put_script.write('#PBS -q HCPput' + os.linesep)
put_script.write('#PBS -o ' + self.log_dir + os.linesep)
put_script.write('#PBS -e ' + self.log_dir + os.linesep)
put_script.write(os.linesep)
put_script.write(self.xnat_pbs_jobs_home + os.sep + 'WorkingDirPut' + os.sep + 'XNAT_working_dir_put.sh \\' + os.linesep)
put_script.write(' --user="******" \\' + os.linesep)
put_script.write(' --password="******" \\' + os.linesep)
put_script.write(' --server="' + str_utils.get_server_name(put_server) + '" \\' + os.linesep)
put_script.write(' --project="' + project + '" \\' + os.linesep)
put_script.write(' --subject="' + subject + '" \\' + os.linesep)
put_script.write(' --session="' + session + '" \\' + os.linesep)
put_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
put_script.write(' --resource-suffix="' + output_resource_name + '" \\' + os.linesep)
put_script.write(' --reason="' + reason + '"' + os.linesep)
put_script.close()
os.chmod(put_script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_freesurfer_assessor_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT_CREATE_FREESURFER_ASSESSOR script to the working directory
freesurfer_assessor_source_path = self.xnat_pbs_jobs_home
freesurfer_assessor_source_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_source_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_source_path += '.XNAT_CREATE_FREESURFER_ASSESSOR'
freesurfer_assessor_dest_path = self.working_directory_name
freesurfer_assessor_dest_path += os.sep + self.PIPELINE_NAME
freesurfer_assessor_dest_path += '.XNAT_CREATE_FREESURFER_ASSESSOR'
shutil.copy(freesurfer_assessor_source_path,
freesurfer_assessor_dest_path)
os.chmod(freesurfer_assessor_dest_path, stat.S_IRWXU | stat.S_IRWXG)
# write the freesurfer assessor submission script (that calls the .XNAT_CREATE_FREESURFER_ASSESSOR script)
script_name = self.freesurfer_assessor_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
self._write_bash_header(script)
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=4gb' +
os.linesep)
script.write('#PBS -o ' + self.working_directory_name + os.linesep)
script.write('#PBS -e ' + self.working_directory_name + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' +
self._get_db_name() + os.linesep)
script.write(os.linesep)
script_line = freesurfer_assessor_dest_path
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
session_classifier_line = ' --session-classifier=' + self.classifier
wdir_line = ' --working-dir=' + self.working_directory_name
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(session_classifier_line + ' \\' + os.linesep)
script.write(wdir_line + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_pre_eddy_script(self):
_debug("_create_pre_eddy_script")
with contextlib.suppress(FileNotFoundError):
os.remove(self._pre_eddy_script_name)
walltime_limit = str(self.pre_eddy_walltime_limit_hours) + ':00:00'
vmem_limit = str(self.pre_eddy_vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=1:ppn=1,walltime=' + walltime_limit
resources_line += ',vmem=' + vmem_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T_PreEddy.XNAT.sh'
user_line = ' --user="******"'
password_line = ' --password="******"'
server_line = ' --server="' + str_utils.get_server_name(
self.server) + '"'
project_line = ' --project="' + self.project + '"'
subject_line = ' --subject="' + self.subject + '"'
session_line = ' --session="' + self.session + '"'
ref_proj_line = ' --structural-reference-project="'
ref_proj_line += self.structural_reference_project + '"'
ref_sess_line = ' --structural-reference-session="'
ref_sess_line += self.structural_reference_session + '"'
wdir_line = ' --working-dir="' + self._working_directory_name + '"'
workflow_line = ' --workflow-id="' + self._workflow_id + '"'
setup_line = ' --setup-script=' + self.setup_script
pe_dirs_line = ' --phase-encoding-dirs=' + self.pe_dirs_spec
pre_eddy_script = open(self._pre_eddy_script_name, 'w')
futils.wl(pre_eddy_script, resources_line)
futils.wl(pre_eddy_script, stdout_line)
futils.wl(pre_eddy_script, stderr_line)
futils.wl(pre_eddy_script, '')
futils.wl(pre_eddy_script, script_line + self._continue)
futils.wl(pre_eddy_script, user_line + self._continue)
futils.wl(pre_eddy_script, password_line + self._continue)
futils.wl(pre_eddy_script, server_line + self._continue)
futils.wl(pre_eddy_script, project_line + self._continue)
futils.wl(pre_eddy_script, subject_line + self._continue)
futils.wl(pre_eddy_script, session_line + self._continue)
futils.wl(pre_eddy_script, ref_proj_line + self._continue)
futils.wl(pre_eddy_script, ref_sess_line + self._continue)
futils.wl(pre_eddy_script, wdir_line + self._continue)
futils.wl(pre_eddy_script, workflow_line + self._continue)
futils.wl(pre_eddy_script, setup_line + self._continue)
futils.wl(pre_eddy_script, pe_dirs_line)
pre_eddy_script.close()
os.chmod(self._pre_eddy_script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_work_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.work_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',vmem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep
script_line += self.PIPELINE_NAME + os.sep + self.PIPELINE_NAME + '.XNAT.sh'
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.setup_script
script = open(script_name, 'w')
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
for group in self._group_list:
script.write(' --group=' + group + ' \\' + os.linesep)
for name in self._concat_name_list:
script.write(' --concat-name=' + name + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_pre_eddy_script(self):
_debug("_create_pre_eddy_script")
with contextlib.suppress(FileNotFoundError):
os.remove(self._pre_eddy_script_name)
walltime_limit = str(self.pre_eddy_walltime_limit_hours) + ':00:00'
vmem_limit = str(self.pre_eddy_vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=1:ppn=1,walltime=' + walltime_limit
resources_line += ',vmem=' + vmem_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T_PreEddy.XNAT.sh'
user_line = ' --user="******"'
password_line = ' --password="******"'
server_line = ' --server="' + str_utils.get_server_name(self.server) + '"'
project_line = ' --project="' + self.project + '"'
subject_line = ' --subject="' + self.subject + '"'
session_line = ' --session="' + self.session + '"'
ref_proj_line = ' --structural-reference-project="'
ref_proj_line += self.structural_reference_project + '"'
ref_sess_line = ' --structural-reference-session="'
ref_sess_line += self.structural_reference_session + '"'
wdir_line = ' --working-dir="' + self._working_directory_name + '"'
workflow_line = ' --workflow-id="' + self._workflow_id + '"'
setup_line = ' --setup-script=' + self.setup_script
pe_dirs_line = ' --phase-encoding-dirs=' + self.pe_dirs_spec
pre_eddy_script = open(self._pre_eddy_script_name, 'w')
futils.wl(pre_eddy_script, resources_line)
futils.wl(pre_eddy_script, stdout_line)
futils.wl(pre_eddy_script, stderr_line)
futils.wl(pre_eddy_script, '')
futils.wl(pre_eddy_script, script_line + self._continue)
futils.wl(pre_eddy_script, user_line + self._continue)
futils.wl(pre_eddy_script, password_line + self._continue)
futils.wl(pre_eddy_script, server_line + self._continue)
futils.wl(pre_eddy_script, project_line + self._continue)
futils.wl(pre_eddy_script, subject_line + self._continue)
futils.wl(pre_eddy_script, session_line + self._continue)
futils.wl(pre_eddy_script, ref_proj_line + self._continue)
futils.wl(pre_eddy_script, ref_sess_line + self._continue)
futils.wl(pre_eddy_script, wdir_line + self._continue)
futils.wl(pre_eddy_script, workflow_line + self._continue)
futils.wl(pre_eddy_script, setup_line + self._continue)
futils.wl(pre_eddy_script, pe_dirs_line)
pre_eddy_script.close()
os.chmod(self._pre_eddy_script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_work_script(self, scan):
logger.debug("_create_work_script - scan: " + scan)
with contextlib.suppress(FileNotFoundError):
os.remove(self._work_script_name(scan))
walltime_limit = str(self.walltime_limit_hours) + ':00:00'
vmem_limit = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=1:ppn=1,walltime=' + walltime_limit
resources_line += ',vmem=' + vmem_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep
script_line += self.PIPELINE_NAME + os.sep + self.PIPELINE_NAME + '.XNAT.sh'
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
ref_proj_line = ' --structural-reference-project=' + self.structural_reference_project
ref_sess_line = ' --structural-reference-session=' + self.structural_reference_session
scan_line = ' --scan=' + scan
wdir_line = ' --working-dir=' + self._working_directory_name
workflow_line = ' --workflow-id=' + self._workflow_id
setup_line = ' --setup-script=' + self.setup_script
work_script = open(self._work_script_name(scan), 'w')
futils.wl(work_script, resources_line)
futils.wl(work_script, stdout_line)
futils.wl(work_script, stderr_line)
futils.wl(work_script, '')
futils.wl(work_script, script_line + self._continue)
futils.wl(work_script, user_line + self._continue)
futils.wl(work_script, password_line + self._continue)
futils.wl(work_script, server_line + self._continue)
futils.wl(work_script, project_line + self._continue)
futils.wl(work_script, subject_line + self._continue)
futils.wl(work_script, session_line + self._continue)
futils.wl(work_script, ref_proj_line + self._continue)
futils.wl(work_script, ref_sess_line + self._continue)
futils.wl(work_script, scan_line + self._continue)
futils.wl(work_script, wdir_line + self._continue)
futils.wl(work_script, workflow_line + self._continue)
futils.wl(work_script, setup_line)
work_script.close()
os.chmod(self._work_script_name(scan), stat.S_IRWXU | stat.S_IRWXG)
def create_put_data_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.put_data_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,mem=12gb' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' + self._get_db_name() + os.linesep)
script.write('module load ' + self._get_xnat_pbs_setup_script_singularity_version() + os.linesep)
script.write(os.linesep)
script.write('mv ' + self.working_directory_name + os.path.sep + '*' + self.PIPELINE_NAME + '* ' + self.working_directory_name + os.path.sep + self.subject + '_' + self.classifier + os.path.sep + 'ProcessingInfo' + os.linesep)
script.write(os.linesep)
script.write('singularity exec -B ' + self._get_xnat_pbs_setup_script_archive_root() + ',' + self._get_xnat_pbs_setup_script_singularity_bind_path() + ' ' + self._get_xnat_pbs_setup_script_singularity_container_xnat_path() + ' ' + self.xnat_pbs_jobs_home + os.sep + 'WorkingDirPut' + os.sep + 'XNAT_working_dir_put.sh \\' + os.linesep)
script.write(' --leave-subject-id-level \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' + str_utils.get_server_name(self.put_server) + '" \\' + os.linesep)
script.write(' --project="' + self.project + '" \\' + os.linesep)
script.write(' --subject="' + self.subject + '" \\' + os.linesep)
script.write(' --session="' + self.session + '" \\' + os.linesep)
script.write(' --working-dir="' + self.working_directory_name + '" \\' + os.linesep)
if self.scan:
script.write(' --scan="' + self.scan + '" \\' + os.linesep)
script.write(' --resource-suffix="' + self.output_resource_suffix + '" \\' + os.linesep)
else:
script.write(' --resource-suffix="' + self.output_resource_name + '" \\' + os.linesep)
script.write(' --reason="' + self.PIPELINE_NAME + '"' + os.linesep)
script.write(os.linesep)
script.write('echo "Run structural QC on hand edited output"' + os.linesep)
script.write('curl -n https://' + str_utils.get_server_name(self.put_server) + '/xapi/structuralQc/project/' + self.project + '/subject/' + self.subject + '/experiment/' + self.session + '/runStructuralQcHandEditingProcessing -X POST' + os.linesep)
script.write('curl -n https://' + str_utils.get_server_name(self.put_server) + '/xapi/structuralQc/project/' + self.project + '/subject/' + self.subject + '/experiment/' + self.session + '/sendCompletionNotification -X POST' + os.linesep)
script.write(os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_put_script(self,
put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
reason,
leave_subject_id_level=False):
with contextlib.suppress(FileNotFoundError):
os.remove(put_script_name)
put_script = open(put_script_name, 'w')
put_script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=12gb' +
os.linesep)
put_script.write('#PBS -q HCPput' + os.linesep)
put_script.write('#PBS -o ' + self.log_dir + os.linesep)
put_script.write('#PBS -e ' + self.log_dir + os.linesep)
put_script.write(os.linesep)
put_script.write(self.xnat_pbs_jobs_home + os.sep + 'WorkingDirPut' +
os.sep + 'XNAT_working_dir_files_put.sh \\' +
os.linesep)
put_script.write(' --user="******" \\' + os.linesep)
put_script.write(' --password="******" \\' + os.linesep)
put_script.write(' --server="' +
str_utils.get_server_name(put_server) + '" \\' +
os.linesep)
put_script.write(' --project="' + project + '" \\' + os.linesep)
put_script.write(' --subject="' + subject + '" \\' + os.linesep)
put_script.write(' --session="' + session + '" \\' + os.linesep)
put_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
put_script.write(' --resource-suffix="' + output_resource_name +
'" \\' + os.linesep)
put_script.write(' --reason="' + self.PIPELINE_NAME + '" \\' +
os.linesep)
if leave_subject_id_level:
put_script.write(' --leave-subject-id-level' + os.linesep)
else:
put_script.write(os.linesep)
put_script.close()
os.chmod(put_script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_work_script(self):
logger.debug(debug_utils.get_name())
script_name = self._work_script_name()
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit = str(self.walltime_limit_hours) + ':00:00'
vmem_limit = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=1:ppn=1,walltime=' + walltime_limit + ',vmem=' + vmem_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.PIPELINE_NAME + '.XNAT.sh'
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
scan_line = ' --scan=' + self.scan
wdir_line = ' --working-dir=' + self._working_directory_name
setup_line = ' --setup-script=' + self.xnat_pbs_jobs_home + os.sep + self.PIPELINE_NAME + os.sep + self.setup_script
reg_name_line = ' --reg-name=' + self.reg_name
work_script = open(self._work_script_name(), 'w')
futils.wl(work_script, resources_line)
futils.wl(work_script, stdout_line)
futils.wl(work_script, stderr_line)
futils.wl(work_script, '')
futils.wl(work_script, script_line + self._continue)
futils.wl(work_script, user_line + self._continue)
futils.wl(work_script, password_line + self._continue)
futils.wl(work_script, server_line + self._continue)
futils.wl(work_script, project_line + self._continue)
futils.wl(work_script, subject_line + self._continue)
futils.wl(work_script, session_line + self._continue)
futils.wl(work_script, scan_line + self._continue)
futils.wl(work_script, wdir_line + self._continue)
if self.reg_name != 'MSMSulc':
futils.wl(work_script, reg_name_line + self._continue)
futils.wl(work_script, setup_line)
work_script.close()
os.chmod(self._work_script_name(), stat.S_IRWXU | stat.S_IRWXG)
def create_put_data_script(self):
module_logger.debug(debug_utils.get_name())
script_name = self.put_data_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
script = open(script_name, 'w')
script.write('#PBS -l nodes=1:ppn=1,walltime=4:00:00,vmem=12gb' +
os.linesep)
script.write('#PBS -q HCPput' + os.linesep)
script.write('#PBS -o ' + self.log_dir + os.linesep)
script.write('#PBS -e ' + self.log_dir + os.linesep)
script.write(os.linesep)
script.write('source ' + self._get_xnat_pbs_setup_script_path() + ' ' +
self._get_db_name() + os.linesep)
script.write(os.linesep)
script.write(self.xnat_pbs_jobs_home + os.sep + 'WorkingDirPut' +
os.sep + 'XNAT_working_dir_put.sh \\' + os.linesep)
script.write(' --leave-subject-id-level \\' + os.linesep)
script.write(' --user="******" \\' + os.linesep)
script.write(' --password="******" \\' + os.linesep)
script.write(' --server="' +
str_utils.get_server_name(self.put_server) + '" \\' +
os.linesep)
script.write(' --project="' + self.project + '" \\' + os.linesep)
script.write(' --subject="' + self.subject + '" \\' + os.linesep)
script.write(' --session="' + self.session + '" \\' + os.linesep)
script.write(' --working-dir="' + self.working_directory_name +
'" \\' + os.linesep)
script.write(' --use-http' + ' \\' + os.linesep)
if self.scan:
script.write(' --scan="' + self.scan + '" \\' + os.linesep)
script.write(' --resource-suffix="' +
self.output_resource_suffix + '" \\' + os.linesep)
else:
script.write(' --resource-suffix="' + self.output_resource_name +
'" \\' + os.linesep)
script.write(' --reason="' + self.PIPELINE_NAME + '"' + os.linesep)
script.close()
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_eddy_script(self):
_debug("_create_eddy_script")
with contextlib.suppress(FileNotFoundError):
os.remove(self._eddy_script_name)
walltime_limit = str(self.eddy_walltime_limit_hours) + ':00:00'
resources_line = '#PBS -l nodes=1:ppn=3:gpus=1,walltime=' + walltime_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T_Eddy.XNAT.sh'
user_line = ' --user="******"'
password_line = ' --password="******"'
server_line = ' --server="' + str_utils.get_server_name(
self.server) + '"'
subject_line = ' --subject="' + self.subject + '"'
wdir_line = ' --working-dir="' + self._working_directory_name + '"'
workflow_line = ' --workflow-id="' + self._workflow_id + '"'
setup_line = ' --setup-script=' + self.setup_script
eddy_script = open(self._eddy_script_name, 'w')
futils.wl(eddy_script, resources_line)
futils.wl(eddy_script, stdout_line)
futils.wl(eddy_script, stderr_line)
futils.wl(eddy_script, '')
futils.wl(eddy_script, script_line + self._continue)
futils.wl(eddy_script, user_line + self._continue)
futils.wl(eddy_script, password_line + self._continue)
futils.wl(eddy_script, server_line + self._continue)
futils.wl(eddy_script, subject_line + self._continue)
futils.wl(eddy_script, wdir_line + self._continue)
futils.wl(eddy_script, workflow_line + self._continue)
futils.wl(eddy_script, setup_line)
eddy_script.close()
os.chmod(self._eddy_script_name, stat.S_IRWXU | stat.S_IRWXG)
def _create_eddy_script(self):
_debug("_create_eddy_script")
with contextlib.suppress(FileNotFoundError):
os.remove(self._eddy_script_name)
walltime_limit = str(self.eddy_walltime_limit_hours) + ':00:00'
resources_line = '#PBS -l nodes=1:ppn=3:gpus=1,walltime=' + walltime_limit
stdout_line = '#PBS -o ' + self._working_directory_name
stderr_line = '#PBS -e ' + self._working_directory_name
script_line = self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T' + os.sep
script_line += 'DiffusionPreprocessingHCP7T_Eddy.XNAT.sh'
user_line = ' --user="******"'
password_line = ' --password="******"'
server_line = ' --server="' + str_utils.get_server_name(self.server) + '"'
subject_line = ' --subject="' + self.subject + '"'
wdir_line = ' --working-dir="' + self._working_directory_name + '"'
workflow_line = ' --workflow-id="' + self._workflow_id + '"'
setup_line = ' --setup-script=' + self.setup_script
eddy_script = open(self._eddy_script_name, 'w')
futils.wl(eddy_script, resources_line)
futils.wl(eddy_script, stdout_line)
futils.wl(eddy_script, stderr_line)
futils.wl(eddy_script, '')
futils.wl(eddy_script, script_line + self._continue)
futils.wl(eddy_script, user_line + self._continue)
futils.wl(eddy_script, password_line + self._continue)
futils.wl(eddy_script, server_line + self._continue)
futils.wl(eddy_script, subject_line + self._continue)
futils.wl(eddy_script, wdir_line + self._continue)
futils.wl(eddy_script, workflow_line + self._continue)
futils.wl(eddy_script, setup_line)
eddy_script.close()
os.chmod(self._eddy_script_name, stat.S_IRWXU | stat.S_IRWXG)
def submit_jobs(
self,
processing_stage=ccf_processing_stage.ProcessingStage.CHECK_DATA):
module_logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
module_logger.info("-----")
module_logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
module_logger.info(" Project: " + self.project)
module_logger.info(" Subject: " + self.subject)
module_logger.info(" Session: " + self.session)
module_logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
os.makedirs(name=self.working_directory_name)
os.makedirs(name=self.check_data_directory_name)
os.makedirs(name=self.mark_completion_directory_name)
module_logger.info("Output Resource Name: " +
self.output_resource_name)
# clean output resource if requested
if self.clean_output_resource_first:
module_logger.info("Deleting resource: " +
self.output_resource_name + " for:")
module_logger.info(" project: " + self.project)
module_logger.info(" subject: " + self.subject)
module_logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self.output_resource_name)
return self.do_job_submissions(processing_stage)
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
mem_spec = 'mem=' + str(self.mem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + ',' + mem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
# work_script.write(' --keep-all' + ' \\' + os.linesep)
# work_script.write(' --prevent-push' + ' \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(
current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password,
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name +
" for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name(
) + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(
completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(
completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(
completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def create_process_data_job_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT_PROCESS script to the working directory
processing_script_source_path = self.xnat_pbs_jobs_home
processing_script_source_path += os.sep + self.PIPELINE_NAME
processing_script_source_path += os.sep + self.PIPELINE_NAME
processing_script_source_path += '.XNAT_PROCESS'
processing_script_dest_path = self.working_directory_name
processing_script_dest_path += os.sep + self.PIPELINE_NAME
processing_script_dest_path += '.XNAT_PROCESS'
shutil.copy(processing_script_source_path, processing_script_dest_path)
os.chmod(processing_script_dest_path, stat.S_IRWXU | stat.S_IRWXG)
# write the process data job script (that calls the .XNAT_PROCESS script)
subject_info = ccf_subject.SubjectInfo(self.project, self.subject,
self.classifier)
script_name = self.process_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',mem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
xnat_pbs_setup_line = 'source ' + self._get_xnat_pbs_setup_script_path(
) + ' ' + self._get_db_name()
script_line = processing_script_dest_path
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
session_classifier_line = ' --session-classifier=' + self.classifier
if self._has_spin_echo_field_maps(subject_info):
fieldmap_type_line = ' --fieldmap-type=' + 'SpinEcho'
else:
fieldmap_type_line = ' --fieldmap-type=' + 'SiemensGradientEcho'
first_t1w_directory_name_line = ' --first-t1w-directory-name=' + self._get_first_t1w_directory_name(
subject_info)
first_t1w_resource_name_line = ' --first-t1w-resource-name=' + self._get_first_t1w_resource_name(
subject_info)
first_t1w_file_name_line = ' --first-t1w-file-name=' + self._get_first_t1w_file_name(
subject_info)
first_t2w_directory_name_line = ' --first-t2w-directory-name=' + self._get_first_t2w_directory_name(
subject_info)
first_t2w_resource_name_line = ' --first-t2w-resource-name=' + self._get_first_t2w_resource_name(
subject_info)
first_t2w_file_name_line = ' --first-t2w-file-name=' + self._get_first_t2w_file_name(
subject_info)
brain_size_line = ' --brainsize=' + str(self.brain_size)
t1template_line = ' --t1template=' + self.T1W_TEMPLATE_NAME
t1templatebrain_line = ' --t1templatebrain=' + self.T1W_TEMPLATE_BRAIN_NAME
t1template2mm_line = ' --t1template2mm=' + self.T1W_TEMPLATE_2MM_NAME
t2template_line = ' --t2template=' + self.T2W_TEMPLATE_NAME
t2templatebrain_line = ' --t2templatebrain=' + self.T2W_TEMPLATE_BRAIN_NAME
t2template2mm_line = ' --t2template2mm=' + self.T2W_TEMPLATE_2MM_NAME
templatemask_line = ' --templatemask=' + self.TEMPLATE_MASK_NAME
template2mmmask_line = ' --template2mmmask=' + self.TEMPLATE_2MM_MASK_NAME
fnirtconfig_line = ' --fnirtconfig=' + self.FNIRT_CONFIG_FILE_NAME
if subject_info.project in OneSubjectJobSubmitter._CONNECTOME_SKYRA_SCANNER_PROJECTS:
gdcoeffs_line = ' --gdcoeffs=' + self.CONNECTOME_GDCOEFFS_FILE_NAME
elif subject_info.project in OneSubjectJobSubmitter._PRISMA_3T_PROJECTS:
gdcoeffs_line = ' --gdcoeffs=' + self.PRISMA_3T_GDCOEFFS_FILE_NAME
else:
raise ValueError(
"Unrecognized project for setting gradient distortion coefficients file: "
+ subject_info.project)
topupconfig_line = ' --topupconfig=' + self.TOPUP_CONFIG_FILE_NAME
if self._has_spin_echo_field_maps(subject_info):
se_phase_pos_line = ' --se-phase-pos=' + self._get_positive_spin_echo_file_name(
subject_info)
se_phase_neg_line = ' --se-phase-neg=' + self._get_negative_spin_echo_file_name(
subject_info)
# mag_line = None
# phase_line = None
else:
se_phase_pos_line = None
se_phase_neg_line = None
# mag_line = ' --fmapmag=' + self._get_fmap_mag_file_name(subject_info)
# phase_line = ' --fmapphase=' + self._get_fmap_phase_file_name(subject_info)
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.setup_file_name
with open(script_name, 'w') as script:
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(xnat_pbs_setup_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(session_classifier_line + ' \\' + os.linesep)
script.write(fieldmap_type_line + ' \\' + os.linesep)
script.write(first_t1w_directory_name_line + ' \\' + os.linesep)
script.write(first_t1w_resource_name_line + ' \\' + os.linesep)
script.write(first_t1w_file_name_line + ' \\' + os.linesep)
script.write(first_t2w_directory_name_line + ' \\' + os.linesep)
script.write(first_t2w_resource_name_line + ' \\' + os.linesep)
script.write(first_t2w_file_name_line + ' \\' + os.linesep)
script.write(brain_size_line + ' \\' + os.linesep)
script.write(t1template_line + ' \\' + os.linesep)
script.write(t1templatebrain_line + ' \\' + os.linesep)
script.write(t1template2mm_line + ' \\' + os.linesep)
script.write(t2template_line + ' \\' + os.linesep)
script.write(t2templatebrain_line + ' \\' + os.linesep)
script.write(t2template2mm_line + ' \\' + os.linesep)
script.write(templatemask_line + ' \\' + os.linesep)
script.write(template2mmmask_line + ' \\' + os.linesep)
script.write(fnirtconfig_line + ' \\' + os.linesep)
script.write(gdcoeffs_line + ' \\' + os.linesep)
script.write(topupconfig_line + ' \\' + os.linesep)
if (se_phase_pos_line):
script.write(se_phase_pos_line + ' \\' + os.linesep)
if (se_phase_neg_line):
script.write(se_phase_neg_line + ' \\' + os.linesep)
# if (mag_line): script.write(mag_line + ' \\' + os.linesep)
# if (phase_line): script.write(phase_line + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def create_process_data_job_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT script to the working directory
processing_script_source_name = self.xnat_pbs_jobs_home
processing_script_source_name += os.sep + '7T'
processing_script_source_name += os.sep + self.PIPELINE_NAME
processing_script_source_name += os.sep + self.PIPELINE_NAME
processing_script_source_name += '.XNAT_PROCESS'
processing_script_dest_name = self.working_directory_name
processing_script_dest_name += os.sep + self.PIPELINE_NAME
processing_script_dest_name += '.XNAT_PROCESS'
shutil.copy(processing_script_source_name, processing_script_dest_name)
os.chmod(processing_script_dest_name, stat.S_IRWXU | stat.S_IRWXG)
# write the process data job script (that calls the .XNAT script)
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project=self.project,
subject_id=self.subject)
script_name = self.process_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
mem_limit_str = str(self.mem_limit_gbs) + 'gb'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',mem=' + mem_limit_str
resources_line += ',vmem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
script_line = processing_script_dest_name
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.subject + '_7T'
avail_retinotopy_task_names = self.archive.available_retinotopy_preproc_names(
subject_info)
# sort available retinotopy task names into the order the
# tasks were presented to the subject
avail_retinotopy_task_names = sorted(
avail_retinotopy_task_names, key=retinotopy_presentation_order_key)
# add the tesla spec to each element of the group
group_names = list(map(add_tesla_spec, avail_retinotopy_task_names))
group_spec = '@'.join(group_names)
group_line = ' --group=' + group_spec
concat_spec = '_'.join(
list(map(remove_scan_type, avail_retinotopy_task_names)))
concat_line = ' --concat-name=tfMRI_7T_' + concat_spec
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.setup_file_name
with open(script_name, 'w') as script:
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(group_line + ' \\' + os.linesep)
script.write(concat_line + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
script.write(os.linesep)
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
def submit_jobs(
self,
processing_stage=one_subject_job_submitter.ProcessingStage.PUT_DATA
):
module_logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
module_logger.info("-----")
module_logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
module_logger.info(" Project: " + self.project)
module_logger.info(" Subject: " + self.subject)
module_logger.info(" Session: " + self.session)
module_logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
os.makedirs(name=self.working_directory_name)
# determine output resource name
self._output_resource_name = self.output_resource_suffix
module_logger.info("Output Resource Name: " +
self._output_resource_name)
# clean output resource if requested
if self.clean_output_resource_first:
module_logger.info("Deleting resource: " +
self._output_resource_name + " for:")
module_logger.info(" project: " + self.project)
module_logger.info(" subject: " + self.subject)
module_logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# build list of groups
self._group_list = []
for group in self.groups:
self._group_list.append(self._expand(group))
module_logger.debug("self._group_list: " + str(self._group_list))
# build list of concat names
self._concat_name_list = []
for group in self._group_list:
self._concat_name_list.append(self._concat(group))
# create scripts for various stages of processing
if processing_stage >= one_subject_job_submitter.ProcessingStage.PREPARE_SCRIPTS:
self.create_get_data_job_script()
self.create_work_script()
self.create_clean_data_script()
self.create_put_data_script()
# Submit the job to get the data
if processing_stage >= one_subject_job_submitter.ProcessingStage.GET_DATA:
get_data_job_no = self.submit_get_data_job()
module_logger.info("get_data_job_no: " + str(get_data_job_no))
else:
module_logger.info("Get data job not submitted")
# Submit the job to process the data (do the work)
if processing_stage >= one_subject_job_submitter.ProcessingStage.PROCESS_DATA:
work_job_no = self.submit_process_data_job(get_data_job_no)
module_logger.info("work_job_no: " + str(work_job_no))
else:
module_logger.info("Process data job not submitted")
# Submit job to clean the data
if processing_stage >= one_subject_job_submitter.ProcessingStage.CLEAN_DATA:
clean_job_no = self.submit_clean_data_job(work_job_no)
module_logger.info("clean_job_no: " + str(clean_job_no))
else:
module_logger.info("Clean data job not submitted")
# Submit job to put the resulting data in the DB
if processing_stage >= one_subject_job_submitter.ProcessingStage.PUT_DATA:
put_job_no = self.submit_put_data_job(clean_job_no)
module_logger.info("put_job_no: " + str(put_job_no))
else:
module_logger.info("Put data job not submitted")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
"""
processing_stage is the last processing stage for which to submit
the corresponding job.
GET_DATA means just get the data.
PROCESS_DATA means get the data and do the processing.
PUT_DATA means get the data, processing it, and put the results
back in the DB
"""
logger.debug("submit_jobs processing_stage: " + str(processing_stage))
if self.validate_parameters():
# determine what scans to run the RestingStateStats pipeline on for this subject
# TBD: Does this get run on every scan for which the ICAFIX pipeline has been run,
# or does it only get run on every resting state scan that has been fix processed.
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
self.project, self.structural_reference_project, self.subject)
fix_processed_scans = self.archive.available_FIX_processed_names(
subject_info)
fix_processed_scans_set = set(fix_processed_scans)
logger.debug("fix_processed_scans_set = " +
str(fix_processed_scans_set))
# resting_state_scans = self.archive.available_resting_state_preproc_names(subject_info)
# resting_state_scans_set = set(resting_state_scans)
# logger.debug("resting_state_scans_set = " + str(resting_state_scans_set))
# scans_to_process_set = resting_state_scans_set & fix_processed_scans_set
scans_to_process_set = fix_processed_scans_set
scans_to_process = list(scans_to_process_set)
scans_to_process.sort()
logger.debug("scans_to_process: " + str(scans_to_process))
incomplete_scans_to_process = list()
for scan in scans_to_process:
if (not is_complete(self.archive, subject_info, scan)):
incomplete_scans_to_process.append(scan)
logger.debug("incomplete_scans_to_process: " +
str(incomplete_scans_to_process))
print("incomplete_scans_to_process:", incomplete_scans_to_process)
# for scan in scans_to_process:
for scan in incomplete_scans_to_process:
logger.info("")
logger.info(
"--------------------------------------------------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Structural Reference Project: " +
self.structural_reference_project)
logger.info(" Structural Reference Session: " +
self.structural_reference_session)
logger.info(" Scan: " + scan)
logger.info(" Stage: " + str(processing_stage))
logger.info(
"--------------------------------------------------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
logger.info("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
logger.info("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, 'https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project,
self.PIPELINE_NAME + '_' + scan, 'Queued')
logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = scan + "_RSS"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " +
self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script(scan)
# create script to do work
self._create_work_script(scan)
# create script to clean data
self._create_clean_data_script(scan)
# create script to put the results into the DB
put_script_name = self._put_data_script_name(scan)
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME + '_' + scan)
# submit job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name(
scan)
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# submit job to process the data
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
scan)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
scan)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
else:
logger.info("Unable to submit jobs")
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(
subject_info)
inform("FIX processed scans available for subject " +
str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker(
)
if completion_checker.is_processing_complete(
self.archive, subject_info, scan_name):
inform("scan: " + scan_name +
" has already completed PostFixHCP7T processing")
inform(
"Only submitting jobs for incomplete scans - skipping "
+ scan_name)
continue
inform("scan_name: " + scan_name)
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password,
project=project,
subject=subject,
session=session)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server,
jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id,
project,
self.PIPELINE_NAME,
'Queued')
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username, password, str_utils.get_server_name(server),
project, subject, session, output_resource_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec +
',' + vmem_spec + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + 'PostFixHCP7T' + os.sep +
'PostFixHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' +
os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, username, password,
put_server, project, subject, session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# subject_info = hcp7t_subject.Hcp7TSubjectInfo(self.project,
# self.structural_reference_project,
# self.subject)
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
logger.info("-----")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Scan: " + self.scan)
logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, self.scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# determine output resource name
self._output_resource_name = self.scan + "_" + self.output_resource_suffix
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
self._create_get_data_script()
self._create_work_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username,
self.password,
self.put_server,
self.project,
self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name()
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# Submit the job to process the data (do the work)
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# Submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# Submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(self.username, self.password,
'https://db.humanconnectome.org', jsession_id)
self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
self.project,
self.PIPELINE_NAME,
'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name() + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name, self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self, username, password, server, project, subject,
session, structural_reference_project,
structural_reference_session, put_server, setup_script,
incomplete_only, scan, walltime_limit_hours, mem_limit_gbs,
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of the preprocessed resting state scans that are
# available for the subject
resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(movie_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = resting_state_scan_names + movie_scan_names
else:
scan_list.append(scan)
# process the specified scans
for scan_name in scan_list:
if incomplete_only and self.archive.FIX_processing_repaired(
subject_info, scan_name):
inform("scan: " + scan_name +
" FIX processing is already repaired")
inform(
"Only submitting jobs for incomplete scans - skipping " +
scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.FIX_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
work_script.write('#PBS -l nodes=1:ppn=1,walltime=' +
str(walltime_limit_hours) + ':00:00,mem=' +
str(mem_limit_gbs) + 'gb,vmem=' +
str(vmem_limit_gbs) + 'gb' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + self.PIPELINE_NAME + os.sep +
self.PIPELINE_NAME + '.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
structural_reference_project + '" \\' +
os.linesep)
work_script.write(' --structural-reference-session="' +
structural_reference_session + '" \\' +
os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# 'https://db.humanconnectome.org', jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._get_data_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError("get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._process_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError("script to submit processing jobs ended with non-zero return code")
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " + postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name()
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs,
):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(subject_info)
inform("Preprocessed resting state scans available for subject: " + str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(subject_info)
inform("Preprocessed movie scans available for subject " + str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(subject_info)
inform("FIX processed scans available for subject " + str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker()
if completion_checker.is_processing_complete(self.archive, subject_info, scan_name):
inform("scan: " + scan_name + " has already completed PostFixHCP7T processing")
inform("Only submitting jobs for incomplete scans - skipping " + scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += "." + subject
working_directory_name += "." + long_scan_name
working_directory_name += "." + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server="db.humanconnectome.org", username=username, password=password
)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server="db.humanconnectome.org",
username=username,
password=password,
project=project,
subject=subject,
session=session,
)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server, jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id, project, self.PIPELINE_NAME, "Queued")
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username,
password,
str_utils.get_server_name(server),
project,
subject,
session,
output_resource_name,
)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += "." + long_scan_name
script_file_start_name += "." + self.PIPELINE_NAME
script_file_start_name += "." + project
script_file_start_name += "." + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + ".XNAT_PBS_job.sh"
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, "w")
nodes_spec = "nodes=1:ppn=1"
walltime_spec = "walltime=" + str(walltime_limit_hours) + ":00:00"
vmem_spec = "vmem=" + str(vmem_limit_gbs) + "gb"
work_script.write("#PBS -l " + nodes_spec + "," + walltime_spec + "," + vmem_spec + os.linesep)
work_script.write("#PBS -o " + working_directory_name + os.linesep)
work_script.write("#PBS -e " + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(
self.xnat_pbs_jobs_home
+ os.sep
+ "7T"
+ os.sep
+ "PostFixHCP7T"
+ os.sep
+ "PostFixHCP7T.XNAT.sh \\"
+ os.linesep
)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(server) + '" \\' + os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(" --setup-script=" + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + ".XNAT_PBS_PUT_job.sh"
self.create_put_script(
put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + "_" + self.PIPELINE_NAME,
)
# Submit the job to do the work
work_submit_cmd = "qsub " + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = "qsub -W depend=afterok:" + work_job_no + " " + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._get_data_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError(
"get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._process_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError(
"script to submit processing jobs ended with non-zero return code"
)
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " +
postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def create_process_data_job_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT_PROCESS script to the working directory
processing_script_source_path = self.xnat_pbs_jobs_home
processing_script_source_path += os.sep + self.PIPELINE_NAME
processing_script_source_path += os.sep + self.PIPELINE_NAME
processing_script_source_path += '.XNAT_PROCESS'
processing_script_dest_path = self.working_directory_name
processing_script_dest_path += os.sep + self.PIPELINE_NAME
processing_script_dest_path += '.XNAT_PROCESS'
shutil.copy(processing_script_source_path, processing_script_dest_path)
os.chmod(processing_script_dest_path, stat.S_IRWXU | stat.S_IRWXG)
# write the process data job script (that calls the .XNAT_PROCESS script)
subject_info = ccf_subject.SubjectInfo(self.project, self.subject,
self.classifier, self.scan)
script_name = self.process_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',mem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
xnat_pbs_setup_line = 'source ' + self._get_xnat_pbs_setup_script_path(
) + ' ' + self._get_db_name()
script_line = processing_script_dest_path
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.session
scan_line = ' --scan=' + self.scan
session_classifier_line = ' --session-classifier=' + self.classifier
dcmethod_line = ' --dcmethod=TOPUP'
topupconfig_line = ' --topupconfig=b02b0.cnf'
gdcoeffs_line = ' --gdcoeffs=Prisma_3T_coeff_AS82.grad'
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.setup_file_name
with open(script_name, 'w') as script:
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(xnat_pbs_setup_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(scan_line + ' \\' + os.linesep)
script.write(session_classifier_line + ' \\' + os.linesep)
script.write(dcmethod_line + ' \\' + os.linesep)
script.write(topupconfig_line + ' \\' + os.linesep)
script.write(gdcoeffs_line + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
