def _simple_interactive_demo():
hcp7t_subject_info = hcp7t_subject.Hcp7TSubjectInfo(
'HCP_1200', 'HCP_500', '102311')
archive = hcp7t_archive.Hcp7T_Archive()
completion_checker = DeDriftAndResampleHCP7T_HighRes_OneSubjectCompletionChecker(
)
_inform("")
_inform("Checking subject: " + hcp7t_subject_info.subject_id)
_inform("")
_inform("hcp7t_subject_info: " + str(hcp7t_subject_info))
resource_exists = completion_checker.does_processed_resource_exist(
archive, hcp7t_subject_info)
_inform("resource_exists: " + str(resource_exists))
scan_name_list = []
scan_name_list.append('rfMRI_REST1_PA')
scan_name_list.append('rfMRI_REST2_AP')
scan_name_list.append('rfMRI_REST3_PA')
scan_name_list.append('rfMRI_REST4_AP')
scan_name_list.append('tfMRI_MOVIE1_AP')
scan_name_list.append('tfMRI_MOVIE2_PA')
scan_name_list.append('tfMRI_MOVIE3_PA')
scan_name_list.append('tfMRI_MOVIE4_AP')
scan_name_list.append('tfMRI_RETBAR1_AP')
scan_name_list.append('tfMRI_RETBAR2_PA')
scan_name_list.append('tfMRI_RETCCW_AP')
scan_name_list.append('tfMRI_RETCON_PA')
scan_name_list.append('tfMRI_RETCW_PA')
scan_name_list.append('tfMRI_RETEXP_AP')
scan_name_list.append(
'tfMRI_7T_RETCCW_AP_RETCW_PA_RETEXP_AP_RETCON_PA_RETBAR1_AP_RETBAR2_PA'
)
for scan_name in scan_name_list:
_inform("scan_name: " + scan_name)
processing_complete = completion_checker.is_processing_complete(
archive, hcp7t_subject_info, scan_name)
_inform("processing_complete: " + str(processing_complete))
# hcp7t_subject_info = hcp7t_subject.Hcp7TSubjectInfo(
# 'HCP_Staging_7T', 'HCP_900', '181636')
hcp7t_subject_info = hcp7t_subject.Hcp7TSubjectInfo(
'HCP_1200', 'HCP_900', '100610')
_inform("")
_inform("Checking subject: " + hcp7t_subject_info.subject_id)
_inform("")
_inform("hcp7t_subject_info: " + str(hcp7t_subject_info))
for scan_name in scan_name_list:
_inform("scan_name: " + scan_name)
processing_complete = completion_checker.is_processing_complete(
archive, hcp7t_subject_info, scan_name)
_inform("processing_complete: " + str(processing_complete))
def _simple_interactive_demo():
archive = hcp7t_archive.Hcp7T_Archive()
completion_checker = OneSubjectCompletionChecker()
# 102311
hcp7t_subject_info = hcp7t_subject.Hcp7TSubjectInfo(
'HCP_Staging_7T', 'HCP_500', '102311')
_inform("")
_inform("Checking subject: " + hcp7t_subject_info.subject_id)
_inform("")
_inform("hcp7t_subject_info: " + str(hcp7t_subject_info))
resource_exists = completion_checker.does_processed_resource_exist(
archive, hcp7t_subject_info)
_inform("resource_exists: " + str(resource_exists))
processing_complete = completion_checker.is_processing_complete(
archive, hcp7t_subject_info, False)
_inform("processing_complete: " + str(processing_complete))
# 100610
hcp7t_subject_info = hcp7t_subject.Hcp7TSubjectInfo(
'HCP_Staging_7T', 'HCP_900', '100610')
_inform("")
_inform("Checking subject: " + hcp7t_subject_info.subject_id)
_inform("")
resource_exists = completion_checker.does_processed_resource_exist(
archive, hcp7t_subject_info)
_inform("resource_exists: " + str(resource_exists))
processing_complete = completion_checker.is_processing_complete(
archive, hcp7t_subject_info, False)
_inform("processing_complete: " + str(processing_complete))
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(
subject_info)
inform("FIX processed scans available for subject " +
str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker(
)
if completion_checker.is_processing_complete(
self.archive, subject_info, scan_name):
inform("scan: " + scan_name +
" has already completed PostFixHCP7T processing")
inform(
"Only submitting jobs for incomplete scans - skipping "
+ scan_name)
continue
inform("scan_name: " + scan_name)
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password,
project=project,
subject=subject,
session=session)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server,
jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id,
project,
self.PIPELINE_NAME,
'Queued')
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username, password, str_utils.get_server_name(server),
project, subject, session, output_resource_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec +
',' + vmem_spec + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + 'PostFixHCP7T' + os.sep +
'PostFixHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' +
os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, username, password,
put_server, project, subject, session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def _simple_interactive_demo():
archive = Hcp7T_Archive()
_inform("archive.FUNCTIONAL_SCAN_MARKER: " +
archive.FUNCTIONAL_SCAN_MARKER)
_inform("archive.RESTING_STATE_SCAN_MARKER: " +
archive.RESTING_STATE_SCAN_MARKER)
_inform("archive.TASK_SCAN_MARKER: " + archive.TASK_SCAN_MARKER)
_inform("archive.UNPROC_SUFFIX: " + archive.UNPROC_SUFFIX)
_inform("archive.PREPROC_SUFFIX: " + archive.PREPROC_SUFFIX)
_inform("archive.FIX_PROCESSED_SUFFIX: " + archive.FIX_PROCESSED_SUFFIX)
_inform("archive.NAME_DELIMITER: " + archive.NAME_DELIMITER)
_inform("archive.TESLA_SPEC: " + archive.TESLA_SPEC)
_inform("archive.build_home: " + archive.build_home)
#subject_info = hcp7t_subject.Hcp7TSubjectInfo('HCP_Staging_7T', 'HCP_500', '102311')
subject_info = hcp7t_subject.Hcp7TSubjectInfo('HCP_1200', 'HCP_900',
'100610')
_inform("created subject_info: " + str(subject_info))
_inform("archive.session_name(subject_info): " +
archive.session_name(subject_info))
_inform("archive.session_dir_fullpath(subject_info): " +
archive.session_dir_fullpath(subject_info))
_inform("archive.subject_resources_dir_fullpath(subject_info): " +
archive.subject_resources_dir_fullpath(subject_info))
_inform("")
_inform("Available functional unproc dirs: ")
for directory in archive.available_functional_unproc_dir_fullpaths(
subject_info):
_inform(directory)
_inform("")
_inform("Available functional unproc scan names: ")
for name in archive.available_functional_unproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available diffusion unproc dirs: ")
for directory in archive.available_diffusion_unproc_dir_fullpaths(
subject_info):
_inform(directory)
_inform("")
_inform("Available diffusion unproc scan names: ")
for name in archive.available_diffusion_unproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available functional preproc dirs: ")
for directory in archive.available_functional_preproc_dir_fullpaths(
subject_info):
_inform(directory)
_inform("")
_inform("Available functional preproc scan names: ")
for name in archive.available_functional_preproc_names(subject_info):
_inform(name)
_inform("")
_inform("Are the following functional scans preprocessed")
for name in archive.available_functional_unproc_names(subject_info):
_inform("scan name: " + name + " " + "\tfunctionally preprocessed: " +
str(archive.functionally_preprocessed(subject_info, name)))
_inform("")
_inform("Available FIX processed dirs: ")
for directory in archive.available_FIX_processed_dir_fullpaths(
subject_info):
_inform(directory)
_inform("")
_inform("Available FIX processed scan names: ")
for name in archive.available_FIX_processed_names(subject_info):
_inform(name)
_inform("")
_inform("Available Multi-Run FIX processed dirs: ")
for directory in archive.available_MultiRun_FIX_processed_dir_fullpaths(
subject_info):
_inform(directory)
_inform("")
_inform("Available Multi-Run FIX processed scan names: ")
for name in archive.available_MultiRun_FIX_processed_names(subject_info):
_inform(name)
_inform("")
_inform("Are the following functional scans FIX processed")
for name in archive.available_functional_unproc_names(subject_info):
_inform('scan name: ' + name + ' ' + '\tFIX processed: ' +
str(archive.FIX_processing_complete(subject_info, name)))
_inform("")
_inform("Available resting state preproc dirs: ")
for directory in archive.available_resting_state_preproc_dirs(
subject_info):
_inform(directory)
_inform("")
_inform("Available resting state preproc names: ")
for name in archive.available_resting_state_preproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available task preproc dirs: ")
for directory in archive.available_task_preproc_dirs(subject_info):
_inform(directory)
_inform("")
_inform("Available task preproc names: ")
for name in archive.available_task_preproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available MOVIE preproc dirs: ")
for directory in archive.available_movie_preproc_dirs(subject_info):
_inform(directory)
_inform("")
_inform("Available MOVIE preproc names: ")
for name in archive.available_movie_preproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available RETINOTOPY preproc dirs: ")
for directory in archive.available_retinotopy_preproc_dirs(subject_info):
_inform(directory)
_inform("")
_inform("Available RETINOTOPY preproc names: ")
for name in archive.available_retinotopy_preproc_names(subject_info):
_inform(name)
_inform("")
_inform("Available functional unprocessed scan names: ")
for name in archive.available_functional_unproc_names(subject_info):
_inform(name + '\t' + '\tprefix: ' +
archive.functional_scan_prefix(name) + '\tbase_name: ' +
archive.functional_scan_base_name(name) + '\tpe_dir: ' +
archive.functional_scan_pe_dir(name) + '\tlong_name: ' +
archive.functional_scan_long_name(name))
_inform("")
_inform("Available session dirs for project: " + subject_info.project)
for session in archive.available_session_dirs(subject_info.project):
_inform(session)
_inform("")
_inform("Available session names for project: " + subject_info.project)
for name in archive.available_session_names(subject_info.project):
_inform(name)
_inform("")
_inform("Available subject ids for project: " + subject_info.project)
for subject_id in archive.available_subject_ids(subject_info.project):
_inform(subject_id)
_inform("")
_inform("Number of available subject ids for project: " +
subject_info.project + " " +
str(archive.subject_count(subject_info.project)))
_inform("")
_inform("Available diffusion scans: ")
for scan in archive.available_diffusion_scan_fullpaths(subject_info):
_inform(scan)
_inform("")
_inform("Available diffusion scan names: ")
for scan_name in archive.available_diffusion_scan_names(subject_info):
_inform(scan_name)
def submit_jobs(self, username, password, server, project, subject,
session, structural_reference_project,
structural_reference_session, put_server, setup_script,
incomplete_only, scan, walltime_limit_hours, mem_limit_gbs,
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of the preprocessed resting state scans that are
# available for the subject
resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(movie_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = resting_state_scan_names + movie_scan_names
else:
scan_list.append(scan)
# process the specified scans
for scan_name in scan_list:
if incomplete_only and self.archive.FIX_processing_repaired(
subject_info, scan_name):
inform("scan: " + scan_name +
" FIX processing is already repaired")
inform(
"Only submitting jobs for incomplete scans - skipping " +
scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.FIX_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
work_script.write('#PBS -l nodes=1:ppn=1,walltime=' +
str(walltime_limit_hours) + ':00:00,mem=' +
str(mem_limit_gbs) + 'gb,vmem=' +
str(vmem_limit_gbs) + 'gb' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + self.PIPELINE_NAME + os.sep +
self.PIPELINE_NAME + '.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
structural_reference_project + '" \\' +
os.linesep)
work_script.write(' --structural-reference-session="' +
structural_reference_session + '" \\' +
os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
required=False,
type=str)
parser.add_argument('-a',
'--check-all',
dest='check_all',
action='store_true',
required=False,
default=False)
# parse the command line arguments
args = parser.parse_args()
# check the specified subject for processing completion
archive = hcp7t_archive.Hcp7T_Archive()
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project=args.project,
subject_id=args.subject,
extra=args.scan)
completion_checker = OneSubjectCompletionChecker()
if args.output:
processing_output = open(args.output, 'w')
else:
processing_output = sys.stdout
if completion_checker.is_processing_complete(
archive=archive,
subject_info=subject_info,
verbose=args.verbose,
output=processing_output,
short_circuit=not args.check_all):
print("Exiting with 0 code - Completion Check Successful")
import hcp.hcp7t.subject as hcp7t_subject
# authorship information
__author__ = "Timothy B. Brown"
__copyright__ = "Copyright 2017, The Human Connectome Project/Connectome Coordination Facility"
__maintainer__ = "Timothy B. Brown"
expected_retinotopy_scans = [
'tfMRI_RETCCW_AP', 'tfMRI_RETCW_PA', 'tfMRI_RETEXP_AP', 'tfMRI_RETCON_PA',
'tfMRI_RETBAR1_AP', 'tfMRI_RETBAR2_PA'
]
if __name__ == '__main__':
project = 'HCP_1200'
archive = hcp7t_archive.Hcp7T_Archive()
subject_ids = archive.available_subject_id_list(project)
for subject_id in subject_ids:
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project=project,
subject_id=subject_id)
available_retinotopy_task_names = archive.available_retinotopy_preproc_names(
subject_info)
# print(subject_info, available_retinotopy_task_names)
for expected_scan in expected_retinotopy_scans:
if expected_scan not in available_retinotopy_task_names:
print(subject_info, "is missing", expected_scan)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
"""
processing_stage is the last processing stage for which to submit
the corresponding job.
GET_DATA means just get the data.
PROCESS_DATA means get the data and do the processing.
PUT_DATA means get the data, processing it, and put the results
back in the DB
"""
logger.debug("submit_jobs processing_stage: " + str(processing_stage))
if self.validate_parameters():
# determine what scans to run the RestingStateStats pipeline on for this subject
# TBD: Does this get run on every scan for which the ICAFIX pipeline has been run,
# or does it only get run on every resting state scan that has been fix processed.
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
self.project, self.structural_reference_project, self.subject)
fix_processed_scans = self.archive.available_FIX_processed_names(
subject_info)
fix_processed_scans_set = set(fix_processed_scans)
logger.debug("fix_processed_scans_set = " +
str(fix_processed_scans_set))
# resting_state_scans = self.archive.available_resting_state_preproc_names(subject_info)
# resting_state_scans_set = set(resting_state_scans)
# logger.debug("resting_state_scans_set = " + str(resting_state_scans_set))
# scans_to_process_set = resting_state_scans_set & fix_processed_scans_set
scans_to_process_set = fix_processed_scans_set
scans_to_process = list(scans_to_process_set)
scans_to_process.sort()
logger.debug("scans_to_process: " + str(scans_to_process))
incomplete_scans_to_process = list()
for scan in scans_to_process:
if (not is_complete(self.archive, subject_info, scan)):
incomplete_scans_to_process.append(scan)
logger.debug("incomplete_scans_to_process: " +
str(incomplete_scans_to_process))
print("incomplete_scans_to_process:", incomplete_scans_to_process)
# for scan in scans_to_process:
for scan in incomplete_scans_to_process:
logger.info("")
logger.info(
"--------------------------------------------------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Structural Reference Project: " +
self.structural_reference_project)
logger.info(" Structural Reference Session: " +
self.structural_reference_session)
logger.info(" Scan: " + scan)
logger.info(" Stage: " + str(processing_stage))
logger.info(
"--------------------------------------------------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
logger.info("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
logger.info("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, 'https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project,
self.PIPELINE_NAME + '_' + scan, 'Queued')
logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = scan + "_RSS"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " +
self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script(scan)
# create script to do work
self._create_work_script(scan)
# create script to clean data
self._create_clean_data_script(scan)
# create script to put the results into the DB
put_script_name = self._put_data_script_name(scan)
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME + '_' + scan)
# submit job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name(
scan)
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# submit job to process the data
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
scan)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
scan)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
else:
logger.info("Unable to submit jobs")
def main():
# create a parser object for getting the command line arguments
parser = my_argparse.MyArgumentParser()
# mandatory arguments
parser.add_argument('-p',
'--project',
dest='project',
required=True,
type=str)
parser.add_argument('-s',
'--subject',
dest='subject',
required=True,
type=str)
parser.add_argument('-d',
'--study-dir',
dest='output_study_dir',
required=True,
type=str)
parser.add_argument('-t',
'--structural-reference-project',
dest='structural_reference_project',
required=True,
type=str)
# optional arguments
parser.add_argument('-c',
'--copy',
dest='copy',
action='store_true',
required=False,
default=False)
parser.add_argument('-l',
'--log',
dest='log',
action='store_true',
required=False,
default=False)
parser.add_argument('-r',
'--remove-non-subdirs',
dest='remove_non_subdirs',
action='store_true',
required=False,
default=False)
parser.add_argument('-j',
'--remove-job-and-catalog-files',
dest='remove_job_and_catalog_files',
action='store_true',
required=False,
default=False)
phase_choices = [
"FULL", "full", "DIFFUSION_PREPROC_VETTING",
"diffusion_preproc_vetting", "MULTIRUNICAFIX_PREREQS",
"multirunicafix_prereqs", "ICAFIX", "icafix", "MULTIRUNICAFIX",
"multirunicafix", "POSTFIX", "postfix"
]
parser.add_argument('-ph',
'--phase',
dest='phase',
required=False,
choices=phase_choices,
default="full")
# parse the command line arguments
args = parser.parse_args()
# show arguments
log.info("Arguments:")
log.info(" Project: " + args.project)
log.info(" Structural Ref Project: " + args.structural_reference_project)
log.info(" Subject: " + args.subject)
log.info(" Output Study Dir: " + args.output_study_dir)
log.info(" Copy: " + str(args.copy))
log.info(" Phase: " + args.phase)
log.info(" Log: " + str(args.log))
log.info(" Remove Non-Subdirs: " + str(args.remove_non_subdirs))
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project=args.project,
structural_reference_project=args.structural_reference_project,
subject_id=args.subject)
archive = hcp7t_archive.Hcp7T_Archive()
reference_archive = hcp3t_archive.Hcp3T_Archive()
# create and configure CinabStyleDataRetriever
data_retriever = CinabStyleDataRetriever(archive, reference_archive)
data_retriever.copy = args.copy
data_retriever.show_log = args.log
# retrieve data based on phase requested
args.phase = args.phase.upper()
if args.phase == "FULL":
data_retriever.get_full_data(subject_info, args.output_study_dir)
elif args.phase == "DIFFUSION_PREPROC_VETTING":
data_retriever.get_diffusion_preproc_vetting_data(
subject_info, args.output_study_dir)
elif args.phase == "MULTIRUNICAFIX_PREREQS":
data_retriever.get_multirunicafix_prereqs(subject_info,
args.output_study_dir)
elif args.phase == "ICAFIX":
data_retriever.get_data_through_ICAFIX(subject_info,
args.output_study_dir)
elif args.phase == "MULTIRUNICAFIX":
data_retriever.get_data_through_multirun_ICAFIX(
subject_info, args.output_study_dir)
elif args.phase == "POSTFIX":
data_retriever.get_data_through_PostFix(subject_info,
args.output_study_dir)
if args.remove_non_subdirs:
# remove any non-subdirectory data at the output study directory level
data_retriever.remove_non_subdirs(args.output_study_dir)
data_retriever.remove_non_subdirs(args.output_study_dir + os.sep +
subject_info.subject_id)
if args.remove_job_and_catalog_files:
# remove any PBS job files and XNAT catalog files
data_retriever.remove_pbs_job_files(args.output_study_dir)
data_retriever.remove_xnat_catalog_files(args.output_study_dir)
def create_process_data_job_script(self):
module_logger.debug(debug_utils.get_name())
# copy the .XNAT script to the working directory
processing_script_source_name = self.xnat_pbs_jobs_home
processing_script_source_name += os.sep + '7T'
processing_script_source_name += os.sep + self.PIPELINE_NAME
processing_script_source_name += os.sep + self.PIPELINE_NAME
processing_script_source_name += '.XNAT_PROCESS'
processing_script_dest_name = self.working_directory_name
processing_script_dest_name += os.sep + self.PIPELINE_NAME
processing_script_dest_name += '.XNAT_PROCESS'
shutil.copy(processing_script_source_name, processing_script_dest_name)
os.chmod(processing_script_dest_name, stat.S_IRWXU | stat.S_IRWXG)
# write the process data job script (that calls the .XNAT script)
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project=self.project,
subject_id=self.subject)
script_name = self.process_data_job_script_name
with contextlib.suppress(FileNotFoundError):
os.remove(script_name)
walltime_limit_str = str(self.walltime_limit_hours) + ':00:00'
mem_limit_str = str(self.mem_limit_gbs) + 'gb'
vmem_limit_str = str(self.vmem_limit_gbs) + 'gb'
resources_line = '#PBS -l nodes=' + str(self.WORK_NODE_COUNT)
resources_line += ':ppn=' + str(self.WORK_PPN)
resources_line += ',walltime=' + walltime_limit_str
resources_line += ',mem=' + mem_limit_str
resources_line += ',vmem=' + vmem_limit_str
stdout_line = '#PBS -o ' + self.working_directory_name
stderr_line = '#PBS -e ' + self.working_directory_name
script_line = processing_script_dest_name
user_line = ' --user='******' --password='******' --server=' + str_utils.get_server_name(self.server)
project_line = ' --project=' + self.project
subject_line = ' --subject=' + self.subject
session_line = ' --session=' + self.subject + '_7T'
avail_retinotopy_task_names = self.archive.available_retinotopy_preproc_names(
subject_info)
# sort available retinotopy task names into the order the
# tasks were presented to the subject
avail_retinotopy_task_names = sorted(
avail_retinotopy_task_names, key=retinotopy_presentation_order_key)
# add the tesla spec to each element of the group
group_names = list(map(add_tesla_spec, avail_retinotopy_task_names))
group_spec = '@'.join(group_names)
group_line = ' --group=' + group_spec
concat_spec = '_'.join(
list(map(remove_scan_type, avail_retinotopy_task_names)))
concat_line = ' --concat-name=tfMRI_7T_' + concat_spec
wdir_line = ' --working-dir=' + self.working_directory_name
setup_line = ' --setup-script=' + self.setup_file_name
with open(script_name, 'w') as script:
script.write(resources_line + os.linesep)
script.write(stdout_line + os.linesep)
script.write(stderr_line + os.linesep)
script.write(os.linesep)
script.write(script_line + ' \\' + os.linesep)
script.write(user_line + ' \\' + os.linesep)
script.write(password_line + ' \\' + os.linesep)
script.write(server_line + ' \\' + os.linesep)
script.write(project_line + ' \\' + os.linesep)
script.write(subject_line + ' \\' + os.linesep)
script.write(session_line + ' \\' + os.linesep)
script.write(group_line + ' \\' + os.linesep)
script.write(concat_line + ' \\' + os.linesep)
script.write(wdir_line + ' \\' + os.linesep)
script.write(setup_line + os.linesep)
script.write(os.linesep)
os.chmod(script_name, stat.S_IRWXU | stat.S_IRWXG)
