def read_subject_info_list(file_name, separator=Hcp3TSubjectInfo.DEFAULT_SEPARATOR()):
"""Reads a subject information list from the specified file.
:param file_name: name of file from which to read
:type file_name: str
"""
subject_info_list = []
input_file = open(file_name, "r")
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != "" and line[0] != "#":
(project, subject_id, extra) = line.split(separator)
# Make the string 'None' in the file translate to a None type instead of just the
# string itself
if extra == "None":
extra = None
subject_info = Hcp3TSubjectInfo(project, subject_id, extra)
subject_info_list.append(subject_info)
return subject_info_list
def submit_process_data_jobs(self, stage, prior_job=None):
module_logger.debug(debug_utils.get_name())
# go ahead and submit the standard process data job and then
# submit an additional freesurfer assessor job
standard_process_data_jobno, all_process_data_jobs = super().submit_process_data_jobs(stage, prior_job)
if OneSubjectJobSubmitter._SUPPRESS_FREESURFER_ASSESSOR_JOB:
module_logger.info("freesufer assessor job not submitted because freesurfer assessor creation has been suppressed")
return standard_process_data_jobno, all_process_data_jobs
if stage >= ccf_processing_stage.ProcessingStage.PROCESS_DATA:
if standard_process_data_jobno:
fs_submit_cmd = 'qsub -W depend=afterok:' + standard_process_data_jobno + ' ' + self.freesurfer_assessor_script_name
else:
fs_submit_cmd = 'qsub ' + self.freesurfer_assessor_script_name
completed_submit_process = subprocess.run(
fs_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)
fs_job_no = str_utils.remove_ending_new_lines(completed_submit_process.stdout)
all_process_data_jobs.append(fs_job_no)
return fs_job_no, all_process_data_jobs
else:
module_logger.info("freesurfer assessor job not submitted because of requested processing stage")
return standard_process_data_jobno, all_process_data_jobs
def read_subject_info_list(file_name,
separator=SubjectInfo.DEFAULT_SEPARATOR()):
"""
Reads a subject information list from the specified file.
"""
subject_info_list = []
input_file = open(file_name, 'r')
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != '' and line[0] != '#':
(project, subject_id, classifier, extra) = line.split(separator)
# Make the string 'None' in the file translate to a None type instead of
# just the string itself
if extra == 'None':
extra = None
subject_info = SubjectInfo(project, subject_id, classifier, extra)
subject_info_list.append(subject_info)
input_file.close()
return subject_info_list
def main():
# create a parser object for getting the command line options
parser = my_argparse.MyArgumentParser()
# mandatory arguments
parser.add_argument('-u', '--user', dest='user', required=True, type=str)
parser.add_argument(dest='input_file')
# optional arguments
parser.add_argument('-ser',
'--server',
dest='server',
required=False,
default='https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
type=str)
parser.add_argument('-p',
'--password',
dest='password',
required=False,
type=str)
# parse the command line arguments
args = parser.parse_args()
if args.password:
password = args.password
else:
password = getpass.getpass("Password: "******"Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Input File: " + args.input_file)
_inform("")
input_file = open(args.input_file, 'r')
for line in input_file:
line = str_utils.remove_ending_new_lines(line)
line = line.strip()
if line != '' and line[0] != '#':
(project, subject, session, resource) = line.split('\t')
_inform("")
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
_inform("")
delete_resource.delete_resource(args.user, password, args.server,
project, subject, session,
resource)
def submit_no_longer_running_jobs(self, stage, prior_job=None): module_logger.debug(debug_utils.get_name()) if prior_job: cmd = 'qsub -W depend=afterany:' + prior_job + ' ' + self.mark_no_longer_running_script_name else: cmd = 'qsub ' + self.mark_no_longer_running_script_name completed_submit_process = subprocess.run( cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True) job_no = str_utils.remove_ending_new_lines(completed_submit_process.stdout) return job_no, [job_no]
def make_all_links_into_copies_ext(full_path):
xnat_pbs_jobs = os_utils.getenv_required('XNAT_PBS_JOBS')
command_str = xnat_pbs_jobs + os.sep + 'lib' + os.sep + 'utils' + os.sep + 'make_symlinks_into_copies.sh' + ' ' + full_path
completed_subprocess = subprocess.run(command_str,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
output = str_utils.remove_ending_new_lines(completed_subprocess.stdout)
print(output)
def read_subject_info_list(file_name,
separator=Hcp7TSubjectInfo.DEFAULT_SEPARATOR()):
"""Reads a subject information list from the specified file.
:param file_name: name of file from which to read
:type file_name: str
"""
subject_info_list = []
input_file = open(file_name, 'r')
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != '' and line[0] != '#':
try:
(project, structural_ref_project, subject_id,
extra) = line.split(separator)
except ValueError as e:
if str(e) == 'not enough values to unpack (expected 4, got 3)':
(project, structural_ref_project,
subject_id) = line.split(separator)
extra = None
else:
raise
# Make the string 'None' in the file translate to a None type instead of just the
# string itself
if project == 'None':
project = None
if structural_ref_project == 'None':
structural_ref_project = None
if subject_id == 'None':
subject_id = None
if extra == 'None':
extra = None
subject_info = Hcp7TSubjectInfo(project, structural_ref_project,
subject_id, extra)
subject_info_list.append(subject_info)
input_file.close()
return subject_info_list
def submit_check_jobs(self, stage, prior_job=None):
module_logger.debug(debug_utils.get_name())
if stage >= ccf_processing_stage.ProcessingStage.CHECK_DATA:
if prior_job:
check_submit_cmd = 'qsub -W depend=afterok:' + prior_job + ' ' + self.check_data_job_script_name
else:
check_submit_cmd = 'qsub ' + self.check_data_job_script_name
completed_submit_process = subprocess.run(
check_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True)
check_job_no = str_utils.remove_ending_new_lines(completed_submit_process.stdout)
return check_job_no, [check_job_no]
else:
module_logger.info("Check data job not submitted")
return None, None
def main():
# create a parser object for getting the command line options
parser = my_argparse.MyArgumentParser()
# mandatory arguments
parser.add_argument("-u", "--user", dest="user", required=True, type=str)
parser.add_argument(dest="input_file")
# optional arguments
parser.add_argument(
"-ser", "--server", dest="server", required=False, default="https://db.humanconnectome.org", type=str
)
parser.add_argument("-p", "--password", dest="password", required=False, type=str)
# parse the command line arguments
args = parser.parse_args()
if args.password:
password = args.password
else:
password = getpass.getpass("Password: "******"Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Input File: " + args.input_file)
_inform("")
input_file = open(args.input_file, "r")
for line in input_file:
line = str_utils.remove_ending_new_lines(line)
line = line.strip()
if line != "" and line[0] != "#":
(project, subject, session, resource) = line.split("\t")
_inform("")
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
_inform("")
delete_resource.delete_resource(args.user, password, args.server, project, subject, session, resource)
def _get_volume_count(self, file_name):
if not os.path.isfile(file_name):
return 0
cmd = 'fslinfo ' + file_name + ' | grep dim4 | grep -v pix'
completed_process = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
# remove new line characters and leading and trailing spaces
output = str_utils.remove_ending_new_lines(completed_process.stdout).strip()
# collapse whitespace to single spaces
output = re.sub('\W+', ' ', output)
log.debug("output: " + output)
(name, value) = output.split(' ')
log.debug("name: " + name)
log.debug("value: " + value)
return int(value)
def read_subject_id_list(file_name):
"""Reads a subject id list from the specified file."""
subject_id_list = []
input_file = open(file_name, 'r')
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != '' and line[0] != '#':
subject_id = line
subject_id_list.append(subject_id)
input_file.close()
return subject_id_list
def read_subject_id_list(file_name):
"""Reads a subject id list from the specified file."""
subject_id_list = []
input_file = open(file_name, 'r')
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != '' and line[0] != '#':
subject_id = line
subject_id_list.append(subject_id)
input_file.close()
return subject_id_list
def _get_expected_volume_count(self, file_name):
sum_col1 = 0
sum_col2 = 0
series_file = open(file_name, 'r')
for line in series_file:
# remove new line characters and leading and trailing spaces
line = str_utils.remove_ending_new_lines(line).strip()
log.debug("line: " + line)
# get 2 values from the line
(num1, num2) = line.split(' ')
log.debug("num1: " + num1)
log.debug("num2: " + num2)
sum_col1 += int(num1)
sum_col2 += int(num2)
series_file.close()
return min(sum_col1, sum_col2)
def read_subject_info_list(file_name, separator=Hcp7TSubjectInfo.DEFAULT_SEPARATOR()):
"""Reads a subject information list from the specified file.
:param file_name: name of file from which to read
:type file_name: str
"""
subject_info_list = []
input_file = open(file_name, 'r')
for line in input_file:
# remove new line characters
line = str_utils.remove_ending_new_lines(line)
# remove leading and trailing spaces
line = line.strip()
# ignore blank lines and comment lines - starting with #
if line != '' and line[0] != '#':
try:
(project, structural_ref_project, subject_id, extra) = line.split(separator)
except ValueError as e:
if str(e) == 'not enough values to unpack (expected 4, got 3)':
(project, structural_ref_project, subject_id) = line.split(separator)
extra = None
else:
raise
# Make the string 'None' in the file translate to a None type instead of just the
# string itself
if extra == 'None':
extra = None
subject_info = Hcp7TSubjectInfo(project, structural_ref_project, subject_id, extra)
subject_info_list.append(subject_info)
input_file.close()
return subject_info_list
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._get_data_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError(
"get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._process_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError(
"script to submit processing jobs ended with non-zero return code"
)
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " +
postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(self, packaging_stage=PackagingStage.CREATE_PACKAGE):
logger.debug("submit_jobs: packaging_stage: " + str(packaging_stage))
logger.info("----------")
logger.info("Submitting " + self.WORK_DESC + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(packaging_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same starting time by sleeping a few seconds
time.sleep(5)
# build the working directory
logger.info("Making working directory: " + self.working_directory_name)
os.makedirs(name=self.working_directory_name)
if packaging_stage >= PackagingStage.PREPARE_SCRIPTS:
self._write_get_data_script()
self._write_create_package_script()
self._write_clean_build_space_script()
# submit the script to get the data
if packaging_stage >= PackagingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self.get_data_script_name
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_process = subprocess.run(get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# submit the script to create the package
if packaging_stage >= PackagingStage.CREATE_PACKAGE:
create_package_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' '
create_package_submit_cmd += self.create_package_script_name
logger.info("create_package_submit_cmd: " +
create_package_submit_cmd)
completed_process = subprocess.run(create_package_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
create_package_job_no = str_utils.remove_ending_new_lines(
completed_process.stdout)
logger.info("create_package_job_no: " + create_package_job_no)
else:
logger.info("Create package job not submitted")
# submit the script to clean the build space
if packaging_stage >= PackagingStage.CLEAN_BUILD_SPACE:
clean_build_space_submit_cmd = 'qsub -W depend=afterok:' + create_package_job_no + ' '
clean_build_space_submit_cmd += self.clean_build_space_script_name
logger.info("clean_build_space_submit_cmd: " +
clean_build_space_submit_cmd)
completed_process = subprocess.run(clean_build_space_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_build_space_job_no = str_utils.remove_ending_new_lines(
completed_process.stdout)
logger.info("clean_build_space_job_no: " +
clean_build_space_job_no)
else:
logger.info("Clean build space job not submitted")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(self.username, self.password,
'https://db.humanconnectome.org', jsession_id)
self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
self.project,
self.PIPELINE_NAME,
'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name() + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name, self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(
subject_info)
inform("FIX processed scans available for subject " +
str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker(
)
if completion_checker.is_processing_complete(
self.archive, subject_info, scan_name):
inform("scan: " + scan_name +
" has already completed PostFixHCP7T processing")
inform(
"Only submitting jobs for incomplete scans - skipping "
+ scan_name)
continue
inform("scan_name: " + scan_name)
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password,
project=project,
subject=subject,
session=session)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server,
jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id,
project,
self.PIPELINE_NAME,
'Queued')
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username, password, str_utils.get_server_name(server),
project, subject, session, output_resource_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec +
',' + vmem_spec + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + 'PostFixHCP7T' + os.sep +
'PostFixHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' +
os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, username, password,
put_server, project, subject, session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# 'https://db.humanconnectome.org', jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._get_data_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError("get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._process_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError("script to submit processing jobs ended with non-zero return code")
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " + postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name()
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
logger.info("-----")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Scan: " + self.scan)
logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, self.scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# determine output resource name
self._output_resource_name = self.scan + "_" + self.output_resource_suffix
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
self._create_get_data_script()
self._create_work_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username,
self.password,
self.put_server,
self.project,
self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name()
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# Submit the job to process the data (do the work)
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# Submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# Submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(self, username, password, server, project, subject,
session, structural_reference_project,
structural_reference_session, put_server, setup_script,
incomplete_only, scan, walltime_limit_hours, mem_limit_gbs,
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of the preprocessed resting state scans that are
# available for the subject
resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(movie_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = resting_state_scan_names + movie_scan_names
else:
scan_list.append(scan)
# process the specified scans
for scan_name in scan_list:
if incomplete_only and self.archive.FIX_processing_repaired(
subject_info, scan_name):
inform("scan: " + scan_name +
" FIX processing is already repaired")
inform(
"Only submitting jobs for incomplete scans - skipping " +
scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.FIX_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
work_script.write('#PBS -l nodes=1:ppn=1,walltime=' +
str(walltime_limit_hours) + ':00:00,mem=' +
str(mem_limit_gbs) + 'gb,vmem=' +
str(vmem_limit_gbs) + 'gb' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + self.PIPELINE_NAME + os.sep +
self.PIPELINE_NAME + '.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
structural_reference_project + '" \\' +
os.linesep)
work_script.write(' --structural-reference-session="' +
structural_reference_session + '" \\' +
os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs,
):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(subject_info)
inform("Preprocessed resting state scans available for subject: " + str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(subject_info)
inform("Preprocessed movie scans available for subject " + str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(subject_info)
inform("FIX processed scans available for subject " + str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker()
if completion_checker.is_processing_complete(self.archive, subject_info, scan_name):
inform("scan: " + scan_name + " has already completed PostFixHCP7T processing")
inform("Only submitting jobs for incomplete scans - skipping " + scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += "." + subject
working_directory_name += "." + long_scan_name
working_directory_name += "." + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server="db.humanconnectome.org", username=username, password=password
)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server="db.humanconnectome.org",
username=username,
password=password,
project=project,
subject=subject,
session=session,
)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server, jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id, project, self.PIPELINE_NAME, "Queued")
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username,
password,
str_utils.get_server_name(server),
project,
subject,
session,
output_resource_name,
)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += "." + long_scan_name
script_file_start_name += "." + self.PIPELINE_NAME
script_file_start_name += "." + project
script_file_start_name += "." + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + ".XNAT_PBS_job.sh"
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, "w")
nodes_spec = "nodes=1:ppn=1"
walltime_spec = "walltime=" + str(walltime_limit_hours) + ":00:00"
vmem_spec = "vmem=" + str(vmem_limit_gbs) + "gb"
work_script.write("#PBS -l " + nodes_spec + "," + walltime_spec + "," + vmem_spec + os.linesep)
work_script.write("#PBS -o " + working_directory_name + os.linesep)
work_script.write("#PBS -e " + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(
self.xnat_pbs_jobs_home
+ os.sep
+ "7T"
+ os.sep
+ "PostFixHCP7T"
+ os.sep
+ "PostFixHCP7T.XNAT.sh \\"
+ os.linesep
)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(server) + '" \\' + os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(" --setup-script=" + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + ".XNAT_PBS_PUT_job.sh"
self.create_put_script(
put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + "_" + self.PIPELINE_NAME,
)
# Submit the job to do the work
work_submit_cmd = "qsub " + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = "qsub -W depend=afterok:" + work_job_no + " " + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
mem_spec = 'mem=' + str(self.mem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + ',' + mem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
# work_script.write(' --keep-all' + ' \\' + os.linesep)
# work_script.write(' --prevent-push' + ' \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# subject_info = hcp7t_subject.Hcp7TSubjectInfo(self.project,
# self.structural_reference_project,
# self.subject)
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
"""
processing_stage is the last processing stage for which to submit
the corresponding job.
GET_DATA means just get the data.
PROCESS_DATA means get the data and do the processing.
PUT_DATA means get the data, processing it, and put the results
back in the DB
"""
logger.debug("submit_jobs processing_stage: " + str(processing_stage))
if self.validate_parameters():
# determine what scans to run the RestingStateStats pipeline on for this subject
# TBD: Does this get run on every scan for which the ICAFIX pipeline has been run,
# or does it only get run on every resting state scan that has been fix processed.
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
self.project, self.structural_reference_project, self.subject)
fix_processed_scans = self.archive.available_FIX_processed_names(
subject_info)
fix_processed_scans_set = set(fix_processed_scans)
logger.debug("fix_processed_scans_set = " +
str(fix_processed_scans_set))
# resting_state_scans = self.archive.available_resting_state_preproc_names(subject_info)
# resting_state_scans_set = set(resting_state_scans)
# logger.debug("resting_state_scans_set = " + str(resting_state_scans_set))
# scans_to_process_set = resting_state_scans_set & fix_processed_scans_set
scans_to_process_set = fix_processed_scans_set
scans_to_process = list(scans_to_process_set)
scans_to_process.sort()
logger.debug("scans_to_process: " + str(scans_to_process))
incomplete_scans_to_process = list()
for scan in scans_to_process:
if (not is_complete(self.archive, subject_info, scan)):
incomplete_scans_to_process.append(scan)
logger.debug("incomplete_scans_to_process: " +
str(incomplete_scans_to_process))
print("incomplete_scans_to_process:", incomplete_scans_to_process)
# for scan in scans_to_process:
for scan in incomplete_scans_to_process:
logger.info("")
logger.info(
"--------------------------------------------------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Structural Reference Project: " +
self.structural_reference_project)
logger.info(" Structural Reference Session: " +
self.structural_reference_session)
logger.info(" Scan: " + scan)
logger.info(" Stage: " + str(processing_stage))
logger.info(
"--------------------------------------------------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
logger.info("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
logger.info("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, 'https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project,
self.PIPELINE_NAME + '_' + scan, 'Queued')
logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = scan + "_RSS"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " +
self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script(scan)
# create script to do work
self._create_work_script(scan)
# create script to clean data
self._create_clean_data_script(scan)
# create script to put the results into the DB
put_script_name = self._put_data_script_name(scan)
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME + '_' + scan)
# submit job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name(
scan)
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# submit job to process the data
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
scan)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
scan)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
else:
logger.info("Unable to submit jobs")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(
current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password,
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name +
" for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name(
) + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(
completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(
completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(
completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
