def main():
# create a parser object for getting the command line options
parser = my_argparse.MyArgumentParser()
# mandatory arguments
parser.add_argument('-u', '--user', dest='user', required=True, type=str)
parser.add_argument(dest='input_file')
# optional arguments
parser.add_argument('-ser',
'--server',
dest='server',
required=False,
default='https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
type=str)
parser.add_argument('-p',
'--password',
dest='password',
required=False,
type=str)
# parse the command line arguments
args = parser.parse_args()
if args.password:
password = args.password
else:
password = getpass.getpass("Password: "******"Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Input File: " + args.input_file)
_inform("")
input_file = open(args.input_file, 'r')
for line in input_file:
line = str_utils.remove_ending_new_lines(line)
line = line.strip()
if line != '' and line[0] != '#':
(project, subject, session, resource) = line.split('\t')
_inform("")
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
_inform("")
delete_resource.delete_resource(args.user, password, args.server,
project, subject, session,
resource)
def main():
# create a parser object for getting the command line arguments
parser = my_argparse.MyArgumentParser(
description="Program to delete all DB resources of a given name for all sessions in a given ConnectomeDB project.")
# mandatory arguments
parser.add_argument('-u', '--user', dest='user', required=True, type=str)
parser.add_argument('-pw', '--password', dest='password', required=True, type=str)
parser.add_argument('-pr', '--project', dest='project', required=True, type=str)
parser.add_argument('-r', '--resource', dest='resource', required=True, type=str)
# optional arguments
parser.add_argument('-ser', '--server', dest='server', required=False,
default='https://' + os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
type=str)
parser.add_argument('-f', '--force', dest='force', action='store_true', required=False, default=False)
# parse the command line arguments
args = parser.parse_args()
# show parsed arguments
_inform("Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Project: " + args.project)
_inform(" Resource: " + args.resource)
_inform(" Force: " + str(args.force))
# find all instances of the specified resource in the specified project
my_xnat_archive = xnat_archive.XNAT_Archive()
archive_root = my_xnat_archive.project_archive_root(args.project)
dir_list = glob.glob(archive_root + os.sep + '*')
for directory in sorted(dir_list):
resource_dir_to_look_for = directory + os.sep + 'RESOURCES' + os.sep + args.resource
if os.path.isdir(resource_dir_to_look_for):
unprefixed = resource_dir_to_look_for.replace(archive_root + os.sep, "")
sep_loc = unprefixed.find(os.sep)
session = unprefixed[:sep_loc]
underscore_loc = session.find('_')
subject = session[:underscore_loc]
_inform("Deleting resource: " + args.resource + " for session: " + session)
delete_resource.delete_resource(args.user, args.password, args.server,
args.project, subject, session,
args.resource, args.force)
def main():
# create a parser object for getting the command line arguments
parser = my_argparse.MyArgumentParser(
description="Program to delete all DB resources of a given name for all sessions in a given ConnectomeDB project.")
# mandatory arguments
parser.add_argument('-u', '--user', dest='user', required=True, type=str)
parser.add_argument('-pw', '--password', dest='password', required=True, type=str)
parser.add_argument('-pr', '--project', dest='project', required=True, type=str)
parser.add_argument('-r', '--resource', dest='resource', required=True, type=str)
# optional arguments
parser.add_argument('-ser', '--server', dest='server', required=False, default='https://db.humanconnectome.org', type=str)
parser.add_argument('-f', '--force', dest='force', action='store_true', required=False, default=False)
# parse the command line arguments
args = parser.parse_args()
# show parsed arguments
_inform("Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Project: " + args.project)
_inform(" Resource: " + args.resource)
_inform(" Force: " + str(args.force))
# find all instances of the specified resource in the specified project
my_xnat_archive = xnat_archive.XNAT_Archive()
archive_root = my_xnat_archive.project_archive_root(args.project)
dir_list = glob.glob(archive_root + os.sep + '*')
for directory in sorted(dir_list):
resource_dir_to_look_for = directory + os.sep + 'RESOURCES' + os.sep + args.resource
if os.path.isdir(resource_dir_to_look_for):
unprefixed = resource_dir_to_look_for.replace(archive_root + os.sep, "")
sep_loc = unprefixed.find(os.sep)
session = unprefixed[:sep_loc]
underscore_loc = session.find('_')
subject = session[:underscore_loc]
_inform("Deleting resource: " + args.resource + " for session: " + session)
delete_resource.delete_resource(args.user, args.password, args.server,
args.project, subject, session,
args.resource, args.force)
def main():
# create a parser object for getting the command line options
parser = my_argparse.MyArgumentParser()
# mandatory arguments
parser.add_argument("-u", "--user", dest="user", required=True, type=str)
parser.add_argument(dest="input_file")
# optional arguments
parser.add_argument(
"-ser", "--server", dest="server", required=False, default="https://db.humanconnectome.org", type=str
)
parser.add_argument("-p", "--password", dest="password", required=False, type=str)
# parse the command line arguments
args = parser.parse_args()
if args.password:
password = args.password
else:
password = getpass.getpass("Password: "******"Parsed arguments:")
_inform(" Username: "******" Password: "******"*** password mask ***")
_inform(" Server: " + args.server)
_inform(" Input File: " + args.input_file)
_inform("")
input_file = open(args.input_file, "r")
for line in input_file:
line = str_utils.remove_ending_new_lines(line)
line = line.strip()
if line != "" and line[0] != "#":
(project, subject, session, resource) = line.split("\t")
_inform("")
_inform(" Project: " + project)
_inform(" Subject: " + subject)
_inform(" Session: " + session)
_inform(" Resource: " + resource)
_inform("")
delete_resource.delete_resource(args.user, password, args.server, project, subject, session, resource)
def submit_jobs(
self,
processing_stage=ccf_processing_stage.ProcessingStage.CHECK_DATA):
module_logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
module_logger.info("-----")
module_logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
module_logger.info(" Project: " + self.project)
module_logger.info(" Subject: " + self.subject)
module_logger.info(" Session: " + self.session)
module_logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
os.makedirs(name=self.working_directory_name)
os.makedirs(name=self.check_data_directory_name)
os.makedirs(name=self.mark_completion_directory_name)
module_logger.info("Output Resource Name: " +
self.output_resource_name)
# clean output resource if requested
if self.clean_output_resource_first:
module_logger.info("Deleting resource: " +
self.output_resource_name + " for:")
module_logger.info(" project: " + self.project)
module_logger.info(" subject: " + self.subject)
module_logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self.output_resource_name)
return self.do_job_submissions(processing_stage)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
# jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._get_data_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError(
"get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(
['bash', self._process_script_name()],
stdout=stdout_file,
stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError(
"script to submit processing jobs ended with non-zero return code"
)
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " +
postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(self.username, self.password,
'https://db.humanconnectome.org', jsession_id)
self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
self.project,
self.PIPELINE_NAME,
'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name() + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name, self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(
self,
processing_stage=one_subject_job_submitter.ProcessingStage.PUT_DATA
):
module_logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
module_logger.info("-----")
module_logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
module_logger.info(" Project: " + self.project)
module_logger.info(" Subject: " + self.subject)
module_logger.info(" Session: " + self.session)
module_logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
os.makedirs(name=self.working_directory_name)
# determine output resource name
self._output_resource_name = self.output_resource_suffix
module_logger.info("Output Resource Name: " +
self._output_resource_name)
# clean output resource if requested
if self.clean_output_resource_first:
module_logger.info("Deleting resource: " +
self._output_resource_name + " for:")
module_logger.info(" project: " + self.project)
module_logger.info(" subject: " + self.subject)
module_logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# build list of groups
self._group_list = []
for group in self.groups:
self._group_list.append(self._expand(group))
module_logger.debug("self._group_list: " + str(self._group_list))
# build list of concat names
self._concat_name_list = []
for group in self._group_list:
self._concat_name_list.append(self._concat(group))
# create scripts for various stages of processing
if processing_stage >= one_subject_job_submitter.ProcessingStage.PREPARE_SCRIPTS:
self.create_get_data_job_script()
self.create_work_script()
self.create_clean_data_script()
self.create_put_data_script()
# Submit the job to get the data
if processing_stage >= one_subject_job_submitter.ProcessingStage.GET_DATA:
get_data_job_no = self.submit_get_data_job()
module_logger.info("get_data_job_no: " + str(get_data_job_no))
else:
module_logger.info("Get data job not submitted")
# Submit the job to process the data (do the work)
if processing_stage >= one_subject_job_submitter.ProcessingStage.PROCESS_DATA:
work_job_no = self.submit_process_data_job(get_data_job_no)
module_logger.info("work_job_no: " + str(work_job_no))
else:
module_logger.info("Process data job not submitted")
# Submit job to clean the data
if processing_stage >= one_subject_job_submitter.ProcessingStage.CLEAN_DATA:
clean_job_no = self.submit_clean_data_job(work_job_no)
module_logger.info("clean_job_no: " + str(clean_job_no))
else:
module_logger.info("Clean data job not submitted")
# Submit job to put the resulting data in the DB
if processing_stage >= one_subject_job_submitter.ProcessingStage.PUT_DATA:
put_job_no = self.submit_put_data_job(clean_job_no)
module_logger.info("put_job_no: " + str(put_job_no))
else:
module_logger.info("Put data job not submitted")
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(
subject_info)
inform("Preprocessed resting state scans available for subject: " +
str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(
subject_info)
inform("Preprocessed movie scans available for subject " +
str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(
subject_info)
inform("FIX processed scans available for subject " +
str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker(
)
if completion_checker.is_processing_complete(
self.archive, subject_info, scan_name):
inform("scan: " + scan_name +
" has already completed PostFixHCP7T processing")
inform(
"Only submitting jobs for incomplete scans - skipping "
+ scan_name)
continue
inform("scan_name: " + scan_name)
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(
scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + subject
working_directory_name += '.' + long_scan_name
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=username,
password=password,
project=project,
subject=subject,
session=session)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server,
jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id,
project,
self.PIPELINE_NAME,
'Queued')
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username, password, str_utils.get_server_name(server),
project, subject, session, output_resource_name)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += '.' + long_scan_name
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + project
script_file_start_name += '.' + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec +
',' + vmem_spec + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
os.sep + 'PostFixHCP7T' + os.sep +
'PostFixHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' +
os.linesep)
work_script.write(' --server="' +
str_utils.get_server_name(server) + '" \\' +
os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' +
os.linesep)
work_script.write(' --working-dir="' + working_directory_name +
'" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' +
os.linesep)
work_script.write(' --setup-script=' + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, username, password,
put_server, project, subject, session,
working_directory_name,
output_resource_name,
scan_name + '_' + self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug(debug_utils.get_name() + ": processing_stage: " +
str(processing_stage))
logger.info("-----")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Scan: " + self.scan)
logger.info(" Stage: " + str(processing_stage))
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, self.scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# determine output resource name
self._output_resource_name = self.scan + "_" + self.output_resource_suffix
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name +
" for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
self._create_get_data_script()
self._create_work_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username,
self.password,
self.put_server,
self.project,
self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME,
leave_subject_id_level=True)
# Submit the job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name()
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# Submit the job to process the data (do the work)
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# Submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# Submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
def submit_jobs(
self,
username,
password,
server,
project,
subject,
session,
structural_reference_project,
structural_reference_session,
put_server,
clean_output_resource_first,
setup_script,
incomplete_only,
scan,
walltime_limit_hours,
mem_limit_gbs, # UNUSED
vmem_limit_gbs,
):
subject_info = hcp7t_subject.Hcp7TSubjectInfo(project, structural_reference_project, subject)
# determine names of preprocessed resting state scans that are
# available for the subject
preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(subject_info)
inform("Preprocessed resting state scans available for subject: " + str(preproc_resting_state_scan_names))
# determine names of the preprocessed MOVIE task scans that are available for the subject
preproc_movie_scan_names = self.archive.available_movie_preproc_names(subject_info)
inform("Preprocessed movie scans available for subject " + str(preproc_movie_scan_names))
# determine names of the FIX processed scans that are available for the subject
fix_processed_scan_names = self.archive.available_FIX_processed_names(subject_info)
inform("FIX processed scans available for subject " + str(fix_processed_scan_names))
# build list of scans to process
scan_list = []
if scan is None:
scan_list = fix_processed_scan_names
else:
scan_list.append(scan)
# process specified scans
for scan_name in scan_list:
if incomplete_only:
completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker()
if completion_checker.is_processing_complete(self.archive, subject_info, scan_name):
inform("scan: " + scan_name + " has already completed PostFixHCP7T processing")
inform("Only submitting jobs for incomplete scans - skipping " + scan_name)
continue
long_scan_name = self.archive.functional_scan_long_name(scan_name)
output_resource_name = self.archive.PostFix_processed_resource_name(scan_name)
inform("")
inform("-------------------------------------------------")
inform("Submitting jobs for scan: " + long_scan_name)
inform("Output resource name: " + output_resource_name)
inform("-------------------------------------------------")
inform("")
# make sure working directories don't have the same name based on the
# same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += "." + subject
working_directory_name += "." + long_scan_name
working_directory_name += "." + str(current_seconds_since_epoch)
# make the working directory
inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server="db.humanconnectome.org", username=username, password=password
)
inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server="db.humanconnectome.org",
username=username,
password=password,
project=project,
subject=subject,
session=session,
)
inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(username, password, server, jsession_id)
workflow_id = workflow_obj.create_workflow(xnat_session_id, project, self.PIPELINE_NAME, "Queued")
inform("workflow_id: " + workflow_id)
# Clean the output resource if requested
if clean_output_resource_first:
inform("Deleting resource: " + output_resource_name + " for:")
inform(" project: " + project)
inform(" subject: " + subject)
inform(" session: " + session)
delete_resource.delete_resource(
username,
password,
str_utils.get_server_name(server),
project,
subject,
session,
output_resource_name,
)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + subject
script_file_start_name += "." + long_scan_name
script_file_start_name += "." + self.PIPELINE_NAME
script_file_start_name += "." + project
script_file_start_name += "." + session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + ".XNAT_PBS_job.sh"
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, "w")
nodes_spec = "nodes=1:ppn=1"
walltime_spec = "walltime=" + str(walltime_limit_hours) + ":00:00"
vmem_spec = "vmem=" + str(vmem_limit_gbs) + "gb"
work_script.write("#PBS -l " + nodes_spec + "," + walltime_spec + "," + vmem_spec + os.linesep)
work_script.write("#PBS -o " + working_directory_name + os.linesep)
work_script.write("#PBS -e " + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(
self.xnat_pbs_jobs_home
+ os.sep
+ "7T"
+ os.sep
+ "PostFixHCP7T"
+ os.sep
+ "PostFixHCP7T.XNAT.sh \\"
+ os.linesep
)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(server) + '" \\' + os.linesep)
work_script.write(' --project="' + project + '" \\' + os.linesep)
work_script.write(' --subject="' + subject + '" \\' + os.linesep)
work_script.write(' --session="' + session + '" \\' + os.linesep)
work_script.write(' --scan="' + long_scan_name + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(" --setup-script=" + setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + ".XNAT_PBS_PUT_job.sh"
self.create_put_script(
put_script_name,
username,
password,
put_server,
project,
subject,
session,
working_directory_name,
output_resource_name,
scan_name + "_" + self.PIPELINE_NAME,
)
# Submit the job to do the work
work_submit_cmd = "qsub " + work_script_name
inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = "qsub -W depend=afterok:" + work_job_no + " " + put_script_name
inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
inform("put_job_no: " + put_job_no)
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
mem_spec = 'mem=' + str(self.mem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + ',' + mem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
# work_script.write(' --keep-all' + ' \\' + os.linesep)
# work_script.write(' --prevent-push' + ' \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
# subject_info = hcp7t_subject.Hcp7TSubjectInfo(self.project,
# self.structural_reference_project,
# self.subject)
# make sure working directories don't have the same name based on the same
# start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
working_directory_name = self.build_home
working_directory_name += os.sep + self.project
working_directory_name += os.sep + self.PIPELINE_NAME
working_directory_name += '.' + self.subject
working_directory_name += '.' + str(current_seconds_since_epoch)
# make the working directory
_inform("Making working directory: " + working_directory_name)
os.makedirs(name=working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server='db.humanconnectome.org',
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, self.server, jsession_id)
workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + workflow_id)
# Determine the output resource name
output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
_inform("output_resource_name: " + output_resource_name)
# Clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resouce: " + output_resource_name + " for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password, str_utils.get_server_name(self.server),
self.project, self.subject, self.session, output_resource_name, True)
script_file_start_name = working_directory_name
script_file_start_name += os.sep + self.subject
script_file_start_name += '.' + self.PIPELINE_NAME
script_file_start_name += '.' + self.project
script_file_start_name += '.' + self.session
# Create script to submit to do the actual work
work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
with contextlib.suppress(FileNotFoundError):
os.remove(work_script_name)
work_script = open(work_script_name, 'w')
nodes_spec = 'nodes=1:ppn=1'
walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + os.linesep)
# work_script.write('#PBS -q HCPput' + os.linesep)
work_script.write('#PBS -o ' + working_directory_name + os.linesep)
work_script.write('#PBS -e ' + working_directory_name + os.linesep)
work_script.write(os.linesep)
work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
work_script.write(' --user="******" \\' + os.linesep)
work_script.write(' --password="******" \\' + os.linesep)
work_script.write(' --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
work_script.write(' --project="' + self.project + '" \\' + os.linesep)
work_script.write(' --subject="' + self.subject + '" \\' + os.linesep)
work_script.write(' --session="' + self.session + '" \\' + os.linesep)
work_script.write(' --structural-reference-project="' +
self.structural_reference_project + '" \\' + os.linesep)
work_script.write(' --structural-reference-session="' +
self.structural_reference_session + '" \\' + os.linesep)
work_script.write(' --working-dir="' + working_directory_name + '" \\' + os.linesep)
work_script.write(' --workflow-id="' + workflow_id + '" \\' + os.linesep)
work_script.write(' --setup-script=' + self.setup_script + os.linesep)
work_script.close()
os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)
# Create script to put the results into the DB
put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
working_directory_name, output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the work
work_submit_cmd = 'qsub ' + work_script_name
_inform("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
_inform("work_job_no: " + work_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
"""
processing_stage is the last processing stage for which to submit
the corresponding job.
GET_DATA means just get the data.
PROCESS_DATA means get the data and do the processing.
PUT_DATA means get the data, processing it, and put the results
back in the DB
"""
logger.debug("submit_jobs processing_stage: " + str(processing_stage))
if self.validate_parameters():
# determine what scans to run the RestingStateStats pipeline on for this subject
# TBD: Does this get run on every scan for which the ICAFIX pipeline has been run,
# or does it only get run on every resting state scan that has been fix processed.
subject_info = hcp7t_subject.Hcp7TSubjectInfo(
self.project, self.structural_reference_project, self.subject)
fix_processed_scans = self.archive.available_FIX_processed_names(
subject_info)
fix_processed_scans_set = set(fix_processed_scans)
logger.debug("fix_processed_scans_set = " +
str(fix_processed_scans_set))
# resting_state_scans = self.archive.available_resting_state_preproc_names(subject_info)
# resting_state_scans_set = set(resting_state_scans)
# logger.debug("resting_state_scans_set = " + str(resting_state_scans_set))
# scans_to_process_set = resting_state_scans_set & fix_processed_scans_set
scans_to_process_set = fix_processed_scans_set
scans_to_process = list(scans_to_process_set)
scans_to_process.sort()
logger.debug("scans_to_process: " + str(scans_to_process))
incomplete_scans_to_process = list()
for scan in scans_to_process:
if (not is_complete(self.archive, subject_info, scan)):
incomplete_scans_to_process.append(scan)
logger.debug("incomplete_scans_to_process: " +
str(incomplete_scans_to_process))
print("incomplete_scans_to_process:", incomplete_scans_to_process)
# for scan in scans_to_process:
for scan in incomplete_scans_to_process:
logger.info("")
logger.info(
"--------------------------------------------------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Structural Reference Project: " +
self.structural_reference_project)
logger.info(" Structural Reference Session: " +
self.structural_reference_session)
logger.info(" Scan: " + scan)
logger.info(" Stage: " + str(processing_stage))
logger.info(
"--------------------------------------------------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, scan)
logger.info("Making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
logger.info("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=self.project,
subject=self.subject,
session=self.session)
logger.info("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password, 'https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project,
self.PIPELINE_NAME + '_' + scan, 'Queued')
logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = scan + "_RSS"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " +
self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script(scan)
# create script to do work
self._create_work_script(scan)
# create script to clean data
self._create_clean_data_script(scan)
# create script to put the results into the DB
put_script_name = self._put_data_script_name(scan)
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject,
self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME + '_' + scan)
# submit job to get the data
if processing_stage >= ProcessingStage.GET_DATA:
get_data_submit_cmd = 'qsub ' + self._get_data_script_name(
scan)
logger.info("get_data_submit_cmd: " + get_data_submit_cmd)
completed_get_data_submit_process = subprocess.run(
get_data_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
get_data_job_no = str_utils.remove_ending_new_lines(
completed_get_data_submit_process.stdout)
logger.info("get_data_job_no: " + get_data_job_no)
else:
logger.info("Get data job not submitted")
# submit job to process the data
if processing_stage >= ProcessingStage.PROCESS_DATA:
work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
scan)
logger.info("work_submit_cmd: " + work_submit_cmd)
completed_work_submit_process = subprocess.run(
work_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
work_job_no = str_utils.remove_ending_new_lines(
completed_work_submit_process.stdout)
logger.info("work_job_no: " + work_job_no)
else:
logger.info("Process data job not submitted")
# submit job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
scan)
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(
completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
else:
logger.info("Unable to submit jobs")
def submit_jobs(self):
_debug("submit_jobs")
if self.validate_parameters():
_inform("")
_inform("--------------------------------------------------")
_inform("Submitting " + self.PIPELINE_NAME + " jobs for")
_inform(" Project: " + self.project)
_inform(" Subject: " + self.subject)
_inform(" Session: " + self.session)
_inform("--------------------------------------------------")
# make sure working directories don't have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
current_seconds_since_epoch = int(time.time())
# build the working directory name
self._working_directory_name = self.build_home
self._working_directory_name += os.sep + self.project
self._working_directory_name += os.sep + self.PIPELINE_NAME
self._working_directory_name += '.' + self.subject
self._working_directory_name += '.' + str(
current_seconds_since_epoch)
# make the working directory
_inform("making working directory: " +
self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# get JSESSION ID
jsession_id = xnat_access.get_jsession_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password)
_inform("jsession_id: " + jsession_id)
# get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
xnat_session_id = xnat_access.get_session_id(
server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
username=self.username,
password=self.password,
project=kself.project,
subject=self.subject,
session=self.session)
_inform("xnat_session_id: " + xnat_session_id)
# get XNAT Workflow ID
workflow_obj = xnat_access.Workflow(
self.username, self.password,
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
jsession_id)
self._workflow_id = workflow_obj.create_workflow(
xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
_inform("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = 'Diffusion_preproc'
# clean the output resource if requested
if self.clean_output_resource_first:
_inform("Deleting resource: " + self._output_resource_name +
" for:")
_inform(" project: " + self.project)
_inform(" subject: " + self.subject)
_inform(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server), self.project,
self.subject, self.session, self._output_resource_name)
# create script to do the PreEddy work
self._create_pre_eddy_script()
# create script to do the Eddy work
self._create_eddy_script()
# create script to do the PostEddy work
self._create_post_eddy_script()
# create script to put the results into the DB
put_script_name = self._get_scripts_start_name(
) + '.XNAT_PBS_PUT_job.sh'
self.create_put_script(put_script_name, self.username,
self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# Submit the job to do the Pre-Eddy work
pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
_inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)
completed_pre_eddy_submit_process = subprocess.run(
pre_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
pre_eddy_job_no = str_utils.remove_ending_new_lines(
completed_pre_eddy_submit_process.stdout)
_inform("pre_eddy_job_no: " + pre_eddy_job_no)
# Submit the job to do the Eddy work
eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
_inform("eddy_submit_cmd: " + eddy_submit_cmd)
completed_eddy_submit_process = subprocess.run(
eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
eddy_job_no = str_utils.remove_ending_new_lines(
completed_eddy_submit_process.stdout)
_inform("eddy_job_no: " + eddy_job_no)
# Submit the job to do the Post-Eddy work
post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
_inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)
completed_post_eddy_submit_process = subprocess.run(
post_eddy_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
post_eddy_job_no = str_utils.remove_ending_new_lines(
completed_post_eddy_submit_process.stdout)
_inform("post_eddy_job_no: " + post_eddy_job_no)
# Submit the job to put the results in the DB
put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
_inform("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd,
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(
completed_put_submit_process.stdout)
_inform("put_job_no: " + put_job_no)
else:
_inform("Unable to submit jobs")
def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
logger.debug("submit_jobs: processing_stage: " + str(processing_stage))
logger.info("----------")
logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
logger.info(" Project: " + self.project)
logger.info(" Subject: " + self.subject)
logger.info(" Session: " + self.session)
logger.info(" Stage: " + str(processing_stage))
logger.info("----------")
# make sure working directories do not have the same name based on
# the same start time by sleeping a few seconds
time.sleep(5)
# build the working directory name
self._working_directory_name = \
self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject)
logger.info("Making working directory: " + self._working_directory_name)
os.makedirs(name=self._working_directory_name)
# # get JSESSION ID
# jsession_id = xnat_access.get_jsession_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password)
# logger.info("jsession_id: " + jsession_id)
# # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
# xnat_session_id = xnat_access.get_session_id(
# server='db.humanconnectome.org',
# username=self.username,
# password=self.password,
# project=self.project,
# subject=self.subject,
# session=self.session)
# logger.info("xnat_session_id: " + xnat_session_id)
# # get XNAT Workflow ID
# workflow_obj = xnat_access.Workflow(self.username, self.password,
# 'https://db.humanconnectome.org', jsession_id)
# self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
# self.project,
# self.PIPELINE_NAME,
# 'Queued')
# logger.info("workflow_id: " + self._workflow_id)
# determine output resource name
self._output_resource_name = "Diffusion_bedpostx"
# clean output resource if requested
if self.clean_output_resource_first:
logger.info("Deleting resource: " + self._output_resource_name + " for:")
logger.info(" project: " + self.project)
logger.info(" subject: " + self.subject)
logger.info(" session: " + self.session)
delete_resource.delete_resource(
self.username, self.password,
str_utils.get_server_name(self.server),
self.project, self.subject, self.session,
self._output_resource_name)
# create scripts for various stages of processing
if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
# create script to get data
self._create_get_data_script()
self._create_process_script()
self._create_clean_data_script()
put_script_name = self._put_data_script_name()
self.create_put_script(put_script_name,
self.username, self.password, self.put_server,
self.project, self.subject, self.session,
self._working_directory_name,
self._output_resource_name,
self.PIPELINE_NAME)
# run the script to get the data
if processing_stage >= ProcessingStage.GET_DATA:
stdout_file = open(self._get_data_script_name() + '.stdout', 'w')
stderr_file = open(self._get_data_script_name() + '.stderr', 'w')
logger.info("Running get data script")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._get_data_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
logger.info(" return code: " + str(proc.returncode))
stdout_file.close()
stderr_file.close()
if proc.returncode != 0:
raise RuntimeError("get data script ended with non-zero return code")
else:
logger.info("Get data script not run")
# run the script to submit processing jobs
if processing_stage >= ProcessingStage.PROCESS_DATA:
stdout_file = open(self._process_script_name() + '.stdout', 'w')
stderr_file = open(self._process_script_name() + '.stderr', 'w')
logger.info("Running script to submit processing jobs")
logger.info(" stdout: " + stdout_file.name)
logger.info(" stderr: " + stderr_file.name)
proc = subprocess.Popen(['bash', self._process_script_name()],
stdout=stdout_file, stderr=stderr_file)
proc.communicate()
stdout_file.close()
stderr_file.close()
logger.info(" return code: " + str(proc.returncode))
if proc.returncode != 0:
raise RuntimeError("script to submit processing jobs ended with non-zero return code")
else:
logger.info("process data job not submitted")
# submit the job to clean the data
if processing_stage >= ProcessingStage.CLEAN_DATA:
# figure out what the job id number is for the bedpostx postprocessing job
postproc_file_name = self._working_directory_name + os.sep
postproc_file_name += self.subject + os.sep
postproc_file_name += 'T1w' + os.sep
postproc_file_name += 'Diffusion.bedpostX' + os.sep
postproc_file_name += 'logs' + os.sep
postproc_file_name += 'postproc_ID'
logger.info("Post-processing job ID file name: " + postproc_file_name)
f = open(postproc_file_name, 'r')
id_str = f.readline().rstrip()
logger.info("Post-processing job ID: " + id_str)
f.close()
clean_submit_cmd = 'qsub -W depend=afterok:' + id_str + ' ' + self._clean_data_script_name()
logger.info("clean_submit_cmd: " + clean_submit_cmd)
completed_clean_submit_process = subprocess.run(
clean_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
clean_job_no = str_utils.remove_ending_new_lines(completed_clean_submit_process.stdout)
logger.info("clean_job_no: " + clean_job_no)
else:
logger.info("Clean data job not submitted")
# submit the job to put the resulting data in the DB
if processing_stage >= ProcessingStage.PUT_DATA:
put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
logger.info("put_submit_cmd: " + put_submit_cmd)
completed_put_submit_process = subprocess.run(
put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
logger.info("put_job_no: " + put_job_no)
else:
logger.info("Put data job not submitted")
