def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs_home + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
pre_eddy_walltime_limit_hrs = config.get_int_value(subject.subject_id, 'PreEddyWalltimeLimit')
pre_eddy_vmem_limit_gbs = config.get_int_value(subject.subject_id, 'PreEddyVmemLimit')
eddy_walltime_limit_hrs = config.get_int_value(subject.subject_id, 'EddyWalltimeLimit')
post_eddy_walltime_limit_hrs = config.get_int_value(subject.subject_id, 'PostEddyWalltimeLimit')
post_eddy_vmem_limit_gbs = config.get_int_value(subject.subject_id, 'PostEddyVmemLimit')
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform(" Submitting DiffusionPreprocessingHCP7T jobs for:")
_inform(" project: " + subject.project)
_inform(" refproject: " + subject.structural_reference_project)
_inform(" subject: " + subject.subject_id)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" clean_output_first: " + str(clean_output_first))
_inform("--------------------------------------------------------------------------------")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://db.humanconnectome.org'
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.pre_eddy_walltime_limit_hours = pre_eddy_walltime_limit_hrs
submitter.pre_eddy_vmem_limit_gbs = pre_eddy_vmem_limit_gbs
submitter.eddy_walltime_limit_hours = eddy_walltime_limit_hrs
submitter.post_eddy_walltime_limit_hours = post_eddy_walltime_limit_hrs
submitter.post_eddy_vmem_limit_gbs = post_eddy_vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.pe_dirs_spec = 'PAAP'
submitter.put_server = put_server
submitter.submit_jobs()
def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
logger.info("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
walltime_limit_hrs = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(processing_stage_str)
logger.info("")
logger.info("--------------------------------------------------------------------------------")
logger.info(" Submitting RestingStateStatsHCP7T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" refproject: " + subject.structural_reference_project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
logger.debug("walltime_limit_hrs: " + str(walltime_limit_hrs))
logger.debug(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info("--------------------------------------------------------------------------------")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER')
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
logger.info("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
walltime_limit_hrs = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(processing_stage_str)
logger.info("")
logger.info("--------------------------------------------------------------------------------")
logger.info(" Submitting RestingStateStatsHCP7T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" refproject: " + subject.structural_reference_project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
logger.debug("walltime_limit_hrs: " + str(walltime_limit_hrs))
logger.debug(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info("--------------------------------------------------------------------------------")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://db.humanconnectome.org'
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def do_submissions(userid, password, subject_list):
# read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
print("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# process the subjects in the list
batch_submitter = BatchSubmitter()
batch_submitter.submit_jobs(userid, password, subject_list, config)
def submit_jobs(self, subject_list):
"""Submit a batch of jobs."""
# read configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.shadow_number) + '.nrg.mir:8080'
# get information for subject from configuration file
setup_file = scripts_home + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
wall_time_limit = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit = config.get_int_value(subject.subject_id, 'VmemLimit')
mem_limit = config.get_int_value(subject.subject_id, 'MemLimit')
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform(" Submitting DeDriftAndResampleHCP7T_HighRes jobs for: ")
_inform(" project: " + subject.project)
_inform(" refproject: " + subject.structural_reference_project)
_inform(" subject: " + subject.subject_id)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" clean_output_first: " + str(clean_output_first))
_inform(" wall_time_limit: " + str(wall_time_limit))
_inform(" vmem_limit: " + str(vmem_limit))
_inform(" mem_limit: " + str(mem_limit))
_inform("--------------------------------------------------------------------------------")
_debug("Create and configure an appropriate 'one subject submitter'")
one_subject_submitter = DeDriftAndResampleHCP7T_HighRes_OneSubjectJobSubmitter.DeDriftAndResampleHCP7T_HighRes_OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
_debug("one_subject_submitter: " + str(one_subject_submitter))
one_subject_submitter.username = userid
one_subject_submitter.password = password
one_subject_submitter.server = 'https://' + os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER')
one_subject_submitter.project = subject.project
one_subject_submitter.subject = subject.subject_id
one_subject_submitter.session = subject.subject_id + '_7T'
one_subject_submitter.structural_reference_project = subject.structural_reference_project
one_subject_submitter.structural_reference_session = subject.subject_id + '_3T'
one_subject_submitter.put_server = put_server
one_subject_submitter.clean_output_resource_first = clean_output_first
one_subject_submitter.setup_script = setup_file
one_subject_submitter.walltime_limit_hours = wall_time_limit
one_subject_submitter.vmem_limit_gbs = vmem_limit
one_subject_submitter.mem_limit_gbs = mem_limit
_debug("Use the 'one subject submitter' to submit the jobs for the current subject")
one_subject_submitter.submit_jobs()
self.increment_shadow_number()
time.sleep(60)
logger.info(" server: " + server)
logger.info(" packaging_stage: " + str(packaging_stage))
logger.info("-----")
submitter = one_subject_package_job_submitter.OneSubjectPackageJobSubmitter(self._archive, self._archive.build_home)
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_3T'
submitter.server = server
submitter.submit_jobs(packaging_stage)
if __name__ == '__main__':
# read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
logger.info("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# get list of subjects to process
subject_file_name = file_utils.get_subjects_file_name(__file__)
logger.info("Retrieving subject list from: " + subject_file_name)
subject_list = hcp3t_subject.read_subject_info_list(subject_file_name, separator='\t')
# process the subjects in the list
batch_submitter = BatchPackagingSubmitter()
batch_submitter.submit_jobs(subject_list, config)
def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(
self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs_home + os.sep + config.get_value(
subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id,
'CleanOutputFirst')
pre_eddy_walltime_limit_hrs = config.get_int_value(
subject.subject_id, 'PreEddyWalltimeLimit')
pre_eddy_vmem_limit_gbs = config.get_int_value(
subject.subject_id, 'PreEddyVmemLimit')
eddy_walltime_limit_hrs = config.get_int_value(
subject.subject_id, 'EddyWalltimeLimit')
post_eddy_walltime_limit_hrs = config.get_int_value(
subject.subject_id, 'PostEddyWalltimeLimit')
post_eddy_vmem_limit_gbs = config.get_int_value(
subject.subject_id, 'PostEddyVmemLimit')
_inform("")
_inform(
"--------------------------------------------------------------------------------"
)
_inform(" Submitting DiffusionPreprocessingHCP7T jobs for:")
_inform(" project: " + subject.project)
_inform(" refproject: " +
subject.structural_reference_project)
_inform(" subject: " + subject.subject_id)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" clean_output_first: " + str(clean_output_first))
_inform(
"--------------------------------------------------------------------------------"
)
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER')
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.pre_eddy_walltime_limit_hours = pre_eddy_walltime_limit_hrs
submitter.pre_eddy_vmem_limit_gbs = pre_eddy_vmem_limit_gbs
submitter.eddy_walltime_limit_hours = eddy_walltime_limit_hrs
submitter.post_eddy_walltime_limit_hours = post_eddy_walltime_limit_hrs
submitter.post_eddy_vmem_limit_gbs = post_eddy_vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.pe_dirs_spec = 'PAAP'
submitter.put_server = put_server
submitter.submit_jobs()
def submit_jobs(self, subject_list):
# read configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("")
_inform(
"--------------------------------------------------------------------------------"
)
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(
self._current_shadow_number) + '.nrg.mir:8080'
setup_file = scripts_home + os.sep + config.get_value(
subject.subject_id, 'SetUpFile')
wall_time_limit = int(
config.get_value(subject.subject_id, 'WalltimeLimit'))
mem_limit = int(config.get_value(subject.subject_id, 'MemLimit'))
vmem_limit = int(config.get_value(subject.subject_id, 'VmemLimit'))
scan = subject.extra
_inform("")
_inform(
"--------------------------------------------------------------------------------"
)
_inform(" Submitting RepairIcaFixProcessingHCP7T jobs for: ")
_inform(" project: " + subject.project)
_inform(" refproject: " +
subject.structural_reference_project)
_inform(" subject: " + subject.subject_id)
_inform(" scan: " + scan)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" wall_time_limit: " + str(wall_time_limit))
_inform(" mem_limit: " + str(mem_limit))
_inform(" vmem_limit: " + str(vmem_limit))
_inform(
"--------------------------------------------------------------------------------"
)
# figure out the specification of the scan(s) to process and whether
# to only process incomplete scans
if scan == 'all':
# want to run them all without regard to whether they are previously complete
scan_spec = None
incomplete_only = False
elif scan == 'incomplete':
# want to look at all of them and run only those that are incomplete
scan_spec = None
incomplete_only = True
else:
# want to run this specific one without regard to whether it is previously complete
scan_spec = scan
incomplete_only = False
# Use the "one subject submitter" to submit the jobs for the current subject
self._one_subject_submitter.submit_jobs(
userid, password, 'https://' +
os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
subject.project, subject.subject_id,
subject.subject_id + '_7T',
subject.structural_reference_project,
subject.subject_id + '_3T', put_server, setup_file,
incomplete_only, scan_spec, wall_time_limit, mem_limit,
vmem_limit)
self.increment_shadow_number()
# submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
if __name__ == '__main__':
# get ConnectomeDB credentials
userid = input("Connectome DB Username: "******"Connectome DB Password: "******"Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# get list of subjects to process
subject_file_name = file_utils.get_subjects_file_name(__file__)
logger.info("Retrieving subject list from: " + subject_file_name)
subject_list = hcp3t_subject.read_subject_info_list(subject_file_name, separator='\t')
# process the subjects in the list
batch_submitter = BatchSubmitter()
batch_submitter.submit_jobs(subject_list, config)
def submit_jobs(self, subject_list):
"""Submit a batch of jobs."""
# read configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self._current_shadow_number) + '.nrg.mir:8080'
# get information for subject from configuration file
setup_file = scripts_home + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
wall_time_limit = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit = config.get_int_value(subject.subject_id, 'VmemLimit')
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform(" Submitting DeDriftAndResampleHCP7T_HighRes jobs for: ")
_inform(" project: " + subject.project)
_inform(" refproject: " + subject.structural_reference_project)
_inform(" subject: " + subject.subject_id)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" clean_output_first: " + str(clean_output_first))
_inform(" wall_time_limit: " + str(wall_time_limit))
_inform(" vmem_limit: " + str(vmem_limit))
_inform("--------------------------------------------------------------------------------")
_debug("Create and configure an appropriate 'one subject submitter'")
one_subject_submitter =
DeDriftAndResampleHCP7T_HighRes_OneSubjectJobSubmitter.DeDriftAndResampleHCP7T_HighRes_OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
_debug("one_subject_submitter: " + str(one_subject_submitter))
one_subject_submitter.username = userid
one_subject_submitter.password = password
one_subject_submitter.server = 'https://db.humanconnectome.org'
one_subject_submitter.project = subject.project
one_subject_submitter.subject = subject.subject_id
one_subject_submitter.session = subject.subject_id + '_7T'
one_subject_submitter.structural_reference_project = subject.structural_reference_project
one_subject_submitter.structural_reference_session = subject.subject_id + '_3T'
one_subject_submitter.put_server = put_server
one_subject_submitter.clean_output_resource_first = clean_output_first
one_subject_submitter.setup_script = setup_file
one_subject_submitter.walltime_limit_hours = wall_time_limit
one_subject_submitter.vmem_limit_gbs = vmem_limit
_debug("Use the 'one subject submitter' to submit the jobs for the current subject")
one_subject_submitter.submit_jobs()
self.increment_shadow_number()
def submit_jobs(self, subject_list):
"""Submit a batch of PostFix processing jobs."""
# read configuration file
config_file_name = file_utils.get_config_file_name(__file__)
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self._current_shadow_number) + '.nrg.mir:8080'
# get information for subject from configuration file
setup_file = scripts_home + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
if config.get_value(subject.subject_id, 'CleanOutputFirst') == 'True':
clean_output_first = True
else:
clean_output_first = False
wall_time_limit = int(config.get_value(subject.subject_id, 'WalltimeLimit'))
vmem_limit = int(config.get_value(subject.subject_id, 'VmemLimit'))
mem_limit = 'UNSPECIFIED'
scan = subject.extra
_inform("")
_inform("--------------------------------------------------------------------------------")
_inform(" Submitting PostFixHCP7T jobs for: ")
_inform(" project: " + subject.project)
_inform(" subject: " + subject.subject_id)
_inform(" scan: " + scan)
_inform(" put_server: " + put_server)
_inform(" setup_file: " + setup_file)
_inform(" clean_output_first: " + str(clean_output_first))
_inform(" wall_time_limit: " + str(wall_time_limit))
_inform(" mem_limit: " + str(mem_limit))
_inform(" vmem_limit: " + str(vmem_limit))
_inform("--------------------------------------------------------------------------------")
# figure out the specification of the scan(s) to process and whether to only
# process incomplete scans
if scan == 'all':
# want to run them all without regard to whether they are previously complete
scan_spec = None
incomplete_only = False
elif scan == 'incomplete':
# want to look at all of them and run only those that are incomplete
scan_spec = None
incomplete_only = True
else:
# want to run this specific one without regard to whether it is previously complete
scan_spec = scan
incomplete_only = False
# Use the "one subject submitter" to submit the jobs for the current subject
self._one_subject_submitter.submit_jobs(
userid, password, 'https://db.humanconnectome.org',
subject.project, subject.subject_id, subject.subject_id + '_7T',
subject.structural_reference_project, subject.subject_id + '_3T',
put_server, clean_output_first, setup_file,
incomplete_only, scan_spec,
wall_time_limit, mem_limit, vmem_limit)
self.increment_shadow_number