def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
logger.info("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
walltime_limit_hrs = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(processing_stage_str)
logger.info("")
logger.info("--------------------------------------------------------------------------------")
logger.info(" Submitting RestingStateStatsHCP7T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" refproject: " + subject.structural_reference_project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
logger.debug("walltime_limit_hrs: " + str(walltime_limit_hrs))
logger.debug(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info("--------------------------------------------------------------------------------")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER')
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def submit_jobs(self, subject_list):
# Read the configuration file
config_file_name = file_utils.get_config_file_name(__file__)
logger.info("Reading configuration from file: " + config_file_name)
config = my_configparser.MyConfigParser()
config.read(config_file_name)
# Submit jobs for listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for subject from the configuration file
setup_file = xnat_pbs_jobs + os.sep + config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
walltime_limit_hrs = config.get_int_value(subject.subject_id, 'WalltimeLimit')
vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(processing_stage_str)
logger.info("")
logger.info("--------------------------------------------------------------------------------")
logger.info(" Submitting RestingStateStatsHCP7T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" refproject: " + subject.structural_reference_project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
logger.debug("walltime_limit_hrs: " + str(walltime_limit_hrs))
logger.debug(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info("--------------------------------------------------------------------------------")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://db.humanconnectome.org'
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_7T'
submitter.structural_reference_project = subject.structural_reference_project
submitter.structural_reference_session = subject.subject_id + '_3T'
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def submit_jobs(self, subject_list, config):
# submit jobs for the listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(
self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for the subject from the configuration
setup_file = config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id,
'CleanOutputFirst')
# walltime_limit_hours = config.get_int_value(subject.subject_id, 'WalltimeLimit')
# vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id,
'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(
processing_stage_str)
logger.info("-----")
logger.info(" Submitting bedpostxHCP3T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
# logger.info(" walltime_limit_hours: " + str(walltime_limit_hours))
# logger.info(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info(" processing_stage: " + str(processing_stage))
logger.info("-----")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER')
submitter.setup_script = setup_file
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_3T'
# submitter.walltime_limit_hours = walltime_limit_hours
# submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def submit_jobs(self, subject_list, config):
# submit jobs for the listed subjects
for subject in subject_list:
put_server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
# get information for the subject from the configuration
setup_file = config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst')
# walltime_limit_hours = config.get_int_value(subject.subject_id, 'WalltimeLimit')
# vmem_limit_gbs = config.get_int_value(subject.subject_id, 'VmemLimit')
processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage')
processing_stage = one_subject_job_submitter.ProcessingStage.from_string(processing_stage_str)
logger.info("-----")
logger.info(" Submitting bedpostxHCP3T jobs for:")
logger.info(" project: " + subject.project)
logger.info(" subject: " + subject.subject_id)
logger.info(" put_server: " + put_server)
logger.info(" setup_file: " + setup_file)
logger.info(" clean_output_first: " + str(clean_output_first))
# logger.info(" walltime_limit_hours: " + str(walltime_limit_hours))
# logger.info(" vmem_limit_gbs: " + str(vmem_limit_gbs))
logger.info(" processing_stage: " + str(processing_stage))
logger.info("-----")
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(self._archive, self._archive.build_home)
submitter.username = userid
submitter.password = password
submitter.server = 'https://db.humanconnectome.org'
submitter.setup_script = setup_file
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_3T'
# submitter.walltime_limit_hours = walltime_limit_hours
# submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.submit_jobs(processing_stage)
def submit_jobs(self, subject_list, config):
for subject in subject_list:
server = 'http://db-shadow' + str(self.get_and_inc_shadow_number()) + '.nrg.mir:8080'
packaging_stage_str = config.get_value(subject.subject_id, 'PackagingStage')
packaging_stage = one_subject_package_job_submitter.PackagingStage.from_string(packaging_stage_str)
logger.info("-----")
logger.info(" Submitting bedpostx packaging jobs for:")
logger.info(" project: " + subject.project)
logger.info(" subject: " + subject.subject_id)
logger.info(" server: " + server)
logger.info(" packaging_stage: " + str(packaging_stage))
logger.info("-----")
submitter = one_subject_package_job_submitter.OneSubjectPackageJobSubmitter(self._archive, self._archive.build_home)
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_3T'
submitter.server = server
submitter.submit_jobs(packaging_stage)
def submit_jobs(self,
username,
password,
subject_list,
config,
force_job_submission=False):
# submit jobs for the listed subjects
for subject in subject_list:
if not force_job_submission:
run_status_checker = one_subject_run_status_checker.OneSubjectRunStatusChecker(
)
if run_status_checker.get_queued_or_running(subject):
print("-----")
print("\t NOT SUBMITTING JOBS FOR")
print("\t project: " + subject.project)
print("\t subject: " + subject.subject_id)
print("\t session classifier: " + subject.classifier)
print("\t JOBS ARE ALREADY QUEUED OR RUNNING")
continue
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
put_server_name = os.environ.get(
"XNAT_PBS_JOBS_PUT_SERVER_LIST").split(" ")
put_server = random.choice(put_server_name)
# get information for the subject from the configuration
clean_output_first = config.get_bool_value(subject.subject_id,
'CleanOutputFirst')
processing_stage_str = config.get_value(subject.subject_id,
'ProcessingStage')
processing_stage = submitter.processing_stage_from_string(
processing_stage_str)
walltime_limit_hrs = config.get_value(subject.subject_id,
'WalltimeLimitHours')
vmem_limit_gbs = config.get_value(subject.subject_id,
'VmemLimitGbs')
output_resource_suffix = config.get_value(subject.subject_id,
'OutputResourceSuffix')
brain_size = config.get_value(subject.subject_id, 'BrainSize')
use_prescan_normalized = config.get_bool_value(
subject.subject_id, 'UsePrescanNormalized')
print("-----")
print("\tSubmitting", submitter.PIPELINE_NAME, "jobs for:")
print("\t project:", subject.project)
print("\t subject:", subject.subject_id)
print("\t session classifier:", subject.classifier)
print("\t put_server:", put_server)
print("\t clean_output_first:", clean_output_first)
print("\t processing_stage:", processing_stage)
print("\t walltime_limit_hrs:", walltime_limit_hrs)
print("\t vmem_limit_gbs:", vmem_limit_gbs)
print("\toutput_resource_suffix:", output_resource_suffix)
print("\t brain_size:", brain_size)
print("\tuse_prescan_normalized:", use_prescan_normalized)
# configure one subject submitter
# user and server information
submitter.username = username
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER')
# subject and project information
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_' + subject.classifier
submitter.classifier = subject.classifier
submitter.brain_size = brain_size
submitter.use_prescan_normalized = use_prescan_normalized
# job parameters
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.output_resource_suffix = output_resource_suffix
# submit jobs
submitted_job_list = submitter.submit_jobs(processing_stage)
for job in submitted_job_list:
print("\tsubmitted jobs:", job)
print("-----")
def submit_jobs(self, username, password, subject_list, config):
# submit jobs for the listed subjects
for subject in subject_list:
submitter = one_subject_job_submitter.OneSubjectJobSubmitter(
self._archive, self._archive.build_home)
put_server = 'http://db-shadow' + str(
self.get_and_inc_shadow_number())
put_server += '.nrg.mir:8080'
# get information for the subject from the configuration
setup_file = config.get_value(subject.subject_id, 'SetUpFile')
clean_output_first = config.get_bool_value(subject.subject_id,
'CleanOutputFirst')
processing_stage_str = config.get_value(subject.subject_id,
'ProcessingStage')
processing_stage = submitter.processing_stage_from_string(
processing_stage_str)
walltime_limit_hrs = config.get_value(subject.subject_id,
'WalltimeLimitHours')
vmem_limit_gbs = config.get_value(subject.subject_id,
'VmemLimitGbs')
reg_name = config.get_value(subject.subject_id, 'RegName')
output_resource_suffix = config.get_value(subject.subject_id,
'OutputResourceSuffix')
scan = subject.extra
module_logger.info("-----")
module_logger.info(" Submitting " + submitter.PIPELINE_NAME +
" jobs for:")
module_logger.info(" project: " + subject.project)
module_logger.info(" subject: " +
subject.subject_id)
module_logger.info(" session classifier: " +
subject.classifier)
module_logger.info(" scan: " + scan)
module_logger.info(" put_server: " + put_server)
module_logger.info(" setup_file: " + setup_file)
module_logger.info(" clean_output_first: " +
str(clean_output_first))
module_logger.info(" processing_stage: " +
str(processing_stage))
module_logger.info(" walltime_limit_hrs: " +
str(walltime_limit_hrs))
module_logger.info(" vmem_limit_gbs: " +
str(vmem_limit_gbs))
module_logger.info(" reg_name: " + str(reg_name))
module_logger.info(" output_resource_suffix: " +
str(output_resource_suffix))
module_logger.info("-----")
# user and server information
submitter.username = username
submitter.password = password
submitter.server = 'https://' + os_utils.getenv_required(
'XNAT_PBS_JOBS_XNAT_SERVER')
# subject and project information
submitter.project = subject.project
submitter.subject = subject.subject_id
submitter.session = subject.subject_id + '_' + subject.classifier
submitter.scan = scan
submitter.reg_name = reg_name
# job parameters
submitter.setup_script = setup_file
submitter.clean_output_resource_first = clean_output_first
submitter.put_server = put_server
submitter.walltime_limit_hours = walltime_limit_hrs
submitter.vmem_limit_gbs = vmem_limit_gbs
submitter.output_resource_suffix = output_resource_suffix
# submit jobs
submitter.submit_jobs(processing_stage)