subject_list = ccf_subject.read_subject_info_list(subject_file_name, separator=":") # open output file output_file = open('FunctionalPreprocessing.status', 'w') _write_header(output_file) # create archive archive = ccf_archive.CcfArchive() # create one subject checkers completion_checker = one_subject_completion_checker.OneSubjectCompletionChecker( ) prereq_checker = one_subject_prereq_checker.OneSubjectPrereqChecker() running_checker = one_subject_run_status_checker.OneSubjectRunStatusChecker( ) for subject in subject_list: project = subject.project subject_id = subject.subject_id classifier = subject.classifier scan = subject.extra prereqs_met = prereq_checker.are_prereqs_met(archive, subject) queued_or_running = running_checker.get_queued_or_running(subject) if completion_checker.does_processed_resource_exist(archive, subject): resource_exists = True fullpath = archive.functional_preproc_dir_full_path(subject) resource = archive.functional_preproc_dir_name(subject)
def submit_jobs(self, username, password, subject_list, config): # submit jobs for the listed subject scans for subject in subject_list: run_status_checker = one_subject_run_status_checker.OneSubjectRunStatusChecker( ) if run_status_checker.get_queued_or_running(subject): print("-----") print("\t NOT SUBMITTING JOBS FOR") print("\t project:", subject.project) print("\t subject:", subject.subject_id) print("\t classifier:", subject.classifier) print("\t JOBS ARE ALREADY QUEUED OR RUNNING") continue submitter = one_subject_job_submitter.OneSubjectJobSubmitter( self._archive, self._archive.build_home) # ####put_server = 'http://intradb-shadow' # ####put_server += str(self.get_and_inc_shadow_number()) # ####put_server += '.nrg.mir:8080' put_server_name = os.environ.get( "XNAT_PBS_JOBS_PUT_SERVER_LIST").split(" ") put_server = random.choice(put_server_name) # get information for the subject/scan from the configuration clean_output_first = config.get_bool_value(subject.subject_id, 'CleanOutputFirst') processing_stage_str = config.get_value(subject.subject_id, 'ProcessingStage') processing_stage = submitter.processing_stage_from_string( processing_stage_str) walltime_limit_hrs = config.get_value(subject.subject_id, 'WalltimeLimitHours') vmem_limit_gbs = config.get_value(subject.subject_id, 'VmemLimitGbs') output_resource_suffix = config.get_value(subject.subject_id, 'OutputResourceSuffix') print("-----") print("\tSubmitting", submitter.PIPELINE_NAME, "jobs for:") print("\t project:", subject.project) print("\t subject:", subject.subject_id) print("\t scan:", subject.extra) print("\t session classifier:", subject.classifier) print("\t put_server:", put_server) print("\t clean_output_first:", clean_output_first) print("\t processing_stage:", processing_stage) print("\t walltime_limit_hrs:", walltime_limit_hrs) print("\t vmem_limit_gbs:", vmem_limit_gbs) print("\toutput_resource_suffix:", output_resource_suffix) # configure one subject submitter # user and server information submitter.username = username submitter.password = password submitter.server = 'https://' + os_utils.getenv_required( 'XNAT_PBS_JOBS_XNAT_SERVER') # subject and project information submitter.project = subject.project submitter.subject = subject.subject_id submitter.classifier = subject.classifier submitter.session = subject.subject_id + '_' + subject.classifier submitter.scan = subject.extra # job parameters submitter.clean_output_resource_first = clean_output_first submitter.put_server = put_server submitter.walltime_limit_hours = walltime_limit_hrs submitter.vmem_limit_gbs = vmem_limit_gbs submitter.output_resource_suffix = output_resource_suffix # submit jobs submitted_job_list = submitter.submit_jobs(processing_stage) for job in submitted_job_list: print("\tsubmitted jobs:", job) print("-----")