def delete_resource(user, password, server, project, subject, session, resource, perform_delete=True):
    # get XNAT session id
    xnat_session_id = xnat_access.get_session_id(server=str_utils.get_server_name(server), username=user, password=password,
                                                 project=project, subject=subject, session=session)

    resource_url = ''
    resource_url += 'https://' + str_utils.get_server_name(server)
    resource_url += '/REST/projects/' + project
    resource_url += '/subjects/' + subject
    resource_url += '/experiments/' + xnat_session_id
    resource_url += '/resources/' + resource

    variable_values = '?removeFiles=true'

    resource_uri = resource_url + variable_values

    delete_cmd = 'java -Xmx1024m -jar /home/HCPpipeline/pipeline/lib/xnat-data-client-1.6.4-SNAPSHOT-jar-with-dependencies.jar'
    delete_cmd += ' -u ' + user
    delete_cmd += ' -p ' + password
    delete_cmd += ' -m DELETE'
    delete_cmd += ' -r ' + resource_uri

    if perform_delete:
        _inform("Deleting")
        _inform("    Server: " + server)
        _inform("   Project: " + project)
        _inform("   Subject: " + subject)
        _inform("   Session: " + session)
        _inform("  Resource: " + resource)

        completed_delete_process = subprocess.run(delete_cmd, shell=True, check=True)

    else:
        _inform("delete_cmd: " + delete_cmd)
        _inform("Deletion not attempted")
Exemplo n.º 2
0
def delete_resource(user,
                    password,
                    server,
                    project,
                    subject,
                    session,
                    resource,
                    perform_delete=True):
    # get XNAT session id
    xnat_session_id = xnat_access.get_session_id(
        server=str_utils.get_server_name(server),
        username=user,
        password=password,
        project=project,
        subject=subject,
        session=session)

    resource_url = ''
    resource_url += 'https://' + str_utils.get_server_name(server)
    resource_url += '/REST/projects/' + project
    resource_url += '/subjects/' + subject
    resource_url += '/experiments/' + xnat_session_id
    resource_url += '/resources/' + resource

    variable_values = '?removeFiles=true'

    resource_uri = resource_url + variable_values

    pipeline_engine = os_utils.getenv_required('XNAT_PBS_JOBS_PIPELINE_ENGINE')

    delete_cmd = 'java -Xmx1024m -jar ' + pipeline_engine + os.sep + 'lib' + os.sep + 'xnat-data-client-1.6.4-SNAPSHOT-jar-with-dependencies.jar'
    delete_cmd += ' -u ' + user
    delete_cmd += ' -p ' + password
    delete_cmd += ' -m DELETE'
    delete_cmd += ' -r ' + resource_uri

    if perform_delete:
        _inform("Deleting")
        _inform("    Server: " + server)
        _inform("   Project: " + project)
        _inform("   Subject: " + subject)
        _inform("   Session: " + session)
        _inform("  Resource: " + resource)

        completed_delete_process = subprocess.run(delete_cmd,
                                                  shell=True,
                                                  check=True)

    else:
        _inform("delete_cmd: " + delete_cmd)
        _inform("Deletion not attempted")
    def submit_jobs(self):
        _debug("submit_jobs")

        if self.validate_parameters():

            _inform("")
            _inform("--------------------------------------------------")
            _inform("Submitting " + self.PIPELINE_NAME + " jobs for")
            _inform("  Project: " + self.project)
            _inform("  Subject: " + self.subject)
            _inform("  Session: " + self.session)
            _inform("--------------------------------------------------")

            # make sure working directories don't have the same name based on
            # the same start time by sleeping a few seconds
            time.sleep(5)
            current_seconds_since_epoch = int(time.time())

            # build the working directory name
            self._working_directory_name = self.build_home
            self._working_directory_name += os.sep + self.project
            self._working_directory_name += os.sep + self.PIPELINE_NAME
            self._working_directory_name += '.' + self.subject
            self._working_directory_name += '.' + str(current_seconds_since_epoch)

            # make the working directory
            _inform("making working directory: " + self._working_directory_name)
            os.makedirs(name=self._working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server='db.humanconnectome.org',
                username=self.username,
                password=self.password)
            _inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server='db.humanconnectome.org',
                username=self.username,
                password=self.password,
                project=kself.project,
                subject=self.subject,
                session=self.session)
            _inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(self.username, self.password,
                                                'https://db.humanconnectome.org', jsession_id)
            self._workflow_id = workflow_obj.create_workflow(xnat_session_id,
                                                             self.project,
                                                             self.PIPELINE_NAME,
                                                             'Queued')
            _inform("workflow_id: " + self._workflow_id)

            # determine output resource name
            self._output_resource_name = 'Diffusion_preproc'

            # clean the output resource if requested
            if self.clean_output_resource_first:
                _inform("Deleting resource: " + self._output_resource_name + " for:")
                _inform("  project: " + self.project)
                _inform("  subject: " + self.subject)
                _inform("  session: " + self.session)

                delete_resource.delete_resource(
                    self.username, self.password,
                    str_utils.get_server_name(self.server),
                    self.project, self.subject, self.session,
                    self._output_resource_name)

            # create script to do the PreEddy work
            self._create_pre_eddy_script()

            # create script to do the Eddy work
            self._create_eddy_script()

            # create script to do the PostEddy work
            self._create_post_eddy_script()

            # create script to put the results into the DB
            put_script_name = self._get_scripts_start_name() + '.XNAT_PBS_PUT_job.sh'
            self.create_put_script(put_script_name,
                                   self.username, self.password, self.put_server,
                                   self.project, self.subject, self.session,
                                   self._working_directory_name, self._output_resource_name,
                                   self.PIPELINE_NAME)

            # Submit the job to do the Pre-Eddy work
            pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
            _inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)

            completed_pre_eddy_submit_process = subprocess.run(
                pre_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            pre_eddy_job_no = str_utils.remove_ending_new_lines(completed_pre_eddy_submit_process.stdout)
            _inform("pre_eddy_job_no: " + pre_eddy_job_no)

            # Submit the job to do the Eddy work
            eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
            _inform("eddy_submit_cmd: " + eddy_submit_cmd)

            completed_eddy_submit_process = subprocess.run(
                eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            eddy_job_no = str_utils.remove_ending_new_lines(completed_eddy_submit_process.stdout)
            _inform("eddy_job_no: " + eddy_job_no)

            # Submit the job to do the Post-Eddy work
            post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
            _inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)

            completed_post_eddy_submit_process = subprocess.run(
                post_eddy_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            post_eddy_job_no = str_utils.remove_ending_new_lines(completed_post_eddy_submit_process.stdout)
            _inform("post_eddy_job_no: " + post_eddy_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
            _inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
            _inform("put_job_no: " + put_job_no)

        else:
            _inform("Unable to submit jobs")
Exemplo n.º 4
0
    def submit_jobs(
            self,
            username,
            password,
            server,
            project,
            subject,
            session,
            structural_reference_project,
            structural_reference_session,
            put_server,
            clean_output_resource_first,
            setup_script,
            incomplete_only,
            scan,
            walltime_limit_hours,
            mem_limit_gbs,  # UNUSED
            vmem_limit_gbs):

        subject_info = hcp7t_subject.Hcp7TSubjectInfo(
            project, structural_reference_project, subject)

        # determine names of preprocessed resting state scans that are
        # available for the subject
        preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(
            subject_info)
        inform("Preprocessed resting state scans available for subject: " +
               str(preproc_resting_state_scan_names))

        # determine names of the preprocessed MOVIE task scans that are available for the subject
        preproc_movie_scan_names = self.archive.available_movie_preproc_names(
            subject_info)
        inform("Preprocessed movie scans available for subject " +
               str(preproc_movie_scan_names))

        # determine names of the FIX processed scans that are available for the subject
        fix_processed_scan_names = self.archive.available_FIX_processed_names(
            subject_info)
        inform("FIX processed scans available for subject " +
               str(fix_processed_scan_names))

        # build list of scans to process
        scan_list = []
        if scan is None:
            scan_list = fix_processed_scan_names
        else:
            scan_list.append(scan)

        # process specified scans
        for scan_name in scan_list:
            if incomplete_only:
                completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker(
                )
                if completion_checker.is_processing_complete(
                        self.archive, subject_info, scan_name):
                    inform("scan: " + scan_name +
                           " has already completed PostFixHCP7T processing")
                    inform(
                        "Only submitting jobs for incomplete scans - skipping "
                        + scan_name)
                    continue

            inform("scan_name: " + scan_name)
            long_scan_name = self.archive.functional_scan_long_name(scan_name)
            output_resource_name = self.archive.PostFix_processed_resource_name(
                scan_name)

            inform("")
            inform("-------------------------------------------------")
            inform("Submitting jobs for scan: " + long_scan_name)
            inform("Output resource name: " + output_resource_name)
            inform("-------------------------------------------------")
            inform("")

            # make sure working directories don't have the same name based on the
            # same start time by sleeping a few seconds
            time.sleep(5)

            current_seconds_since_epoch = int(time.time())

            working_directory_name = self.build_home
            working_directory_name += os.sep + project
            working_directory_name += os.sep + self.PIPELINE_NAME
            working_directory_name += '.' + subject
            working_directory_name += '.' + long_scan_name
            working_directory_name += '.' + str(current_seconds_since_epoch)

            # make the working directory
            inform("Making working directory: " + working_directory_name)
            os.makedirs(name=working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=username,
                password=password)
            inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=username,
                password=password,
                project=project,
                subject=subject,
                session=session)

            inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(username, password, server,
                                                jsession_id)
            workflow_id = workflow_obj.create_workflow(xnat_session_id,
                                                       project,
                                                       self.PIPELINE_NAME,
                                                       'Queued')
            inform("workflow_id: " + workflow_id)

            # Clean the output resource if requested
            if clean_output_resource_first:
                inform("Deleting resource: " + output_resource_name + " for:")
                inform("  project: " + project)
                inform("  subject: " + subject)
                inform("  session: " + session)

                delete_resource.delete_resource(
                    username, password, str_utils.get_server_name(server),
                    project, subject, session, output_resource_name)

            script_file_start_name = working_directory_name
            script_file_start_name += os.sep + subject
            script_file_start_name += '.' + long_scan_name
            script_file_start_name += '.' + self.PIPELINE_NAME
            script_file_start_name += '.' + project
            script_file_start_name += '.' + session

            # Create script to submit to do the actual work
            work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
            with contextlib.suppress(FileNotFoundError):
                os.remove(work_script_name)

            work_script = open(work_script_name, 'w')

            nodes_spec = 'nodes=1:ppn=1'
            walltime_spec = 'walltime=' + str(walltime_limit_hours) + ':00:00'
            vmem_spec = 'vmem=' + str(vmem_limit_gbs) + 'gb'

            work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec +
                              ',' + vmem_spec + os.linesep)
            work_script.write('#PBS -o ' + working_directory_name + os.linesep)
            work_script.write('#PBS -e ' + working_directory_name + os.linesep)
            work_script.write(os.linesep)
            work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' +
                              os.sep + 'PostFixHCP7T' + os.sep +
                              'PostFixHCP7T.XNAT.sh \\' + os.linesep)
            work_script.write('  --user="******" \\' + os.linesep)
            work_script.write('  --password="******" \\' +
                              os.linesep)
            work_script.write('  --server="' +
                              str_utils.get_server_name(server) + '" \\' +
                              os.linesep)
            work_script.write('  --project="' + project + '" \\' + os.linesep)
            work_script.write('  --subject="' + subject + '" \\' + os.linesep)
            work_script.write('  --session="' + session + '" \\' + os.linesep)
            work_script.write('  --scan="' + long_scan_name + '" \\' +
                              os.linesep)
            work_script.write('  --working-dir="' + working_directory_name +
                              '" \\' + os.linesep)
            work_script.write('  --workflow-id="' + workflow_id + '" \\' +
                              os.linesep)
            work_script.write('  --setup-script=' + setup_script + os.linesep)

            work_script.close()
            os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)

            # Create script to put the results into the DB
            put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
            self.create_put_script(put_script_name, username, password,
                                   put_server, project, subject, session,
                                   working_directory_name,
                                   output_resource_name,
                                   scan_name + '_' + self.PIPELINE_NAME)

            # Submit the job to do the work
            work_submit_cmd = 'qsub ' + work_script_name
            inform("work_submit_cmd: " + work_submit_cmd)

            completed_work_submit_process = subprocess.run(
                work_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            work_job_no = str_utils.remove_ending_new_lines(
                completed_work_submit_process.stdout)
            inform("work_job_no: " + work_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
            inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            put_job_no = str_utils.remove_ending_new_lines(
                completed_put_submit_process.stdout)
            inform("put_job_no: " + put_job_no)
    def submit_jobs(
        self,
        username,
        password,
        server,
        project,
        subject,
        session,
        structural_reference_project,
        structural_reference_session,
        put_server,
        clean_output_resource_first,
        setup_script,
        incomplete_only,
        scan,
        walltime_limit_hours,
        mem_limit_gbs,  # UNUSED
        vmem_limit_gbs,
    ):

        subject_info = hcp7t_subject.Hcp7TSubjectInfo(project, structural_reference_project, subject)

        # determine names of preprocessed resting state scans that are
        # available for the subject
        preproc_resting_state_scan_names = self.archive.available_resting_state_preproc_names(subject_info)
        inform("Preprocessed resting state scans available for subject: " + str(preproc_resting_state_scan_names))

        # determine names of the preprocessed MOVIE task scans that are available for the subject
        preproc_movie_scan_names = self.archive.available_movie_preproc_names(subject_info)
        inform("Preprocessed movie scans available for subject " + str(preproc_movie_scan_names))

        # determine names of the FIX processed scans that are available for the subject
        fix_processed_scan_names = self.archive.available_FIX_processed_names(subject_info)
        inform("FIX processed scans available for subject " + str(fix_processed_scan_names))

        # build list of scans to process
        scan_list = []
        if scan is None:
            scan_list = fix_processed_scan_names
        else:
            scan_list.append(scan)

        # process specified scans
        for scan_name in scan_list:
            if incomplete_only:
                completion_checker = PostFixHCP7T_OneSubjectCompletionChecker.PostFixHCP7T_OneSubjectCompletionChecker()
                if completion_checker.is_processing_complete(self.archive, subject_info, scan_name):
                    inform("scan: " + scan_name + " has already completed PostFixHCP7T processing")
                    inform("Only submitting jobs for incomplete scans - skipping " + scan_name)
                    continue

            long_scan_name = self.archive.functional_scan_long_name(scan_name)
            output_resource_name = self.archive.PostFix_processed_resource_name(scan_name)

            inform("")
            inform("-------------------------------------------------")
            inform("Submitting jobs for scan: " + long_scan_name)
            inform("Output resource name: " + output_resource_name)
            inform("-------------------------------------------------")
            inform("")

            # make sure working directories don't have the same name based on the
            # same start time by sleeping a few seconds
            time.sleep(5)

            current_seconds_since_epoch = int(time.time())

            working_directory_name = self.build_home
            working_directory_name += os.sep + project
            working_directory_name += os.sep + self.PIPELINE_NAME
            working_directory_name += "." + subject
            working_directory_name += "." + long_scan_name
            working_directory_name += "." + str(current_seconds_since_epoch)

            # make the working directory
            inform("Making working directory: " + working_directory_name)
            os.makedirs(name=working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server="db.humanconnectome.org", username=username, password=password
            )
            inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server="db.humanconnectome.org",
                username=username,
                password=password,
                project=project,
                subject=subject,
                session=session,
            )

            inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(username, password, server, jsession_id)
            workflow_id = workflow_obj.create_workflow(xnat_session_id, project, self.PIPELINE_NAME, "Queued")
            inform("workflow_id: " + workflow_id)

            # Clean the output resource if requested
            if clean_output_resource_first:
                inform("Deleting resource: " + output_resource_name + " for:")
                inform("  project: " + project)
                inform("  subject: " + subject)
                inform("  session: " + session)

                delete_resource.delete_resource(
                    username,
                    password,
                    str_utils.get_server_name(server),
                    project,
                    subject,
                    session,
                    output_resource_name,
                )

            script_file_start_name = working_directory_name
            script_file_start_name += os.sep + subject
            script_file_start_name += "." + long_scan_name
            script_file_start_name += "." + self.PIPELINE_NAME
            script_file_start_name += "." + project
            script_file_start_name += "." + session

            # Create script to submit to do the actual work
            work_script_name = script_file_start_name + ".XNAT_PBS_job.sh"
            with contextlib.suppress(FileNotFoundError):
                os.remove(work_script_name)

            work_script = open(work_script_name, "w")

            nodes_spec = "nodes=1:ppn=1"
            walltime_spec = "walltime=" + str(walltime_limit_hours) + ":00:00"
            vmem_spec = "vmem=" + str(vmem_limit_gbs) + "gb"

            work_script.write("#PBS -l " + nodes_spec + "," + walltime_spec + "," + vmem_spec + os.linesep)
            work_script.write("#PBS -o " + working_directory_name + os.linesep)
            work_script.write("#PBS -e " + working_directory_name + os.linesep)
            work_script.write(os.linesep)
            work_script.write(
                self.xnat_pbs_jobs_home
                + os.sep
                + "7T"
                + os.sep
                + "PostFixHCP7T"
                + os.sep
                + "PostFixHCP7T.XNAT.sh \\"
                + os.linesep
            )
            work_script.write('  --user="******" \\' + os.linesep)
            work_script.write('  --password="******" \\' + os.linesep)
            work_script.write('  --server="' + str_utils.get_server_name(server) + '" \\' + os.linesep)
            work_script.write('  --project="' + project + '" \\' + os.linesep)
            work_script.write('  --subject="' + subject + '" \\' + os.linesep)
            work_script.write('  --session="' + session + '" \\' + os.linesep)
            work_script.write('  --scan="' + long_scan_name + '" \\' + os.linesep)
            work_script.write('  --working-dir="' + working_directory_name + '" \\' + os.linesep)
            work_script.write('  --workflow-id="' + workflow_id + '" \\' + os.linesep)
            work_script.write("  --setup-script=" + setup_script + os.linesep)

            work_script.close()
            os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)

            # Create script to put the results into the DB
            put_script_name = script_file_start_name + ".XNAT_PBS_PUT_job.sh"
            self.create_put_script(
                put_script_name,
                username,
                password,
                put_server,
                project,
                subject,
                session,
                working_directory_name,
                output_resource_name,
                scan_name + "_" + self.PIPELINE_NAME,
            )

            # Submit the job to do the work
            work_submit_cmd = "qsub " + work_script_name
            inform("work_submit_cmd: " + work_submit_cmd)

            completed_work_submit_process = subprocess.run(
                work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
            )
            work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
            inform("work_job_no: " + work_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = "qsub -W depend=afterok:" + work_job_no + " " + put_script_name
            inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True
            )
            put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
            inform("put_job_no: " + put_job_no)
    def submit_jobs(self):
        _debug("submit_jobs")

        if self.validate_parameters():

            # make sure working directories don't have the same name based on the same
            # start time by sleeping a few seconds
            time.sleep(5)

            current_seconds_since_epoch = int(time.time())

            working_directory_name = self.build_home
            working_directory_name += os.sep + self.project
            working_directory_name += os.sep + self.PIPELINE_NAME
            working_directory_name += '.' + self.subject
            working_directory_name += '.' + str(current_seconds_since_epoch)

            # make the working directory
            _inform("Making working directory: " + working_directory_name)
            os.makedirs(name=working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=self.username,
                password=self.password)
            _inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=self.username,
                password=self.password,
                project=self.project,
                subject=self.subject,
                session=self.session)
            _inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(
                self.username, self.password, self.server, jsession_id)
            workflow_id = workflow_obj.create_workflow(
                xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
            _inform("workflow_id: " + workflow_id)

            # Determine the output resource name
            output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
            _inform("output_resource_name: " + output_resource_name)

            # Clean the output resource if requested
            if self.clean_output_resource_first:
                _inform("Deleting resouce: " + output_resource_name + " for:")
                _inform("  project: " + self.project)
                _inform("  subject: " + self.subject)
                _inform("  session: " + self.session)

                delete_resource.delete_resource(
                    self.username, self.password, str_utils.get_server_name(self.server),
                    self.project, self.subject, self.session, output_resource_name, True)

            script_file_start_name = working_directory_name
            script_file_start_name += os.sep + self.subject
            script_file_start_name += '.' + self.PIPELINE_NAME
            script_file_start_name += '.' + self.project
            script_file_start_name += '.' + self.session

            # Create script to submit to do the actual work
            work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
            with contextlib.suppress(FileNotFoundError):
                os.remove(work_script_name)

            work_script = open(work_script_name, 'w')

            nodes_spec = 'nodes=1:ppn=1'
            walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
            vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'
            mem_spec = 'mem=' + str(self.mem_limit_gbs) + 'gb'

            work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + ',' + mem_spec + os.linesep)
            # work_script.write('#PBS -q HCPput' + os.linesep)
            work_script.write('#PBS -o ' + working_directory_name + os.linesep)
            work_script.write('#PBS -e ' + working_directory_name + os.linesep)
            work_script.write(os.linesep)
            work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
                              'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
            work_script.write('  --user="******" \\' + os.linesep)
            work_script.write('  --password="******" \\' + os.linesep)
            work_script.write('  --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
            work_script.write('  --project="' + self.project + '" \\' + os.linesep)
            work_script.write('  --subject="' + self.subject + '" \\' + os.linesep)
            work_script.write('  --session="' + self.session + '" \\' + os.linesep)
            work_script.write('  --structural-reference-project="' +
                              self.structural_reference_project + '" \\' + os.linesep)
            work_script.write('  --structural-reference-session="' +
                              self.structural_reference_session + '" \\' + os.linesep)
            work_script.write('  --working-dir="' + working_directory_name + '" \\' + os.linesep)
            work_script.write('  --workflow-id="' + workflow_id + '" \\' + os.linesep)

            # work_script.write('  --keep-all' + ' \\' + os.linesep)
            # work_script.write('  --prevent-push' + ' \\' + os.linesep)

            work_script.write('  --setup-script=' + self.setup_script + os.linesep)
            
            work_script.close()
            os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)

            # Create script to put the results into the DB
            put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
            self.create_put_script(put_script_name,
                                   self.username, self.password, self.put_server,
                                   self.project, self.subject, self.session,
                                   working_directory_name, output_resource_name,
                                   self.PIPELINE_NAME)

            # Submit the job to do the work
            work_submit_cmd = 'qsub ' + work_script_name
            _inform("work_submit_cmd: " + work_submit_cmd)

            completed_work_submit_process = subprocess.run(
                work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
            _inform("work_job_no: " + work_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
            _inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
            _inform("put_job_no: " + put_job_no)

        else:
            _inform("Unable to submit jobs")
    def submit_jobs(self):
        _debug("submit_jobs")

        if self.validate_parameters():
            # subject_info = hcp7t_subject.Hcp7TSubjectInfo(self.project,
            #                                               self.structural_reference_project,
            #                                               self.subject)

            # make sure working directories don't have the same name based on the same
            # start time by sleeping a few seconds
            time.sleep(5)

            current_seconds_since_epoch = int(time.time())

            working_directory_name = self.build_home
            working_directory_name += os.sep + self.project
            working_directory_name += os.sep + self.PIPELINE_NAME
            working_directory_name += '.' + self.subject
            working_directory_name += '.' + str(current_seconds_since_epoch)

            # make the working directory
            _inform("Making working directory: " + working_directory_name)
            os.makedirs(name=working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server='db.humanconnectome.org',
                username=self.username,
                password=self.password)
            _inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server='db.humanconnectome.org',
                username=self.username,
                password=self.password,
                project=self.project,
                subject=self.subject,
                session=self.session)
            _inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(
                self.username, self.password, self.server, jsession_id)
            workflow_id = workflow_obj.create_workflow(
                xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
            _inform("workflow_id: " + workflow_id)

            # Determine the output resource name
            output_resource_name = self.archive.DEDRIFT_AND_RESAMPLE_RESOURCE_NAME
            _inform("output_resource_name: " + output_resource_name)

            # Clean the output resource if requested
            if self.clean_output_resource_first:
                _inform("Deleting resouce: " + output_resource_name + " for:")
                _inform("  project: " + self.project)
                _inform("  subject: " + self.subject)
                _inform("  session: " + self.session)

                delete_resource.delete_resource(
                    self.username, self.password, str_utils.get_server_name(self.server),
                    self.project, self.subject, self.session, output_resource_name, True)

            script_file_start_name = working_directory_name
            script_file_start_name += os.sep + self.subject
            script_file_start_name += '.' + self.PIPELINE_NAME
            script_file_start_name += '.' + self.project
            script_file_start_name += '.' + self.session

            # Create script to submit to do the actual work
            work_script_name = script_file_start_name + '.XNAT_PBS_job.sh'
            with contextlib.suppress(FileNotFoundError):
                os.remove(work_script_name)

            work_script = open(work_script_name, 'w')

            nodes_spec = 'nodes=1:ppn=1'
            walltime_spec = 'walltime=' + str(self.walltime_limit_hours) + ':00:00'
            vmem_spec = 'vmem=' + str(self.vmem_limit_gbs) + 'gb'

            work_script.write('#PBS -l ' + nodes_spec + ',' + walltime_spec + ',' + vmem_spec + os.linesep)
            # work_script.write('#PBS -q HCPput' + os.linesep)
            work_script.write('#PBS -o ' + working_directory_name + os.linesep)
            work_script.write('#PBS -e ' + working_directory_name + os.linesep)
            work_script.write(os.linesep)
            work_script.write(self.xnat_pbs_jobs_home + os.sep + '7T' + os.sep + 'DeDriftAndResampleHCP7T' + os.sep +
                              'DeDriftAndResampleHCP7T.XNAT.sh \\' + os.linesep)
            work_script.write('  --user="******" \\' + os.linesep)
            work_script.write('  --password="******" \\' + os.linesep)
            work_script.write('  --server="' + str_utils.get_server_name(self.server) + '" \\' + os.linesep)
            work_script.write('  --project="' + self.project + '" \\' + os.linesep)
            work_script.write('  --subject="' + self.subject + '" \\' + os.linesep)
            work_script.write('  --session="' + self.session + '" \\' + os.linesep)
            work_script.write('  --structural-reference-project="' +
                              self.structural_reference_project + '" \\' + os.linesep)
            work_script.write('  --structural-reference-session="' +
                              self.structural_reference_session + '" \\' + os.linesep)
            work_script.write('  --working-dir="' + working_directory_name + '" \\' + os.linesep)
            work_script.write('  --workflow-id="' + workflow_id + '" \\' + os.linesep)
            work_script.write('  --setup-script=' + self.setup_script + os.linesep)

            work_script.close()
            os.chmod(work_script_name, stat.S_IRWXU | stat.S_IRWXG)

            # Create script to put the results into the DB
            put_script_name = script_file_start_name + '.XNAT_PBS_PUT_job.sh'
            self.create_put_script(put_script_name,
                                   self.username, self.password, self.put_server,
                                   self.project, self.subject, self.session,
                                   working_directory_name, output_resource_name,
                                   self.PIPELINE_NAME)

            # Submit the job to do the work
            work_submit_cmd = 'qsub ' + work_script_name
            _inform("work_submit_cmd: " + work_submit_cmd)

            completed_work_submit_process = subprocess.run(
                work_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            work_job_no = str_utils.remove_ending_new_lines(completed_work_submit_process.stdout)
            _inform("work_job_no: " + work_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + put_script_name
            _inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd, shell=True, check=True, stdout=subprocess.PIPE,
                universal_newlines=True)
            put_job_no = str_utils.remove_ending_new_lines(completed_put_submit_process.stdout)
            _inform("put_job_no: " + put_job_no)

        else:
            _inform("Unable to submit jobs")
Exemplo n.º 8
0
    def submit_jobs(self, processing_stage=ProcessingStage.PUT_DATA):
        """
        processing_stage is the last processing stage for which to submit
        the corresponding job.
        GET_DATA means just get the data.
        PROCESS_DATA means get the data and do the processing.
        PUT_DATA means get the data, processing it, and put the results
         back in the DB
        """
        logger.debug("submit_jobs processing_stage: " + str(processing_stage))

        if self.validate_parameters():

            # determine what scans to run the RestingStateStats pipeline on for this subject
            # TBD: Does this get run on every scan for which the ICAFIX pipeline has been run,
            #      or does it only get run on every resting state scan that has been fix processed.

            subject_info = hcp7t_subject.Hcp7TSubjectInfo(
                self.project, self.structural_reference_project, self.subject)

            fix_processed_scans = self.archive.available_FIX_processed_names(
                subject_info)
            fix_processed_scans_set = set(fix_processed_scans)
            logger.debug("fix_processed_scans_set = " +
                         str(fix_processed_scans_set))

            # resting_state_scans = self.archive.available_resting_state_preproc_names(subject_info)
            # resting_state_scans_set = set(resting_state_scans)
            # logger.debug("resting_state_scans_set = " + str(resting_state_scans_set))

            # scans_to_process_set = resting_state_scans_set & fix_processed_scans_set
            scans_to_process_set = fix_processed_scans_set
            scans_to_process = list(scans_to_process_set)
            scans_to_process.sort()
            logger.debug("scans_to_process: " + str(scans_to_process))

            incomplete_scans_to_process = list()
            for scan in scans_to_process:
                if (not is_complete(self.archive, subject_info, scan)):
                    incomplete_scans_to_process.append(scan)

            logger.debug("incomplete_scans_to_process: " +
                         str(incomplete_scans_to_process))
            print("incomplete_scans_to_process:", incomplete_scans_to_process)

            # for scan in scans_to_process:
            for scan in incomplete_scans_to_process:

                logger.info("")
                logger.info(
                    "--------------------------------------------------")
                logger.info("Submitting " + self.PIPELINE_NAME + " jobs for")
                logger.info("  Project: " + self.project)
                logger.info("  Subject: " + self.subject)
                logger.info("  Session: " + self.session)
                logger.info("  Structural Reference Project: " +
                            self.structural_reference_project)
                logger.info("  Structural Reference Session: " +
                            self.structural_reference_session)
                logger.info("     Scan: " + scan)
                logger.info("    Stage: " + str(processing_stage))
                logger.info(
                    "--------------------------------------------------")

                # make sure working directories do not have the same name based on
                # the same start time by sleeping a few seconds
                time.sleep(5)

                # build the working directory name
                self._working_directory_name = \
                    self.build_working_directory_name(self.project, self.PIPELINE_NAME, self.subject, scan)
                logger.info("Making working directory: " +
                            self._working_directory_name)
                os.makedirs(name=self._working_directory_name)

                # get JSESSION ID
                jsession_id = xnat_access.get_jsession_id(
                    server=os_utils.getenv_required(
                        'XNAT_PBS_JOBS_XNAT_SERVER'),
                    username=self.username,
                    password=self.password)
                logger.info("jsession_id: " + jsession_id)

                # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
                xnat_session_id = xnat_access.get_session_id(
                    server=os_utils.getenv_required(
                        'XNAT_PBS_JOBS_XNAT_SERVER'),
                    username=self.username,
                    password=self.password,
                    project=self.project,
                    subject=self.subject,
                    session=self.session)
                logger.info("xnat_session_id: " + xnat_session_id)

                # get XNAT Workflow ID
                workflow_obj = xnat_access.Workflow(
                    self.username, self.password, 'https://' +
                    os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                    jsession_id)
                self._workflow_id = workflow_obj.create_workflow(
                    xnat_session_id, self.project,
                    self.PIPELINE_NAME + '_' + scan, 'Queued')
                logger.info("workflow_id: " + self._workflow_id)

                # determine output resource name
                self._output_resource_name = scan + "_RSS"

                # clean output resource if requested
                if self.clean_output_resource_first:
                    logger.info("Deleting resource: " +
                                self._output_resource_name + " for:")
                    logger.info("  project: " + self.project)
                    logger.info("  subject: " + self.subject)
                    logger.info("  session: " + self.session)

                    delete_resource.delete_resource(
                        self.username, self.password,
                        str_utils.get_server_name(self.server), self.project,
                        self.subject, self.session, self._output_resource_name)

                # create scripts for various stages of processing
                if processing_stage >= ProcessingStage.PREPARE_SCRIPTS:
                    # create script to get data
                    self._create_get_data_script(scan)

                    # create script to do work
                    self._create_work_script(scan)

                    # create script to clean data
                    self._create_clean_data_script(scan)

                    # create script to put the results into the DB
                    put_script_name = self._put_data_script_name(scan)
                    self.create_put_script(put_script_name, self.username,
                                           self.password, self.put_server,
                                           self.project, self.subject,
                                           self.session,
                                           self._working_directory_name,
                                           self._output_resource_name,
                                           self.PIPELINE_NAME + '_' + scan)

                # submit job to get the data
                if processing_stage >= ProcessingStage.GET_DATA:

                    get_data_submit_cmd = 'qsub ' + self._get_data_script_name(
                        scan)
                    logger.info("get_data_submit_cmd: " + get_data_submit_cmd)

                    completed_get_data_submit_process = subprocess.run(
                        get_data_submit_cmd,
                        shell=True,
                        check=True,
                        stdout=subprocess.PIPE,
                        universal_newlines=True)
                    get_data_job_no = str_utils.remove_ending_new_lines(
                        completed_get_data_submit_process.stdout)
                    logger.info("get_data_job_no: " + get_data_job_no)

                else:
                    logger.info("Get data job not submitted")

                # submit job to process the data
                if processing_stage >= ProcessingStage.PROCESS_DATA:

                    work_submit_cmd = 'qsub -W depend=afterok:' + get_data_job_no + ' ' + self._work_script_name(
                        scan)
                    logger.info("work_submit_cmd: " + work_submit_cmd)

                    completed_work_submit_process = subprocess.run(
                        work_submit_cmd,
                        shell=True,
                        check=True,
                        stdout=subprocess.PIPE,
                        universal_newlines=True)
                    work_job_no = str_utils.remove_ending_new_lines(
                        completed_work_submit_process.stdout)
                    logger.info("work_job_no: " + work_job_no)

                else:
                    logger.info("Process data job not submitted")

                # submit job to clean the data
                if processing_stage >= ProcessingStage.CLEAN_DATA:

                    clean_submit_cmd = 'qsub -W depend=afterok:' + work_job_no + ' ' + self._clean_data_script_name(
                        scan)
                    logger.info("clean_submit_cmd: " + clean_submit_cmd)

                    completed_clean_submit_process = subprocess.run(
                        clean_submit_cmd,
                        shell=True,
                        check=True,
                        stdout=subprocess.PIPE,
                        universal_newlines=True)
                    clean_job_no = str_utils.remove_ending_new_lines(
                        completed_clean_submit_process.stdout)
                    logger.info("clean_job_no: " + clean_job_no)

                else:
                    logger.info("Clean data job not submitted")

                # submit job to put the resulting data in the DB
                if processing_stage >= ProcessingStage.PUT_DATA:

                    put_submit_cmd = 'qsub -W depend=afterok:' + clean_job_no + ' ' + put_script_name
                    logger.info("put_submit_cmd: " + put_submit_cmd)

                    completed_put_submit_process = subprocess.run(
                        put_submit_cmd,
                        shell=True,
                        check=True,
                        stdout=subprocess.PIPE,
                        universal_newlines=True)
                    put_job_no = str_utils.remove_ending_new_lines(
                        completed_put_submit_process.stdout)
                    logger.info("put_job_no: " + put_job_no)

                else:
                    logger.info("Put data job not submitted")

        else:
            logger.info("Unable to submit jobs")
    def submit_jobs(self):
        _debug("submit_jobs")

        if self.validate_parameters():

            _inform("")
            _inform("--------------------------------------------------")
            _inform("Submitting " + self.PIPELINE_NAME + " jobs for")
            _inform("  Project: " + self.project)
            _inform("  Subject: " + self.subject)
            _inform("  Session: " + self.session)
            _inform("--------------------------------------------------")

            # make sure working directories don't have the same name based on
            # the same start time by sleeping a few seconds
            time.sleep(5)
            current_seconds_since_epoch = int(time.time())

            # build the working directory name
            self._working_directory_name = self.build_home
            self._working_directory_name += os.sep + self.project
            self._working_directory_name += os.sep + self.PIPELINE_NAME
            self._working_directory_name += '.' + self.subject
            self._working_directory_name += '.' + str(
                current_seconds_since_epoch)

            # make the working directory
            _inform("making working directory: " +
                    self._working_directory_name)
            os.makedirs(name=self._working_directory_name)

            # get JSESSION ID
            jsession_id = xnat_access.get_jsession_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=self.username,
                password=self.password)
            _inform("jsession_id: " + jsession_id)

            # get XNAT Session ID (a.k.a. the experiment ID, e.g. ConnectomeDB_E1234)
            xnat_session_id = xnat_access.get_session_id(
                server=os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                username=self.username,
                password=self.password,
                project=kself.project,
                subject=self.subject,
                session=self.session)
            _inform("xnat_session_id: " + xnat_session_id)

            # get XNAT Workflow ID
            workflow_obj = xnat_access.Workflow(
                self.username, self.password,
                os_utils.getenv_required('XNAT_PBS_JOBS_XNAT_SERVER'),
                jsession_id)
            self._workflow_id = workflow_obj.create_workflow(
                xnat_session_id, self.project, self.PIPELINE_NAME, 'Queued')
            _inform("workflow_id: " + self._workflow_id)

            # determine output resource name
            self._output_resource_name = 'Diffusion_preproc'

            # clean the output resource if requested
            if self.clean_output_resource_first:
                _inform("Deleting resource: " + self._output_resource_name +
                        " for:")
                _inform("  project: " + self.project)
                _inform("  subject: " + self.subject)
                _inform("  session: " + self.session)

                delete_resource.delete_resource(
                    self.username, self.password,
                    str_utils.get_server_name(self.server), self.project,
                    self.subject, self.session, self._output_resource_name)

            # create script to do the PreEddy work
            self._create_pre_eddy_script()

            # create script to do the Eddy work
            self._create_eddy_script()

            # create script to do the PostEddy work
            self._create_post_eddy_script()

            # create script to put the results into the DB
            put_script_name = self._get_scripts_start_name(
            ) + '.XNAT_PBS_PUT_job.sh'
            self.create_put_script(put_script_name, self.username,
                                   self.password, self.put_server,
                                   self.project, self.subject, self.session,
                                   self._working_directory_name,
                                   self._output_resource_name,
                                   self.PIPELINE_NAME)

            # Submit the job to do the Pre-Eddy work
            pre_eddy_submit_cmd = 'qsub ' + self._pre_eddy_script_name
            _inform("pre_eddy_submit_cmd: " + pre_eddy_submit_cmd)

            completed_pre_eddy_submit_process = subprocess.run(
                pre_eddy_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            pre_eddy_job_no = str_utils.remove_ending_new_lines(
                completed_pre_eddy_submit_process.stdout)
            _inform("pre_eddy_job_no: " + pre_eddy_job_no)

            # Submit the job to do the Eddy work
            eddy_submit_cmd = 'qsub -W depend=afterok:' + pre_eddy_job_no + ' ' + self._eddy_script_name
            _inform("eddy_submit_cmd: " + eddy_submit_cmd)

            completed_eddy_submit_process = subprocess.run(
                eddy_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            eddy_job_no = str_utils.remove_ending_new_lines(
                completed_eddy_submit_process.stdout)
            _inform("eddy_job_no: " + eddy_job_no)

            # Submit the job to do the Post-Eddy work
            post_eddy_submit_cmd = 'qsub -W depend=afterok:' + eddy_job_no + ' ' + self._post_eddy_script_name
            _inform("post_eddy_submit_cmd: " + post_eddy_submit_cmd)

            completed_post_eddy_submit_process = subprocess.run(
                post_eddy_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            post_eddy_job_no = str_utils.remove_ending_new_lines(
                completed_post_eddy_submit_process.stdout)
            _inform("post_eddy_job_no: " + post_eddy_job_no)

            # Submit the job to put the results in the DB
            put_submit_cmd = 'qsub -W depend=afterok:' + post_eddy_job_no + ' ' + put_script_name
            _inform("put_submit_cmd: " + put_submit_cmd)

            completed_put_submit_process = subprocess.run(
                put_submit_cmd,
                shell=True,
                check=True,
                stdout=subprocess.PIPE,
                universal_newlines=True)
            put_job_no = str_utils.remove_ending_new_lines(
                completed_put_submit_process.stdout)
            _inform("put_job_no: " + put_job_no)

        else:
            _inform("Unable to submit jobs")