Example #1
0
 def machine_job(self, machine_name, command, bg_flag, post_data = None):
     "Run a non-package job on a remote machine"
     if post_data == None:
         post_data = {}
     url = URL_LOOKUP[command] % machine_name
     job_name = self.get_job(url, post_data)
     if bg_flag:
         return OK, ["Started job: %s" % job_name]
     libUi.info("Watching job progress. Press ^C to abort or disconnect.")
     return self.watch_jobs([job_name])
Example #2
0
    def watch_jobs(self, job_names):
        "Given a list of jobs, watch their progress."
        summary_output = []
        summary_status = OK
        jobs = set(job_names)
        try:
            while job_names:
                jobs = jobs.union(job_names)
                time.sleep(0.25)
                url = "json/job/poll/"
                post_data = {"yaml": yaml.dump({"job_names": job_names})}
                output = self.service_yaml_request(url, post_data=post_data)
                if type(output) != type({}):
                    msg = "Request to %s returned and invalid type (%s)"
                    libUi.error(msg % (url, output))
                    return FAIL, output
                for job_name in output:
                    job_output = output[job_name]
                    if "traceback" in job_output:
                        raise MachineTraceback(url, job_output["traceback"])
                    new_output = job_output.get("new_output", '')
                    if new_output:
                        libUi.process_cnm(new_output)
                    alive = job_output.get("alive")
                    if alive == None:
                        if job_output.get("command_status") == "QUEUED":
                            pending_job = job_output.get("pending_job")
                            msg = "%s is pending %s." % (job_name, pending_job)
                            libUi.warning(msg)
                            libUi.info("Watching %s..." % pending_job)
                            job_names = [pending_job]
                            break
                    if alive == False:
                        out = system_state.cnm_connector.join_job(job_name)
                        summary_output.append(out.get("command_output"))
                        status = out.get("command_status", OK)
                        if status != OK:
                            summary_status = FAIL
                        job_names +=  out.get("children", [])
                        job_names.remove(job_name)
                        comments = out.get("jobs_requiring_comment", 0)
                        if comments:
                            system_state.comment_counter = len(comments)
        except exceptions.KeyboardInterrupt:
            term = libUi.ask_yes_no("Terminate job", libUi.YES)
            if term == libUi.NO:
                msg = ["Disconnected from %s" % job_names,
                       "To re-connect, use 'job view'"]
                return OK, msg
            else:
                matcher = re.compile("(\S+)@(\S+)\-(\d+)")
                machine_names = set()
                for job_name in job_names:
                    _user, machine_name, _counter = matcher.findall(job_name)[0]
                    machine_names.update([machine_name])
                output = {}
                for machine_name in machine_names:
                    _status, output = self.stop_jobs(machine_name)
                    print ">>> Halting all jobs on %s..." % machine_name
                    libUi.user_output(output, FAIL)
                raise exceptions.KeyboardInterrupt

        libUi.info("Finishing processing of %d jobs..." % len(jobs))
        return summary_status, summary_output