예제 #1
0
    def manager_thread_main(self):
        """ Checks for workers that died unexpectedly and listens to their
            status update messages.
        """
        while True:
            # wait up to 60 seconds
            try:
                command, argument = self.manager_thread_queue.get(True, 60)

                if command == 'quit':
                    break
                elif command == 'start':
                    with self._lock:
                        self.now_building[argument] = True
                elif command == 'end':
                    with self._lock:
                        self.now_building[argument] = False
                else:
                    logger.warn("Unknown command to manager thread: %s" % command)

            except Queue.Empty:
                pass

            # this one is to remove zombie processes
            multiprocessing.active_children()

            with self._lock:
                ids_to_restart = []
                for id, worker in self.workers.items():
                    if not worker.is_alive():
                        ids_to_restart.append(id)

                for id in ids_to_restart:
                    self.restart(id)
예제 #2
0
    def main(self):
        """ Process main function """
        # to make sure we do not use inherited descriptor
        # from the parent process
        close_db_connection()
        self.connection = get_db_connection(self.config)
        self.watch = Watch.load(self.id, self.connection)


        if self.watch:
            logger.info("Starting worker for watch `%s` (id: %s)" % (self.watch.name, self.id))
            self.tick_interval = self.watch.interval
            twillmanager.async.Worker.main(self)
        else:
            logger.warn("Failed to start worker for watch (id: %s) - no such watch" % self.id)
예제 #3
0
    def execute(self):
        """ Called by `tick` and when `execute` schedules immediate script execution."""
        try: # large try block to ensure on_end is called
            if self.on_start:
                self.on_start()

            new_status, output = self.execute_script()

            old_status = self.watch.status
            self.watch.status = new_status
            self.watch.time = time.time()
            self.watch.update_status(self.connection)

            msg = "Status for watch `%s` (id: %s): %s" % (self.watch.name, self.id, new_status)

            if new_status != STATUS_OK:
                logger.warn(msg)
            else:
                logger.info(msg)

            # when was last e-mail alert sent
            if self.watch.last_alert is None:
                time_since_last_alert = None
            else:
                time_since_last_alert = self.watch.time - self.watch.last_alert

            status_has_changed = (old_status != new_status)

            # whether last alert was sent long ago enough to send a failure reminder
            # (normally e-mails are sent only on change, but a reminder is sent
            # if the watch keeps failing)
            if self.watch.reminder_interval:
                last_alert_was_long_ago = time_since_last_alert is None or time_since_last_alert > self.watch.reminder_interval
            else:
                last_alert_was_long_ago = False


            if status_has_changed or (last_alert_was_long_ago and new_status == STATUS_FAILED):
                logger.info("Sending notification for watch `%s` (id: %s)" % (self.watch.name, self.id))
                self.status_notify(old_status, new_status, output)
                self.watch.last_alert = time.time()
                self.watch.update_status(self.connection)
        except Exception, e:
            logger.error("Worker `%s` (id: %s) failed with exception: %s" % (self.watch.name, self.id, e.message))
            raise