def run(self):
     """ Coordinate the job. """
     self.logger.info("%s started" % (self.prog,))
     self._running = True
     action = None
     while self._running:
         self.supervise()
         self.save_status()
         if self._config.get("command") == "single":
             self.logger.debug("single mode, exiting")
             return
         wake_time = self.sleep_interval() + time.time()
         self.logger.debug("sleeping for %d seconds" % self.sleep_interval())
         while wake_time >= time.time():
             if self._config.get("pidfile"):
                 pid_touch(self._config["pidfile"])
                 action = pid_check(self._config["pidfile"])
                 if action in ["quit", "stop_supervisor"]:
                     self.logger.info("asked to %s" % action)
                     self._running = False
                     break
                 elif action == "stop_children":
                     self.logger.info("stopping all the children")
                     self._child.stop()
                     pid_write(self._config["pidfile"], os.getpid())
                 elif action != "":
                     self.logger.warning("unknown action: %s" % action)
             if not self._running:
                 break
             time.sleep(0.2)
     if action != "stop_supervisor":
         self.logger.info("stopping all the children")
         self._child.stop()
     self.logger.info("stopping %s" % (self.prog,))
     self.save_status()
 def run(self):
     """ Coordinate the job. """
     self.logger.info("started")
     self._running = True
     action = None
     while self._running:
         self.supervise()
         self.save_status()
         wake_time = self.sleep_interval() + time.time()
         self.logger.debug(
             "sleeping for %d seconds", self.sleep_interval())
         while wake_time >= time.time():
             if self._config.get("pidfile"):
                 pid_touch(self._config["pidfile"])
                 action = pid_check(self._config["pidfile"])
                 if action != "":
                     self.logger.info("asked to %s", action)
                     pid_write(self._config["pidfile"], os.getpid())
                 if action in ["quit", "stop_supervisor"]:
                     self._running = False
                 elif action == "stop_children":
                     self._child.stop()
                 elif action == "wake_up":
                     break
                 elif action[:14] == "restart_child ":
                     target = self.get_child(action[14:])
                     target.restart()
                 elif action != "":
                     self.logger.warning("unknown action: %s", action)
             if not self._running:
                 break
             time.sleep(0.5)
     if action != "stop_supervisor":
         self.logger.info("stopping all the children")
         self._child.stop()
     self.logger.info("stopping")
     self.save_status()
     self.logger.info("stopped")
Beispiel #3
0
 def work(self):
     """ Do it! """
     pending = dict()
     put_list = list()
     timek = dict()
     timek["start"] = time.time()
     self.logger.debug("starting")
     signal.signal(signal.SIGINT, self._handle_sig)
     signal.signal(signal.SIGTERM, self._handle_sig)
     signal.signal(signal.SIGHUP, self._handle_sig)
     if self._config.get("incoming-broker") is not None:
         mtype = self._config.get("incoming-broker-module") or "pika"
         if mtype == "kombu":
             incoming = amqpclt.kombu.KombuIncomingBroker(self._config)
         elif mtype == "pika":
             incoming = amqpclt.pika.PikaIncomingBroker(self._config)
         else:
             raise AmqpcltError(
                 "invalid incoming broker module: %s" % (mtype, ))
     else:
         incoming = amqpclt.queue.IncomingQueue(self._config)
     if self._config.get("outgoing-broker") is not None:
         mtype = self._config.get("outgoing-broker-module") or "pika"
         if mtype == "kombu":
             outgoing = amqpclt.kombu.KombuOutgoingBroker(self._config)
         elif mtype == "pika":
             outgoing = amqpclt.pika.PikaOutgoingBroker(self._config)
         else:
             raise AmqpcltError(
                 "invalid outgoing broker module: %s" % (mtype, ))
     else:
         outgoing = amqpclt.queue.OutgoingQueue(self._config)
     incoming.start()
     if not self._config["lazy"]:
         outgoing.start()
     if self._config.get("callback", None) is not None:
         self._callback.start(*self._config["callback"]["data"])
     self.logger.debug("running")
     count = size = 0
     if self._config.get("duration") is not None:
         timek["max"] = time.time() + self._config["duration"]
     else:
         timek["max"] = 0
     tina = self._config.get("timeout-inactivity")
     if tina is not None:
         timek["ina"] = time.time() + tina
     else:
         timek["ina"] = 0
     self._running = True
     while self._running:
         # are we done
         if self._config.get("count") is not None and \
                 count >= self._config["count"]:
             break
         if timek["max"] and time.time() > timek["max"]:
             break
         # get message
         if self._config["reliable"]:
             if self._config.get("window") >= 0 and \
                     len(pending) > self._config("window"):
                 incoming.idle()
                 (msg, msg_id) = ("too many pending acks", None)
             else:
                 (msg, msg_id) = incoming.get()
                 if type(msg) != str:
                     if msg_id in pending:
                         self.logger.debug(
                             "duplicate ack id: %s" % (msg_id, ))
                         sys.exit(1)
                     else:
                         pending[msg_id] = True
         else:
             (msg, msg_id) = incoming.get()
         # check inactivity
         if timek.get("ina"):
             if isinstance(msg, Message):
                 timek["ina"] = time.time() + tina
             elif time.time() >= timek["ina"]:
                 break
         # count and statistics
         if isinstance(msg, Message):
             count += 1
             if self._config["statistics"]:
                 size += msg.size()
                 if count == 1:
                     timek["first"] = time.time()
         # callback
         if self._config.get("callback") is not None:
             if type(msg) != str:
                 msg = self._callback.check(msg)
                 if not isinstance(msg, Message):
                     self.logger.debug(
                         "message discarded by callback: %s" % (msg, ))
                     # message discarded by callback
                     if self._config["reliable"]:
                         if msg_id not in pending:
                             raise AmqpcltError(
                                 "unexpected ack id: %s" % (msg_id, ))
                         del(pending[msg_id])
                         incoming.ack(msg_id)
                     if self._config["statistics"]:
                         timek["last"] = time.time()
             else:
                 self._callback.idle()
         # put | idle
         if isinstance(msg, Message):
             self.logger.debug("loop got new message")
             if self._config["lazy"]:
                 outgoing.start()
                 self._config["lazy"] = False
             put_list = outgoing.put(msg, msg_id)
             if self._config["statistics"]:
                 timek["last"] = time.time()
         else:
             if msg:
                 self.logger.debug("loop %s" % (msg, ))
             else:
                 self.logger.debug("loop end")
                 self._running = False
             if self._config["lazy"]:
                 put_list = list()
             else:
                 put_list = outgoing.idle()
         # ack
         for msg_id in put_list:
             if msg_id not in pending:
                 raise AmqpcltError("unexpected ack id: %s" % (msg_id, ))
             del(pending[msg_id])
             incoming.ack(msg_id)
         # check --quit and show that we are alive
         if self._config.get("pidfile"):
             action = pid_check(self._config["pidfile"])
             if action == "quit":
                 self.logger.debug("asked to quit")
                 break
             pid_touch(self._config["pidfile"])
     # linger
     self.logger.debug("linger")
     timeout_linger = self._config.get("timeout-linger")
     if timeout_linger:
         timek["max"] = time.time() + timeout_linger
     else:
         timek["max"] = 0
     self._running = True
     while self._running:
         if not pending:
             break
         if "max" in timek and time.time() >= timek["max"]:
             break
         put_list = outgoing.idle()
         if put_list:
             for msg_id in put_list:
                 if msg_id not in pending:
                     raise AmqpcltError(
                         "unexpected ack id: %s" % (msg_id, ))
                 del(pending[msg_id])
                 incoming.ack(msg_id)
         else:
             incoming.idle()
     if pending:
         raise AmqpcltError("%d pending messages" % len(pending))
     # report statistics
     if self._config.get("statistics"):
         if count == 0:
             print("no messages processed")
         elif count == 1:
             print("only 1 message processed")
         else:
             timek["elapsed"] = timek["last"] - timek["first"]
             print(("processed %d messages in %.3f seconds"
                    " (%.3f k messages/second)") %
                   (count, timek["elapsed"],
                    count / timek["elapsed"] / 1000))
             print("troughput is around %.3f MB/second" %
                   (size / timek["elapsed"] / 1024 / 1024))
             print("average message size is around %d bytes" %
                   (int(size / count + 0.5)))
     # stop
     self.logger.debug("stopping")
     if self._config.get("callback", None) is not None:
         self.logger.debug("stopping callback")
         self._callback.stop()
     if not self._config.get("lazy"):
         self.logger.debug("stopping outgoing")
         outgoing.stop()
     self.logger.debug("stopping incoming")
     incoming.stop()
     self.logger.debug("incoming stopped")
     timek["stop"] = time.time()
     timek["elapsed"] = timek["stop"] - timek["start"]
     self.logger.debug(
         "work processed %d messages in %.3f seconds" %
         (count, timek["elapsed"]))
Beispiel #4
0
 def work(self):
     """ Do it! """
     pending = dict()
     put_list = list()
     timek = dict()
     timek["start"] = time.time()
     self.logger.debug("starting")
     signal.signal(signal.SIGINT, self._handle_sig)
     signal.signal(signal.SIGTERM, self._handle_sig)
     signal.signal(signal.SIGHUP, self._handle_sig)
     if self._config.get("incoming-broker") is not None:
         mtype = self._config.get("incoming-broker-module") or "pika"
         if mtype == "kombu":
             incoming = amqpclt.kombu.KombuIncomingBroker(self._config)
         elif mtype == "pika":
             incoming = amqpclt.pika.PikaIncomingBroker(self._config)
         else:
             raise AmqpcltError("invalid incoming broker module: %s" %
                                (mtype, ))
     else:
         incoming = amqpclt.queue.IncomingQueue(self._config)
     if self._config.get("outgoing-broker") is not None:
         mtype = self._config.get("outgoing-broker-module") or "pika"
         if mtype == "kombu":
             outgoing = amqpclt.kombu.KombuOutgoingBroker(self._config)
         elif mtype == "pika":
             outgoing = amqpclt.pika.PikaOutgoingBroker(self._config)
         else:
             raise AmqpcltError("invalid outgoing broker module: %s" %
                                (mtype, ))
     else:
         outgoing = amqpclt.queue.OutgoingQueue(self._config)
     incoming.start()
     if not self._config["lazy"]:
         outgoing.start()
     if self._config.get("callback", None) is not None:
         self._callback.start(*self._config["callback"]["data"])
     self.logger.debug("running")
     count = size = 0
     if self._config.get("duration") is not None:
         timek["max"] = time.time() + self._config["duration"]
     else:
         timek["max"] = 0
     tina = self._config.get("timeout-inactivity")
     if tina is not None:
         timek["ina"] = time.time() + tina
     else:
         timek["ina"] = 0
     self._running = True
     while self._running:
         # are we done
         if self._config.get("count") is not None and \
                 count >= self._config["count"]:
             break
         if timek["max"] and time.time() > timek["max"]:
             break
         # get message
         if self._config["reliable"]:
             if self._config.get("window") >= 0 and \
                     len(pending) > self._config("window"):
                 incoming.idle()
                 (msg, msg_id) = ("too many pending acks", None)
             else:
                 (msg, msg_id) = incoming.get()
                 if type(msg) != str:
                     if msg_id in pending:
                         self.logger.debug("duplicate ack id: %s" %
                                           (msg_id, ))
                         sys.exit(1)
                     else:
                         pending[msg_id] = True
         else:
             (msg, msg_id) = incoming.get()
         # check inactivity
         if timek.get("ina"):
             if isinstance(msg, Message):
                 timek["ina"] = time.time() + tina
             elif time.time() >= timek["ina"]:
                 break
         # count and statistics
         if isinstance(msg, Message):
             count += 1
             if self._config["statistics"]:
                 size += msg.size()
                 if count == 1:
                     timek["first"] = time.time()
         # callback
         if self._config.get("callback") is not None:
             if type(msg) != str:
                 msg = self._callback.check(msg)
                 if not isinstance(msg, Message):
                     self.logger.debug("message discarded by callback: %s" %
                                       (msg, ))
                     # message discarded by callback
                     if self._config["reliable"]:
                         if msg_id not in pending:
                             raise AmqpcltError("unexpected ack id: %s" %
                                                (msg_id, ))
                         del (pending[msg_id])
                         incoming.ack(msg_id)
                     if self._config["statistics"]:
                         timek["last"] = time.time()
             else:
                 self._callback.idle()
         # put | idle
         if isinstance(msg, Message):
             self.logger.debug("loop got new message")
             if self._config["lazy"]:
                 outgoing.start()
                 self._config["lazy"] = False
             put_list = outgoing.put(msg, msg_id)
             if self._config["statistics"]:
                 timek["last"] = time.time()
         else:
             if msg:
                 self.logger.debug("loop %s" % (msg, ))
             else:
                 self.logger.debug("loop end")
                 self._running = False
             if self._config["lazy"]:
                 put_list = list()
             else:
                 put_list = outgoing.idle()
         # ack
         for msg_id in put_list:
             if msg_id not in pending:
                 raise AmqpcltError("unexpected ack id: %s" % (msg_id, ))
             del (pending[msg_id])
             incoming.ack(msg_id)
         # check --quit and show that we are alive
         if self._config.get("pidfile"):
             action = pid_check(self._config["pidfile"])
             if action == "quit":
                 self.logger.debug("asked to quit")
                 break
             pid_touch(self._config["pidfile"])
     # linger
     self.logger.debug("linger")
     timeout_linger = self._config.get("timeout-linger")
     if timeout_linger:
         timek["max"] = time.time() + timeout_linger
     else:
         timek["max"] = 0
     self._running = True
     while self._running:
         if not pending:
             break
         if "max" in timek and time.time() >= timek["max"]:
             break
         put_list = outgoing.idle()
         if put_list:
             for msg_id in put_list:
                 if msg_id not in pending:
                     raise AmqpcltError("unexpected ack id: %s" %
                                        (msg_id, ))
                 del (pending[msg_id])
                 incoming.ack(msg_id)
         else:
             incoming.idle()
     if pending:
         raise AmqpcltError("%d pending messages" % len(pending))
     # report statistics
     if self._config.get("statistics"):
         if count == 0:
             print("no messages processed")
         elif count == 1:
             print("only 1 message processed")
         else:
             timek["elapsed"] = timek["last"] - timek["first"]
             print(
                 ("processed %d messages in %.3f seconds"
                  " (%.3f k messages/second)") %
                 (count, timek["elapsed"], count / timek["elapsed"] / 1000))
             print("troughput is around %.3f MB/second" %
                   (size / timek["elapsed"] / 1024 / 1024))
             print("average message size is around %d bytes" %
                   (int(size / count + 0.5)))
     # stop
     self.logger.debug("stopping")
     if self._config.get("callback", None) is not None:
         self.logger.debug("stopping callback")
         self._callback.stop()
     if not self._config.get("lazy"):
         self.logger.debug("stopping outgoing")
         outgoing.stop()
     self.logger.debug("stopping incoming")
     incoming.stop()
     self.logger.debug("incoming stopped")
     timek["stop"] = time.time()
     timek["elapsed"] = timek["stop"] - timek["start"]
     self.logger.debug("work processed %d messages in %.3f seconds" %
                       (count, timek["elapsed"]))