def run(self):
""" Coordinate the job. """
self.logger.info("%s started" % (self.prog,))
self._running = True
action = None
while self._running:
self.supervise()
self.save_status()
if self._config.get("command") == "single":
self.logger.debug("single mode, exiting")
return
wake_time = self.sleep_interval() + time.time()
self.logger.debug("sleeping for %d seconds" % self.sleep_interval())
while wake_time >= time.time():
if self._config.get("pidfile"):
pid_touch(self._config["pidfile"])
action = pid_check(self._config["pidfile"])
if action in ["quit", "stop_supervisor"]:
self.logger.info("asked to %s" % action)
self._running = False
break
elif action == "stop_children":
self.logger.info("stopping all the children")
self._child.stop()
pid_write(self._config["pidfile"], os.getpid())
elif action != "":
self.logger.warning("unknown action: %s" % action)
if not self._running:
break
time.sleep(0.2)
if action != "stop_supervisor":
self.logger.info("stopping all the children")
self._child.stop()
self.logger.info("stopping %s" % (self.prog,))
self.save_status()
def run(self):
""" Coordinate the job. """
self.logger.info("started")
self._running = True
action = None
while self._running:
self.supervise()
self.save_status()
wake_time = self.sleep_interval() + time.time()
self.logger.debug(
"sleeping for %d seconds", self.sleep_interval())
while wake_time >= time.time():
if self._config.get("pidfile"):
pid_touch(self._config["pidfile"])
action = pid_check(self._config["pidfile"])
if action != "":
self.logger.info("asked to %s", action)
pid_write(self._config["pidfile"], os.getpid())
if action in ["quit", "stop_supervisor"]:
self._running = False
elif action == "stop_children":
self._child.stop()
elif action == "wake_up":
break
elif action[:14] == "restart_child ":
target = self.get_child(action[14:])
target.restart()
elif action != "":
self.logger.warning("unknown action: %s", action)
if not self._running:
break
time.sleep(0.5)
if action != "stop_supervisor":
self.logger.info("stopping all the children")
self._child.stop()
self.logger.info("stopping")
self.save_status()
self.logger.info("stopped")
def work(self):
""" Do it! """
pending = dict()
put_list = list()
timek = dict()
timek["start"] = time.time()
self.logger.debug("starting")
signal.signal(signal.SIGINT, self._handle_sig)
signal.signal(signal.SIGTERM, self._handle_sig)
signal.signal(signal.SIGHUP, self._handle_sig)
if self._config.get("incoming-broker") is not None:
mtype = self._config.get("incoming-broker-module") or "pika"
if mtype == "kombu":
incoming = amqpclt.kombu.KombuIncomingBroker(self._config)
elif mtype == "pika":
incoming = amqpclt.pika.PikaIncomingBroker(self._config)
else:
raise AmqpcltError(
"invalid incoming broker module: %s" % (mtype, ))
else:
incoming = amqpclt.queue.IncomingQueue(self._config)
if self._config.get("outgoing-broker") is not None:
mtype = self._config.get("outgoing-broker-module") or "pika"
if mtype == "kombu":
outgoing = amqpclt.kombu.KombuOutgoingBroker(self._config)
elif mtype == "pika":
outgoing = amqpclt.pika.PikaOutgoingBroker(self._config)
else:
raise AmqpcltError(
"invalid outgoing broker module: %s" % (mtype, ))
else:
outgoing = amqpclt.queue.OutgoingQueue(self._config)
incoming.start()
if not self._config["lazy"]:
outgoing.start()
if self._config.get("callback", None) is not None:
self._callback.start(*self._config["callback"]["data"])
self.logger.debug("running")
count = size = 0
if self._config.get("duration") is not None:
timek["max"] = time.time() + self._config["duration"]
else:
timek["max"] = 0
tina = self._config.get("timeout-inactivity")
if tina is not None:
timek["ina"] = time.time() + tina
else:
timek["ina"] = 0
self._running = True
while self._running:
# are we done
if self._config.get("count") is not None and \
count >= self._config["count"]:
break
if timek["max"] and time.time() > timek["max"]:
break
# get message
if self._config["reliable"]:
if self._config.get("window") >= 0 and \
len(pending) > self._config("window"):
incoming.idle()
(msg, msg_id) = ("too many pending acks", None)
else:
(msg, msg_id) = incoming.get()
if type(msg) != str:
if msg_id in pending:
self.logger.debug(
"duplicate ack id: %s" % (msg_id, ))
sys.exit(1)
else:
pending[msg_id] = True
else:
(msg, msg_id) = incoming.get()
# check inactivity
if timek.get("ina"):
if isinstance(msg, Message):
timek["ina"] = time.time() + tina
elif time.time() >= timek["ina"]:
break
# count and statistics
if isinstance(msg, Message):
count += 1
if self._config["statistics"]:
size += msg.size()
if count == 1:
timek["first"] = time.time()
# callback
if self._config.get("callback") is not None:
if type(msg) != str:
msg = self._callback.check(msg)
if not isinstance(msg, Message):
self.logger.debug(
"message discarded by callback: %s" % (msg, ))
# message discarded by callback
if self._config["reliable"]:
if msg_id not in pending:
raise AmqpcltError(
"unexpected ack id: %s" % (msg_id, ))
del(pending[msg_id])
incoming.ack(msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
self._callback.idle()
# put | idle
if isinstance(msg, Message):
self.logger.debug("loop got new message")
if self._config["lazy"]:
outgoing.start()
self._config["lazy"] = False
put_list = outgoing.put(msg, msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
if msg:
self.logger.debug("loop %s" % (msg, ))
else:
self.logger.debug("loop end")
self._running = False
if self._config["lazy"]:
put_list = list()
else:
put_list = outgoing.idle()
# ack
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" % (msg_id, ))
del(pending[msg_id])
incoming.ack(msg_id)
# check --quit and show that we are alive
if self._config.get("pidfile"):
action = pid_check(self._config["pidfile"])
if action == "quit":
self.logger.debug("asked to quit")
break
pid_touch(self._config["pidfile"])
# linger
self.logger.debug("linger")
timeout_linger = self._config.get("timeout-linger")
if timeout_linger:
timek["max"] = time.time() + timeout_linger
else:
timek["max"] = 0
self._running = True
while self._running:
if not pending:
break
if "max" in timek and time.time() >= timek["max"]:
break
put_list = outgoing.idle()
if put_list:
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError(
"unexpected ack id: %s" % (msg_id, ))
del(pending[msg_id])
incoming.ack(msg_id)
else:
incoming.idle()
if pending:
raise AmqpcltError("%d pending messages" % len(pending))
# report statistics
if self._config.get("statistics"):
if count == 0:
print("no messages processed")
elif count == 1:
print("only 1 message processed")
else:
timek["elapsed"] = timek["last"] - timek["first"]
print(("processed %d messages in %.3f seconds"
" (%.3f k messages/second)") %
(count, timek["elapsed"],
count / timek["elapsed"] / 1000))
print("troughput is around %.3f MB/second" %
(size / timek["elapsed"] / 1024 / 1024))
print("average message size is around %d bytes" %
(int(size / count + 0.5)))
# stop
self.logger.debug("stopping")
if self._config.get("callback", None) is not None:
self.logger.debug("stopping callback")
self._callback.stop()
if not self._config.get("lazy"):
self.logger.debug("stopping outgoing")
outgoing.stop()
self.logger.debug("stopping incoming")
incoming.stop()
self.logger.debug("incoming stopped")
timek["stop"] = time.time()
timek["elapsed"] = timek["stop"] - timek["start"]
self.logger.debug(
"work processed %d messages in %.3f seconds" %
(count, timek["elapsed"]))
def work(self):
""" Do it! """
pending = dict()
put_list = list()
timek = dict()
timek["start"] = time.time()
self.logger.debug("starting")
signal.signal(signal.SIGINT, self._handle_sig)
signal.signal(signal.SIGTERM, self._handle_sig)
signal.signal(signal.SIGHUP, self._handle_sig)
if self._config.get("incoming-broker") is not None:
mtype = self._config.get("incoming-broker-module") or "pika"
if mtype == "kombu":
incoming = amqpclt.kombu.KombuIncomingBroker(self._config)
elif mtype == "pika":
incoming = amqpclt.pika.PikaIncomingBroker(self._config)
else:
raise AmqpcltError("invalid incoming broker module: %s" %
(mtype, ))
else:
incoming = amqpclt.queue.IncomingQueue(self._config)
if self._config.get("outgoing-broker") is not None:
mtype = self._config.get("outgoing-broker-module") or "pika"
if mtype == "kombu":
outgoing = amqpclt.kombu.KombuOutgoingBroker(self._config)
elif mtype == "pika":
outgoing = amqpclt.pika.PikaOutgoingBroker(self._config)
else:
raise AmqpcltError("invalid outgoing broker module: %s" %
(mtype, ))
else:
outgoing = amqpclt.queue.OutgoingQueue(self._config)
incoming.start()
if not self._config["lazy"]:
outgoing.start()
if self._config.get("callback", None) is not None:
self._callback.start(*self._config["callback"]["data"])
self.logger.debug("running")
count = size = 0
if self._config.get("duration") is not None:
timek["max"] = time.time() + self._config["duration"]
else:
timek["max"] = 0
tina = self._config.get("timeout-inactivity")
if tina is not None:
timek["ina"] = time.time() + tina
else:
timek["ina"] = 0
self._running = True
while self._running:
# are we done
if self._config.get("count") is not None and \
count >= self._config["count"]:
break
if timek["max"] and time.time() > timek["max"]:
break
# get message
if self._config["reliable"]:
if self._config.get("window") >= 0 and \
len(pending) > self._config("window"):
incoming.idle()
(msg, msg_id) = ("too many pending acks", None)
else:
(msg, msg_id) = incoming.get()
if type(msg) != str:
if msg_id in pending:
self.logger.debug("duplicate ack id: %s" %
(msg_id, ))
sys.exit(1)
else:
pending[msg_id] = True
else:
(msg, msg_id) = incoming.get()
# check inactivity
if timek.get("ina"):
if isinstance(msg, Message):
timek["ina"] = time.time() + tina
elif time.time() >= timek["ina"]:
break
# count and statistics
if isinstance(msg, Message):
count += 1
if self._config["statistics"]:
size += msg.size()
if count == 1:
timek["first"] = time.time()
# callback
if self._config.get("callback") is not None:
if type(msg) != str:
msg = self._callback.check(msg)
if not isinstance(msg, Message):
self.logger.debug("message discarded by callback: %s" %
(msg, ))
# message discarded by callback
if self._config["reliable"]:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" %
(msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
self._callback.idle()
# put | idle
if isinstance(msg, Message):
self.logger.debug("loop got new message")
if self._config["lazy"]:
outgoing.start()
self._config["lazy"] = False
put_list = outgoing.put(msg, msg_id)
if self._config["statistics"]:
timek["last"] = time.time()
else:
if msg:
self.logger.debug("loop %s" % (msg, ))
else:
self.logger.debug("loop end")
self._running = False
if self._config["lazy"]:
put_list = list()
else:
put_list = outgoing.idle()
# ack
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" % (msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
# check --quit and show that we are alive
if self._config.get("pidfile"):
action = pid_check(self._config["pidfile"])
if action == "quit":
self.logger.debug("asked to quit")
break
pid_touch(self._config["pidfile"])
# linger
self.logger.debug("linger")
timeout_linger = self._config.get("timeout-linger")
if timeout_linger:
timek["max"] = time.time() + timeout_linger
else:
timek["max"] = 0
self._running = True
while self._running:
if not pending:
break
if "max" in timek and time.time() >= timek["max"]:
break
put_list = outgoing.idle()
if put_list:
for msg_id in put_list:
if msg_id not in pending:
raise AmqpcltError("unexpected ack id: %s" %
(msg_id, ))
del (pending[msg_id])
incoming.ack(msg_id)
else:
incoming.idle()
if pending:
raise AmqpcltError("%d pending messages" % len(pending))
# report statistics
if self._config.get("statistics"):
if count == 0:
print("no messages processed")
elif count == 1:
print("only 1 message processed")
else:
timek["elapsed"] = timek["last"] - timek["first"]
print(
("processed %d messages in %.3f seconds"
" (%.3f k messages/second)") %
(count, timek["elapsed"], count / timek["elapsed"] / 1000))
print("troughput is around %.3f MB/second" %
(size / timek["elapsed"] / 1024 / 1024))
print("average message size is around %d bytes" %
(int(size / count + 0.5)))
# stop
self.logger.debug("stopping")
if self._config.get("callback", None) is not None:
self.logger.debug("stopping callback")
self._callback.stop()
if not self._config.get("lazy"):
self.logger.debug("stopping outgoing")
outgoing.stop()
self.logger.debug("stopping incoming")
incoming.stop()
self.logger.debug("incoming stopped")
timek["stop"] = time.time()
timek["elapsed"] = timek["stop"] - timek["start"]
self.logger.debug("work processed %d messages in %.3f seconds" %
(count, timek["elapsed"]))