class StreamConsumer(BaseConsumer):
"""
A consumer intended to control stream harvests using Supervisor.
When it receives a harvest start message, it starts a supervisor
process for harvesting the message.
When it receives a harvest stop message, it removes the supervisor process
for the harvest.
Logs for the supervisor processes are in /var/log/sfm.
"""
def __init__(self,
script,
working_path,
debug=False,
mq_config=None,
debug_warcprox=False,
tries=3):
BaseConsumer.__init__(self,
working_path=working_path,
mq_config=mq_config)
# Add routing keys for harvest stop messages
# The queue will be unique to this instance of StreamServer so that it
# will receive all stop requests
if mq_config:
for queue, routing_keys in list(mq_config.queues.items()):
mq_config.queues["_".join([queue, socket.gethostname()])] = [
routing_key.replace("start", "stop")
for routing_key in routing_keys
]
log.debug("Queues are now %s", mq_config.queues)
self.message = None
self.debug = debug
self.debug_warcprox = debug_warcprox
self.tries = tries
self._supervisor = HarvestSupervisor(script,
mq_config.host,
mq_config.username,
mq_config.password,
working_path,
debug=debug,
process_owner="sfm")
# Shutdown Supervisor.
def shutdown(signal_number, stack_frame):
log.debug("Shutdown triggered")
self._supervisor.pause_all()
self.should_stop = True
log.debug("Registering shutdown signal")
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
def on_message(self):
harvest_id = self.message["id"]
if self.routing_key.startswith("harvest.start."):
# Start
log.info("Starting %s", harvest_id)
log.debug("Message for %s is %s", harvest_id,
json.dumps(self.message, indent=4))
self._supervisor.start(self.message,
self.routing_key,
debug=self.debug,
debug_warcprox=self.debug_warcprox,
tries=self.tries)
else:
# Stop
log.info("Stopping %s", harvest_id)
self._supervisor.remove(harvest_id)
