def get_mon_data(self):
"""
Returns monitoring data
"""
r = {
"status": self.get_mon_status(),
"service": self.name,
"instance": str(self.service_id),
"node": config.node,
"pid": self.pid,
# Current process uptime
"uptime": perf_counter() - self.start_time,
}
if self.pooled:
r["pool"] = config.pool
if self.executors:
for x in self.executors:
self.executors[x].apply_metrics(r)
apply_metrics(r)
for topic in self.topic_queues:
self.topic_queues[topic].apply_metrics(r)
if self.publish_queue:
self.publish_queue.apply_metrics(r)
apply_hists(r)
apply_quantiles(r)
return r
def deactivate(self):
if not self.is_active:
raise tornado.gen.Return()
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
self.logger.info("Stopping API")
self.server.stop()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
yield self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
yield self.scheduler.shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown executors
if self.executors:
self.logger.info("Shutting down executors")
for x in self.executors:
try:
self.logger.info("Shutting down %s", x)
yield self.executors[x].shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down %s", x)
# Custom deactivation
yield self.on_deactivate()
# Shutdown NSQ topics
yield self.shutdown_topic_queues()
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping IOLoop")
self.ioloop.stop()
m = {}
apply_metrics(m)
apply_hists(m)
apply_quantiles(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")
async def deactivate(self):
if not self.is_active:
return
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
await self.shutdown_api()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
await self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
await self.scheduler.shutdown()
except asyncio.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown subscriptions
await self.shutdown_subscriptions()
# Shutdown executors
await self.shutdown_executors()
# Custom deactivation
await self.on_deactivate()
# Shutdown NSQ topics
await self.shutdown_topic_queues()
# Shutdown Liftbridge publisher
await self.shutdown_publisher()
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping EventLoop")
self.loop.stop()
m = {}
apply_metrics(m)
apply_hists(m)
apply_quantiles(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")
def deactivate(self):
if not self.is_active:
raise tornado.gen.Return()
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
self.logger.info("Stopping API")
self.server.stop()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
yield self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
yield self.scheduler.shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown executors
if self.executors:
self.logger.info("Shutting down executors")
for x in self.executors:
try:
self.logger.info("Shutting down %s", x)
yield self.executors[x].shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down %s", x)
# Custom deactivation
yield self.on_deactivate()
# Flush pending NSQ messages
if self.nsq_writer:
conns = list(self.nsq_writer.conns)
n = self.NSQ_WRITER_CLOSE_RETRIES
while conns:
self.logger.info("Waiting for %d NSQ connections to finish",
len(conns))
waiting = []
for conn_id in conns:
connect = self.nsq_writer.conns.get(conn_id)
if not connect:
continue
if connect.callback_queue:
# Pending operations
waiting += [conn_id]
elif not connect.closed():
# Unclosed connection
connect.close()
waiting += [conn_id]
conns = waiting
if conns:
n -= 1
if n <= 0:
self.logger.info(
"Failed to close NSQ writer properly. Giving up. Pending messages may be lost"
)
break
# Wait
yield tornado.gen.sleep(self.NSQ_WRITER_CLOSE_TRY_TIMEOUT)
else:
self.logger.info("NSQ writer is shut down clearly")
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping IOLoop")
self.ioloop.stop()
m = {}
apply_metrics(m)
apply_hists(m)
apply_quantiles(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")