def get_mon_data(self):
"""
Returns monitoring data
"""
r = {
"status": self.get_mon_status(),
"service": self.name,
"instance": str(self.service_id),
"node": config.node,
"pid": self.pid,
# Current process uptime
"uptime": perf_counter() - self.start_time
}
if self.pooled:
r["pool"] = config.pool
if self.executors:
for x in self.executors:
self.executors[x].apply_metrics(r)
apply_metrics(r)
return r
async def deactivate(self):
if not self.is_active:
return
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
await self.shutdown_api()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
await self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
await self.scheduler.shutdown()
except asyncio.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown subscriptions
await self.shutdown_subscriptions()
# Shutdown executors
await self.shutdown_executors()
# Custom deactivation
await self.on_deactivate()
# Shutdown NSQ topics
await self.shutdown_topic_queues()
# Shutdown Liftbridge publisher
await self.shutdown_publisher()
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping EventLoop")
self.loop.stop()
m = {}
apply_metrics(m)
apply_hists(m)
apply_quantiles(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")
def deactivate(self):
if not self.is_active:
raise tornado.gen.Return()
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
self.logger.info("Stopping API")
self.server.stop()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
yield self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
yield self.scheduler.shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown executors
yield self.shutdown_executors()
# Custom deactivation
yield self.on_deactivate()
# Shutdown NSQ topics
yield self.shutdown_topic_queues()
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping IOLoop")
self.ioloop.stop()
m = {}
apply_metrics(m)
apply_hists(m)
apply_quantiles(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")
def deactivate(self):
if not self.is_active:
raise tornado.gen.Return()
self.is_active = False
self.logger.info("Deactivating")
# Shutdown API
self.logger.info("Stopping API")
self.server.stop()
# Release registration
if self.dcs:
self.logger.info("Deregistration")
yield self.dcs.deregister()
# Shutdown schedulers
if self.scheduler:
try:
self.logger.info("Shutting down scheduler")
yield self.scheduler.shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down scheduler")
# Shutdown executors
if self.executors:
self.logger.info("Shutting down executors")
for x in self.executors:
try:
self.logger.info("Shutting down %s", x)
yield self.executors[x].shutdown()
except tornado.gen.TimeoutError:
self.logger.info("Timed out when shutting down %s", x)
# Custom deactivation
yield self.on_deactivate()
# Flush pending NSQ messages
if self.nsq_writer:
conns = list(self.nsq_writer.conns)
n = self.NSQ_WRITER_CLOSE_RETRIES
while conns:
self.logger.info("Waiting for %d NSQ connections to finish",
len(conns))
waiting = []
for conn_id in conns:
connect = self.nsq_writer.conns.get(conn_id)
if not connect:
continue
if connect.callback_queue:
# Pending operations
waiting += [conn_id]
elif not connect.closed():
# Unclosed connection
connect.close()
waiting += [conn_id]
conns = waiting
if conns:
n -= 1
if n <= 0:
self.logger.info(
"Failed to close NSQ writer properly. Giving up. Pending messages may be lost"
)
break
# Wait
yield tornado.gen.sleep(self.NSQ_WRITER_CLOSE_TRY_TIMEOUT)
else:
self.logger.info("NSQ writer is shut down clearly")
# Continue deactivation
# Finally stop ioloop
self.dcs = None
self.logger.info("Stopping IOLoop")
self.ioloop.stop()
m = {}
apply_metrics(m)
self.logger.info("Post-mortem metrics: %s", m)
self.die("")
def run_from_argv(self, argv):
"""
Execute command. Usually from script
if __name__ == "__main__":
import sys
sys.exit(Command.run_from_argv())
"""
parser = self.create_parser()
self.add_default_arguments(parser)
self.add_arguments(parser)
options = parser.parse_args(argv)
cmd_options = vars(options)
args = cmd_options.pop("args", ())
loglevel = cmd_options.pop("loglevel")
if loglevel:
self.setup_logging(loglevel)
enable_profiling = cmd_options.pop("enable_profiling", False)
show_metrics = cmd_options.pop("show_metrics", False)
self.no_progressbar = cmd_options.pop("no_progressbar", False)
if enable_profiling:
# Start profiler
import yappi
yappi.start()
try:
return self.handle(*args, **cmd_options) or 0
except CommandError as e:
self.print(str(e))
return 1
except KeyboardInterrupt:
self.print("Ctrl+C")
return 3
except AssertionError as e:
if e.args and e.args[0]:
self.print("ERROR: %s" % e.args[0])
else:
self.print("Assertion error: %s" % e)
return 4
except Exception:
from noc.core.debug import error_report
error_report()
return 2
finally:
if enable_profiling:
i = yappi.get_func_stats()
i.print_all(
out=self.stdout,
columns={
0: ("name", 80),
1: ("ncall", 10),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8),
},
)
if show_metrics:
from noc.core.perf import apply_metrics
d = apply_metrics({})
self.print("Internal metrics:")
for k in d:
self.print("%40s : %s" % (k, d[k]))
