def run(self, message_bus, timeout): """ send verb to all nodes, waits for max timeout returns True if all nodes behaved as expected and False otherwise - including in case of KeyboardInterrupt """ nodes = [ Node(cmc_name, message_bus) for cmc_name in self.selector.cmc_names() ] jobs = [ Job(self.get_and_show_verb(node, self.verb), critical=True) for node in nodes ] display = Display(nodes, message_bus) scheduler = Scheduler(Job(display.run(), forever=True, critical=True), *jobs, timeout=timeout, critical=False) try: if scheduler.run(): return True else: scheduler.debrief() print(f"rhubarbe-{self.verb} failed: {scheduler.why()}") return False except KeyboardInterrupt: print(f"rhubarbe-{self.verb} : keyboard interrupt - exiting") return False
def monitornodes(*argv): # pylint: disable=r0914 # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all selected nodes, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-c', "--cycle", default=config.value('monitor', 'cycle_nodes'), type=float, help="Delay to wait between 2 probes of each node") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) logger.info({'selected_nodes': selector}) monitornodes = MonitorNodes(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, sidecar_url=args.sidecar_url, report_wlan=args.report_wlan, verbose=args.verbose) async def async_main(): # run both the core and the log loop in parallel await asyncio.gather(monitornodes.run_forever(), display.run()) MonitorLoop("monitornodes").run(async_main(), logger) return 0
def cmc_verb(verb, check_resa, *argv): """ check_resa can be either (*) enforce: refuse to send the message if the lease is not there (*) warn: issue a warning when the lease is not there (*) none: does not check the leases """ usage = """ Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb) if check_resa == 'enforce': usage += "\n {resa}".format(resa=reservation_required) the_config = Config() default_timeout = the_config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() if check_resa in ('warn', 'enforce'): reserved = check_reservation(verbose=False) if not reserved: if check_resa == 'enforce': return 1 verb_to_method = {'status': 'get_status', 'on': 'turn_on', 'off': 'turn_off', 'reset': 'do_reset', 'info': 'get_info', 'usrpstatus': 'get_usrpstatus', 'usrpon': 'turn_usrpon', 'usrpoff': 'turn_usrpoff', } async def get_and_show_verb(node, verb): assert verb in verb_to_method # send the 'verb' method on node method = getattr(node, verb_to_method[verb]) # bound methods must not be passed the subject ! await method() result = getattr(node, verb) result = result if result is not None else "{} N/A".format(verb) for line in result.split("\n"): if line: print("{}:{}".format(node.cmc_name, line)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] jobs = [Job(get_and_show_verb(node, verb)) for node in nodes] display = Display(nodes, message_bus) scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: if scheduler.orchestrate(timeout=args.timeout): return 0 else: print("rhubarbe-{} failed: {}".format(verb, scheduler.why())) return 1 except KeyboardInterrupt as e: print("rhubarbe-{} : keyboard interrupt - exiting".format(verb)) return 1
def monitor(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger # xxx hacky usage = """ Cyclic probe all selected nodes to report real-time status at a sidecar service over socketIO """ the_config = Config() default_cycle = the_config.value('monitor', 'cycle_status') parser = ArgumentParser(usage=usage) parser.add_argument('-c', "--cycle", default=default_cycle, type=float, help="Delay to wait between 2 probes of each node, default ={}" .format(default_cycle)) parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-H", "--sidecar-hostname", dest="sidecar_hostname", default=None) parser.add_argument("-P", "--sidecar-port", dest="sidecar_port", default=None) parser.add_argument("-d", "--debug", dest="debug", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) loop = asyncio.get_event_loop() message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) from rhubarbe.logger import logger logger.info({'selected_nodes': selector}) monitor = Monitor(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, report_wlan=args.report_wlan, sidecar_hostname=args.sidecar_hostname, sidecar_port=args.sidecar_port, debug=args.debug) # trap signals so we get a nice message in monitor.log import signal import functools def exiting(signame): logger.info("Received signal {} - exiting".format(signame)) loop.stop() exit(1) for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), functools.partial(exiting, signame)) async def run(): # run both the core and the log loop in parallel await asyncio.gather(monitor.run(), monitor.log()) await display.stop() t1 = util.self_manage(run()) t2 = util.self_manage(display.run()) tasks = asyncio.gather(t1, t2) wrapper = asyncio.gather(tasks) try: loop.run_until_complete(wrapper) return 0 except KeyboardInterrupt as e: logger.info("rhubarbe-monitor : keyboard interrupt - exiting") tasks.cancel() loop.run_forever() tasks.exception() return 1 except asyncio.TimeoutError as e: logger.info( "rhubarbe-monitor : timeout expired after {}s".format(args.timeout)) return 1 finally: loop.close()