def load(*argv): usage = """ Load an image on selected nodes in parallel {resa} """.format(resa=reservation_required) the_config = Config() the_imagesrepo = ImagesRepo() default_image = the_imagesrepo.default() default_timeout = the_config.value('nodes', 'load_default_timeout') default_bandwidth = the_config.value('networking', 'bandwidth') parser = ArgumentParser(usage=usage) parser.add_argument("-i", "--image", action='store', default=default_image, help="Specify image to load (default is {})" .format(default_image)) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--bandwidth", action='store', default=default_bandwidth, type=int, help="Set bandwidth in Mibps for frisbee uploading - default={}" .format(default_bandwidth)) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if not selector.how_many(): parser.print_help() return 1 nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("timeout is {}s".format(args.timeout)) logger.info("bandwidth is {} Mibps".format(args.bandwidth)) actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print("Image file {} not found - emergency exit".format(args.image)) exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)
def main(*argv): parser = ArgumentParser(usage=USAGE) parser.add_argument("-v", "--verbose", action='store_true', default=False, help="more verbose output") parser.add_argument("-n", "--dry-run", action='store_true', default=False, help="DEBUG ONLY: minimal interactions with node," " that are expected to be ON;" "won't check on and off and reset; " "won't load any image on node") parser.add_argument("-s", "--speedy", action='store_true', default=False, help="DEBUG ONLY: will only load one image") add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args, defaults_to_all=True) nightly = Nightly(selector, dry_run=args.dry_run, verbose=args.verbose, speedy=args.speedy) # turn off asyncssh info message unless verbose if not args.verbose: set_log_level(logging.ERROR) return 0 if nightly.run() else 1
def nodes(*argv): usage = """ Just display the hostnames of selected nodes """ parser = ArgumentParser(usage=usage) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) print(" ".join(selector.node_names()))
def nodes(*argv): usage = """ Just display the hostnames of selected nodes """ parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) print(" ".join(selector.node_names()))
def monitornodes(*argv): # pylint: disable=r0914 # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all selected nodes, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-c', "--cycle", default=config.value('monitor', 'cycle_nodes'), type=float, help="Delay to wait between 2 probes of each node") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) logger.info({'selected_nodes': selector}) monitornodes = MonitorNodes(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, sidecar_url=args.sidecar_url, report_wlan=args.report_wlan, verbose=args.verbose) async def async_main(): # run both the core and the log loop in parallel await asyncio.gather(monitornodes.run_forever(), display.run()) MonitorLoop("monitornodes").run(async_main(), logger) return 0
def bye(*argv): """ An alternative implementation of the previous 'all-off' utility Switch the lights off when you leave """ usage = """ Turn off whole testbed """ config = Config() default_timeout = config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify timeout for each phase") add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args, defaults_to_all=True) if selector.is_empty(): selector.use_all_scope() bus = asyncio.Queue() Action('usrpoff', selector).run(bus, args.timeout) # keep it simple for now import time time.sleep(1) Action('off', selector).run(bus, args.timeout) # even simpler import os phones_inventory = InventoryPhones() for phone in phones_inventory.all_phones(): command = (f"ssh -i {phone['gw_key']} " f"{phone['gw_user']}@{phone['gw_host']} phone-off") print(command) os.system(command)
def cmc_verb(verb, resa_policy, *argv): """ resa_policy can be either (*) 'enforce': refuse to send the message if the lease is not there (*) 'warn': issue a warning when the lease is not there (*) 'none' - or anything else really: does not check the leases """ usage = f""" Send verb '{verb}' to the CMC interface of selected nodes""" if resa_policy == 'enforce': usage += f"\n {RESERVATION_REQUIRED}" config = Config() default_timeout = config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() leases = Leases(message_bus) # pylint: disable=w0621 if resa_policy in ('warn', 'enforce'): reserved = check_reservation(leases, verbose=False) if not reserved: if resa_policy == 'enforce': return 1 selector = selected_selector(args) action = Action(verb, selector) return 0 if action.run(message_bus, args.timeout) else 1
def cmc_verb(verb, check_resa, *argv): """ check_resa can be either (*) enforce: refuse to send the message if the lease is not there (*) warn: issue a warning when the lease is not there (*) none: does not check the leases """ usage = """ Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb) if check_resa == 'enforce': usage += "\n {resa}".format(resa=reservation_required) the_config = Config() default_timeout = the_config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() if check_resa in ('warn', 'enforce'): reserved = check_reservation(verbose=False) if not reserved: if check_resa == 'enforce': return 1 verb_to_method = {'status': 'get_status', 'on': 'turn_on', 'off': 'turn_off', 'reset': 'do_reset', 'info': 'get_info', 'usrpstatus': 'get_usrpstatus', 'usrpon': 'turn_usrpon', 'usrpoff': 'turn_usrpoff', } async def get_and_show_verb(node, verb): assert verb in verb_to_method # send the 'verb' method on node method = getattr(node, verb_to_method[verb]) # bound methods must not be passed the subject ! await method() result = getattr(node, verb) result = result if result is not None else "{} N/A".format(verb) for line in result.split("\n"): if line: print("{}:{}".format(node.cmc_name, line)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] jobs = [Job(get_and_show_verb(node, verb)) for node in nodes] display = Display(nodes, message_bus) scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: if scheduler.orchestrate(timeout=args.timeout): return 0 else: print("rhubarbe-{} failed: {}".format(verb, scheduler.why())) return 1 except KeyboardInterrupt as e: print("rhubarbe-{} : keyboard interrupt - exiting".format(verb)) return 1
def monitor(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger # xxx hacky usage = """ Cyclic probe all selected nodes to report real-time status at a sidecar service over socketIO """ the_config = Config() default_cycle = the_config.value('monitor', 'cycle_status') parser = ArgumentParser(usage=usage) parser.add_argument('-c', "--cycle", default=default_cycle, type=float, help="Delay to wait between 2 probes of each node, default ={}" .format(default_cycle)) parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-H", "--sidecar-hostname", dest="sidecar_hostname", default=None) parser.add_argument("-P", "--sidecar-port", dest="sidecar_port", default=None) parser.add_argument("-d", "--debug", dest="debug", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) loop = asyncio.get_event_loop() message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) from rhubarbe.logger import logger logger.info({'selected_nodes': selector}) monitor = Monitor(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, report_wlan=args.report_wlan, sidecar_hostname=args.sidecar_hostname, sidecar_port=args.sidecar_port, debug=args.debug) # trap signals so we get a nice message in monitor.log import signal import functools def exiting(signame): logger.info("Received signal {} - exiting".format(signame)) loop.stop() exit(1) for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), functools.partial(exiting, signame)) async def run(): # run both the core and the log loop in parallel await asyncio.gather(monitor.run(), monitor.log()) await display.stop() t1 = util.self_manage(run()) t2 = util.self_manage(display.run()) tasks = asyncio.gather(t1, t2) wrapper = asyncio.gather(tasks) try: loop.run_until_complete(wrapper) return 0 except KeyboardInterrupt as e: logger.info("rhubarbe-monitor : keyboard interrupt - exiting") tasks.cancel() loop.run_forever() tasks.exception() return 1 except asyncio.TimeoutError as e: logger.info( "rhubarbe-monitor : timeout expired after {}s".format(args.timeout)) return 1 finally: loop.close()
def wait(*argv): usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ the_config = Config() default_timeout = the_config.value('nodes', 'wait_default_timeout') default_backoff = the_config.value('networking', 'ssh_backoff') parser = ArgumentParser(usage=usage) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--backoff", action='store', default=default_backoff, type=float, help="Specify backoff average between " "attempts to ssh connect, default={}" .format(default_backoff)) # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("wait: backoff is {} and global timeout is {}" .format(args.backoff, args.timeout)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] sshs = [SshProxy(node, verbose=args.verbose) for node in nodes] jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: orchestration = scheduler.orchestrate(timeout=args.timeout) if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt as e: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print("{}:ssh {}".format(ssh.node, "OK" if ssh.status else "KO"))
def wait(*argv): # pylint: disable=r0914 usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ # suppress info log messages from asyncssh asyncssh_set_log_level(logging.WARNING) config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'wait_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--backoff", action='store', default=config.value('networking', 'ssh_backoff'), type=float, help="Specify backoff average between " "attempts to ssh connect") parser.add_argument("-u", "--user", default="root", help="select other username") # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"wait: backoff is {args.backoff} " f"and global timeout is {args.timeout}") nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] sshs = [ SshProxy(node, username=args.user, verbose=args.verbose) for node in nodes ] jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True, critical=True), *jobs, timeout=args.timeout, critical=False) try: orchestration = scheduler.run() if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
def load(*argv): usage = f""" Load an image on selected nodes in parallel {RESERVATION_REQUIRED} """ config = Config() config.check_binaries() imagesrepo = ImagesRepo() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--image", action='store', default=imagesrepo.default(), help="Specify image to load") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'load_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--bandwidth", action='store', default=config.value('networking', 'bandwidth'), type=int, help="Set bandwidth in Mibps for frisbee uploading") parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if selector.is_empty(): parser.print_help() return 1 nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"timeout is {args.timeout}s") logger.info(f"bandwidth is {args.bandwidth} Mibps") actual_image = imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print(f"Image file {args.image} not found - emergency exit") exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)