def load(*argv): usage = """ Load an image on selected nodes in parallel {resa} """.format(resa=reservation_required) the_config = Config() the_imagesrepo = ImagesRepo() default_image = the_imagesrepo.default() default_timeout = the_config.value('nodes', 'load_default_timeout') default_bandwidth = the_config.value('networking', 'bandwidth') parser = ArgumentParser(usage=usage) parser.add_argument("-i", "--image", action='store', default=default_image, help="Specify image to load (default is {})" .format(default_image)) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--bandwidth", action='store', default=default_bandwidth, type=int, help="Set bandwidth in Mibps for frisbee uploading - default={}" .format(default_bandwidth)) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if not selector.how_many(): parser.print_help() return 1 nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("timeout is {}s".format(args.timeout)) logger.info("bandwidth is {} Mibps".format(args.bandwidth)) actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print("Image file {} not found - emergency exit".format(args.image)) exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)
def monitorphones(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all known phones, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--cycle", default=config.value('monitor', 'cycle_phones'), type=float, help="Delay to wait between 2 probes of each phone") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-v", "--verbose", action='store_true') args = parser.parse_args(argv) logger.info("Using all phones") monitorphones = MonitorPhones(**vars(args)) MonitorLoop("monitorphones").run(monitorphones.run_forever(), logger) return 0
def dispatch(self, message): timestamp = time.strftime("%H:%M:%S") # in case the message is sent before the event loop has started duration = (f"+{int(time.time()-self._start_time):03}s" if self._start_time is not None else 5 * '-') if isinstance(message, dict) and 'ip' in message: ipaddr = message['ip'] node = self.get_display_node(ipaddr) if node is None: logger.info( f"Unexpected message gave node=None in dispatch: {message}" ) elif 'tick' in message: self.dispatch_ip_tick_hook(ipaddr, node, message, timestamp, duration) elif 'percent' in message: # compute delta, update node.percent and self.total_percent node_previous_percent = node.percent node_current_percent = message['percent'] delta = node_current_percent - node_previous_percent node.percent = node_current_percent self.total_percent += delta logger.info( f"{node.name} percent: {node_current_percent}/100 " f"(was {node_previous_percent}), " f"total {self.total_percent}/{100*len(self.nodes)}") self.dispatch_ip_percent_hook(ipaddr, node, message, timestamp, duration) else: self.dispatch_ip_hook(ipaddr, node, message, timestamp, duration) else: self.dispatch_hook(message, timestamp, duration)
def manage_nextboot_symlink(self, action): """ Messes with the symlink in /tftpboot/pxelinux.cfg/ Depending on 'action' * 'cleanup' or 'harddrive' : clear the symlink corresponding to this CMC * 'frisbee' : define a symlink so that next boot will run the frisbee image see rhubarbe.conf for configurable options """ the_config = Config() root = the_config.value('pxelinux', 'config_dir') frisbee = the_config.value('pxelinux', 'frisbee_image') # of the form 01-00-03-1d-0e-03-53 mylink = "01-" + self.control_mac_address().replace(':', '-') source = os.path.join(root, mylink) if action in ('cleanup', 'harddrive'): if os.path.exists(source): logger.info(f"Removing {source}") os.remove(source) elif action in ('frisbee', ): if os.path.exists(source): os.remove(source) logger.info(f"Creating {source}") os.symlink(frisbee, source) else: logger.critical( f"manage_nextboot_symlink : unknown action {action}")
def dispatch(self, message): timestamp = time.strftime("%H:%M:%S") # in case the message is sent before the event loop has started duration = "+{:03}s".format(int(time.time()-self._start_time)) \ if self._start_time is not None \ else 5*'-' if isinstance(message, dict) and 'ip' in message: ip = message['ip'] node = self.get_display_node(ip) if 'tick' in message: self.dispatch_ip_tick_hook(ip, node, message, timestamp, duration) elif 'percent' in message: # compute delta, update node.percent and self.total_percent node_previous_percent = node.percent node_current_percent = message['percent'] delta = node_current_percent - node_previous_percent node.percent = node_current_percent self.total_percent += delta logger.info("{} percent: {}/100 (was {}), total {}/{}" .format(node.name, node_current_percent, node_previous_percent, self.total_percent, 100*len(self.nodes))) self.dispatch_ip_percent_hook(ip, node, message, timestamp, duration) else: self.dispatch_ip_hook(ip, node, message, timestamp, duration) else: self.dispatch_hook(message, timestamp, duration)
def manage_nextboot_symlink(self, action): """ Messes with the symlink in /tftpboot/pxelinux.cfg/ Depending on 'action' * 'cleanup' or 'harddrive' : clear the symlink corresponding to this CMC * 'frisbee' : define a symlink so that next boot will run the frisbee image see rhubarbe.conf for configurable options """ the_config = Config() root = the_config.value("pxelinux", "config_dir") frisbee = the_config.value("pxelinux", "frisbee_image") # of the form 01-00-03-1d-0e-03-53 mylink = "01-" + self.control_mac_address().replace(":", "-") source = os.path.join(root, mylink) dest = os.path.join(root, frisbee) if action in ("cleanup", "harddrive"): if os.path.exists(source): logger.info("Removing {}".format(source)) os.remove(source) elif action in ("frisbee"): if os.path.exists(source): os.remove(source) logger.info("Creating {}".format(source)) os.symlink(frisbee, source) else: logger.critical("manage_nextboot_symlink : unknown action {}".format(action))
def monitornodes(*argv): # pylint: disable=r0914 # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all selected nodes, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-c', "--cycle", default=config.value('monitor', 'cycle_nodes'), type=float, help="Delay to wait between 2 probes of each node") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) logger.info({'selected_nodes': selector}) monitornodes = MonitorNodes(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, sidecar_url=args.sidecar_url, report_wlan=args.report_wlan, verbose=args.verbose) async def async_main(): # run both the core and the log loop in parallel await asyncio.gather(monitornodes.run_forever(), display.run()) MonitorLoop("monitornodes").run(async_main(), logger) return 0
async def start(self): # pylint: disable=r0914 """ Start a collector instance; returns a port_number """ the_config = Config() netcat = the_config.value('frisbee', 'netcat') local_ip = the_config.local_control_ip() # should use default.ndz if not provided # use shell-style as we rather have bash handle the redirection # we instruct bash to exec nc; # otherwise when cleaning up we just kill the bash process # but nc is left lingering behind # WARNING: it is intended that format contains {port} # for future formatting command_format_ubuntu = ( f"exec {netcat} -d -l {local_ip} {{port}} > {self.image}" f" 2> /dev/null") command_format_fedora = ( f"exec {netcat} -l {local_ip} {{port}} > {self.image}" f" 2> /dev/null") netcat_style = the_config.value('frisbee', 'netcat_style') if netcat_style not in ('fedora', 'ubuntu'): message = f"wrong netcat_style {netcat_style}" print(message) raise Exception(message) command_format = (command_format_fedora if netcat_style == 'fedora' else command_format_ubuntu) nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts + 1): pat = str(i) port = str( eval( # pylint: disable=w0123 pat_port.replace('*', pat))) command = command_format.format(port=port) self.subprocess = await asyncio.create_subprocess_shell(command) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if is has, we try our luck on another couple (ip, port) command_line = command if self.subprocess.returncode is None: logger.info(f"collector started: {command_line}") await self.feedback('info', f"collector started on {self.image}") self.port = port return port else: logger.warning( f"failed to start collector with {command_line}") logger.critical("Could not find a free port to start collector") raise Exception("Could not start collector server")
def stop_nowait(self): # make it idempotent if self.subprocess: # when everything is running fine, nc will exit on its own try: self.subprocess.kill() except: pass self.subprocess = None logger.info("collector (on port {}) stopped".format(self.port)) self.feedback_nowait('info', "image collector server (on port {}) stopped" .format(self.port))
def __init__(self): self.parser = configparser.ConfigParser() self.files = [] # load all configurations when they exist for location, mandatory in locations: if os.path.exists(location): self.files.append(location) self.parser.read(location) logger.info("Loaded config from {}".format(location)) elif mandatory: raise ConfigException("Missing mandatory config file {}".format(location)) # self._hostname = None
def stop_nowait(self): # make it idempotent if self.subprocess: # when everything is running fine, nc will exit on its own try: self.subprocess.kill() except Exception: # pylint: disable=w0703 pass self.subprocess = None logger.info(f"collector (on port {self.port}) stopped") self.feedback_nowait( 'info', f"image collector server (on port {self.port}) stopped")
def __init__(self): self.parser = configparser.ConfigParser() self.files = [] # load all configurations when they exist for location, exists, mandatory in LOCATIONS: if exists is None: exists = Path(location).exists() if exists: self.files.append(location) self.parser.read(location) logger.info(f"Loaded config from {location}") elif mandatory: raise ConfigException( f"Missing mandatory config file {location}") # self._hostname = None
async def run(self, multicast_ip, port): control_ip = self.control_ip the_config = Config() client = the_config.value('frisbee', 'client') hdd = the_config.value('frisbee', 'hard_drive') self.command = ( f"{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}") logger.info(f"on {self.control_ip} : running command {self.command}") await self.feedback('frisbee_status', "starting frisbee client") retcod = await self.session([self.command]) logger.info(f"frisbee on {self.control_ip} returned {retcod}") return retcod
def parse_line(self, line): # match = self.matcher_new_style_progress.match(line) if match: if self.total_chunks == 0: logger.error( "ip={self.ip()}: new frisbee: cannot report progress, " "missing total chunks") return percent = int(100 * (1 - int(match.group('remaining_chunks')) / self.total_chunks)) self.send_percent(percent) # match = self.matcher_total_chunks.match(line) if match: self.total_chunks = int(match.group('total_chunks')) self.send_percent(0) return # match = self.matcher_old_style_progress.match(line) if match: self.send_percent(match.group('percent')) return # match = self.matcher_final_report.match(line) if match: logger.info(f"ip={self.ip()} FRISBEE END: " f"total = {match.group('total')} bytes, " f"actual = {match.group('actual')} bytes") self.send_percent(100) return # match = self.matcher_short_write.match(line) if match: self.feedback('frisbee_error', "Something went wrong with frisbee (short write...)") return # match = self.matcher_status.match(line) if match: status = int(match.group('status')) self.feedback('frisbee_retcod', status)
def parse_line(self): line = self.bytes_line.decode().strip() logger.debug("line from frisbee:" + line) # m = self.matcher_new_style_progress.match(line) if m: if self.total_chunks == 0: logger.error("ip={}: new frisbee: cannot report progress, missing total chunks" .format(self.ip())) return percent = int ( 100 * (1 - int(m.group('remaining_chunks'))/self.total_chunks)) self.send_percent(percent) # m = self.matcher_total_chunks.match(line) if m: self.total_chunks = int(m.group('total_chunks')) self.send_percent(0) return # m = self.matcher_old_style_progress.match(line) if m: self.send_percent(m.group('percent')) return # m = self.matcher_final_report.match(line) if m: logger.info("ip={ip} FRISBEE END: total = {total} bytes, actual = {actual} bytes" .format(ip=self.ip(), total=m.group('total'), actual=m.group('actual'))) self.send_percent(100) return # m = self.matcher_short_write.match(line) if m: self.feedback('frisbee_error', "Something went wrong with frisbee (short write...)...") return # m = self.matcher_status.match(line) if m: status = int(m.group('status')) self.feedback('frisbee_retcod', status)
async def _get_cmc_verb(self, verb, strip_result=True): """ verb typically is 'status', 'on', 'off' or 'info' """ url = f"http://{self.cmc_name}/{verb}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as response: text = await response.text(encoding='utf-8') if strip_result: text = text.strip() setattr(self, verb, text) except aiohttp.client_exceptions.ClientConnectorError: logger.info(f"cannot connect to {url}") setattr(self, verb, None) return None except Exception: import traceback traceback.print_exc() setattr(self, verb, None) return None return getattr(self, verb)
async def try_to_connect(self): # a little closure to capture our ip and expose it to the parser def client_factory(): return TelnetClient(proxy=self, encoding='utf-8') await self.feedback('frisbee_status', "trying to telnet..") logger.info(f"Trying to telnet on {self.control_ip}") try: self._reader, self._writer = await asyncio.wait_for( telnetlib3.open_connection( self.control_ip, 23, shell=None, log=logger, connect_minwait=self.connect_minwait, connect_maxwait=self.connect_maxwait), timeout=self.connect_timeout) except (asyncio.TimeoutError, OSError) as exc: self._reader, self._writer = None, None except Exception as exc: logger.exception(f"telnet connect: unexpected exception {exc}")
async def run(self, multicast_ip, port): control_ip = self.control_ip the_config = Config() client = the_config.value('frisbee', 'client') hdd = the_config.value('frisbee', 'hard_drive') self.command = \ "{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}".format(**locals()) logger.info("on {} : running command {}".format(self.control_ip, self.command)) await self.feedback('frisbee_status', "starting frisbee client") EOF = chr(4) EOL = '\n' # print out exit status so the parser can catch it and expose it command = self.command command = command + "; echo FRISBEE-STATUS=$?" # make sure the command is sent (EOL) and that the session terminates afterwards (exit + EOF) command = command + "; exit" + EOL + EOF self._protocol.stream.write(self._protocol.shell.encode(command)) # wait for telnet to terminate await self._protocol.waiter_closed return True
async def start(self): """ Start a collector instance; returns a port_number """ the_config = Config() netcat = the_config.value('frisbee', 'netcat') local_ip = the_config.local_control_ip() # should use default.ndz if not provided # use shell-style as we rather have bash handle the redirection # we instruct bash to exec nc; otherwise when cleaning up we just kill the bash process # but nc is left lingering behind # WARNING: it is intended that format contains {port} for future formatting command_format = "exec {} -d -l {} {{port}} > {} 2> /dev/null"\ .format(netcat, local_ip, self.image) nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts+1): pat = str(i) port = str(eval(pat_port.replace('*', pat))) command = command_format.format(port=port) self.subprocess = \ await asyncio.create_subprocess_shell(command) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if is has, we try our luck on another couple (ip, port) command_line = command if self.subprocess.returncode is None: logger.info("collector started: {}".format(command_line)) await self.feedback('info', "collector started on {}".format(self.image)) self.port = port return port else: logger.warning("failed to start collector with {}".format(command_line)) logger.critical("Could not find a free port to start collector") raise Exception("Could not start collector server")
async def _try_to_connect(self, shell=telnetlib3.TerminalShell): # a little closure to capture our ip and expose it to the parser def client_factory(): return TelnetClient(proxy = self, encoding='utf-8', shell=shell) await self.feedback('frisbee_status', "trying to telnet..") logger.info("Trying to telnet to {}".format(self.control_ip)) loop = asyncio.get_event_loop() try: self._transport, self._protocol = \ await asyncio.wait_for( loop.create_connection(client_factory, self.control_ip, self.port), self.timeout) logger.info("{}: telnet connected".format(self.control_ip)) return True except asyncio.TimeoutError as e: await self.feedback('frisbee_status', "timed out..") self._transport, self._protocol = None, None except Exception as e: # import traceback # traceback.print_exc() logger.info("telnet connect: unexpected exception {}".format(e)) self._transport, self._protocol = None, None
def exiting(signame): logger.info("Received signal {} - exiting".format(signame)) loop.stop() exit(1)
def monitor(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger # xxx hacky usage = """ Cyclic probe all selected nodes to report real-time status at a sidecar service over socketIO """ the_config = Config() default_cycle = the_config.value('monitor', 'cycle_status') parser = ArgumentParser(usage=usage) parser.add_argument('-c', "--cycle", default=default_cycle, type=float, help="Delay to wait between 2 probes of each node, default ={}" .format(default_cycle)) parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-H", "--sidecar-hostname", dest="sidecar_hostname", default=None) parser.add_argument("-P", "--sidecar-port", dest="sidecar_port", default=None) parser.add_argument("-d", "--debug", dest="debug", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) loop = asyncio.get_event_loop() message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) from rhubarbe.logger import logger logger.info({'selected_nodes': selector}) monitor = Monitor(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, report_wlan=args.report_wlan, sidecar_hostname=args.sidecar_hostname, sidecar_port=args.sidecar_port, debug=args.debug) # trap signals so we get a nice message in monitor.log import signal import functools def exiting(signame): logger.info("Received signal {} - exiting".format(signame)) loop.stop() exit(1) for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), functools.partial(exiting, signame)) async def run(): # run both the core and the log loop in parallel await asyncio.gather(monitor.run(), monitor.log()) await display.stop() t1 = util.self_manage(run()) t2 = util.self_manage(display.run()) tasks = asyncio.gather(t1, t2) wrapper = asyncio.gather(tasks) try: loop.run_until_complete(wrapper) return 0 except KeyboardInterrupt as e: logger.info("rhubarbe-monitor : keyboard interrupt - exiting") tasks.cancel() loop.run_forever() tasks.exception() return 1 except asyncio.TimeoutError as e: logger.info( "rhubarbe-monitor : timeout expired after {}s".format(args.timeout)) return 1 finally: loop.close()
def load(*argv): usage = f""" Load an image on selected nodes in parallel {RESERVATION_REQUIRED} """ config = Config() config.check_binaries() imagesrepo = ImagesRepo() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--image", action='store', default=imagesrepo.default(), help="Specify image to load") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'load_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--bandwidth", action='store', default=config.value('networking', 'bandwidth'), type=int, help="Set bandwidth in Mibps for frisbee uploading") parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if selector.is_empty(): parser.print_help() return 1 nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"timeout is {args.timeout}s") logger.info(f"bandwidth is {args.bandwidth} Mibps") actual_image = imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print(f"Image file {args.image} not found - emergency exit") exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)
def wait(*argv): # pylint: disable=r0914 usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ # suppress info log messages from asyncssh asyncssh_set_log_level(logging.WARNING) config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'wait_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--backoff", action='store', default=config.value('networking', 'ssh_backoff'), type=float, help="Specify backoff average between " "attempts to ssh connect") parser.add_argument("-u", "--user", default="root", help="select other username") # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"wait: backoff is {args.backoff} " f"and global timeout is {args.timeout}") nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] sshs = [ SshProxy(node, username=args.user, verbose=args.verbose) for node in nodes ] jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True, critical=True), *jobs, timeout=args.timeout, critical=False) try: orchestration = scheduler.run() if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
def wait(*argv): usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ the_config = Config() default_timeout = the_config.value('nodes', 'wait_default_timeout') default_backoff = the_config.value('networking', 'ssh_backoff') parser = ArgumentParser(usage=usage) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--backoff", action='store', default=default_backoff, type=float, help="Specify backoff average between " "attempts to ssh connect, default={}" .format(default_backoff)) # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("wait: backoff is {} and global timeout is {}" .format(args.backoff, args.timeout)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] sshs = [SshProxy(node, verbose=args.verbose) for node in nodes] jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: orchestration = scheduler.orchestrate(timeout=args.timeout) if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt as e: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print("{}:ssh {}".format(ssh.node, "OK" if ssh.status else "KO"))