def monitorleases(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic check of leases; also reacts to 'request' messages on the sidecar channel, which triggers leases acquisition right away. See config for defaults. """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-v", "--verbose", default=False, action='store_true') args = parser.parse_args(argv) message_bus = asyncio.Queue() monitorleases = MonitorLeases(message_bus, args.sidecar_url, args.verbose) MonitorLoop("monitorleases").run(monitorleases.run_forever(), logger) return 0
def __init__( self, cmc_names, message_bus, # pylint: disable=r0913 sidecar_url, cycle, verbose=False): self.cycle = cycle self.verbose = verbose # get miscell config self.ping_timeout = float(Config().value('networking', 'ping_timeout')) self.ssh_timeout = float(Config().value('networking', 'ssh_timeout')) self.log_period = float(Config().value('monitor', 'log_period')) # websockets self.reconnectable = \ ReconnectableSidecar(sidecar_url, 'nodes') # the nodes part nodes = [Node(cmc_name, message_bus) for cmc_name in cmc_names] self.monitor_nodes = [ MonitorNode(node=node, reconnectable=self.reconnectable, verbose=verbose) for node in nodes ]
def __init__(self): conf = Config() try: with open(conf.value('testbed', 'inventory_phones_path')) as feed: self._phones = json.load(feed) except FileNotFoundError: self._phones = []
def main(self, cycle): """ cycle is the duration in seconds of one cycle Corner cases: * cycle = None : fetch value from config_bases * cycle = 0 : run just once (for debug mostly) """ if cycle is None: cycle = Config().value('accounts', 'cycle') cycle = int(cycle) policy = Config().value('accounts', 'access_policy') if policy not in ('open', 'leased', 'closed'): logger.error("Unknown policy {} - using 'closed'" .format(policy)) policy = 'closed' # trick is if cycle != 0: self.run_forever(cycle, policy) else: logger.info("---------- rhubarbe accounts manager oneshot " "policy = {}" .format(policy)) self.manage_accounts(policy)
def __init__(self): the_config = Config() self.plcapiurl = the_config.value('plcapi', 'url') self.email = the_config.value('plcapi', 'admin_email') self.password = the_config.value('plcapi', 'admin_password') self._proxy = None
def manage_nextboot_symlink(self, action): """ Messes with the symlink in /tftpboot/pxelinux.cfg/ Depending on 'action' * 'cleanup' or 'harddrive' : clear the symlink corresponding to this CMC * 'frisbee' : define a symlink so that next boot will run the frisbee image see rhubarbe.conf for configurable options """ the_config = Config() root = the_config.value('pxelinux', 'config_dir') frisbee = the_config.value('pxelinux', 'frisbee_image') # of the form 01-00-03-1d-0e-03-53 mylink = "01-" + self.control_mac_address().replace(':', '-') source = os.path.join(root, mylink) if action in ('cleanup', 'harddrive'): if os.path.exists(source): logger.info(f"Removing {source}") os.remove(source) elif action in ('frisbee', ): if os.path.exists(source): os.remove(source) logger.info(f"Creating {source}") os.symlink(frisbee, source) else: logger.critical( f"manage_nextboot_symlink : unknown action {action}")
def load(*argv): usage = """ Load an image on selected nodes in parallel {resa} """.format(resa=reservation_required) the_config = Config() the_imagesrepo = ImagesRepo() default_image = the_imagesrepo.default() default_timeout = the_config.value('nodes', 'load_default_timeout') default_bandwidth = the_config.value('networking', 'bandwidth') parser = ArgumentParser(usage=usage) parser.add_argument("-i", "--image", action='store', default=default_image, help="Specify image to load (default is {})" .format(default_image)) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--bandwidth", action='store', default=default_bandwidth, type=int, help="Set bandwidth in Mibps for frisbee uploading - default={}" .format(default_bandwidth)) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if not selector.how_many(): parser.print_help() return 1 nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("timeout is {}s".format(args.timeout)) logger.info("bandwidth is {} Mibps".format(args.bandwidth)) actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print("Image file {} not found - emergency exit".format(args.image)) exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)
def manage_nextboot_symlink(self, action): """ Messes with the symlink in /tftpboot/pxelinux.cfg/ Depending on 'action' * 'cleanup' or 'harddrive' : clear the symlink corresponding to this CMC * 'frisbee' : define a symlink so that next boot will run the frisbee image see rhubarbe.conf for configurable options """ the_config = Config() root = the_config.value("pxelinux", "config_dir") frisbee = the_config.value("pxelinux", "frisbee_image") # of the form 01-00-03-1d-0e-03-53 mylink = "01-" + self.control_mac_address().replace(":", "-") source = os.path.join(root, mylink) dest = os.path.join(root, frisbee) if action in ("cleanup", "harddrive"): if os.path.exists(source): logger.info("Removing {}".format(source)) os.remove(source) elif action in ("frisbee"): if os.path.exists(source): os.remove(source) logger.info("Creating {}".format(source)) os.symlink(frisbee, source) else: logger.critical("manage_nextboot_symlink : unknown action {}".format(action))
def monitorphones(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all known phones, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--cycle", default=config.value('monitor', 'cycle_phones'), type=float, help="Delay to wait between 2 probes of each phone") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-v", "--verbose", action='store_true') args = parser.parse_args(argv) logger.info("Using all phones") monitorphones = MonitorPhones(**vars(args)) MonitorLoop("monitorphones").run(monitorphones.run_forever(), logger) return 0
def __init__(self, control_ip, message_bus): self.control_ip = control_ip self.message_bus = message_bus the_config = Config() self.port = int(the_config.value('networking', 'telnet_port')) self.backoff = float(the_config.value('networking', 'telnet_backoff')) self.timeout = float(the_config.value('networking', 'telnet_timeout')) # internals self._transport = None self._protocol = None
def monitornodes(*argv): # pylint: disable=r0914 # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger from rhubarbe.logger import logger usage = """ Cyclic probe all selected nodes, and reports real-time status at a sidecar service over websockets """ config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-c', "--cycle", default=config.value('monitor', 'cycle_nodes'), type=float, help="Delay to wait between 2 probes of each node") parser.add_argument("-u", "--sidecar-url", dest="sidecar_url", default=Config().value('sidecar', 'url'), help="url for the sidecar server") parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) logger.info({'selected_nodes': selector}) monitornodes = MonitorNodes(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, sidecar_url=args.sidecar_url, report_wlan=args.report_wlan, verbose=args.verbose) async def async_main(): # run both the core and the log loop in parallel await asyncio.gather(monitornodes.run_forever(), display.run()) MonitorLoop("monitornodes").run(async_main(), logger) return 0
def config(*argv): usage = """ Display global configuration """ parser = ArgumentParser(usage=usage) parser.add_argument("sections", nargs='*', type=str, help="config section(s) to display") args = parser.parse_args(argv) the_config = Config() the_config.display(args.sections) return 0
def save(*argv): usage = """ Save an image from a node Mandatory radical needs to be provided with --output This info, together with nodename and date, is stored on resulting image in /etc/rhubarbe-image {resa} """.format(resa=reservation_required) the_config = Config() default_timeout = the_config.value('nodes', 'save_default_timeout') parser = ArgumentParser(usage=usage) parser.add_argument("-o", "--output", action='store', dest='radical', default=None, required=True, help="Mandatory radical to name resulting image") parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-c", "--comment", dest='comment', default=None, help="one-liner comment to insert in " "/etc/rhubarbe-image together") parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with a node that is already running a frisbee image. It won't get reset, neither before or after the frisbee session""") parser.add_argument("node") args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = Selector() selector.add_range(args.node) # in case there was one argument but it was not found in inventory if selector.how_many() != 1: parser.print_help() cmc_name = next(selector.cmc_names()) node = Node(cmc_name, message_bus) nodename = node.control_hostname() the_imagesrepo = ImagesRepo() actual_image = the_imagesrepo.where_to_save(nodename, args.radical) message_bus.put_nowait({'info': "Saving image {}".format(actual_image)}) # curses has no interest here since we focus on one node display_class = Display display = display_class([node], message_bus) saver = ImageSaver(node, image=actual_image, radical=args.radical, message_bus=message_bus, display=display, comment=args.comment) return saver.main(reset=args.reset, timeout=args.timeout)
def __init__(self): the_config = Config() self.public = Path(the_config.value('frisbee', 'images_dir')) self.default_name = the_config.value('frisbee', 'default_image') # for building hostname patterns regularname = the_config.value('testbed', 'regularname') self._radical_re_matcher = re.compile( SEP.join([ f"{SAVING}", f"{regularname}[0-9][0-9]", f"{DATE_RE_PATTERN}", f"(?P<radical>.+)", ]))
def config(*argv): usage = """ Display global configuration """ parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("sections", nargs='*', type=str, help="config section(s) to display") args = parser.parse_args(argv) config = Config() config.display(args.sections) return 0
def __init__( self, message_bus, sidecar_url, # pylint: disable=r0913 verbose=False): self.message_bus = message_bus self.sidecar_url = sidecar_url self.verbose = verbose # websockets self.reconnectable = \ ReconnectableSidecar(sidecar_url, 'leases') self.cycle = float(Config().value('monitor', 'cycle_leases')) self.step = float(Config().value('monitor', 'step_leases'))
async def run(self, multicast_ip, port): control_ip = self.control_ip the_config = Config() client = the_config.value('frisbee', 'client') hdd = the_config.value('frisbee', 'hard_drive') self.command = ( f"{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}") logger.info(f"on {self.control_ip} : running command {self.command}") await self.feedback('frisbee_status', "starting frisbee client") retcod = await self.session([self.command]) logger.info(f"frisbee on {self.control_ip} returned {retcod}") return retcod
async def start(self): # pylint: disable=r0914 """ Start a frisbeed instance returns a tuple multicast_group, port_number """ the_config = Config() server = the_config.value('frisbee', 'server') server_options = the_config.value('frisbee', 'server_options') local_ip = the_config.local_control_ip() # in Mibps bandwidth = self.bandwidth * 2**20 # should use default.ndz if not provided command_common = [ server, "-i", local_ip, "-W", str(bandwidth), self.image ] # add configured extra options command_common += server_options.split() nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_ip = the_config.value('networking', 'pattern_multicast') pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts + 1): pat = str(i) multicast_group = pat_ip.replace('*', pat) multicast_port = str( eval( # pylint: disable=w0123 pat_port.replace('*', pat))) command = command_common + [ "-m", multicast_group, "-p", multicast_port, ] self.subprocess = await asyncio.create_subprocess_exec( *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if it has, we try our luck on another couple (ip, port) command_line = " ".join(command) if self.subprocess.returncode is None: self.multicast_group = multicast_group self.multicast_port = multicast_port await self.feedback('info', f"started {self}") return multicast_group, multicast_port else: logger.warning( f"failed to start frisbeed with `{command_line}`" f" -> {self.subprocess.returncode}") logger.critical(f"could not start frisbee server !!! on {self.image}") raise Exception(f"could not start frisbee server !!! on {self.image}")
def bye(*argv): """ An alternative implementation of the previous 'all-off' utility Switch the lights off when you leave """ usage = """ Turn off whole testbed """ config = Config() default_timeout = config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify timeout for each phase") add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args, defaults_to_all=True) if selector.is_empty(): selector.use_all_scope() bus = asyncio.Queue() Action('usrpoff', selector).run(bus, args.timeout) # keep it simple for now import time time.sleep(1) Action('off', selector).run(bus, args.timeout) # even simpler import os phones_inventory = InventoryPhones() for phone in phones_inventory.all_phones(): command = (f"ssh -i {phone['gw_key']} " f"{phone['gw_user']}@{phone['gw_host']} phone-off") print(command) os.system(command)
def __init__(self, selector, *, verbose, dry_run, speedy): # work selector; will remove nodes as they fail self.selector = selector self.verbose = verbose self.dry_run = dry_run self.speedy = speedy # # keep a backup of initial scope for proper cleanup self.all_names = list(selector.node_names()) # the list of failed nodes together with the reason why self.failures = {} # # from rhubarbe config, retrieve bandwidth and other details config = Config() self.bandwidth = int(config.value('networking', 'bandwidth')) self.backoff = int(config.value('networking', 'ssh_backoff')) self.load_timeout = float(config.value('nodes', 'load_nightly_timeout')) self.wait_timeout = float(config.value('nodes', 'wait_nightly_timeout')) # # accessories self.bus = asyncio.Queue() self.nodes = {Node(x, self.bus) for x in self.selector.cmc_names()} self.display = NoProgressBarDisplay(self.nodes, self.bus)
async def start(self): # pylint: disable=r0914 """ Start a collector instance; returns a port_number """ the_config = Config() netcat = the_config.value('frisbee', 'netcat') local_ip = the_config.local_control_ip() # should use default.ndz if not provided # use shell-style as we rather have bash handle the redirection # we instruct bash to exec nc; # otherwise when cleaning up we just kill the bash process # but nc is left lingering behind # WARNING: it is intended that format contains {port} # for future formatting command_format_ubuntu = ( f"exec {netcat} -d -l {local_ip} {{port}} > {self.image}" f" 2> /dev/null") command_format_fedora = ( f"exec {netcat} -l {local_ip} {{port}} > {self.image}" f" 2> /dev/null") netcat_style = the_config.value('frisbee', 'netcat_style') if netcat_style not in ('fedora', 'ubuntu'): message = f"wrong netcat_style {netcat_style}" print(message) raise Exception(message) command_format = (command_format_fedora if netcat_style == 'fedora' else command_format_ubuntu) nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts + 1): pat = str(i) port = str( eval( # pylint: disable=w0123 pat_port.replace('*', pat))) command = command_format.format(port=port) self.subprocess = await asyncio.create_subprocess_shell(command) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if is has, we try our luck on another couple (ip, port) command_line = command if self.subprocess.returncode is None: logger.info(f"collector started: {command_line}") await self.feedback('info', f"collector started on {self.image}") self.port = port return port else: logger.warning( f"failed to start collector with {command_line}") logger.critical("Could not find a free port to start collector") raise Exception("Could not start collector server")
def cmc_verb(verb, resa_policy, *argv): """ resa_policy can be either (*) 'enforce': refuse to send the message if the lease is not there (*) 'warn': issue a warning when the lease is not there (*) 'none' - or anything else really: does not check the leases """ usage = f""" Send verb '{verb}' to the CMC interface of selected nodes""" if resa_policy == 'enforce': usage += f"\n {RESERVATION_REQUIRED}" config = Config() default_timeout = config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() leases = Leases(message_bus) # pylint: disable=w0621 if resa_policy in ('warn', 'enforce'): reserved = check_reservation(leases, verbose=False) if not reserved: if resa_policy == 'enforce': return 1 selector = selected_selector(args) action = Action(verb, selector) return 0 if action.run(message_bus, args.timeout) else 1
async def run(self, multicast_ip, port): control_ip = self.control_ip the_config = Config() client = the_config.value('frisbee', 'client') hdd = the_config.value('frisbee', 'hard_drive') self.command = \ "{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}".format(**locals()) logger.info("on {} : running command {}".format(self.control_ip, self.command)) await self.feedback('frisbee_status', "starting frisbee client") EOF = chr(4) EOL = '\n' # print out exit status so the parser can catch it and expose it command = self.command command = command + "; echo FRISBEE-STATUS=$?" # make sure the command is sent (EOL) and that the session terminates afterwards (exit + EOF) command = command + "; exit" + EOL + EOF self._protocol.stream.write(self._protocol.shell.encode(command)) # wait for telnet to terminate await self._protocol.waiter_closed return True
async def start(self): """ Start a collector instance; returns a port_number """ the_config = Config() netcat = the_config.value('frisbee', 'netcat') local_ip = the_config.local_control_ip() # should use default.ndz if not provided # use shell-style as we rather have bash handle the redirection # we instruct bash to exec nc; otherwise when cleaning up we just kill the bash process # but nc is left lingering behind # WARNING: it is intended that format contains {port} for future formatting command_format = "exec {} -d -l {} {{port}} > {} 2> /dev/null"\ .format(netcat, local_ip, self.image) nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts+1): pat = str(i) port = str(eval(pat_port.replace('*', pat))) command = command_format.format(port=port) self.subprocess = \ await asyncio.create_subprocess_shell(command) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if is has, we try our luck on another couple (ip, port) command_line = command if self.subprocess.returncode is None: logger.info("collector started: {}".format(command_line)) await self.feedback('info', "collector started on {}".format(self.image)) self.port = port return port else: logger.warning("failed to start collector with {}".format(command_line)) logger.critical("Could not find a free port to start collector") raise Exception("Could not start collector server")
async def start(self): """ Start a frisbeed instance returns a tuple multicast_group, port_number """ the_config = Config() server = the_config.value('frisbee', 'server') server_options = the_config.value('frisbee', 'server_options') local_ip = the_config.local_control_ip() # in Mibps bandwidth = self.bandwidth * 2**20 # should use default.ndz if not provided command_common = [ server, "-i", local_ip, "-W", str(bandwidth), self.image ] # add configured extra options command_common += server_options.split() nb_attempts = int(the_config.value('networking', 'pattern_size')) pat_ip = the_config.value('networking', 'pattern_multicast') pat_port = the_config.value('networking', 'pattern_port') for i in range(1, nb_attempts+1): pat = str(i) multicast_group = pat_ip.replace('*', pat) multicast_port = str(eval(pat_port.replace('*', pat))) command = command_common + [ "-m", multicast_group, "-p", multicast_port, ] self.subprocess = await asyncio.create_subprocess_exec( *command, stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.STDOUT ) await asyncio.sleep(1) # after such a short time, frisbeed should not have returned yet # if is has, we try our luck on another couple (ip, port) command_line = " ".join(command) if self.subprocess.returncode is None: self.multicast_group = multicast_group self.multicast_port = multicast_port await self.feedback('info', "started {}".format(self)) return multicast_group, multicast_port else: logger.warning("failed to start frisbeed with {}".format(command_line)) logger.critical("Could not find a free IP multicast address + port to start frisbeed") raise Exception("Could not start frisbee server")
def __init__(self, control_ip, message_bus): self.control_ip = control_ip self.message_bus = message_bus # config the_config = Config() self.port = int(the_config.value('networking', 'telnet_port')) self.backoff = float(the_config.value('networking', 'telnet_backoff')) self.connect_timeout = float( the_config.value('networking', 'telnet_timeout')) self.connect_minwait = float( the_config.value('networking', 'telnet_connect_minwait')) self.connect_maxwait = float( the_config.value('networking', 'telnet_connect_maxwait')) # internals self.running = False self._reader = None self._writer = None
def wait(*argv): # pylint: disable=r0914 usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ # suppress info log messages from asyncssh asyncssh_set_log_level(logging.WARNING) config = Config() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'wait_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--backoff", action='store', default=config.value('networking', 'ssh_backoff'), type=float, help="Specify backoff average between " "attempts to ssh connect") parser.add_argument("-u", "--user", default="root", help="select other username") # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"wait: backoff is {args.backoff} " f"and global timeout is {args.timeout}") nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] sshs = [ SshProxy(node, username=args.user, verbose=args.verbose) for node in nodes ] jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True, critical=True), *jobs, timeout=args.timeout, critical=False) try: orchestration = scheduler.run() if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
async def stage1(self): the_config = Config() idle = int(the_config.value('nodes', 'idle_after_reset')) await asyncio.gather(*[node.reboot_on_frisbee(idle) for node in self.nodes])
def __init__(self): the_config = Config() self.repo = the_config.value('frisbee', 'images_dir') self.default_name = the_config.value('frisbee', 'default_image') # for building hostname patterns self.regularname = the_config.value('testbed', 'regularname')
async def stage1(self): the_config = Config() idle = int(the_config.value('nodes', 'idle_after_reset')) await self.node.reboot_on_frisbee(idle)
def cmc_verb(verb, check_resa, *argv): """ check_resa can be either (*) enforce: refuse to send the message if the lease is not there (*) warn: issue a warning when the lease is not there (*) none: does not check the leases """ usage = """ Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb) if check_resa == 'enforce': usage += "\n {resa}".format(resa=reservation_required) the_config = Config() default_timeout = the_config.value('nodes', 'cmc_default_timeout') parser = ArgumentParser(usage=usage) parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) message_bus = asyncio.Queue() if check_resa in ('warn', 'enforce'): reserved = check_reservation(verbose=False) if not reserved: if check_resa == 'enforce': return 1 verb_to_method = {'status': 'get_status', 'on': 'turn_on', 'off': 'turn_off', 'reset': 'do_reset', 'info': 'get_info', 'usrpstatus': 'get_usrpstatus', 'usrpon': 'turn_usrpon', 'usrpoff': 'turn_usrpoff', } async def get_and_show_verb(node, verb): assert verb in verb_to_method # send the 'verb' method on node method = getattr(node, verb_to_method[verb]) # bound methods must not be passed the subject ! await method() result = getattr(node, verb) result = result if result is not None else "{} N/A".format(verb) for line in result.split("\n"): if line: print("{}:{}".format(node.cmc_name, line)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] jobs = [Job(get_and_show_verb(node, verb)) for node in nodes] display = Display(nodes, message_bus) scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: if scheduler.orchestrate(timeout=args.timeout): return 0 else: print("rhubarbe-{} failed: {}".format(verb, scheduler.why())) return 1 except KeyboardInterrupt as e: print("rhubarbe-{} : keyboard interrupt - exiting".format(verb)) return 1
def __init__(self): the_config = Config() with open(the_config.value('testbed', 'inventory_nodes_path')) as feed: self._nodes = json.load(feed)
def monitor(*argv): # xxx hacky - do a side effect in the logger module import rhubarbe.logger rhubarbe.logger.logger = rhubarbe.logger.monitor_logger # xxx hacky usage = """ Cyclic probe all selected nodes to report real-time status at a sidecar service over socketIO """ the_config = Config() default_cycle = the_config.value('monitor', 'cycle_status') parser = ArgumentParser(usage=usage) parser.add_argument('-c', "--cycle", default=default_cycle, type=float, help="Delay to wait between 2 probes of each node, default ={}" .format(default_cycle)) parser.add_argument("-w", "--wlan", dest="report_wlan", default=False, action='store_true', help="ask for probing of wlan traffic rates") parser.add_argument("-H", "--sidecar-hostname", dest="sidecar_hostname", default=None) parser.add_argument("-P", "--sidecar-port", dest="sidecar_port", default=None) parser.add_argument("-d", "--debug", dest="debug", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) selector = selected_selector(args) loop = asyncio.get_event_loop() message_bus = asyncio.Queue() # xxx having to feed a Display instance with nodes # at creation time is a nuisance display = Display([], message_bus) from rhubarbe.logger import logger logger.info({'selected_nodes': selector}) monitor = Monitor(selector.cmc_names(), message_bus=message_bus, cycle=args.cycle, report_wlan=args.report_wlan, sidecar_hostname=args.sidecar_hostname, sidecar_port=args.sidecar_port, debug=args.debug) # trap signals so we get a nice message in monitor.log import signal import functools def exiting(signame): logger.info("Received signal {} - exiting".format(signame)) loop.stop() exit(1) for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, signame), functools.partial(exiting, signame)) async def run(): # run both the core and the log loop in parallel await asyncio.gather(monitor.run(), monitor.log()) await display.stop() t1 = util.self_manage(run()) t2 = util.self_manage(display.run()) tasks = asyncio.gather(t1, t2) wrapper = asyncio.gather(tasks) try: loop.run_until_complete(wrapper) return 0 except KeyboardInterrupt as e: logger.info("rhubarbe-monitor : keyboard interrupt - exiting") tasks.cancel() loop.run_forever() tasks.exception() return 1 except asyncio.TimeoutError as e: logger.info( "rhubarbe-monitor : timeout expired after {}s".format(args.timeout)) return 1 finally: loop.close()
def save(*argv): usage = f""" Save an image from a node Mandatory radical needs to be provided with --output This info, together with nodename and date, is stored on resulting image in /etc/rhubarbe-image {RESERVATION_REQUIRED} """ config = Config() config.check_binaries() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-o", "--output", action='store', dest='radical', default=None, required=True, help="Mandatory radical to name resulting image") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'save_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-c", "--comment", dest='comment', default=None, help="one-liner comment to insert in " "/etc/rhubarbe-image") parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with a node that is already running a frisbee image. It won't get reset, neither before or after the frisbee session""") parser.add_argument("node") args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = Selector() selector.add_range(args.node) # in case there was one argument but it was not found in inventory if len(selector) != 1: parser.print_help() cmc_name = next(selector.cmc_names()) node = Node(cmc_name, message_bus) nodename = node.control_hostname() imagesrepo = ImagesRepo() actual_image = imagesrepo.where_to_save(nodename, args.radical) message_bus.put_nowait({'info': f"Saving image {actual_image}"}) # curses has no interest here since we focus on one node display_class = Display display = display_class([node], message_bus) saver = ImageSaver(node, image=actual_image, radical=args.radical, message_bus=message_bus, display=display, comment=args.comment) return saver.main(reset=args.reset, timeout=args.timeout)
def __init__(self): the_config = Config() with open(the_config.value('testbed', 'inventory_path')) as feed: self._nodes = json.load(feed)
def __init__(self): the_config = Config() self.regularname = the_config.value('testbed', 'regularname') self.rebootname = the_config.value('testbed', 'rebootname') self.set = set()
def use_all_scope(self): the_config = Config() self.add_range(the_config.value('testbed', 'all_scope'))
def wait(*argv): usage = """ Wait for selected nodes to be reachable by ssh Returns 0 if all nodes indeed are reachable """ the_config = Config() default_timeout = the_config.value('nodes', 'wait_default_timeout') default_backoff = the_config.value('networking', 'ssh_backoff') parser = ArgumentParser(usage=usage) parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") parser.add_argument("-t", "--timeout", action='store', default=default_timeout, type=float, help="Specify global timeout for the whole process, default={}" .format(default_timeout)) parser.add_argument("-b", "--backoff", action='store', default=default_backoff, type=float, help="Specify backoff average between " "attempts to ssh connect, default={}" .format(default_backoff)) # really dont' write anything parser.add_argument("-s", "--silent", action='store_true', default=False) parser.add_argument("-v", "--verbose", action='store_true', default=False) add_selector_arguments(parser) args = parser.parse_args(argv) # --curses implies --verbose otherwise nothing shows up if args.curses: args.verbose = True selector = selected_selector(args) message_bus = asyncio.Queue() if args.verbose: message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info("wait: backoff is {} and global timeout is {}" .format(args.backoff, args.timeout)) nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()] sshs = [SshProxy(node, verbose=args.verbose) for node in nodes] jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs] display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) # have the display class run forever until the other ones are done scheduler = Scheduler(Job(display.run(), forever=True), *jobs) try: orchestration = scheduler.orchestrate(timeout=args.timeout) if orchestration: return 0 else: if args.verbose: scheduler.debrief() return 1 except KeyboardInterrupt as e: print("rhubarbe-wait : keyboard interrupt - exiting") # xxx return 1 finally: display.epilogue() if not args.silent: for ssh in sshs: print("{}:ssh {}".format(ssh.node, "OK" if ssh.status else "KO"))
async def stage1(self): the_config = Config() idle = int(the_config.value('nodes', 'idle_after_reset')) await asyncio.gather( *[node.reboot_on_frisbee(idle) for node in self.nodes])
def load(*argv): usage = f""" Load an image on selected nodes in parallel {RESERVATION_REQUIRED} """ config = Config() config.check_binaries() imagesrepo = ImagesRepo() parser = ArgumentParser(usage=usage, formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-i", "--image", action='store', default=imagesrepo.default(), help="Specify image to load") parser.add_argument("-t", "--timeout", action='store', default=config.value('nodes', 'load_default_timeout'), type=float, help="Specify global timeout for the whole process") parser.add_argument("-b", "--bandwidth", action='store', default=config.value('networking', 'bandwidth'), type=int, help="Set bandwidth in Mibps for frisbee uploading") parser.add_argument("-c", "--curses", action='store_true', default=False, help="Use curses to provide term-based animation") # this is more for debugging parser.add_argument("-n", "--no-reset", dest='reset', action='store_false', default=True, help="""use this with nodes that are already running a frisbee image. They won't get reset, neither before or after the frisbee session""") add_selector_arguments(parser) args = parser.parse_args(argv) message_bus = asyncio.Queue() selector = selected_selector(args) if selector.is_empty(): parser.print_help() return 1 nodes = [ Node(cmc_name, message_bus) # pylint: disable=w0621 for cmc_name in selector.cmc_names() ] # send feedback message_bus.put_nowait({'selected_nodes': selector}) from rhubarbe.logger import logger logger.info(f"timeout is {args.timeout}s") logger.info(f"bandwidth is {args.bandwidth} Mibps") actual_image = imagesrepo.locate_image(args.image, look_in_global=True) if not actual_image: print(f"Image file {args.image} not found - emergency exit") exit(1) # send feedback message_bus.put_nowait({'loading_image': actual_image}) display_class = Display if not args.curses else DisplayCurses display = display_class(nodes, message_bus) loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth, message_bus=message_bus, display=display) return loader.main(reset=args.reset, timeout=args.timeout)