Example #1
0
def load(*argv):
    usage = """
    Load an image on selected nodes in parallel
    {resa}
    """.format(resa=reservation_required)
    the_config = Config()
    the_imagesrepo = ImagesRepo()
    default_image = the_imagesrepo.default()
    default_timeout = the_config.value('nodes', 'load_default_timeout')
    default_bandwidth = the_config.value('networking', 'bandwidth')

    parser = ArgumentParser(usage=usage)
    parser.add_argument("-i", "--image", action='store', default=default_image,
                        help="Specify image to load (default is {})"
                             .format(default_image))
    parser.add_argument("-t", "--timeout", action='store',
                        default=default_timeout, type=float,
                        help="Specify global timeout for the whole process, default={}"
                              .format(default_timeout))
    parser.add_argument("-b", "--bandwidth", action='store',
                        default=default_bandwidth, type=int,
                        help="Set bandwidth in Mibps for frisbee uploading - default={}"
                              .format(default_bandwidth))
    parser.add_argument("-c", "--curses", action='store_true', default=False,
                        help="Use curses to provide term-based animation")
    # this is more for debugging
    parser.add_argument("-n", "--no-reset", dest='reset',
                        action='store_false', default=True,
                        help="""use this with nodes that are already
                        running a frisbee image. They won't get reset,
                        neither before or after the frisbee session""")
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    message_bus = asyncio.Queue()

    selector = selected_selector(args)
    if not selector.how_many():
        parser.print_help()
        return 1
    nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]

    # send feedback
    message_bus.put_nowait({'selected_nodes': selector})
    from rhubarbe.logger import logger
    logger.info("timeout is {}s".format(args.timeout))
    logger.info("bandwidth is {} Mibps".format(args.bandwidth))

    actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True)
    if not actual_image:
        print("Image file {} not found - emergency exit".format(args.image))
        exit(1)

    # send feedback
    message_bus.put_nowait({'loading_image': actual_image})
    display_class = Display if not args.curses else DisplayCurses
    display = display_class(nodes, message_bus)
    loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth,
                         message_bus=message_bus, display=display)
    return loader.main(reset=args.reset, timeout=args.timeout)
Example #2
0
def main(*argv):
    parser = ArgumentParser(usage=USAGE)
    parser.add_argument("-v",
                        "--verbose",
                        action='store_true',
                        default=False,
                        help="more verbose output")
    parser.add_argument("-n",
                        "--dry-run",
                        action='store_true',
                        default=False,
                        help="DEBUG ONLY: minimal interactions with node,"
                        " that are expected to be ON;"
                        "won't check on and off and reset; "
                        "won't load any image on node")
    parser.add_argument("-s",
                        "--speedy",
                        action='store_true',
                        default=False,
                        help="DEBUG ONLY: will only load one image")
    add_selector_arguments(parser)

    args = parser.parse_args(argv)

    selector = selected_selector(args, defaults_to_all=True)
    nightly = Nightly(selector,
                      dry_run=args.dry_run,
                      verbose=args.verbose,
                      speedy=args.speedy)

    # turn off asyncssh info message unless verbose
    if not args.verbose:
        set_log_level(logging.ERROR)

    return 0 if nightly.run() else 1
Example #3
0
def nodes(*argv):
    usage = """
    Just display the hostnames of selected nodes
    """
    parser = ArgumentParser(usage=usage)
    add_selector_arguments(parser)
    args = parser.parse_args(argv)
    selector = selected_selector(args)
    print(" ".join(selector.node_names()))
Example #4
0
def nodes(*argv):
    usage = """
    Just display the hostnames of selected nodes
    """
    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    add_selector_arguments(parser)
    args = parser.parse_args(argv)
    selector = selected_selector(args)
    print(" ".join(selector.node_names()))
Example #5
0
def monitornodes(*argv):  # pylint: disable=r0914

    # xxx hacky - do a side effect in the logger module
    import rhubarbe.logger
    rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
    from rhubarbe.logger import logger

    usage = """
    Cyclic probe all selected nodes, and reports
    real-time status at a sidecar service over websockets
    """
    config = Config()
    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('-c',
                        "--cycle",
                        default=config.value('monitor', 'cycle_nodes'),
                        type=float,
                        help="Delay to wait between 2 probes of each node")
    parser.add_argument("-u",
                        "--sidecar-url",
                        dest="sidecar_url",
                        default=Config().value('sidecar', 'url'),
                        help="url for the sidecar server")
    parser.add_argument("-w",
                        "--wlan",
                        dest="report_wlan",
                        default=False,
                        action='store_true',
                        help="ask for probing of wlan traffic rates")
    parser.add_argument("-v", "--verbose", action='store_true', default=False)
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    selector = selected_selector(args)
    message_bus = asyncio.Queue()

    # xxx having to feed a Display instance with nodes
    # at creation time is a nuisance
    display = Display([], message_bus)

    logger.info({'selected_nodes': selector})
    monitornodes = MonitorNodes(selector.cmc_names(),
                                message_bus=message_bus,
                                cycle=args.cycle,
                                sidecar_url=args.sidecar_url,
                                report_wlan=args.report_wlan,
                                verbose=args.verbose)

    async def async_main():
        # run both the core and the log loop in parallel
        await asyncio.gather(monitornodes.run_forever(), display.run())

    MonitorLoop("monitornodes").run(async_main(), logger)
    return 0
Example #6
0
def bye(*argv):
    """
    An alternative implementation of the previous 'all-off' utility
    Switch the lights off when you leave
    """

    usage = """
    Turn off whole testbed
    """
    config = Config()
    default_timeout = config.value('nodes', 'cmc_default_timeout')
    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-t",
                        "--timeout",
                        action='store',
                        default=default_timeout,
                        type=float,
                        help="Specify timeout for each phase")
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    selector = selected_selector(args, defaults_to_all=True)
    if selector.is_empty():
        selector.use_all_scope()

    bus = asyncio.Queue()
    Action('usrpoff', selector).run(bus, args.timeout)

    # keep it simple for now
    import time
    time.sleep(1)
    Action('off', selector).run(bus, args.timeout)

    # even simpler
    import os
    phones_inventory = InventoryPhones()
    for phone in phones_inventory.all_phones():
        command = (f"ssh -i {phone['gw_key']} "
                   f"{phone['gw_user']}@{phone['gw_host']} phone-off")
        print(command)
        os.system(command)
Example #7
0
def cmc_verb(verb, resa_policy, *argv):
    """
    resa_policy can be either
    (*) 'enforce': refuse to send the message if the lease is not there
    (*) 'warn': issue a warning when the lease is not there
    (*) 'none' - or anything else really: does not check the leases
    """
    usage = f"""
    Send verb '{verb}' to the CMC interface of selected nodes"""
    if resa_policy == 'enforce':
        usage += f"\n    {RESERVATION_REQUIRED}"
    config = Config()
    default_timeout = config.value('nodes', 'cmc_default_timeout')

    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-t",
                        "--timeout",
                        action='store',
                        default=default_timeout,
                        type=float,
                        help="Specify global timeout for the whole process")
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    message_bus = asyncio.Queue()
    leases = Leases(message_bus)  # pylint: disable=w0621

    if resa_policy in ('warn', 'enforce'):
        reserved = check_reservation(leases, verbose=False)
        if not reserved:
            if resa_policy == 'enforce':
                return 1

    selector = selected_selector(args)
    action = Action(verb, selector)

    return 0 if action.run(message_bus, args.timeout) else 1
Example #8
0
def cmc_verb(verb, check_resa, *argv):
    """
    check_resa can be either
    (*) enforce: refuse to send the message if the lease is not there
    (*) warn: issue a warning when the lease is not there
    (*) none: does not check the leases
    """
    usage = """
    Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb)
    if check_resa == 'enforce':
        usage += "\n    {resa}".format(resa=reservation_required)
    the_config = Config()
    default_timeout = the_config.value('nodes', 'cmc_default_timeout')

    parser = ArgumentParser(usage=usage)
    parser.add_argument("-t", "--timeout", action='store',
                        default=default_timeout, type=float,
                        help="Specify global timeout for the whole process, default={}"
                             .format(default_timeout))
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    selector = selected_selector(args)
    message_bus = asyncio.Queue()

    if check_resa in ('warn', 'enforce'):
        reserved = check_reservation(verbose=False)
        if not reserved:
            if check_resa == 'enforce':
                return 1

    verb_to_method = {'status': 'get_status',
                      'on': 'turn_on',
                      'off': 'turn_off',
                      'reset': 'do_reset',
                      'info': 'get_info',
                      'usrpstatus': 'get_usrpstatus',
                      'usrpon': 'turn_usrpon',
                      'usrpoff': 'turn_usrpoff',
                      }

    async def get_and_show_verb(node, verb):
        assert verb in verb_to_method
        # send the 'verb' method on node
        method = getattr(node, verb_to_method[verb])
        # bound methods must not be passed the subject !
        await method()
        result = getattr(node, verb)
        result = result if result is not None else "{} N/A".format(verb)
        for line in result.split("\n"):
            if line:
                print("{}:{}".format(node.cmc_name, line))

    nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
    jobs = [Job(get_and_show_verb(node, verb)) for node in nodes]
    display = Display(nodes, message_bus)
    scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
    try:
        if scheduler.orchestrate(timeout=args.timeout):
            return 0
        else:
            print("rhubarbe-{} failed: {}".format(verb, scheduler.why()))
            return 1
    except KeyboardInterrupt as e:
        print("rhubarbe-{} : keyboard interrupt - exiting".format(verb))
        return 1
Example #9
0
def monitor(*argv):

    # xxx hacky - do a side effect in the logger module
    import rhubarbe.logger
    rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
    # xxx hacky

    usage = """
    Cyclic probe all selected nodes to report real-time status
    at a sidecar service over socketIO
    """
    the_config = Config()
    default_cycle = the_config.value('monitor', 'cycle_status')
    parser = ArgumentParser(usage=usage)
    parser.add_argument('-c', "--cycle", default=default_cycle, type=float,
                        help="Delay to wait between 2 probes of each node, default ={}"
                        .format(default_cycle))
    parser.add_argument("-w", "--wlan", dest="report_wlan",
                        default=False, action='store_true',
                        help="ask for probing of wlan traffic rates")
    parser.add_argument("-H", "--sidecar-hostname",
                        dest="sidecar_hostname", default=None)
    parser.add_argument("-P", "--sidecar-port",
                        dest="sidecar_port", default=None)
    parser.add_argument("-d", "--debug", dest="debug",
                        action='store_true', default=False)
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    selector = selected_selector(args)
    loop = asyncio.get_event_loop()
    message_bus = asyncio.Queue()

    # xxx having to feed a Display instance with nodes
    # at creation time is a nuisance
    display = Display([], message_bus)

    from rhubarbe.logger import logger
    logger.info({'selected_nodes': selector})
    monitor = Monitor(selector.cmc_names(),
                      message_bus=message_bus,
                      cycle=args.cycle,
                      report_wlan=args.report_wlan,
                      sidecar_hostname=args.sidecar_hostname,
                      sidecar_port=args.sidecar_port,
                      debug=args.debug)

    # trap signals so we get a nice message in monitor.log
    import signal
    import functools

    def exiting(signame):
        logger.info("Received signal {} - exiting".format(signame))
        loop.stop()
        exit(1)
    for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'):
        loop.add_signal_handler(getattr(signal, signame),
                                functools.partial(exiting, signame))

    async def run():
        # run both the core and the log loop in parallel
        await asyncio.gather(monitor.run(), monitor.log())
        await display.stop()

    t1 = util.self_manage(run())
    t2 = util.self_manage(display.run())
    tasks = asyncio.gather(t1, t2)
    wrapper = asyncio.gather(tasks)
    try:
        loop.run_until_complete(wrapper)
        return 0
    except KeyboardInterrupt as e:
        logger.info("rhubarbe-monitor : keyboard interrupt - exiting")
        tasks.cancel()
        loop.run_forever()
        tasks.exception()
        return 1
    except asyncio.TimeoutError as e:
        logger.info(
            "rhubarbe-monitor : timeout expired after {}s".format(args.timeout))
        return 1
    finally:
        loop.close()
Example #10
0
def wait(*argv):
    usage = """
    Wait for selected nodes to be reachable by ssh
    Returns 0 if all nodes indeed are reachable
    """
    the_config = Config()
    default_timeout = the_config.value('nodes', 'wait_default_timeout')
    default_backoff = the_config.value('networking', 'ssh_backoff')

    parser = ArgumentParser(usage=usage)
    parser.add_argument("-c", "--curses", action='store_true', default=False,
                        help="Use curses to provide term-based animation")
    parser.add_argument("-t", "--timeout", action='store',
                        default=default_timeout, type=float,
                        help="Specify global timeout for the whole process, default={}"
                              .format(default_timeout))
    parser.add_argument("-b", "--backoff", action='store',
                        default=default_backoff, type=float,
                        help="Specify backoff average between "
                        "attempts to ssh connect, default={}"
                        .format(default_backoff))
    # really dont' write anything
    parser.add_argument("-s", "--silent", action='store_true', default=False)
    parser.add_argument("-v", "--verbose", action='store_true', default=False)

    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    # --curses implies --verbose otherwise nothing shows up
    if args.curses:
        args.verbose = True

    selector = selected_selector(args)
    message_bus = asyncio.Queue()

    if args.verbose:
        message_bus.put_nowait({'selected_nodes': selector})
    from rhubarbe.logger import logger
    logger.info("wait: backoff is {} and global timeout is {}"
                .format(args.backoff, args.timeout))

    nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
    sshs = [SshProxy(node, verbose=args.verbose) for node in nodes]
    jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs]

    display_class = Display if not args.curses else DisplayCurses
    display = display_class(nodes, message_bus)

    # have the display class run forever until the other ones are done
    scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
    try:
        orchestration = scheduler.orchestrate(timeout=args.timeout)
        if orchestration:
            return 0
        else:
            if args.verbose:
                scheduler.debrief()
            return 1
    except KeyboardInterrupt as e:
        print("rhubarbe-wait : keyboard interrupt - exiting")
        # xxx
        return 1
    finally:
        display.epilogue()
        if not args.silent:
            for ssh in sshs:
                print("{}:ssh {}".format(ssh.node,
                                         "OK" if ssh.status else "KO"))
Example #11
0
def wait(*argv):  # pylint: disable=r0914
    usage = """
    Wait for selected nodes to be reachable by ssh
    Returns 0 if all nodes indeed are reachable
    """
    # suppress info log messages from asyncssh
    asyncssh_set_log_level(logging.WARNING)

    config = Config()
    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-c",
                        "--curses",
                        action='store_true',
                        default=False,
                        help="Use curses to provide term-based animation")
    parser.add_argument("-t",
                        "--timeout",
                        action='store',
                        default=config.value('nodes', 'wait_default_timeout'),
                        type=float,
                        help="Specify global timeout for the whole process")
    parser.add_argument("-b",
                        "--backoff",
                        action='store',
                        default=config.value('networking', 'ssh_backoff'),
                        type=float,
                        help="Specify backoff average between "
                        "attempts to ssh connect")
    parser.add_argument("-u",
                        "--user",
                        default="root",
                        help="select other username")
    # really dont' write anything
    parser.add_argument("-s", "--silent", action='store_true', default=False)
    parser.add_argument("-v", "--verbose", action='store_true', default=False)

    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    # --curses implies --verbose otherwise nothing shows up
    if args.curses:
        args.verbose = True

    selector = selected_selector(args)
    message_bus = asyncio.Queue()

    if args.verbose:
        message_bus.put_nowait({'selected_nodes': selector})
    from rhubarbe.logger import logger
    logger.info(f"wait: backoff is {args.backoff} "
                f"and global timeout is {args.timeout}")

    nodes = [
        Node(cmc_name, message_bus)  # pylint: disable=w0621
        for cmc_name in selector.cmc_names()
    ]
    sshs = [
        SshProxy(node, username=args.user, verbose=args.verbose)
        for node in nodes
    ]
    jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs]

    display_class = Display if not args.curses else DisplayCurses
    display = display_class(nodes, message_bus)

    # have the display class run forever until the other ones are done
    scheduler = Scheduler(Job(display.run(), forever=True, critical=True),
                          *jobs,
                          timeout=args.timeout,
                          critical=False)
    try:
        orchestration = scheduler.run()
        if orchestration:
            return 0
        else:
            if args.verbose:
                scheduler.debrief()
            return 1
    except KeyboardInterrupt:
        print("rhubarbe-wait : keyboard interrupt - exiting")
        # xxx
        return 1
    finally:
        display.epilogue()
        if not args.silent:
            for ssh in sshs:
                print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
Example #12
0
def load(*argv):
    usage = f"""
    Load an image on selected nodes in parallel
    {RESERVATION_REQUIRED}
    """
    config = Config()
    config.check_binaries()
    imagesrepo = ImagesRepo()

    parser = ArgumentParser(usage=usage,
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument("-i",
                        "--image",
                        action='store',
                        default=imagesrepo.default(),
                        help="Specify image to load")
    parser.add_argument("-t",
                        "--timeout",
                        action='store',
                        default=config.value('nodes', 'load_default_timeout'),
                        type=float,
                        help="Specify global timeout for the whole process")
    parser.add_argument("-b",
                        "--bandwidth",
                        action='store',
                        default=config.value('networking', 'bandwidth'),
                        type=int,
                        help="Set bandwidth in Mibps for frisbee uploading")
    parser.add_argument("-c",
                        "--curses",
                        action='store_true',
                        default=False,
                        help="Use curses to provide term-based animation")
    # this is more for debugging
    parser.add_argument("-n",
                        "--no-reset",
                        dest='reset',
                        action='store_false',
                        default=True,
                        help="""use this with nodes that are already
                        running a frisbee image. They won't get reset,
                        neither before or after the frisbee session""")
    add_selector_arguments(parser)
    args = parser.parse_args(argv)

    message_bus = asyncio.Queue()

    selector = selected_selector(args)
    if selector.is_empty():
        parser.print_help()
        return 1
    nodes = [
        Node(cmc_name, message_bus)  # pylint: disable=w0621
        for cmc_name in selector.cmc_names()
    ]

    # send feedback
    message_bus.put_nowait({'selected_nodes': selector})
    from rhubarbe.logger import logger
    logger.info(f"timeout is {args.timeout}s")
    logger.info(f"bandwidth is {args.bandwidth} Mibps")

    actual_image = imagesrepo.locate_image(args.image, look_in_global=True)
    if not actual_image:
        print(f"Image file {args.image} not found - emergency exit")
        exit(1)

    # send feedback
    message_bus.put_nowait({'loading_image': actual_image})
    display_class = Display if not args.curses else DisplayCurses
    display = display_class(nodes, message_bus)
    loader = ImageLoader(nodes,
                         image=actual_image,
                         bandwidth=args.bandwidth,
                         message_bus=message_bus,
                         display=display)
    return loader.main(reset=args.reset, timeout=args.timeout)