def load(*argv):
usage = """
Load an image on selected nodes in parallel
{resa}
""".format(resa=reservation_required)
the_config = Config()
the_imagesrepo = ImagesRepo()
default_image = the_imagesrepo.default()
default_timeout = the_config.value('nodes', 'load_default_timeout')
default_bandwidth = the_config.value('networking', 'bandwidth')
parser = ArgumentParser(usage=usage)
parser.add_argument("-i", "--image", action='store', default=default_image,
help="Specify image to load (default is {})"
.format(default_image))
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
parser.add_argument("-b", "--bandwidth", action='store',
default=default_bandwidth, type=int,
help="Set bandwidth in Mibps for frisbee uploading - default={}"
.format(default_bandwidth))
parser.add_argument("-c", "--curses", action='store_true', default=False,
help="Use curses to provide term-based animation")
# this is more for debugging
parser.add_argument("-n", "--no-reset", dest='reset',
action='store_false', default=True,
help="""use this with nodes that are already
running a frisbee image. They won't get reset,
neither before or after the frisbee session""")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = selected_selector(args)
if not selector.how_many():
parser.print_help()
return 1
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
# send feedback
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info("timeout is {}s".format(args.timeout))
logger.info("bandwidth is {} Mibps".format(args.bandwidth))
actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True)
if not actual_image:
print("Image file {} not found - emergency exit".format(args.image))
exit(1)
# send feedback
message_bus.put_nowait({'loading_image': actual_image})
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth,
message_bus=message_bus, display=display)
return loader.main(reset=args.reset, timeout=args.timeout)
def main(*argv):
parser = ArgumentParser(usage=USAGE)
parser.add_argument("-v",
"--verbose",
action='store_true',
default=False,
help="more verbose output")
parser.add_argument("-n",
"--dry-run",
action='store_true',
default=False,
help="DEBUG ONLY: minimal interactions with node,"
" that are expected to be ON;"
"won't check on and off and reset; "
"won't load any image on node")
parser.add_argument("-s",
"--speedy",
action='store_true',
default=False,
help="DEBUG ONLY: will only load one image")
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args, defaults_to_all=True)
nightly = Nightly(selector,
dry_run=args.dry_run,
verbose=args.verbose,
speedy=args.speedy)
# turn off asyncssh info message unless verbose
if not args.verbose:
set_log_level(logging.ERROR)
return 0 if nightly.run() else 1
def nodes(*argv):
usage = """
Just display the hostnames of selected nodes
"""
parser = ArgumentParser(usage=usage)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
print(" ".join(selector.node_names()))
def nodes(*argv):
usage = """
Just display the hostnames of selected nodes
"""
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
print(" ".join(selector.node_names()))
def monitornodes(*argv): # pylint: disable=r0914
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
from rhubarbe.logger import logger
usage = """
Cyclic probe all selected nodes, and reports
real-time status at a sidecar service over websockets
"""
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',
"--cycle",
default=config.value('monitor', 'cycle_nodes'),
type=float,
help="Delay to wait between 2 probes of each node")
parser.add_argument("-u",
"--sidecar-url",
dest="sidecar_url",
default=Config().value('sidecar', 'url'),
help="url for the sidecar server")
parser.add_argument("-w",
"--wlan",
dest="report_wlan",
default=False,
action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
logger.info({'selected_nodes': selector})
monitornodes = MonitorNodes(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
sidecar_url=args.sidecar_url,
report_wlan=args.report_wlan,
verbose=args.verbose)
async def async_main():
# run both the core and the log loop in parallel
await asyncio.gather(monitornodes.run_forever(), display.run())
MonitorLoop("monitornodes").run(async_main(), logger)
return 0
def bye(*argv):
"""
An alternative implementation of the previous 'all-off' utility
Switch the lights off when you leave
"""
usage = """
Turn off whole testbed
"""
config = Config()
default_timeout = config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-t",
"--timeout",
action='store',
default=default_timeout,
type=float,
help="Specify timeout for each phase")
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args, defaults_to_all=True)
if selector.is_empty():
selector.use_all_scope()
bus = asyncio.Queue()
Action('usrpoff', selector).run(bus, args.timeout)
# keep it simple for now
import time
time.sleep(1)
Action('off', selector).run(bus, args.timeout)
# even simpler
import os
phones_inventory = InventoryPhones()
for phone in phones_inventory.all_phones():
command = (f"ssh -i {phone['gw_key']} "
f"{phone['gw_user']}@{phone['gw_host']} phone-off")
print(command)
os.system(command)
def cmc_verb(verb, resa_policy, *argv):
"""
resa_policy can be either
(*) 'enforce': refuse to send the message if the lease is not there
(*) 'warn': issue a warning when the lease is not there
(*) 'none' - or anything else really: does not check the leases
"""
usage = f"""
Send verb '{verb}' to the CMC interface of selected nodes"""
if resa_policy == 'enforce':
usage += f"\n {RESERVATION_REQUIRED}"
config = Config()
default_timeout = config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-t",
"--timeout",
action='store',
default=default_timeout,
type=float,
help="Specify global timeout for the whole process")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
leases = Leases(message_bus) # pylint: disable=w0621
if resa_policy in ('warn', 'enforce'):
reserved = check_reservation(leases, verbose=False)
if not reserved:
if resa_policy == 'enforce':
return 1
selector = selected_selector(args)
action = Action(verb, selector)
return 0 if action.run(message_bus, args.timeout) else 1
def cmc_verb(verb, check_resa, *argv):
"""
check_resa can be either
(*) enforce: refuse to send the message if the lease is not there
(*) warn: issue a warning when the lease is not there
(*) none: does not check the leases
"""
usage = """
Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb)
if check_resa == 'enforce':
usage += "\n {resa}".format(resa=reservation_required)
the_config = Config()
default_timeout = the_config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage)
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
if check_resa in ('warn', 'enforce'):
reserved = check_reservation(verbose=False)
if not reserved:
if check_resa == 'enforce':
return 1
verb_to_method = {'status': 'get_status',
'on': 'turn_on',
'off': 'turn_off',
'reset': 'do_reset',
'info': 'get_info',
'usrpstatus': 'get_usrpstatus',
'usrpon': 'turn_usrpon',
'usrpoff': 'turn_usrpoff',
}
async def get_and_show_verb(node, verb):
assert verb in verb_to_method
# send the 'verb' method on node
method = getattr(node, verb_to_method[verb])
# bound methods must not be passed the subject !
await method()
result = getattr(node, verb)
result = result if result is not None else "{} N/A".format(verb)
for line in result.split("\n"):
if line:
print("{}:{}".format(node.cmc_name, line))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
jobs = [Job(get_and_show_verb(node, verb)) for node in nodes]
display = Display(nodes, message_bus)
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
if scheduler.orchestrate(timeout=args.timeout):
return 0
else:
print("rhubarbe-{} failed: {}".format(verb, scheduler.why()))
return 1
except KeyboardInterrupt as e:
print("rhubarbe-{} : keyboard interrupt - exiting".format(verb))
return 1
def monitor(*argv):
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
# xxx hacky
usage = """
Cyclic probe all selected nodes to report real-time status
at a sidecar service over socketIO
"""
the_config = Config()
default_cycle = the_config.value('monitor', 'cycle_status')
parser = ArgumentParser(usage=usage)
parser.add_argument('-c', "--cycle", default=default_cycle, type=float,
help="Delay to wait between 2 probes of each node, default ={}"
.format(default_cycle))
parser.add_argument("-w", "--wlan", dest="report_wlan",
default=False, action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-H", "--sidecar-hostname",
dest="sidecar_hostname", default=None)
parser.add_argument("-P", "--sidecar-port",
dest="sidecar_port", default=None)
parser.add_argument("-d", "--debug", dest="debug",
action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
loop = asyncio.get_event_loop()
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
from rhubarbe.logger import logger
logger.info({'selected_nodes': selector})
monitor = Monitor(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
report_wlan=args.report_wlan,
sidecar_hostname=args.sidecar_hostname,
sidecar_port=args.sidecar_port,
debug=args.debug)
# trap signals so we get a nice message in monitor.log
import signal
import functools
def exiting(signame):
logger.info("Received signal {} - exiting".format(signame))
loop.stop()
exit(1)
for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, signame),
functools.partial(exiting, signame))
async def run():
# run both the core and the log loop in parallel
await asyncio.gather(monitor.run(), monitor.log())
await display.stop()
t1 = util.self_manage(run())
t2 = util.self_manage(display.run())
tasks = asyncio.gather(t1, t2)
wrapper = asyncio.gather(tasks)
try:
loop.run_until_complete(wrapper)
return 0
except KeyboardInterrupt as e:
logger.info("rhubarbe-monitor : keyboard interrupt - exiting")
tasks.cancel()
loop.run_forever()
tasks.exception()
return 1
except asyncio.TimeoutError as e:
logger.info(
"rhubarbe-monitor : timeout expired after {}s".format(args.timeout))
return 1
finally:
loop.close()
def wait(*argv):
usage = """
Wait for selected nodes to be reachable by ssh
Returns 0 if all nodes indeed are reachable
"""
the_config = Config()
default_timeout = the_config.value('nodes', 'wait_default_timeout')
default_backoff = the_config.value('networking', 'ssh_backoff')
parser = ArgumentParser(usage=usage)
parser.add_argument("-c", "--curses", action='store_true', default=False,
help="Use curses to provide term-based animation")
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
parser.add_argument("-b", "--backoff", action='store',
default=default_backoff, type=float,
help="Specify backoff average between "
"attempts to ssh connect, default={}"
.format(default_backoff))
# really dont' write anything
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
# --curses implies --verbose otherwise nothing shows up
if args.curses:
args.verbose = True
selector = selected_selector(args)
message_bus = asyncio.Queue()
if args.verbose:
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info("wait: backoff is {} and global timeout is {}"
.format(args.backoff, args.timeout))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
sshs = [SshProxy(node, verbose=args.verbose) for node in nodes]
jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs]
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
# have the display class run forever until the other ones are done
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
orchestration = scheduler.orchestrate(timeout=args.timeout)
if orchestration:
return 0
else:
if args.verbose:
scheduler.debrief()
return 1
except KeyboardInterrupt as e:
print("rhubarbe-wait : keyboard interrupt - exiting")
# xxx
return 1
finally:
display.epilogue()
if not args.silent:
for ssh in sshs:
print("{}:ssh {}".format(ssh.node,
"OK" if ssh.status else "KO"))
def wait(*argv): # pylint: disable=r0914
usage = """
Wait for selected nodes to be reachable by ssh
Returns 0 if all nodes indeed are reachable
"""
# suppress info log messages from asyncssh
asyncssh_set_log_level(logging.WARNING)
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-c",
"--curses",
action='store_true',
default=False,
help="Use curses to provide term-based animation")
parser.add_argument("-t",
"--timeout",
action='store',
default=config.value('nodes', 'wait_default_timeout'),
type=float,
help="Specify global timeout for the whole process")
parser.add_argument("-b",
"--backoff",
action='store',
default=config.value('networking', 'ssh_backoff'),
type=float,
help="Specify backoff average between "
"attempts to ssh connect")
parser.add_argument("-u",
"--user",
default="root",
help="select other username")
# really dont' write anything
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
# --curses implies --verbose otherwise nothing shows up
if args.curses:
args.verbose = True
selector = selected_selector(args)
message_bus = asyncio.Queue()
if args.verbose:
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info(f"wait: backoff is {args.backoff} "
f"and global timeout is {args.timeout}")
nodes = [
Node(cmc_name, message_bus) # pylint: disable=w0621
for cmc_name in selector.cmc_names()
]
sshs = [
SshProxy(node, username=args.user, verbose=args.verbose)
for node in nodes
]
jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs]
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
# have the display class run forever until the other ones are done
scheduler = Scheduler(Job(display.run(), forever=True, critical=True),
*jobs,
timeout=args.timeout,
critical=False)
try:
orchestration = scheduler.run()
if orchestration:
return 0
else:
if args.verbose:
scheduler.debrief()
return 1
except KeyboardInterrupt:
print("rhubarbe-wait : keyboard interrupt - exiting")
# xxx
return 1
finally:
display.epilogue()
if not args.silent:
for ssh in sshs:
print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
def load(*argv):
usage = f"""
Load an image on selected nodes in parallel
{RESERVATION_REQUIRED}
"""
config = Config()
config.check_binaries()
imagesrepo = ImagesRepo()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-i",
"--image",
action='store',
default=imagesrepo.default(),
help="Specify image to load")
parser.add_argument("-t",
"--timeout",
action='store',
default=config.value('nodes', 'load_default_timeout'),
type=float,
help="Specify global timeout for the whole process")
parser.add_argument("-b",
"--bandwidth",
action='store',
default=config.value('networking', 'bandwidth'),
type=int,
help="Set bandwidth in Mibps for frisbee uploading")
parser.add_argument("-c",
"--curses",
action='store_true',
default=False,
help="Use curses to provide term-based animation")
# this is more for debugging
parser.add_argument("-n",
"--no-reset",
dest='reset',
action='store_false',
default=True,
help="""use this with nodes that are already
running a frisbee image. They won't get reset,
neither before or after the frisbee session""")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = selected_selector(args)
if selector.is_empty():
parser.print_help()
return 1
nodes = [
Node(cmc_name, message_bus) # pylint: disable=w0621
for cmc_name in selector.cmc_names()
]
# send feedback
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info(f"timeout is {args.timeout}s")
logger.info(f"bandwidth is {args.bandwidth} Mibps")
actual_image = imagesrepo.locate_image(args.image, look_in_global=True)
if not actual_image:
print(f"Image file {args.image} not found - emergency exit")
exit(1)
# send feedback
message_bus.put_nowait({'loading_image': actual_image})
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
loader = ImageLoader(nodes,
image=actual_image,
bandwidth=args.bandwidth,
message_bus=message_bus,
display=display)
return loader.main(reset=args.reset, timeout=args.timeout)