def monitorleases(*argv):
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
from rhubarbe.logger import logger
usage = """
Cyclic check of leases; also reacts to 'request' messages on
the sidecar channel, which triggers leases acquisition right away.
See config for defaults.
"""
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-u",
"--sidecar-url",
dest="sidecar_url",
default=Config().value('sidecar', 'url'),
help="url for the sidecar server")
parser.add_argument("-v", "--verbose", default=False, action='store_true')
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
monitorleases = MonitorLeases(message_bus, args.sidecar_url, args.verbose)
MonitorLoop("monitorleases").run(monitorleases.run_forever(), logger)
return 0
def __init__(
self,
cmc_names,
message_bus, # pylint: disable=r0913
sidecar_url,
cycle,
verbose=False):
self.cycle = cycle
self.verbose = verbose
# get miscell config
self.ping_timeout = float(Config().value('networking', 'ping_timeout'))
self.ssh_timeout = float(Config().value('networking', 'ssh_timeout'))
self.log_period = float(Config().value('monitor', 'log_period'))
# websockets
self.reconnectable = \
ReconnectableSidecar(sidecar_url, 'nodes')
# the nodes part
nodes = [Node(cmc_name, message_bus) for cmc_name in cmc_names]
self.monitor_nodes = [
MonitorNode(node=node,
reconnectable=self.reconnectable,
verbose=verbose) for node in nodes
]
def __init__(self):
conf = Config()
try:
with open(conf.value('testbed', 'inventory_phones_path')) as feed:
self._phones = json.load(feed)
except FileNotFoundError:
self._phones = []
def main(self, cycle):
"""
cycle is the duration in seconds of one cycle
Corner cases:
* cycle = None : fetch value from config_bases
* cycle = 0 : run just once (for debug mostly)
"""
if cycle is None:
cycle = Config().value('accounts', 'cycle')
cycle = int(cycle)
policy = Config().value('accounts', 'access_policy')
if policy not in ('open', 'leased', 'closed'):
logger.error("Unknown policy {} - using 'closed'"
.format(policy))
policy = 'closed'
# trick is
if cycle != 0:
self.run_forever(cycle, policy)
else:
logger.info("---------- rhubarbe accounts manager oneshot "
"policy = {}"
.format(policy))
self.manage_accounts(policy)
def __init__(self):
the_config = Config()
self.plcapiurl = the_config.value('plcapi', 'url')
self.email = the_config.value('plcapi', 'admin_email')
self.password = the_config.value('plcapi', 'admin_password')
self._proxy = None
def manage_nextboot_symlink(self, action):
"""
Messes with the symlink in /tftpboot/pxelinux.cfg/
Depending on 'action'
* 'cleanup' or 'harddrive' : clear the symlink
corresponding to this CMC
* 'frisbee' : define a symlink so that next boot
will run the frisbee image
see rhubarbe.conf for configurable options
"""
the_config = Config()
root = the_config.value('pxelinux', 'config_dir')
frisbee = the_config.value('pxelinux', 'frisbee_image')
# of the form 01-00-03-1d-0e-03-53
mylink = "01-" + self.control_mac_address().replace(':', '-')
source = os.path.join(root, mylink)
if action in ('cleanup', 'harddrive'):
if os.path.exists(source):
logger.info(f"Removing {source}")
os.remove(source)
elif action in ('frisbee', ):
if os.path.exists(source):
os.remove(source)
logger.info(f"Creating {source}")
os.symlink(frisbee, source)
else:
logger.critical(
f"manage_nextboot_symlink : unknown action {action}")
def load(*argv):
usage = """
Load an image on selected nodes in parallel
{resa}
""".format(resa=reservation_required)
the_config = Config()
the_imagesrepo = ImagesRepo()
default_image = the_imagesrepo.default()
default_timeout = the_config.value('nodes', 'load_default_timeout')
default_bandwidth = the_config.value('networking', 'bandwidth')
parser = ArgumentParser(usage=usage)
parser.add_argument("-i", "--image", action='store', default=default_image,
help="Specify image to load (default is {})"
.format(default_image))
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
parser.add_argument("-b", "--bandwidth", action='store',
default=default_bandwidth, type=int,
help="Set bandwidth in Mibps for frisbee uploading - default={}"
.format(default_bandwidth))
parser.add_argument("-c", "--curses", action='store_true', default=False,
help="Use curses to provide term-based animation")
# this is more for debugging
parser.add_argument("-n", "--no-reset", dest='reset',
action='store_false', default=True,
help="""use this with nodes that are already
running a frisbee image. They won't get reset,
neither before or after the frisbee session""")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = selected_selector(args)
if not selector.how_many():
parser.print_help()
return 1
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
# send feedback
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info("timeout is {}s".format(args.timeout))
logger.info("bandwidth is {} Mibps".format(args.bandwidth))
actual_image = the_imagesrepo.locate_image(args.image, look_in_global=True)
if not actual_image:
print("Image file {} not found - emergency exit".format(args.image))
exit(1)
# send feedback
message_bus.put_nowait({'loading_image': actual_image})
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
loader = ImageLoader(nodes, image=actual_image, bandwidth=args.bandwidth,
message_bus=message_bus, display=display)
return loader.main(reset=args.reset, timeout=args.timeout)
def manage_nextboot_symlink(self, action):
"""
Messes with the symlink in /tftpboot/pxelinux.cfg/
Depending on 'action'
* 'cleanup' or 'harddrive' : clear the symlink corresponding to this CMC
* 'frisbee' : define a symlink so that next boot will run the frisbee image
see rhubarbe.conf for configurable options
"""
the_config = Config()
root = the_config.value("pxelinux", "config_dir")
frisbee = the_config.value("pxelinux", "frisbee_image")
# of the form 01-00-03-1d-0e-03-53
mylink = "01-" + self.control_mac_address().replace(":", "-")
source = os.path.join(root, mylink)
dest = os.path.join(root, frisbee)
if action in ("cleanup", "harddrive"):
if os.path.exists(source):
logger.info("Removing {}".format(source))
os.remove(source)
elif action in ("frisbee"):
if os.path.exists(source):
os.remove(source)
logger.info("Creating {}".format(source))
os.symlink(frisbee, source)
else:
logger.critical("manage_nextboot_symlink : unknown action {}".format(action))
def monitorphones(*argv):
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
from rhubarbe.logger import logger
usage = """
Cyclic probe all known phones, and reports real-time status
at a sidecar service over websockets
"""
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-c",
"--cycle",
default=config.value('monitor', 'cycle_phones'),
type=float,
help="Delay to wait between 2 probes of each phone")
parser.add_argument("-u",
"--sidecar-url",
dest="sidecar_url",
default=Config().value('sidecar', 'url'),
help="url for the sidecar server")
parser.add_argument("-v", "--verbose", action='store_true')
args = parser.parse_args(argv)
logger.info("Using all phones")
monitorphones = MonitorPhones(**vars(args))
MonitorLoop("monitorphones").run(monitorphones.run_forever(), logger)
return 0
def __init__(self, control_ip, message_bus):
self.control_ip = control_ip
self.message_bus = message_bus
the_config = Config()
self.port = int(the_config.value('networking', 'telnet_port'))
self.backoff = float(the_config.value('networking', 'telnet_backoff'))
self.timeout = float(the_config.value('networking', 'telnet_timeout'))
# internals
self._transport = None
self._protocol = None
def monitornodes(*argv): # pylint: disable=r0914
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
from rhubarbe.logger import logger
usage = """
Cyclic probe all selected nodes, and reports
real-time status at a sidecar service over websockets
"""
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',
"--cycle",
default=config.value('monitor', 'cycle_nodes'),
type=float,
help="Delay to wait between 2 probes of each node")
parser.add_argument("-u",
"--sidecar-url",
dest="sidecar_url",
default=Config().value('sidecar', 'url'),
help="url for the sidecar server")
parser.add_argument("-w",
"--wlan",
dest="report_wlan",
default=False,
action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
logger.info({'selected_nodes': selector})
monitornodes = MonitorNodes(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
sidecar_url=args.sidecar_url,
report_wlan=args.report_wlan,
verbose=args.verbose)
async def async_main():
# run both the core and the log loop in parallel
await asyncio.gather(monitornodes.run_forever(), display.run())
MonitorLoop("monitornodes").run(async_main(), logger)
return 0
def config(*argv):
usage = """
Display global configuration
"""
parser = ArgumentParser(usage=usage)
parser.add_argument("sections", nargs='*',
type=str,
help="config section(s) to display")
args = parser.parse_args(argv)
the_config = Config()
the_config.display(args.sections)
return 0
def save(*argv):
usage = """
Save an image from a node
Mandatory radical needs to be provided with --output
This info, together with nodename and date, is stored
on resulting image in /etc/rhubarbe-image
{resa}
""".format(resa=reservation_required)
the_config = Config()
default_timeout = the_config.value('nodes', 'save_default_timeout')
parser = ArgumentParser(usage=usage)
parser.add_argument("-o", "--output", action='store', dest='radical',
default=None, required=True,
help="Mandatory radical to name resulting image")
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
parser.add_argument("-c", "--comment", dest='comment', default=None,
help="one-liner comment to insert in "
"/etc/rhubarbe-image together")
parser.add_argument("-n", "--no-reset", dest='reset',
action='store_false', default=True,
help="""use this with a node that is already
running a frisbee image. It won't get reset,
neither before or after the frisbee session""")
parser.add_argument("node")
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = Selector()
selector.add_range(args.node)
# in case there was one argument but it was not found in inventory
if selector.how_many() != 1:
parser.print_help()
cmc_name = next(selector.cmc_names())
node = Node(cmc_name, message_bus)
nodename = node.control_hostname()
the_imagesrepo = ImagesRepo()
actual_image = the_imagesrepo.where_to_save(nodename, args.radical)
message_bus.put_nowait({'info': "Saving image {}".format(actual_image)})
# curses has no interest here since we focus on one node
display_class = Display
display = display_class([node], message_bus)
saver = ImageSaver(node, image=actual_image, radical=args.radical,
message_bus=message_bus, display=display,
comment=args.comment)
return saver.main(reset=args.reset, timeout=args.timeout)
def __init__(self):
the_config = Config()
self.public = Path(the_config.value('frisbee', 'images_dir'))
self.default_name = the_config.value('frisbee', 'default_image')
# for building hostname patterns
regularname = the_config.value('testbed', 'regularname')
self._radical_re_matcher = re.compile(
SEP.join([
f"{SAVING}",
f"{regularname}[0-9][0-9]",
f"{DATE_RE_PATTERN}",
f"(?P<radical>.+)",
]))
def config(*argv):
usage = """
Display global configuration
"""
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("sections",
nargs='*',
type=str,
help="config section(s) to display")
args = parser.parse_args(argv)
config = Config()
config.display(args.sections)
return 0
def __init__(
self,
message_bus,
sidecar_url, # pylint: disable=r0913
verbose=False):
self.message_bus = message_bus
self.sidecar_url = sidecar_url
self.verbose = verbose
# websockets
self.reconnectable = \
ReconnectableSidecar(sidecar_url, 'leases')
self.cycle = float(Config().value('monitor', 'cycle_leases'))
self.step = float(Config().value('monitor', 'step_leases'))
async def run(self, multicast_ip, port):
control_ip = self.control_ip
the_config = Config()
client = the_config.value('frisbee', 'client')
hdd = the_config.value('frisbee', 'hard_drive')
self.command = (
f"{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}")
logger.info(f"on {self.control_ip} : running command {self.command}")
await self.feedback('frisbee_status', "starting frisbee client")
retcod = await self.session([self.command])
logger.info(f"frisbee on {self.control_ip} returned {retcod}")
return retcod
async def start(self): # pylint: disable=r0914
"""
Start a frisbeed instance
returns a tuple multicast_group, port_number
"""
the_config = Config()
server = the_config.value('frisbee', 'server')
server_options = the_config.value('frisbee', 'server_options')
local_ip = the_config.local_control_ip()
# in Mibps
bandwidth = self.bandwidth * 2**20
# should use default.ndz if not provided
command_common = [
server, "-i", local_ip, "-W",
str(bandwidth), self.image
]
# add configured extra options
command_common += server_options.split()
nb_attempts = int(the_config.value('networking', 'pattern_size'))
pat_ip = the_config.value('networking', 'pattern_multicast')
pat_port = the_config.value('networking', 'pattern_port')
for i in range(1, nb_attempts + 1):
pat = str(i)
multicast_group = pat_ip.replace('*', pat)
multicast_port = str(
eval( # pylint: disable=w0123
pat_port.replace('*', pat)))
command = command_common + [
"-m",
multicast_group,
"-p",
multicast_port,
]
self.subprocess = await asyncio.create_subprocess_exec(
*command,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT)
await asyncio.sleep(1)
# after such a short time, frisbeed should not have returned yet
# if it has, we try our luck on another couple (ip, port)
command_line = " ".join(command)
if self.subprocess.returncode is None:
self.multicast_group = multicast_group
self.multicast_port = multicast_port
await self.feedback('info', f"started {self}")
return multicast_group, multicast_port
else:
logger.warning(
f"failed to start frisbeed with `{command_line}`"
f" -> {self.subprocess.returncode}")
logger.critical(f"could not start frisbee server !!! on {self.image}")
raise Exception(f"could not start frisbee server !!! on {self.image}")
def bye(*argv):
"""
An alternative implementation of the previous 'all-off' utility
Switch the lights off when you leave
"""
usage = """
Turn off whole testbed
"""
config = Config()
default_timeout = config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-t",
"--timeout",
action='store',
default=default_timeout,
type=float,
help="Specify timeout for each phase")
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args, defaults_to_all=True)
if selector.is_empty():
selector.use_all_scope()
bus = asyncio.Queue()
Action('usrpoff', selector).run(bus, args.timeout)
# keep it simple for now
import time
time.sleep(1)
Action('off', selector).run(bus, args.timeout)
# even simpler
import os
phones_inventory = InventoryPhones()
for phone in phones_inventory.all_phones():
command = (f"ssh -i {phone['gw_key']} "
f"{phone['gw_user']}@{phone['gw_host']} phone-off")
print(command)
os.system(command)
def __init__(self, selector, *, verbose, dry_run, speedy):
# work selector; will remove nodes as they fail
self.selector = selector
self.verbose = verbose
self.dry_run = dry_run
self.speedy = speedy
#
# keep a backup of initial scope for proper cleanup
self.all_names = list(selector.node_names())
# the list of failed nodes together with the reason why
self.failures = {}
#
# from rhubarbe config, retrieve bandwidth and other details
config = Config()
self.bandwidth = int(config.value('networking', 'bandwidth'))
self.backoff = int(config.value('networking', 'ssh_backoff'))
self.load_timeout = float(config.value('nodes',
'load_nightly_timeout'))
self.wait_timeout = float(config.value('nodes',
'wait_nightly_timeout'))
#
# accessories
self.bus = asyncio.Queue()
self.nodes = {Node(x, self.bus) for x in self.selector.cmc_names()}
self.display = NoProgressBarDisplay(self.nodes, self.bus)
async def start(self): # pylint: disable=r0914
"""
Start a collector instance; returns a port_number
"""
the_config = Config()
netcat = the_config.value('frisbee', 'netcat')
local_ip = the_config.local_control_ip()
# should use default.ndz if not provided
# use shell-style as we rather have bash handle the redirection
# we instruct bash to exec nc;
# otherwise when cleaning up we just kill the bash process
# but nc is left lingering behind
# WARNING: it is intended that format contains {port}
# for future formatting
command_format_ubuntu = (
f"exec {netcat} -d -l {local_ip} {{port}} > {self.image}"
f" 2> /dev/null")
command_format_fedora = (
f"exec {netcat} -l {local_ip} {{port}} > {self.image}"
f" 2> /dev/null")
netcat_style = the_config.value('frisbee', 'netcat_style')
if netcat_style not in ('fedora', 'ubuntu'):
message = f"wrong netcat_style {netcat_style}"
print(message)
raise Exception(message)
command_format = (command_format_fedora if netcat_style == 'fedora'
else command_format_ubuntu)
nb_attempts = int(the_config.value('networking', 'pattern_size'))
pat_port = the_config.value('networking', 'pattern_port')
for i in range(1, nb_attempts + 1):
pat = str(i)
port = str(
eval( # pylint: disable=w0123
pat_port.replace('*', pat)))
command = command_format.format(port=port)
self.subprocess = await asyncio.create_subprocess_shell(command)
await asyncio.sleep(1)
# after such a short time, frisbeed should not have returned yet
# if is has, we try our luck on another couple (ip, port)
command_line = command
if self.subprocess.returncode is None:
logger.info(f"collector started: {command_line}")
await self.feedback('info',
f"collector started on {self.image}")
self.port = port
return port
else:
logger.warning(
f"failed to start collector with {command_line}")
logger.critical("Could not find a free port to start collector")
raise Exception("Could not start collector server")
def cmc_verb(verb, resa_policy, *argv):
"""
resa_policy can be either
(*) 'enforce': refuse to send the message if the lease is not there
(*) 'warn': issue a warning when the lease is not there
(*) 'none' - or anything else really: does not check the leases
"""
usage = f"""
Send verb '{verb}' to the CMC interface of selected nodes"""
if resa_policy == 'enforce':
usage += f"\n {RESERVATION_REQUIRED}"
config = Config()
default_timeout = config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-t",
"--timeout",
action='store',
default=default_timeout,
type=float,
help="Specify global timeout for the whole process")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
leases = Leases(message_bus) # pylint: disable=w0621
if resa_policy in ('warn', 'enforce'):
reserved = check_reservation(leases, verbose=False)
if not reserved:
if resa_policy == 'enforce':
return 1
selector = selected_selector(args)
action = Action(verb, selector)
return 0 if action.run(message_bus, args.timeout) else 1
async def run(self, multicast_ip, port):
control_ip = self.control_ip
the_config = Config()
client = the_config.value('frisbee', 'client')
hdd = the_config.value('frisbee', 'hard_drive')
self.command = \
"{client} -i {control_ip} -m {multicast_ip} -p {port} {hdd}".format(**locals())
logger.info("on {} : running command {}".format(self.control_ip, self.command))
await self.feedback('frisbee_status', "starting frisbee client")
EOF = chr(4)
EOL = '\n'
# print out exit status so the parser can catch it and expose it
command = self.command
command = command + "; echo FRISBEE-STATUS=$?"
# make sure the command is sent (EOL) and that the session terminates afterwards (exit + EOF)
command = command + "; exit" + EOL + EOF
self._protocol.stream.write(self._protocol.shell.encode(command))
# wait for telnet to terminate
await self._protocol.waiter_closed
return True
async def start(self):
"""
Start a collector instance; returns a port_number
"""
the_config = Config()
netcat = the_config.value('frisbee', 'netcat')
local_ip = the_config.local_control_ip()
# should use default.ndz if not provided
# use shell-style as we rather have bash handle the redirection
# we instruct bash to exec nc; otherwise when cleaning up we just kill the bash process
# but nc is left lingering behind
# WARNING: it is intended that format contains {port} for future formatting
command_format = "exec {} -d -l {} {{port}} > {} 2> /dev/null"\
.format(netcat, local_ip, self.image)
nb_attempts = int(the_config.value('networking', 'pattern_size'))
pat_port = the_config.value('networking', 'pattern_port')
for i in range(1, nb_attempts+1):
pat = str(i)
port = str(eval(pat_port.replace('*', pat)))
command = command_format.format(port=port)
self.subprocess = \
await asyncio.create_subprocess_shell(command)
await asyncio.sleep(1)
# after such a short time, frisbeed should not have returned yet
# if is has, we try our luck on another couple (ip, port)
command_line = command
if self.subprocess.returncode is None:
logger.info("collector started: {}".format(command_line))
await self.feedback('info', "collector started on {}".format(self.image))
self.port = port
return port
else:
logger.warning("failed to start collector with {}".format(command_line))
logger.critical("Could not find a free port to start collector")
raise Exception("Could not start collector server")
async def start(self):
"""
Start a frisbeed instance
returns a tuple multicast_group, port_number
"""
the_config = Config()
server = the_config.value('frisbee', 'server')
server_options = the_config.value('frisbee', 'server_options')
local_ip = the_config.local_control_ip()
# in Mibps
bandwidth = self.bandwidth * 2**20
# should use default.ndz if not provided
command_common = [
server, "-i", local_ip, "-W", str(bandwidth), self.image
]
# add configured extra options
command_common += server_options.split()
nb_attempts = int(the_config.value('networking', 'pattern_size'))
pat_ip = the_config.value('networking', 'pattern_multicast')
pat_port = the_config.value('networking', 'pattern_port')
for i in range(1, nb_attempts+1):
pat = str(i)
multicast_group = pat_ip.replace('*', pat)
multicast_port = str(eval(pat_port.replace('*', pat)))
command = command_common + [
"-m", multicast_group, "-p", multicast_port,
]
self.subprocess = await asyncio.create_subprocess_exec(
*command,
stdout = asyncio.subprocess.PIPE,
stderr = asyncio.subprocess.STDOUT
)
await asyncio.sleep(1)
# after such a short time, frisbeed should not have returned yet
# if is has, we try our luck on another couple (ip, port)
command_line = " ".join(command)
if self.subprocess.returncode is None:
self.multicast_group = multicast_group
self.multicast_port = multicast_port
await self.feedback('info', "started {}".format(self))
return multicast_group, multicast_port
else:
logger.warning("failed to start frisbeed with {}".format(command_line))
logger.critical("Could not find a free IP multicast address + port to start frisbeed")
raise Exception("Could not start frisbee server")
def __init__(self, control_ip, message_bus):
self.control_ip = control_ip
self.message_bus = message_bus
# config
the_config = Config()
self.port = int(the_config.value('networking', 'telnet_port'))
self.backoff = float(the_config.value('networking', 'telnet_backoff'))
self.connect_timeout = float(
the_config.value('networking', 'telnet_timeout'))
self.connect_minwait = float(
the_config.value('networking', 'telnet_connect_minwait'))
self.connect_maxwait = float(
the_config.value('networking', 'telnet_connect_maxwait'))
# internals
self.running = False
self._reader = None
self._writer = None
def wait(*argv): # pylint: disable=r0914
usage = """
Wait for selected nodes to be reachable by ssh
Returns 0 if all nodes indeed are reachable
"""
# suppress info log messages from asyncssh
asyncssh_set_log_level(logging.WARNING)
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-c",
"--curses",
action='store_true',
default=False,
help="Use curses to provide term-based animation")
parser.add_argument("-t",
"--timeout",
action='store',
default=config.value('nodes', 'wait_default_timeout'),
type=float,
help="Specify global timeout for the whole process")
parser.add_argument("-b",
"--backoff",
action='store',
default=config.value('networking', 'ssh_backoff'),
type=float,
help="Specify backoff average between "
"attempts to ssh connect")
parser.add_argument("-u",
"--user",
default="root",
help="select other username")
# really dont' write anything
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
# --curses implies --verbose otherwise nothing shows up
if args.curses:
args.verbose = True
selector = selected_selector(args)
message_bus = asyncio.Queue()
if args.verbose:
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info(f"wait: backoff is {args.backoff} "
f"and global timeout is {args.timeout}")
nodes = [
Node(cmc_name, message_bus) # pylint: disable=w0621
for cmc_name in selector.cmc_names()
]
sshs = [
SshProxy(node, username=args.user, verbose=args.verbose)
for node in nodes
]
jobs = [Job(ssh.wait_for(args.backoff), critical=True) for ssh in sshs]
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
# have the display class run forever until the other ones are done
scheduler = Scheduler(Job(display.run(), forever=True, critical=True),
*jobs,
timeout=args.timeout,
critical=False)
try:
orchestration = scheduler.run()
if orchestration:
return 0
else:
if args.verbose:
scheduler.debrief()
return 1
except KeyboardInterrupt:
print("rhubarbe-wait : keyboard interrupt - exiting")
# xxx
return 1
finally:
display.epilogue()
if not args.silent:
for ssh in sshs:
print(f"{ssh.node}:ssh {'OK' if ssh.status else 'KO'}")
async def stage1(self):
the_config = Config()
idle = int(the_config.value('nodes', 'idle_after_reset'))
await asyncio.gather(*[node.reboot_on_frisbee(idle) for node in self.nodes])
def __init__(self):
the_config = Config()
self.repo = the_config.value('frisbee', 'images_dir')
self.default_name = the_config.value('frisbee', 'default_image')
# for building hostname patterns
self.regularname = the_config.value('testbed', 'regularname')
async def stage1(self):
the_config = Config()
idle = int(the_config.value('nodes', 'idle_after_reset'))
await self.node.reboot_on_frisbee(idle)
def cmc_verb(verb, check_resa, *argv):
"""
check_resa can be either
(*) enforce: refuse to send the message if the lease is not there
(*) warn: issue a warning when the lease is not there
(*) none: does not check the leases
"""
usage = """
Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb)
if check_resa == 'enforce':
usage += "\n {resa}".format(resa=reservation_required)
the_config = Config()
default_timeout = the_config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage)
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
if check_resa in ('warn', 'enforce'):
reserved = check_reservation(verbose=False)
if not reserved:
if check_resa == 'enforce':
return 1
verb_to_method = {'status': 'get_status',
'on': 'turn_on',
'off': 'turn_off',
'reset': 'do_reset',
'info': 'get_info',
'usrpstatus': 'get_usrpstatus',
'usrpon': 'turn_usrpon',
'usrpoff': 'turn_usrpoff',
}
async def get_and_show_verb(node, verb):
assert verb in verb_to_method
# send the 'verb' method on node
method = getattr(node, verb_to_method[verb])
# bound methods must not be passed the subject !
await method()
result = getattr(node, verb)
result = result if result is not None else "{} N/A".format(verb)
for line in result.split("\n"):
if line:
print("{}:{}".format(node.cmc_name, line))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
jobs = [Job(get_and_show_verb(node, verb)) for node in nodes]
display = Display(nodes, message_bus)
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
if scheduler.orchestrate(timeout=args.timeout):
return 0
else:
print("rhubarbe-{} failed: {}".format(verb, scheduler.why()))
return 1
except KeyboardInterrupt as e:
print("rhubarbe-{} : keyboard interrupt - exiting".format(verb))
return 1
def __init__(self):
the_config = Config()
with open(the_config.value('testbed', 'inventory_nodes_path')) as feed:
self._nodes = json.load(feed)
def monitor(*argv):
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
# xxx hacky
usage = """
Cyclic probe all selected nodes to report real-time status
at a sidecar service over socketIO
"""
the_config = Config()
default_cycle = the_config.value('monitor', 'cycle_status')
parser = ArgumentParser(usage=usage)
parser.add_argument('-c', "--cycle", default=default_cycle, type=float,
help="Delay to wait between 2 probes of each node, default ={}"
.format(default_cycle))
parser.add_argument("-w", "--wlan", dest="report_wlan",
default=False, action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-H", "--sidecar-hostname",
dest="sidecar_hostname", default=None)
parser.add_argument("-P", "--sidecar-port",
dest="sidecar_port", default=None)
parser.add_argument("-d", "--debug", dest="debug",
action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
loop = asyncio.get_event_loop()
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
from rhubarbe.logger import logger
logger.info({'selected_nodes': selector})
monitor = Monitor(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
report_wlan=args.report_wlan,
sidecar_hostname=args.sidecar_hostname,
sidecar_port=args.sidecar_port,
debug=args.debug)
# trap signals so we get a nice message in monitor.log
import signal
import functools
def exiting(signame):
logger.info("Received signal {} - exiting".format(signame))
loop.stop()
exit(1)
for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, signame),
functools.partial(exiting, signame))
async def run():
# run both the core and the log loop in parallel
await asyncio.gather(monitor.run(), monitor.log())
await display.stop()
t1 = util.self_manage(run())
t2 = util.self_manage(display.run())
tasks = asyncio.gather(t1, t2)
wrapper = asyncio.gather(tasks)
try:
loop.run_until_complete(wrapper)
return 0
except KeyboardInterrupt as e:
logger.info("rhubarbe-monitor : keyboard interrupt - exiting")
tasks.cancel()
loop.run_forever()
tasks.exception()
return 1
except asyncio.TimeoutError as e:
logger.info(
"rhubarbe-monitor : timeout expired after {}s".format(args.timeout))
return 1
finally:
loop.close()
def save(*argv):
usage = f"""
Save an image from a node
Mandatory radical needs to be provided with --output
This info, together with nodename and date, is stored
on resulting image in /etc/rhubarbe-image
{RESERVATION_REQUIRED}
"""
config = Config()
config.check_binaries()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-o",
"--output",
action='store',
dest='radical',
default=None,
required=True,
help="Mandatory radical to name resulting image")
parser.add_argument("-t",
"--timeout",
action='store',
default=config.value('nodes', 'save_default_timeout'),
type=float,
help="Specify global timeout for the whole process")
parser.add_argument("-c",
"--comment",
dest='comment',
default=None,
help="one-liner comment to insert in "
"/etc/rhubarbe-image")
parser.add_argument("-n",
"--no-reset",
dest='reset',
action='store_false',
default=True,
help="""use this with a node that is already
running a frisbee image. It won't get reset,
neither before or after the frisbee session""")
parser.add_argument("node")
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = Selector()
selector.add_range(args.node)
# in case there was one argument but it was not found in inventory
if len(selector) != 1:
parser.print_help()
cmc_name = next(selector.cmc_names())
node = Node(cmc_name, message_bus)
nodename = node.control_hostname()
imagesrepo = ImagesRepo()
actual_image = imagesrepo.where_to_save(nodename, args.radical)
message_bus.put_nowait({'info': f"Saving image {actual_image}"})
# curses has no interest here since we focus on one node
display_class = Display
display = display_class([node], message_bus)
saver = ImageSaver(node,
image=actual_image,
radical=args.radical,
message_bus=message_bus,
display=display,
comment=args.comment)
return saver.main(reset=args.reset, timeout=args.timeout)
def __init__(self):
the_config = Config()
with open(the_config.value('testbed', 'inventory_path')) as feed:
self._nodes = json.load(feed)
def __init__(self):
the_config = Config()
self.regularname = the_config.value('testbed', 'regularname')
self.rebootname = the_config.value('testbed', 'rebootname')
self.set = set()
async def stage1(self):
the_config = Config()
idle = int(the_config.value('nodes', 'idle_after_reset'))
await self.node.reboot_on_frisbee(idle)
def use_all_scope(self):
the_config = Config()
self.add_range(the_config.value('testbed', 'all_scope'))
def wait(*argv):
usage = """
Wait for selected nodes to be reachable by ssh
Returns 0 if all nodes indeed are reachable
"""
the_config = Config()
default_timeout = the_config.value('nodes', 'wait_default_timeout')
default_backoff = the_config.value('networking', 'ssh_backoff')
parser = ArgumentParser(usage=usage)
parser.add_argument("-c", "--curses", action='store_true', default=False,
help="Use curses to provide term-based animation")
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
parser.add_argument("-b", "--backoff", action='store',
default=default_backoff, type=float,
help="Specify backoff average between "
"attempts to ssh connect, default={}"
.format(default_backoff))
# really dont' write anything
parser.add_argument("-s", "--silent", action='store_true', default=False)
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
# --curses implies --verbose otherwise nothing shows up
if args.curses:
args.verbose = True
selector = selected_selector(args)
message_bus = asyncio.Queue()
if args.verbose:
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info("wait: backoff is {} and global timeout is {}"
.format(args.backoff, args.timeout))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
sshs = [SshProxy(node, verbose=args.verbose) for node in nodes]
jobs = [Job(ssh.wait_for(args.backoff)) for ssh in sshs]
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
# have the display class run forever until the other ones are done
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
orchestration = scheduler.orchestrate(timeout=args.timeout)
if orchestration:
return 0
else:
if args.verbose:
scheduler.debrief()
return 1
except KeyboardInterrupt as e:
print("rhubarbe-wait : keyboard interrupt - exiting")
# xxx
return 1
finally:
display.epilogue()
if not args.silent:
for ssh in sshs:
print("{}:ssh {}".format(ssh.node,
"OK" if ssh.status else "KO"))
async def stage1(self):
the_config = Config()
idle = int(the_config.value('nodes', 'idle_after_reset'))
await asyncio.gather(
*[node.reboot_on_frisbee(idle) for node in self.nodes])
def load(*argv):
usage = f"""
Load an image on selected nodes in parallel
{RESERVATION_REQUIRED}
"""
config = Config()
config.check_binaries()
imagesrepo = ImagesRepo()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-i",
"--image",
action='store',
default=imagesrepo.default(),
help="Specify image to load")
parser.add_argument("-t",
"--timeout",
action='store',
default=config.value('nodes', 'load_default_timeout'),
type=float,
help="Specify global timeout for the whole process")
parser.add_argument("-b",
"--bandwidth",
action='store',
default=config.value('networking', 'bandwidth'),
type=int,
help="Set bandwidth in Mibps for frisbee uploading")
parser.add_argument("-c",
"--curses",
action='store_true',
default=False,
help="Use curses to provide term-based animation")
# this is more for debugging
parser.add_argument("-n",
"--no-reset",
dest='reset',
action='store_false',
default=True,
help="""use this with nodes that are already
running a frisbee image. They won't get reset,
neither before or after the frisbee session""")
add_selector_arguments(parser)
args = parser.parse_args(argv)
message_bus = asyncio.Queue()
selector = selected_selector(args)
if selector.is_empty():
parser.print_help()
return 1
nodes = [
Node(cmc_name, message_bus) # pylint: disable=w0621
for cmc_name in selector.cmc_names()
]
# send feedback
message_bus.put_nowait({'selected_nodes': selector})
from rhubarbe.logger import logger
logger.info(f"timeout is {args.timeout}s")
logger.info(f"bandwidth is {args.bandwidth} Mibps")
actual_image = imagesrepo.locate_image(args.image, look_in_global=True)
if not actual_image:
print(f"Image file {args.image} not found - emergency exit")
exit(1)
# send feedback
message_bus.put_nowait({'loading_image': actual_image})
display_class = Display if not args.curses else DisplayCurses
display = display_class(nodes, message_bus)
loader = ImageLoader(nodes,
image=actual_image,
bandwidth=args.bandwidth,
message_bus=message_bus,
display=display)
return loader.main(reset=args.reset, timeout=args.timeout)
