def run(self, message_bus, timeout):
"""
send verb to all nodes, waits for max timeout
returns True if all nodes behaved as expected
and False otherwise - including in case of KeyboardInterrupt
"""
nodes = [
Node(cmc_name, message_bus)
for cmc_name in self.selector.cmc_names()
]
jobs = [
Job(self.get_and_show_verb(node, self.verb), critical=True)
for node in nodes
]
display = Display(nodes, message_bus)
scheduler = Scheduler(Job(display.run(), forever=True, critical=True),
*jobs,
timeout=timeout,
critical=False)
try:
if scheduler.run():
return True
else:
scheduler.debrief()
print(f"rhubarbe-{self.verb} failed: {scheduler.why()}")
return False
except KeyboardInterrupt:
print(f"rhubarbe-{self.verb} : keyboard interrupt - exiting")
return False
def monitornodes(*argv): # pylint: disable=r0914
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
from rhubarbe.logger import logger
usage = """
Cyclic probe all selected nodes, and reports
real-time status at a sidecar service over websockets
"""
config = Config()
parser = ArgumentParser(usage=usage,
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',
"--cycle",
default=config.value('monitor', 'cycle_nodes'),
type=float,
help="Delay to wait between 2 probes of each node")
parser.add_argument("-u",
"--sidecar-url",
dest="sidecar_url",
default=Config().value('sidecar', 'url'),
help="url for the sidecar server")
parser.add_argument("-w",
"--wlan",
dest="report_wlan",
default=False,
action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-v", "--verbose", action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
logger.info({'selected_nodes': selector})
monitornodes = MonitorNodes(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
sidecar_url=args.sidecar_url,
report_wlan=args.report_wlan,
verbose=args.verbose)
async def async_main():
# run both the core and the log loop in parallel
await asyncio.gather(monitornodes.run_forever(), display.run())
MonitorLoop("monitornodes").run(async_main(), logger)
return 0
def cmc_verb(verb, check_resa, *argv):
"""
check_resa can be either
(*) enforce: refuse to send the message if the lease is not there
(*) warn: issue a warning when the lease is not there
(*) none: does not check the leases
"""
usage = """
Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb)
if check_resa == 'enforce':
usage += "\n {resa}".format(resa=reservation_required)
the_config = Config()
default_timeout = the_config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage)
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
if check_resa in ('warn', 'enforce'):
reserved = check_reservation(verbose=False)
if not reserved:
if check_resa == 'enforce':
return 1
verb_to_method = {'status': 'get_status',
'on': 'turn_on',
'off': 'turn_off',
'reset': 'do_reset',
'info': 'get_info',
'usrpstatus': 'get_usrpstatus',
'usrpon': 'turn_usrpon',
'usrpoff': 'turn_usrpoff',
}
async def get_and_show_verb(node, verb):
assert verb in verb_to_method
# send the 'verb' method on node
method = getattr(node, verb_to_method[verb])
# bound methods must not be passed the subject !
await method()
result = getattr(node, verb)
result = result if result is not None else "{} N/A".format(verb)
for line in result.split("\n"):
if line:
print("{}:{}".format(node.cmc_name, line))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
jobs = [Job(get_and_show_verb(node, verb)) for node in nodes]
display = Display(nodes, message_bus)
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
if scheduler.orchestrate(timeout=args.timeout):
return 0
else:
print("rhubarbe-{} failed: {}".format(verb, scheduler.why()))
return 1
except KeyboardInterrupt as e:
print("rhubarbe-{} : keyboard interrupt - exiting".format(verb))
return 1
def monitor(*argv):
# xxx hacky - do a side effect in the logger module
import rhubarbe.logger
rhubarbe.logger.logger = rhubarbe.logger.monitor_logger
# xxx hacky
usage = """
Cyclic probe all selected nodes to report real-time status
at a sidecar service over socketIO
"""
the_config = Config()
default_cycle = the_config.value('monitor', 'cycle_status')
parser = ArgumentParser(usage=usage)
parser.add_argument('-c', "--cycle", default=default_cycle, type=float,
help="Delay to wait between 2 probes of each node, default ={}"
.format(default_cycle))
parser.add_argument("-w", "--wlan", dest="report_wlan",
default=False, action='store_true',
help="ask for probing of wlan traffic rates")
parser.add_argument("-H", "--sidecar-hostname",
dest="sidecar_hostname", default=None)
parser.add_argument("-P", "--sidecar-port",
dest="sidecar_port", default=None)
parser.add_argument("-d", "--debug", dest="debug",
action='store_true', default=False)
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
loop = asyncio.get_event_loop()
message_bus = asyncio.Queue()
# xxx having to feed a Display instance with nodes
# at creation time is a nuisance
display = Display([], message_bus)
from rhubarbe.logger import logger
logger.info({'selected_nodes': selector})
monitor = Monitor(selector.cmc_names(),
message_bus=message_bus,
cycle=args.cycle,
report_wlan=args.report_wlan,
sidecar_hostname=args.sidecar_hostname,
sidecar_port=args.sidecar_port,
debug=args.debug)
# trap signals so we get a nice message in monitor.log
import signal
import functools
def exiting(signame):
logger.info("Received signal {} - exiting".format(signame))
loop.stop()
exit(1)
for signame in ('SIGHUP', 'SIGQUIT', 'SIGINT', 'SIGTERM'):
loop.add_signal_handler(getattr(signal, signame),
functools.partial(exiting, signame))
async def run():
# run both the core and the log loop in parallel
await asyncio.gather(monitor.run(), monitor.log())
await display.stop()
t1 = util.self_manage(run())
t2 = util.self_manage(display.run())
tasks = asyncio.gather(t1, t2)
wrapper = asyncio.gather(tasks)
try:
loop.run_until_complete(wrapper)
return 0
except KeyboardInterrupt as e:
logger.info("rhubarbe-monitor : keyboard interrupt - exiting")
tasks.cancel()
loop.run_forever()
tasks.exception()
return 1
except asyncio.TimeoutError as e:
logger.info(
"rhubarbe-monitor : timeout expired after {}s".format(args.timeout))
return 1
finally:
loop.close()
