def all_off(slice, verbose, debug):
"""
expects a slice name, and turns off faraday completely
"""
# what argparse knows as a slice actually is a gateway (user + host)
try:
gwuser, gwhost = slice.split('@')
except:
gwuser, gwhost = slice, "faraday.inria.fr"
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=ColonFormatter(verbose=verbose),
debug=debug)
scheduler = Scheduler(
SshJob(
node=gwnode,
command=Run("rhubarbe", "bye"),
label="turn off",
))
result = scheduler.orchestrate()
if not result:
print("RUN KO : {}".format(scheduler.why()))
sched.debrief()
else:
print("faraday turned off OK")
return 0 if result else 1
def main(self, reset, timeout):
mainjob = Job(self.run(reset))
displayjob = Job(self.display.run(), forever=True)
scheduler = Scheduler (mainjob, displayjob)
try:
ok = scheduler.orchestrate(timeout = timeout)
if not ok:
self.display.set_goodbye("rhubarbe-load failed: {}".format(scheduler.why()))
return 1
return 0 if mainjob.result() else 1
except KeyboardInterrupt as e:
self.display.set_goodbye("rhubarbe-load : keyboard interrupt - exiting")
return 1
finally:
self.frisbeed and self.frisbeed.stop_nowait()
self.nextboot_cleanup()
self.display.epilogue()
def all_off(slice, verbose, debug):
"""
expects a slice name, and turns off faraday completely
"""
# what argparse knows as a slice actually is a gateway (user + host)
try:
gwuser, gwhost = slice.split('@')
except:
gwuser, gwhost = slice, "faraday.inria.fr"
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=ColonFormatter(verbose=verbose),
debug=debug)
scheduler = Scheduler(
Sequence(
SshJob(
node=gwnode,
command=Run("rhubarbe", "leases", "--check"),
label="check we have a current lease",
),
SshJob(
node=gwnode,
command=Run("rhubarbe", "bye"),
label="turn off",
)))
result = scheduler.orchestrate()
if not result:
if check_for_lease.raised_exception():
print(
"slice {} does not appear to hold a valid lease".format(slice))
else:
print("RUN KO : {}".format(scheduler.why()))
sched.debrief()
else:
print("faraday turned off OK")
return 0 if result else 1
def run(
*,
# the pieces to use
slice,
hss,
epc,
enb,
phones,
e3372_ues,
oai_ues,
gnuradios,
e3372_ue_xterms,
oai_ue_xterms,
gnuradio_xterms,
# boolean flags
load_nodes,
skip_reset_usb,
oscillo,
# the images to load
image_gw,
image_enb,
image_oai_ue,
image_e3372_ue,
image_gnuradio,
# miscell
n_rb,
verbose,
dry_run):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slice : s.t like [email protected]
* hss : 04
* epc : 03
* enb : 23
* phones: list of indices of phones to use
* e3372_ues : list of nodes to use as a UE using e3372
* oai_ues : list of nodes to use as a UE using OAI
* gnuradios : list of nodes to load with a gnuradio image
* image_* : the name of the images to load on the various nodes
Plus
* load_nodes: whether to load images or not - in which case
image_gw, image_enb and image_*
are used to tell the image names
* skip_reset_usb : the USRP board will be reset as well unless this is set
"""
# what argparse knows as a slice actually is a gateway (user + host)
gwuser, gwhost = r2lab_parse_slice(slice)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
hostnames = hssname, epcname, enbname = [
r2lab_hostname(x) for x in (hss, epc, enb)
]
optional_ids = e3372_ues + oai_ues + gnuradios + \
e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms
hssnode, epcnode, enbnode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
sched = Scheduler(verbose=verbose)
########## preparation
job_check_for_lease = SshJob(
node=gwnode,
command=["rhubarbe", "leases", "--check"],
label="check we have a current lease",
scheduler=sched,
)
# turn off all nodes
turn_off_command = ["rhubarbe", "off", "-a"]
# except our 3 nodes and the optional ones
turn_off_command += [
"~{}".format(x) for x in [hss, epc, enb] + optional_ids
]
# only do the turn-off thing if load_nodes
if load_nodes:
job_off_nodes = SshJob(
node=gwnode,
# switch off all nodes but the ones we use
command=turn_off_command,
label="turn off unused nodes",
required=job_check_for_lease,
scheduler=sched,
)
# actually run this in the gateway, not on the macphone
# the ssh keys are stored in the gateway and we do not yet have
# the tools to leverage such remote keys
job_stop_phones = [
SshJob(
node=gwnode,
command=RunScript(
# script
find_local_embedded_script("faraday.sh"),
# arguments
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-off",
# options
includes=includes),
label="put phone{} in airplane mode".format(id),
required=job_check_for_lease,
scheduler=sched,
) for id in phones
]
########## prepare the image-loading phase
# this will be a dict of items imagename -> ids
to_load = defaultdict(list)
to_load[image_gw] += [hss, epc]
to_load[image_enb] += [enb]
if e3372_ues:
to_load[image_e3372_ue] += e3372_ues
if e3372_ue_xterms:
to_load[image_e3372_ue] += e3372_ue_xterms
if oai_ues:
to_load[image_oai_ue] += oai_ues
if oai_ue_xterms:
to_load[image_oai_ue] += oai_ue_xterms
if gnuradios:
to_load[image_gnuradio] += gnuradios
if gnuradio_xterms:
to_load[image_gnuradio] += gnuradio_xterms
prep_job_by_node = {}
for image, nodes in to_load.items():
commands = []
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", *nodes))
commands.append(Run("rhubarbe", "load", "-i", image, *nodes))
commands.append(Run("rhubarbe", "usrpon", *nodes))
# always do this
commands.append(Run("rhubarbe", "wait", "-t", 120, *nodes))
job = SshJob(
node=gwnode,
commands=commands,
label="Prepare node(s) {}".format(nodes),
required=job_check_for_lease,
scheduler=sched,
)
for node in nodes:
prep_job_by_node[node] = job
# start services
job_service_hss = SshJob(
node=hssnode,
command=RunScript(find_local_embedded_script("oai-hss.sh"),
"run-hss",
epc,
includes=includes),
label="start HSS service",
required=prep_job_by_node[hss],
scheduler=sched,
)
delay = 15
job_service_epc = SshJob(
node=epcnode,
commands=[
Run("echo giving HSS a headstart {delay}s to warm up; sleep {delay}"
.format(delay=delay)),
RunScript(find_local_embedded_script("oai-epc.sh"),
"run-epc",
hss,
includes=includes),
],
label="start EPC services",
required=prep_job_by_node[epc],
scheduler=sched,
)
########## enodeb
job_warm_enb = SshJob(
node=enbnode,
commands=[
RunScript(find_local_embedded_script("oai-enb.sh"),
"warm-enb",
epc,
n_rb,
not skip_reset_usb,
includes=includes),
],
label="Warm eNB",
required=prep_job_by_node[enb],
scheduler=sched,
)
enb_requirements = (job_warm_enb, job_service_hss, job_service_epc)
# wait for everything to be ready, and add an extra grace delay
grace = 30 if load_nodes else 10
grace_delay = SshJob(
node = LocalNode(formatter=TimeColonFormatter()),
command = "echo Allowing grace of {grace} seconds; sleep {grace}"\
.format(grace=grace),
required = enb_requirements,
scheduler = sched,
)
# start services
job_service_enb = SshJob(
node=enbnode,
# run-enb expects the id of the epc as a parameter
# n_rb means number of resource blocks for DL, set to either 25 or 50.
commands=[
RunScript(find_local_embedded_script("oai-enb.sh"),
"run-enb",
oscillo,
includes=includes,
x11=oscillo),
],
label="start softmodem on eNB",
required=grace_delay,
scheduler=sched,
)
########## run experiment per se
# Manage phone(s)
# this starts at the same time as the eNB, but some
# headstart is needed so that eNB actually is ready to serve
delay = 12
msg = "wait for {delay}s for enodeb to start up"\
.format(delay=delay)
wait_command = "echo {msg}; sleep {delay}".format(msg=msg, delay=delay)
job_start_phones = [
SshJob(
node=gwnode,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-on",
includes=includes),
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-start-app",
includes=includes),
],
label="start Nexus phone and speedtest app",
required=grace_delay,
scheduler=sched,
) for id in phones
]
job_ping_phones_from_epc = [
SshJob(
node=epcnode,
commands=[
Run("sleep 10"),
Run("ping -c 100 -s 100 -i .05 172.16.0.{ip} &> /root/ping-phone"
.format(ip=id + 1)),
],
label="ping Nexus phone from EPC",
critical=False,
required=job_start_phones,
) for id in phones
]
########## xterm nodes
colors = ["wheat", "gray", "white", "darkolivegreen"]
xterms = e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms
for xterm, color in zip(xterms, itertools.cycle(colors)):
xterm_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(xterm),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
SshJob(
node=xterm_node,
command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*"
" -bg {} -geometry 90x10".format(color),
x11=True),
label="xterm on node {}".format(xterm_node.hostname),
required=prep_job_by_node[xterm],
scheduler=sched,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
# # remove dangling requirements - if any - should not be needed but won't hurt either
sched.sanitize()
print(20 * "*", "nodes usage summary")
if load_nodes:
for image, nodes in to_load.items():
for node in nodes:
print("node {node} : {image}".format(node=node, image=image))
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
print(10 * "*", "phones usage summary")
if phones:
for phone in phones:
print("Using phone{phone}".format(phone=phone))
else:
print("No phone involved")
sched.rain_check()
# Update the .dot and .png file for illustration purposes
if verbose or dry_run:
sched.list()
name = "scenario-load" if load_nodes else \
"scenario"
sched.export_as_dotfile("{name}.dot".format(name=name))
os.system("dot -Tpng {name}.dot -o {name}.png".format(name=name))
print("(Over)wrote {name}.png".format(name=name))
if dry_run:
return False
if verbose:
input('OK ? - press control C to abort ? ')
if not sched.orchestrate():
print("RUN KO : {}".format(sched.why()))
sched.debrief()
return False
else:
print("RUN OK")
return True
def cmc_verb(verb, check_resa, *argv):
"""
check_resa can be either
(*) enforce: refuse to send the message if the lease is not there
(*) warn: issue a warning when the lease is not there
(*) none: does not check the leases
"""
usage = """
Send verb '{verb}' to the CMC interface of selected nodes""".format(verb=verb)
if check_resa == 'enforce':
usage += "\n {resa}".format(resa=reservation_required)
the_config = Config()
default_timeout = the_config.value('nodes', 'cmc_default_timeout')
parser = ArgumentParser(usage=usage)
parser.add_argument("-t", "--timeout", action='store',
default=default_timeout, type=float,
help="Specify global timeout for the whole process, default={}"
.format(default_timeout))
add_selector_arguments(parser)
args = parser.parse_args(argv)
selector = selected_selector(args)
message_bus = asyncio.Queue()
if check_resa in ('warn', 'enforce'):
reserved = check_reservation(verbose=False)
if not reserved:
if check_resa == 'enforce':
return 1
verb_to_method = {'status': 'get_status',
'on': 'turn_on',
'off': 'turn_off',
'reset': 'do_reset',
'info': 'get_info',
'usrpstatus': 'get_usrpstatus',
'usrpon': 'turn_usrpon',
'usrpoff': 'turn_usrpoff',
}
async def get_and_show_verb(node, verb):
assert verb in verb_to_method
# send the 'verb' method on node
method = getattr(node, verb_to_method[verb])
# bound methods must not be passed the subject !
await method()
result = getattr(node, verb)
result = result if result is not None else "{} N/A".format(verb)
for line in result.split("\n"):
if line:
print("{}:{}".format(node.cmc_name, line))
nodes = [Node(cmc_name, message_bus) for cmc_name in selector.cmc_names()]
jobs = [Job(get_and_show_verb(node, verb)) for node in nodes]
display = Display(nodes, message_bus)
scheduler = Scheduler(Job(display.run(), forever=True), *jobs)
try:
if scheduler.orchestrate(timeout=args.timeout):
return 0
else:
print("rhubarbe-{} failed: {}".format(verb, scheduler.why()))
return 1
except KeyboardInterrupt as e:
print("rhubarbe-{} : keyboard interrupt - exiting".format(verb))
return 1
def run(slice, hss, epc, enb, extras, load_nodes, image_gw, image_enb,
image_extra, reset_nodes, reset_usrp, spawn_xterms, verbose):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slice : s.t like [email protected]
* hss : 04
* epc : 03
* enb : 23
* extras : a list of ids that will be loaded with the gnuradio image
Plus
* load_nodes: whether to load images or not - in which case
image_gw, image_enb and image_extra
are used to tell the image names
* reset_nodes: if load_nodes is false and reset_nodes is true, the nodes are reset - i.e. rebooted
* otherwise (both False): do nothing
* reset_usrp : if not False, the USRP board won't be reset
* spawn_xterms : if set, starts xterm on all extra nodes
* image_* : the name of the images to load on the various nodes
"""
# what argparse knows as a slice actually is a gateway (user + host)
gwuser, gwhost = parse_slice(slice)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=ColonFormatter(verbose=verbose),
debug=verbose)
hostnames = hssname, epcname, enbname = [
r2lab_hostname(x) for x in (hss, epc, enb)
]
extra_hostnames = [r2lab_hostname(x) for x in extras]
hssnode, epcnode, enbnode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=ColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
extra_nodes = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=ColonFormatter(verbose=verbose),
debug=verbose) for hostname in extra_hostnames
]
########## preparation
job_check_for_lease = SshJob(
node=gwnode,
command=["rhubarbe", "leases", "--check"],
label="check we have a current lease",
)
# turn off all nodes
turn_off_command = ["rhubarbe", "off", "-a"]
# except our 3 nodes and the optional extras
turn_off_command += [
"~{}".format(x) for x in [hss, epc, enb] + extras + [20]
]
job_off_nodes = SshJob(
node=gwnode,
# switch off all nodes but the ones we use
command=turn_off_command,
label="turn off unused nodes",
required=job_check_for_lease,
)
# actually run this in the gateway, not on the mac
# the ssh keys are stored in the gateway and we do not yet have
# the tools to leverage such remote keys
job_stop_phone = SshJob(
node=gwnode,
command=RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-off",
includes=includes),
label="stop phone",
required=job_check_for_lease,
)
jobs_prepare = [job_check_for_lease, job_stop_phone]
# turn off nodes only when --load or --reset is set
if load_nodes or reset_nodes:
jobs_prepare.append(job_off_nodes)
########## infra nodes hss + epc
# prepare nodes
commands = []
if load_nodes:
commands.append(
Run("rhubarbe", "load", "-i", image_gw, hssname, epcname))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", hssname, epcname))
# always do this
commands.append(Run("rhubarbe", "wait", "-t", 120, hssname, epcname))
job_load_infra = SshJob(
node=gwnode,
commands=commands,
label="load and wait HSS and EPC nodes",
required=jobs_prepare,
)
# start services
job_service_hss = SshJob(
node=hssnode,
command=RunScript(locate_local_script("oai-hss.sh"),
"run-hss",
epc,
includes=includes),
label="start HSS service",
required=job_load_infra,
)
msg = "wait for HSS to warm up"
job_service_epc = Sequence(
# let 15 seconds to HSS
Job(
verbose_delay(15, msg),
label=msg,
),
SshJob(
node=epcnode,
command=RunScript(locate_local_script("oai-epc.sh"),
"run-epc",
hss,
includes=includes),
label="start EPC services",
),
required=job_load_infra,
)
jobs_infra = job_load_infra, job_service_hss, job_service_epc
########## enodeb
# prepare node
commands = []
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", enb))
commands.append(Run("rhubarbe", "load", "-i", image_enb, enb))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", enb))
commands.append(Run("rhubarbe", "wait", "-t", "120", enb))
job_load_enb = SshJob(
node=gwnode,
commands=commands,
label="load and wait ENB",
required=jobs_prepare,
)
# start service
msg = "wait for EPC to warm up"
job_service_enb = Sequence(
Job(verbose_delay(15, msg), label=msg),
SshJob(
node=enbnode,
# run-enb expects the id of the epc as a parameter
command=RunScript(locate_local_script("oai-enb.sh"),
"run-enb",
epc,
reset_usrp,
includes=includes),
label="start softmodem on ENB",
),
required=(job_load_enb, job_service_hss, job_service_epc),
)
jobs_enb = job_load_enb, job_service_enb
########## run experiment per se
# the phone
# we need to wait for the USB firmware to be loaded
duration = 30 if reset_usrp is not False else 8
msg = "wait for enodeb firmware to load on USRP".format(duration)
job_wait_enb = Job(verbose_delay(duration, msg),
label=msg,
required=job_service_enb)
job_start_phone = SshJob(
node=gwnode,
commands=[
RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-on",
includes=includes),
RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-start-app",
includes=includes),
],
label="start phone 4g and speedtest app",
required=job_wait_enb,
)
job_ping_phone_from_epc = SshJob(
node=epcnode,
commands=[
Run("sleep 10"),
Run("ping -c 100 -s 100 -i .05 172.16.0.2 &> /root/ping-phone"),
],
label="ping phone from EPC",
critical=False,
required=job_wait_enb,
)
jobs_exp = job_wait_enb, job_start_phone, job_ping_phone_from_epc
########## extra nodes
# ssh -X not yet supported in apssh, so one option is to start them using
# a local process
# xxx to update: The following code kind of works, but it needs to be
# turned off, because the process in question would be killed
# at the end of the Scheduler orchestration (at the end of the run function)
# which is the exact time where it would be useful :)
# however the code for LocalJob appears to work fine, it would be nice to
# move it around - maybe in apssh ?
commands = []
if not extras:
commands.append(Run("echo no extra nodes specified - ignored"))
else:
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", *extra_hostnames))
commands.append(
Run("rhubarbe", "load", "-i", image_extra, *extra_hostnames))
commands.append(
Run("rhubarbe", "wait", "-t", 120, *extra_hostnames))
commands.append(Run("rhubarbe", "usrpon", *extra_hostnames))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", extra_hostnames))
commands.append(Run("rhubarbe", "wait", "-t", "120", *extra_hostnames))
job_load_extras = SshJob(
node=gwnode,
commands=commands,
label="load and wait extra nodes",
required=job_check_for_lease,
)
jobs_extras = [job_load_extras]
colors = ["wheat", "gray", "white"]
if spawn_xterms:
jobs_xterms_extras = [
SshJob(
node=extra_node,
command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*"
" -bg {} -geometry 90x10".format(color),
x11=True),
label="xterm on node {}".format(extra_node.hostname),
required=job_load_extras,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
for extra_node, color in zip(extra_nodes, itertools.cycle(colors))
]
jobs_extras += jobs_xterms_extras
# schedule the load phases only if required
sched = Scheduler(verbose=verbose)
# this is just a way to add a collection of jobs to the scheduler
sched.update(jobs_prepare)
sched.update(jobs_infra)
sched.update(jobs_enb)
sched.update(jobs_exp)
sched.update(jobs_extras)
# remove dangling requirements - if any - should not be needed but won't hurt either
sched.sanitize()
print(40 * "*")
if load_nodes:
print("LOADING IMAGES: (gw->{}, enb->{}, extras->{})".format(
load_nodes, image_gw, image_enb, image_extra))
elif reset_nodes:
print("RESETTING NODES")
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
sched.rain_check()
# Update the .dot and .png file for illustration purposes
if verbose:
sched.list()
name = "scenario-load" if load_nodes else \
"scenario-reset" if reset_nodes else \
"scenario"
sched.export_as_dotfile("{}.dot".format(name))
os.system("dot -Tpng {}.dot -o {}.png".format(name, name))
sched.list()
if not sched.orchestrate():
print("RUN KO : {}".format(sched.why()))
sched.debrief()
return False
else:
print("RUN OK")
return True
