def test_mixed_commands(self):
includes = ["tests/inclusion.sh"]
self.run_one_job(
SshJob(node=self.gateway(),
commands=[
RunScript("tests/needsinclude.sh",
"run1",
includes=includes),
Run("echo +++++; cat /etc/lsb-release; echo +++++"),
RunScript("tests/needsinclude.sh",
"another",
"run",
includes=includes)
],
label='script_commands'))
def test_commands_verbose(self):
dummy_path = "tests/dummy-10"
dummy_file = Path(dummy_path).name
scheduler = Scheduler()
Sequence(SshJob(
node=self.gateway(),
verbose=True,
commands=[
Run("hostname"),
RunScript("tests/script-with-args.sh", "arg1", "arg2"),
RunString("for i in $(seq 3); do echo verbose$i; done"),
Push(localpaths=dummy_path, remotepath="."),
Pull(remotepaths=dummy_file, localpath=dummy_path + ".loop"),
]),
SshJob(node=LocalNode(),
critical=True,
commands=Run("diff {x} {x}.loop".format(x=dummy_path),
verbose=True)),
scheduler=scheduler)
ok = scheduler.run()
ok or scheduler.debrief()
self.assertTrue(ok)
def test_local_script_includes(self):
self.run_one_job(
SshJob(node=self.gateway(),
command=RunScript("tests/needsinclude.sh",
includes=["tests/inclusion.sh"]),
label='script_includes'))
def test_local_script(self):
self.run_one_job(
SshJob(node=self.gateway(),
command=RunScript("tests/script.sh"),
label='script'))
def run(slice, hss, epc, enb, extras, load_nodes, image_gw, image_enb,
image_extra, reset_nodes, reset_usrp, spawn_xterms, verbose):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slice : s.t like [email protected]
* hss : 04
* epc : 03
* enb : 23
* extras : a list of ids that will be loaded with the gnuradio image
Plus
* load_nodes: whether to load images or not - in which case
image_gw, image_enb and image_extra
are used to tell the image names
* reset_nodes: if load_nodes is false and reset_nodes is true, the nodes are reset - i.e. rebooted
* otherwise (both False): do nothing
* reset_usrp : if not False, the USRP board won't be reset
* spawn_xterms : if set, starts xterm on all extra nodes
* image_* : the name of the images to load on the various nodes
"""
# what argparse knows as a slice actually is a gateway (user + host)
gwuser, gwhost = parse_slice(slice)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=ColonFormatter(verbose=verbose),
debug=verbose)
hostnames = hssname, epcname, enbname = [
r2lab_hostname(x) for x in (hss, epc, enb)
]
extra_hostnames = [r2lab_hostname(x) for x in extras]
hssnode, epcnode, enbnode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=ColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
extra_nodes = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=ColonFormatter(verbose=verbose),
debug=verbose) for hostname in extra_hostnames
]
########## preparation
job_check_for_lease = SshJob(
node=gwnode,
command=["rhubarbe", "leases", "--check"],
label="check we have a current lease",
)
# turn off all nodes
turn_off_command = ["rhubarbe", "off", "-a"]
# except our 3 nodes and the optional extras
turn_off_command += [
"~{}".format(x) for x in [hss, epc, enb] + extras + [20]
]
job_off_nodes = SshJob(
node=gwnode,
# switch off all nodes but the ones we use
command=turn_off_command,
label="turn off unused nodes",
required=job_check_for_lease,
)
# actually run this in the gateway, not on the mac
# the ssh keys are stored in the gateway and we do not yet have
# the tools to leverage such remote keys
job_stop_phone = SshJob(
node=gwnode,
command=RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-off",
includes=includes),
label="stop phone",
required=job_check_for_lease,
)
jobs_prepare = [job_check_for_lease, job_stop_phone]
# turn off nodes only when --load or --reset is set
if load_nodes or reset_nodes:
jobs_prepare.append(job_off_nodes)
########## infra nodes hss + epc
# prepare nodes
commands = []
if load_nodes:
commands.append(
Run("rhubarbe", "load", "-i", image_gw, hssname, epcname))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", hssname, epcname))
# always do this
commands.append(Run("rhubarbe", "wait", "-t", 120, hssname, epcname))
job_load_infra = SshJob(
node=gwnode,
commands=commands,
label="load and wait HSS and EPC nodes",
required=jobs_prepare,
)
# start services
job_service_hss = SshJob(
node=hssnode,
command=RunScript(locate_local_script("oai-hss.sh"),
"run-hss",
epc,
includes=includes),
label="start HSS service",
required=job_load_infra,
)
msg = "wait for HSS to warm up"
job_service_epc = Sequence(
# let 15 seconds to HSS
Job(
verbose_delay(15, msg),
label=msg,
),
SshJob(
node=epcnode,
command=RunScript(locate_local_script("oai-epc.sh"),
"run-epc",
hss,
includes=includes),
label="start EPC services",
),
required=job_load_infra,
)
jobs_infra = job_load_infra, job_service_hss, job_service_epc
########## enodeb
# prepare node
commands = []
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", enb))
commands.append(Run("rhubarbe", "load", "-i", image_enb, enb))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", enb))
commands.append(Run("rhubarbe", "wait", "-t", "120", enb))
job_load_enb = SshJob(
node=gwnode,
commands=commands,
label="load and wait ENB",
required=jobs_prepare,
)
# start service
msg = "wait for EPC to warm up"
job_service_enb = Sequence(
Job(verbose_delay(15, msg), label=msg),
SshJob(
node=enbnode,
# run-enb expects the id of the epc as a parameter
command=RunScript(locate_local_script("oai-enb.sh"),
"run-enb",
epc,
reset_usrp,
includes=includes),
label="start softmodem on ENB",
),
required=(job_load_enb, job_service_hss, job_service_epc),
)
jobs_enb = job_load_enb, job_service_enb
########## run experiment per se
# the phone
# we need to wait for the USB firmware to be loaded
duration = 30 if reset_usrp is not False else 8
msg = "wait for enodeb firmware to load on USRP".format(duration)
job_wait_enb = Job(verbose_delay(duration, msg),
label=msg,
required=job_service_enb)
job_start_phone = SshJob(
node=gwnode,
commands=[
RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-on",
includes=includes),
RunScript(locate_local_script("faraday.sh"),
"macphone",
"r2lab/infra/user-env/macphone.sh",
"phone-start-app",
includes=includes),
],
label="start phone 4g and speedtest app",
required=job_wait_enb,
)
job_ping_phone_from_epc = SshJob(
node=epcnode,
commands=[
Run("sleep 10"),
Run("ping -c 100 -s 100 -i .05 172.16.0.2 &> /root/ping-phone"),
],
label="ping phone from EPC",
critical=False,
required=job_wait_enb,
)
jobs_exp = job_wait_enb, job_start_phone, job_ping_phone_from_epc
########## extra nodes
# ssh -X not yet supported in apssh, so one option is to start them using
# a local process
# xxx to update: The following code kind of works, but it needs to be
# turned off, because the process in question would be killed
# at the end of the Scheduler orchestration (at the end of the run function)
# which is the exact time where it would be useful :)
# however the code for LocalJob appears to work fine, it would be nice to
# move it around - maybe in apssh ?
commands = []
if not extras:
commands.append(Run("echo no extra nodes specified - ignored"))
else:
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", *extra_hostnames))
commands.append(
Run("rhubarbe", "load", "-i", image_extra, *extra_hostnames))
commands.append(
Run("rhubarbe", "wait", "-t", 120, *extra_hostnames))
commands.append(Run("rhubarbe", "usrpon", *extra_hostnames))
elif reset_nodes:
commands.append(Run("rhubarbe", "reset", extra_hostnames))
commands.append(Run("rhubarbe", "wait", "-t", "120", *extra_hostnames))
job_load_extras = SshJob(
node=gwnode,
commands=commands,
label="load and wait extra nodes",
required=job_check_for_lease,
)
jobs_extras = [job_load_extras]
colors = ["wheat", "gray", "white"]
if spawn_xterms:
jobs_xterms_extras = [
SshJob(
node=extra_node,
command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*"
" -bg {} -geometry 90x10".format(color),
x11=True),
label="xterm on node {}".format(extra_node.hostname),
required=job_load_extras,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
for extra_node, color in zip(extra_nodes, itertools.cycle(colors))
]
jobs_extras += jobs_xterms_extras
# schedule the load phases only if required
sched = Scheduler(verbose=verbose)
# this is just a way to add a collection of jobs to the scheduler
sched.update(jobs_prepare)
sched.update(jobs_infra)
sched.update(jobs_enb)
sched.update(jobs_exp)
sched.update(jobs_extras)
# remove dangling requirements - if any - should not be needed but won't hurt either
sched.sanitize()
print(40 * "*")
if load_nodes:
print("LOADING IMAGES: (gw->{}, enb->{}, extras->{})".format(
load_nodes, image_gw, image_enb, image_extra))
elif reset_nodes:
print("RESETTING NODES")
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
sched.rain_check()
# Update the .dot and .png file for illustration purposes
if verbose:
sched.list()
name = "scenario-load" if load_nodes else \
"scenario-reset" if reset_nodes else \
"scenario"
sched.export_as_dotfile("{}.dot".format(name))
os.system("dot -Tpng {}.dot -o {}.png".format(name, name))
sched.list()
if not sched.orchestrate():
print("RUN KO : {}".format(sched.why()))
sched.debrief()
return False
else:
print("RUN OK")
return True
def test_graphics1(self):
scheduler = Scheduler(critical=False)
gateway = SshNode(hostname=localhostname(), username=localuser())
Sequence(
SshJob(
node=gateway,
command='hostname',
),
SshJob(node=gateway,
command=[
Run('ls /etc/passwd'),
Run('wc -l /etc/passwd'),
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo with RunString on $(hostname) at $(date)"),
]),
SshJob(node=gateway, commands=[
RunScript("tests/testbasic.sh"),
]),
SshJob(node=gateway,
commands=[
Run('wc -l /etc/passwd'),
RunString(
"#!/usr/bin/env bash\n"
"echo with RunsString on $(hostname) at $(date)",
remote_name="show-host-date"),
RunScript("tests/testbasic.sh"),
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo first arg is $1\n", 10)
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo first arg is $1\n",
10,
remote_name='short-show-args')
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo first arg is $1\n"
"echo second arg is $2\n"
"echo third arg is $3\n"
"echo fourth arg is $4\n", 100, 200, 300, 400)
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo first arg is $1\n"
"echo second arg is $2\n"
"echo third arg is $3\n"
"echo fourth arg is $4\n",
1000,
2000,
3000,
4000,
remote_name='long-show-args')
]),
SshJob(node=gateway,
commands=[
RunString(
"#!/usr/bin/env bash\n"
"echo first arg is $1\n"
"echo second arg is $2\n"
"echo third arg is $3\n"
"echo fourth arg is $4\n",
1000,
2000,
3000,
4000,
remote_name='long-show-args',
label='snip')
]),
SshJob(node=gateway,
commands=[
Run("hostname", label="Run()"),
RunScript("foobar", label="RunScript()"),
RunString("foobar", label="RunString()"),
Push("foobar", remotepath="remote", label="Push()"),
Pull("remote", localpath="foobar", label="Pull()"),
Run("hostname", label=None),
RunScript("foobar", label=[]),
RunString("foobar", label=0),
Push("foobar", remotepath="remote", label={}),
Pull("remote", localpath="foobar", label=""),
]),
scheduler=scheduler,
)
print("NO DETAILS")
scheduler.list()
print("WITH DETAILS")
scheduler.list(details=True)
produce_png(scheduler, "test_graphics1")
ok = scheduler.run()
self.assertFalse(ok)
def one_run(*, protocol, interference,
run_name=default_run_name, slicename=default_slicename,
tx_power, phy_rate, antenna_mask, channel,
load_images=False,
node_ids=DEFAULT_NODE_IDS,
src_ids=DEFAULT_SRC_IDS, dest_ids=DEFAULT_DEST_IDS,
scrambler_id=DEFAULT_SCRAMBLER_ID,
tshark=False, map=False, warmup=False,
route_sampling=False, iperf=False,
verbose_ssh=False, verbose_jobs=False, dry_run=False,
run_number=None):
"""
Performs data acquisition on all nodes with the following settings
Arguments:
tx_power: in dBm, a string like 5, 10 or 14.
Corresponds to the transmission power.
phy_rate: a string among 1, 54. Correspond to the wifi rate.
antenna_mask: a string among 1, 3, 7.
channel: a string like e.g. 1 or 40. Correspond to the channel.
protocol: a string among batman , olsr. Correspond to the protocol
interference : in amplitude percentage, a string like 15 or 20.
Correspond to the power of the noise generated in the spectrum.
Can be either None or "None" to mean no interference.
run_name: the name for a subdirectory where all data will be kept
successive runs should use the same name for further visualization
slicename: the Unix login name (slice name) to enter the gateway
load_images: a boolean specifying whether nodes should be re-imaged first
node_ids: a list of node ids to run the scenario against;
strings or ints are OK;
tshark: a boolean specifying wether we should format/parse the .pcap.
map: a boolean specifying wether we should fetch/parse
the route tables of the nodes.
warmup: a boolean specifying whether we should run a ping before
the experiment to be certain of the stabilisation on the network.
src_ids: a list of nodes from which we will launch the ping from.
strings or ints are OK.
ping_messages : the number of ping packets that will be generated
"""
# set default for the nodes parameter
node_ids = ([int(id) for id in node_ids]
if node_ids is not None else DEFAULT_NODE_IDS)
src_ids = ([int(id) for id in src_ids]
if src_ids is not None else DEFAULT_SRC_IDS)
dest_ids = ([int(id) for id in dest_ids]
if dest_ids is not None else DEFAULT_NODE_IDS)
# all nodes - i.e. including sources and destinations -
# need to run the protocol
node_ids = list(set(node_ids).union(set(src_ids).union(set(dest_ids))))
if interference == "None":
interference = None
# open result dir no matter what
run_root = naming_scheme(
run_name=run_name, protocol=protocol,
interference=interference, autocreate=True)
# fix me trace = run_root / f"trace-{%m-%d-%H-%M}"
ref_time = apssh_time()
trace = run_root / f"trace-{ref_time}"
try:
with trace.open('w') as feed:
def log_line(line):
time_line(line, file=feed)
load_msg = f"{'WITH' if load_images else 'NO'} image loading"
interference_msg = (f"interference={interference} "
f"from scrambler={scrambler_id}")
nodes = " ".join(str(n) for n in node_ids)
srcs = " ".join(str(n) for n in src_ids)
dests = " ".join(str(n) for n in dest_ids)
ping_labels = [
f"PING {s} ➡︎ {d}"
for s in src_ids
# and on the destination
for d in dest_ids
if d != s
]
log_line(f"output in {run_root}")
log_line(f"trace in {trace}")
log_line(f"protocol={protocol}")
log_line(f"{load_msg}")
log_line(f"{interference_msg}")
log_line("----")
log_line(f"Selected nodes : {nodes}")
log_line(f"Sources : {srcs}")
log_line(f"Destinations : {dests}")
for label in ping_labels:
log_line(f"{label}")
log_line("----")
for feature in ('warmup', 'tshark', 'map',
'route_sampling', 'iperf'):
log_line(f"Feature {feature}: {locals()[feature]}")
except Exception as exc:
print(f"Cannot write into {trace} - aborting this run")
print(f"Found exception {type(exc)} - {exc}")
return False
#
# dry-run mode
# just display a one-liner with parameters
#
prelude = "" if not dry_run else "dry_run:"
with trace.open() as feed:
print(f"**************** {ref_time} one_run #{run_number}:")
for line in feed:
print(prelude, line, sep='', end='')
if dry_run:
return True
# the nodes involved
faraday = SshNode(hostname=default_gateway, username=slicename,
formatter=TimeColonFormatter(), verbose=verbose_ssh)
# this is a python dictionary that allows to retrieve a node object
# from an id
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id), username="******",
formatter=TimeColonFormatter(), verbose=verbose_ssh)
for id in node_ids
}
# extracts for sources and destinations
src_index = {id:node for (id, node) in node_index.items()
if id in src_ids}
dest_index = {id:node for (id, node) in node_index.items()
if id in dest_ids}
if interference:
node_scrambler = SshNode(
gateway=faraday, hostname=fitname(scrambler_id), username="******",
formatter=TimeColonFormatter(), verbose=verbose_ssh)
# the global scheduler
scheduler = Scheduler(verbose=verbose_jobs)
##########
check_lease = SshJob(
scheduler=scheduler,
node=faraday,
verbose=verbose_jobs,
label="rhubarbe check lease",
command=Run("rhubarbe leases --check", label="rlease"),
)
# load images if requested
green_light = check_lease
# at some point we did not load the scrambler if interference was None
# and that was a way to run faster loads with no interference
# but now we always load the scrambler node with gnuradio
# this is because when we do runs.py -i None 15 30 ...
# then the first call to one_run is with interference being None
# but it is still important to load the scrambler
if load_images:
# copy node_ids
load_ids = node_ids[:]
load_ids.append(scrambler_id)
# the nodes that we **do not** use should be turned off
# so if we have selected e.g. nodes 10 12 and 15, we will do
# rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15
negated_node_ids = [f"~{id}" for id in load_ids]
# we can do these three things in parallel
ready_jobs = [
SshJob(node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
command=Run("rhubarbe", "off", "-a", *negated_node_ids,
label="turn off unused nodes")),
SshJob(node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
label="load batman image",
command=Run("rhubarbe", "load", "-i",
"batman-olsr",
*node_ids,
label=f"load ubuntu on {node_ids}")),
SshJob(
node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
label="load gnuradio image",
command=Run("rhubarbe", "load", "-i",
"batman-olsr-gnuradio",
scrambler_id,
label=f"load gnuradio on {scrambler_id}")),
]
# replace green_light in this case
green_light = SshJob(
node=faraday, required=ready_jobs,
scheduler=scheduler, verbose=verbose_jobs,
label="wait for nodes to come up",
command=Run("rhubarbe", "wait", *load_ids))
##########
# setting up the wireless interface on all nodes
#
# provide node-utilities with the ranges/units it expects
frequency = channel_frequency[int(channel)]
# tx_power_in_mBm not in dBm
tx_power_driver = tx_power * 100
#just in case somme services failed in the previous experiment
reset_failed_services_job = [
SshJob(
node=node,
verbose=verbose_jobs,
label="reset failed services",
command=Run("systemctl reset-failed",
label="reset-failed services"))
for id, node in node_index.items()
]
reset_failed_services = Scheduler(
*reset_failed_services_job,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Reset failed services")
init_wireless_sshjobs = [
SshJob(
node=node,
verbose=verbose_jobs,
label=f"init {id}",
command=RunScript(
"node-utilities.sh",
f"init-ad-hoc-network-{WIRELESS_DRIVER}",
WIRELESS_DRIVER, "foobar", frequency, phy_rate,
antenna_mask, tx_power_driver,
label="init add-hoc network"),
)
for id, node in node_index.items()]
init_wireless_jobs = Scheduler(
*init_wireless_sshjobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Initialisation of wireless chips")
if interference:
# Run uhd_siggen with the chosen power
init_scrambler_job = SshJob(
scheduler=scheduler,
required=green_light,
forever=True,
node=node_scrambler,
verbose=verbose_jobs,
#TODO : If exit-signal patch is done add exit-signal=["TERM"]
# to this run object and call uhd_siggen directly
commands=[RunScript("node-utilities.sh",
"init-scrambler",
label="init scrambler"),
Run(f"systemd-run --unit=uhd_siggen -t ",
f"uhd_siggen -a usrp -f {frequency}M",
f"--sine --amplitude 0.{interference}",
label="systemctl start uhd_siggen")
]
)
green_light = [init_wireless_jobs, reset_failed_services]
# then install and run batman on fit nodes
run_protocol_job = [
SshJob(
# scheduler=scheduler,
node=node,
label=f"init and run {protocol} on fit node {id}",
verbose=verbose_jobs,
# CAREFUL : These ones use sytemd-run
# with the ----service-type=forking option!
command=RunScript("node-utilities.sh",
f"run-{protocol}",
label=f"run {protocol}"),
)
for id, node in node_index.items()]
run_protocol = Scheduler(
*run_protocol_job,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="init and run routing protocols")
green_light = run_protocol
# after that, run tcpdump on fit nodes, this job never ends...
if tshark:
run_tcpdump_job = [
SshJob(
# scheduler=scheduler_monitoring,
node=node,
forever=True,
label=f"run tcpdump on fit node {id}",
verbose=verbose_jobs,
command=[
Run("systemd-run -t --unit=tcpdump",
f"tcpdump -U -i moni-{WIRELESS_DRIVER}",
f"-y ieee802_11_radio -w /tmp/fit{id}.pcap",
label=f"tcpdump {id}")
]
)
for id, node in node_index.items()
]
run_tcpdump = Scheduler(
*run_tcpdump_job,
scheduler=scheduler,
required=green_light,
forever=True,
verbose=verbose_jobs,
label="Monitoring - tcpdumps")
# let the wireless network settle
settle_scheduler = Scheduler(
scheduler=scheduler,
required=green_light,
)
if warmup:
# warmup pings don't need to be sequential, so let's
# do all the nodes at the same time
# on a given node though, we'll ping the other ends sequentially
# see the graph for more
warmup_jobs = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
RunScript("node-utilities.sh",
"my-ping", f"10.0.0.{d}",
warmup_ping_timeout,
warmup_ping_interval,
warmup_ping_size,
warmup_ping_messages,
f"warmup {s} ➡︎ {d}",
label=f"warmup {s} ➡︎ {d}")
for d in dest_index.keys()
if s != d
]
)
# for each selected experiment nodes
for s, node_s in src_index.items()
]
warmup_scheduler = Scheduler(
*warmup_jobs,
scheduler=settle_scheduler,
verbose=verbose_jobs,
label="Warmup pings")
settle_wireless_job2 = PrintJob(
"Let the wireless network settle after warmup",
sleep=settle_delay_shorter,
scheduler=settle_scheduler,
required=warmup_scheduler,
label=f"settling-warmup for {settle_delay_shorter} sec")
# this is a little cheating; could have gone before the bloc above
# but produces a nicer graphical output
# we might want to help asynciojobs if it offered a means
# to specify entry and exit jobs in a scheduler
settle_wireless_job = PrintJob(
"Let the wireless network settle",
sleep=settle_delay_long,
scheduler=settle_scheduler,
label=f"settling for {settle_delay_long} sec")
green_light = settle_scheduler
if iperf:
iperf_service_jobs = [
SshJob(
node=node_d,
verbose=verbose_jobs,
forever=True,
commands=[
Run("systemd-run -t --unit=iperf",
"iperf -s -p 1234 -u",
label=f"iperf serv on {d}"),
],
)
for d, node_d in dest_index.items()
]
iperf_serv_sched = Scheduler(
*iperf_service_jobs,
verbose=verbose_jobs,
label="Iperf Servers",
# for a nicer graphical output
# otherwise the exit arrow
# from scheduler 'iperf mode'
# to job 'settling for 60s'
# gets to start from this box
forever=True,
)
iperf_cli = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
Run("sleep 7", label=""),
Run(f"iperf",
f"-c 10.0.0.{d} -p 1234",
f"-u -b {phy_rate}M -t 60",
f"-l 1024 > IPERF-{s:02d}-{d:02d}",
label=f"run iperf {s} ➡︎ {d}")
]
)
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
iperf_cli_sched = Scheduler(
Sequence(*iperf_cli),
verbose=verbose_jobs,
label="Iperf Clients")
iperf_stop = [
SshJob(node=node_d,
verbose=verbose_jobs,
label=f"Stop iperf on {d}",
command=Run("systemctl stop iperf"))
for d, node_d in dest_index.items()
]
iperf_stop_sched = Scheduler(
*iperf_stop,
required=iperf_cli_sched,
verbose=verbose_jobs,
label="Iperf server stop")
iperf_fetch = [
SshJob(node=node_s,
verbose=verbose_jobs,
command=Pull(
remotepaths=[f"IPERF-{s:02d}-{d:02d}"],
localpath=str(run_root),
label="fetch iperf {s} ➡︎ {d}")
)
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
iperf_fetch_sched = Scheduler(
*iperf_fetch,
required=iperf_stop_sched,
verbose=verbose_jobs,
label="Iperf fetch report")
iperf_jobs = [iperf_serv_sched, iperf_cli_sched,
iperf_stop_sched, iperf_fetch_sched]
iperf_sched = Scheduler(
*iperf_jobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Iperf Module")
settle_wireless_job_iperf = PrintJob(
"Let the wireless network settle",
sleep=settle_delay_shorter,
scheduler=scheduler,
required=iperf_sched,
label=f"settling-iperf for {settle_delay_shorter} sec")
green_light = settle_wireless_job_iperf
# create all the tracepath jobs from the first node in the list
if map:
map_jobs = [
SshJob(
node=node,
label=f"Generating ROUTE file for proto {protocol} on node {id}",
verbose=verbose_jobs,
commands=[
RunScript(f"node-utilities.sh",
f"route-{protocol}",
f"> ROUTE-TABLE-{id:02d}",
label="get route table"),
Pull(remotepaths=[f"ROUTE-TABLE-{id:02d}"],
localpath=str(run_root),
label="")
],
)
for id, node in node_index.items()
]
map_scheduler = Scheduler(
*map_jobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Snapshoting route files")
green_light = map_scheduler
if route_sampling:
route_sampling_jobs = [
SshJob(
node=node,
label=f"Route sampling service for proto {protocol} on node {id}",
verbose=False,
forever=True,
commands=[
Push(localpaths=["route-sample-service.sh"],
remotepath=".", label=""),
Run("chmod +x route-sample-service.sh", label=""),
Run("systemd-run -t --unit=route-sample",
"/root/route-sample-service.sh",
"route-sample",
f"ROUTE-TABLE-{id:02d}-SAMPLED",
protocol,
label="start route-sampling"),
],
)
for id, node in node_index.items()
]
route_sampling_scheduler = Scheduler(
*route_sampling_jobs,
scheduler=scheduler,
verbose=False,
forever=True,
label="Route Sampling services launch",
required=green_light)
##########
# create all the ping jobs, i.e. max*(max-1)/2
# this again is a python list comprehension
# see the 2 for instructions at the bottom
#
# notice that these SshJob instances are not yet added
# to the scheduler, we will add them later on
# depending on the sequential/parallel strategy
pings_job = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
Run(f"echo actual ping {s} ➡︎ {d} using {protocol}",
label=f"ping {s} ➡︎ {d}"),
RunScript("node-utilities.sh", "my-ping",
f"10.0.0.{d}",
ping_timeout, ping_interval,
ping_size, ping_messages,
f"actual {s} ➡︎ {d}",
">", f"PING-{s:02d}-{d:02d}",
label=""),
Pull(remotepaths=[f"PING-{s:02d}-{d:02d}"],
localpath=str(run_root),
label=""),
],
)
# for each selected experiment nodes
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
pings = Scheduler(
scheduler=scheduler,
label="PINGS",
verbose=verbose_jobs,
required=green_light)
# retrieve all pcap files from fit nodes
stop_protocol_job = [
SshJob(
# scheduler=scheduler,
node=node,
# required=pings,
label=f"kill routing protocol on {id}",
verbose=verbose_jobs,
command=RunScript(f"node-utilities.sh",
f"kill-{protocol}",
label=f"kill-{protocol}"),
)
for id, node in node_index.items()
]
stop_protocol = Scheduler(
*stop_protocol_job,
scheduler=scheduler,
required=pings,
label="Stop routing protocols",
)
if tshark:
retrieve_tcpdump_job = [
SshJob(
# scheduler=scheduler,
node=nodei,
# required=pings,
label=f"retrieve pcap trace from fit{i:02d}",
verbose=verbose_jobs,
commands=[
Run("systemctl stop tcpdump",
label="stop tcpdump"),
#Run("systemctl reset-failed tcpdump"),
#RunScript("node-utilities.sh", "kill-tcpdump",
# label="kill-tcpdump"),
Run(
f"echo retrieving pcap trace and result-{i}.txt from fit{i:02d}",
label=""),
Pull(remotepaths=[f"/tmp/fit{i}.pcap"],
localpath=str(run_root), label=""),
],
)
for i, nodei in node_index.items()
]
retrieve_tcpdump = Scheduler(
*retrieve_tcpdump_job,
scheduler=scheduler,
required=pings,
label="Retrieve tcpdump",
)
if route_sampling:
retrieve_sampling_job = [
SshJob(
# scheduler=scheduler,
node=nodei,
# required=pings,
label=f"retrieve sampling trace from fit{i:02d}",
verbose=verbose_jobs,
commands=[
# RunScript("node-utilities.sh", "kill-route-sample", protocol,
# label = "kill route sample"),
#RunScript("route-sample-service.sh", "kill-route-sample",
# label="kill route sample"),
Run("systemctl stop route-sample",
label="stop route-sample"),
Run(
f"echo retrieving sampling trace from fit{i:02d}",
label=""),
Pull(remotepaths=[f"ROUTE-TABLE-{i:02d}-SAMPLED"],
localpath=str(run_root), label=""),
],
)
for i, nodei in node_index.items()
]
retrieve_sampling = Scheduler(
*retrieve_sampling_job,
scheduler=scheduler,
required=pings,
verbose=verbose_jobs,
label="Stop & retrieve route sampling",
)
if tshark:
parse_pcaps_job = [
SshJob(
# scheduler=scheduler,
node=LocalNode(),
# required=retrieve_tcpdump,
label=f"parse pcap trace {run_root}/fit{i}.pcap",
verbose=verbose_jobs,
#commands = [RunScript("parsepcap.sh", run_root, i)]
command=Run("tshark", "-2", "-r",
f"{run_root}/fit{i}.pcap",
"-R",
f"'(ip.dst==10.0.0.{i} && icmp) && radiotap.dbm_antsignal'",
"-Tfields",
"-e", "'ip.src'",
"-e" "'ip.dst'",
"-e", "'radiotap.dbm_antsignal'",
">", f"{run_root}/result-{i}.txt",
label=f"parsing pcap from {i}"),
)
for i in node_ids
]
parse_pcaps = Scheduler(
*parse_pcaps_job,
scheduler=scheduler,
required=retrieve_tcpdump,
label="Parse pcap",
)
if interference:
kill_uhd_siggen = SshJob(
scheduler=scheduler,
node=node_scrambler,
required=pings,
label=f"killing uhd_siggen on the scrambler node {scrambler_id}",
verbose=verbose_jobs,
commands=[Run("systemctl", "stop", "uhd_siggen"),
#Run("systemctl reset-failed tcpdump"),
],
)
kill_2_uhd_siggen = SshJob(
scheduler=scheduler,
node=faraday,
required=kill_uhd_siggen,
label=f"turning off usrp on the scrambler node {scrambler_id}",
verbose=verbose_jobs,
command=Run("rhubarbe", "usrpoff", scrambler_id),
)
pings.add(Sequence(*pings_job))
# for running sequentially we impose no limit on the scheduler
# that will be limitied anyways by the very structure
# of the required graph
# safety check
scheduler.export_as_pngfile(run_root / "experiment-graph")
if dry_run:
scheduler.list()
return True
# if not in dry-run mode, let's proceed to the actual experiment
ok = scheduler.run() # jobs_window=jobs_window)
# close all ssh connections
close_ssh_in_scheduler(scheduler)
# give details if it failed
if not ok:
scheduler.debrief()
scheduler.export_as_pngfile("debug")
if ok and map:
time_line("Creation of MAP files")
post_processor = ProcessRoutes(run_root, src_ids, node_ids)
post_processor.run()
if ok and route_sampling:
time_line("Creation of ROUTE SAMPLING files")
post_processor = ProcessRoutes(run_root, src_ids, node_ids)
post_processor.run_sampled()
# data acquisition is done, let's aggregate results
# i.e. compute averages
#if ok and tshark:
#post_processor = Aggregator(run_root, node_ids, antenna_mask)
#post_processor.run()
time_line("one_run done")
return ok
def run(
*,
# the pieces to use
slice,
hss,
epc,
enb,
phones,
e3372_ues,
oai_ues,
gnuradios,
e3372_ue_xterms,
oai_ue_xterms,
gnuradio_xterms,
# boolean flags
load_nodes,
skip_reset_usb,
oscillo,
# the images to load
image_gw,
image_enb,
image_oai_ue,
image_e3372_ue,
image_gnuradio,
# miscell
n_rb,
verbose,
dry_run):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slice : s.t like [email protected]
* hss : 04
* epc : 03
* enb : 23
* phones: list of indices of phones to use
* e3372_ues : list of nodes to use as a UE using e3372
* oai_ues : list of nodes to use as a UE using OAI
* gnuradios : list of nodes to load with a gnuradio image
* image_* : the name of the images to load on the various nodes
Plus
* load_nodes: whether to load images or not - in which case
image_gw, image_enb and image_*
are used to tell the image names
* skip_reset_usb : the USRP board will be reset as well unless this is set
"""
# what argparse knows as a slice actually is a gateway (user + host)
gwuser, gwhost = r2lab_parse_slice(slice)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
hostnames = hssname, epcname, enbname = [
r2lab_hostname(x) for x in (hss, epc, enb)
]
optional_ids = e3372_ues + oai_ues + gnuradios + \
e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms
hssnode, epcnode, enbnode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
sched = Scheduler(verbose=verbose)
########## preparation
job_check_for_lease = SshJob(
node=gwnode,
command=["rhubarbe", "leases", "--check"],
label="check we have a current lease",
scheduler=sched,
)
# turn off all nodes
turn_off_command = ["rhubarbe", "off", "-a"]
# except our 3 nodes and the optional ones
turn_off_command += [
"~{}".format(x) for x in [hss, epc, enb] + optional_ids
]
# only do the turn-off thing if load_nodes
if load_nodes:
job_off_nodes = SshJob(
node=gwnode,
# switch off all nodes but the ones we use
command=turn_off_command,
label="turn off unused nodes",
required=job_check_for_lease,
scheduler=sched,
)
# actually run this in the gateway, not on the macphone
# the ssh keys are stored in the gateway and we do not yet have
# the tools to leverage such remote keys
job_stop_phones = [
SshJob(
node=gwnode,
command=RunScript(
# script
find_local_embedded_script("faraday.sh"),
# arguments
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-off",
# options
includes=includes),
label="put phone{} in airplane mode".format(id),
required=job_check_for_lease,
scheduler=sched,
) for id in phones
]
########## prepare the image-loading phase
# this will be a dict of items imagename -> ids
to_load = defaultdict(list)
to_load[image_gw] += [hss, epc]
to_load[image_enb] += [enb]
if e3372_ues:
to_load[image_e3372_ue] += e3372_ues
if e3372_ue_xterms:
to_load[image_e3372_ue] += e3372_ue_xterms
if oai_ues:
to_load[image_oai_ue] += oai_ues
if oai_ue_xterms:
to_load[image_oai_ue] += oai_ue_xterms
if gnuradios:
to_load[image_gnuradio] += gnuradios
if gnuradio_xterms:
to_load[image_gnuradio] += gnuradio_xterms
prep_job_by_node = {}
for image, nodes in to_load.items():
commands = []
if load_nodes:
commands.append(Run("rhubarbe", "usrpoff", *nodes))
commands.append(Run("rhubarbe", "load", "-i", image, *nodes))
commands.append(Run("rhubarbe", "usrpon", *nodes))
# always do this
commands.append(Run("rhubarbe", "wait", "-t", 120, *nodes))
job = SshJob(
node=gwnode,
commands=commands,
label="Prepare node(s) {}".format(nodes),
required=job_check_for_lease,
scheduler=sched,
)
for node in nodes:
prep_job_by_node[node] = job
# start services
job_service_hss = SshJob(
node=hssnode,
command=RunScript(find_local_embedded_script("oai-hss.sh"),
"run-hss",
epc,
includes=includes),
label="start HSS service",
required=prep_job_by_node[hss],
scheduler=sched,
)
delay = 15
job_service_epc = SshJob(
node=epcnode,
commands=[
Run("echo giving HSS a headstart {delay}s to warm up; sleep {delay}"
.format(delay=delay)),
RunScript(find_local_embedded_script("oai-epc.sh"),
"run-epc",
hss,
includes=includes),
],
label="start EPC services",
required=prep_job_by_node[epc],
scheduler=sched,
)
########## enodeb
job_warm_enb = SshJob(
node=enbnode,
commands=[
RunScript(find_local_embedded_script("oai-enb.sh"),
"warm-enb",
epc,
n_rb,
not skip_reset_usb,
includes=includes),
],
label="Warm eNB",
required=prep_job_by_node[enb],
scheduler=sched,
)
enb_requirements = (job_warm_enb, job_service_hss, job_service_epc)
# wait for everything to be ready, and add an extra grace delay
grace = 30 if load_nodes else 10
grace_delay = SshJob(
node = LocalNode(formatter=TimeColonFormatter()),
command = "echo Allowing grace of {grace} seconds; sleep {grace}"\
.format(grace=grace),
required = enb_requirements,
scheduler = sched,
)
# start services
job_service_enb = SshJob(
node=enbnode,
# run-enb expects the id of the epc as a parameter
# n_rb means number of resource blocks for DL, set to either 25 or 50.
commands=[
RunScript(find_local_embedded_script("oai-enb.sh"),
"run-enb",
oscillo,
includes=includes,
x11=oscillo),
],
label="start softmodem on eNB",
required=grace_delay,
scheduler=sched,
)
########## run experiment per se
# Manage phone(s)
# this starts at the same time as the eNB, but some
# headstart is needed so that eNB actually is ready to serve
delay = 12
msg = "wait for {delay}s for enodeb to start up"\
.format(delay=delay)
wait_command = "echo {msg}; sleep {delay}".format(msg=msg, delay=delay)
job_start_phones = [
SshJob(
node=gwnode,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-on",
includes=includes),
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(id),
"r2lab-embedded/shell/macphone.sh",
"phone-start-app",
includes=includes),
],
label="start Nexus phone and speedtest app",
required=grace_delay,
scheduler=sched,
) for id in phones
]
job_ping_phones_from_epc = [
SshJob(
node=epcnode,
commands=[
Run("sleep 10"),
Run("ping -c 100 -s 100 -i .05 172.16.0.{ip} &> /root/ping-phone"
.format(ip=id + 1)),
],
label="ping Nexus phone from EPC",
critical=False,
required=job_start_phones,
) for id in phones
]
########## xterm nodes
colors = ["wheat", "gray", "white", "darkolivegreen"]
xterms = e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms
for xterm, color in zip(xterms, itertools.cycle(colors)):
xterm_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(xterm),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
SshJob(
node=xterm_node,
command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*"
" -bg {} -geometry 90x10".format(color),
x11=True),
label="xterm on node {}".format(xterm_node.hostname),
required=prep_job_by_node[xterm],
scheduler=sched,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
# # remove dangling requirements - if any - should not be needed but won't hurt either
sched.sanitize()
print(20 * "*", "nodes usage summary")
if load_nodes:
for image, nodes in to_load.items():
for node in nodes:
print("node {node} : {image}".format(node=node, image=image))
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
print(10 * "*", "phones usage summary")
if phones:
for phone in phones:
print("Using phone{phone}".format(phone=phone))
else:
print("No phone involved")
sched.rain_check()
# Update the .dot and .png file for illustration purposes
if verbose or dry_run:
sched.list()
name = "scenario-load" if load_nodes else \
"scenario"
sched.export_as_dotfile("{name}.dot".format(name=name))
os.system("dot -Tpng {name}.dot -o {name}.png".format(name=name))
print("(Over)wrote {name}.png".format(name=name))
if dry_run:
return False
if verbose:
input('OK ? - press control C to abort ? ')
if not sched.orchestrate():
print("RUN KO : {}".format(sched.why()))
sched.debrief()
return False
else:
print("RUN OK")
return True
def run(self, verbose, no_load, no_save):
"""
can skip the load or save phases
"""
print("Using node {} through gateway {}".format(
self.node, self.gateway))
print("In order to produce {} from {}".format(self.to_image,
self.from_image))
print("The following scripts will be run:")
for i, script in enumerate(self.scripts, 1):
print("{:03d}:{}".format(i, " ".join(script)))
items = []
if no_load: items.append("skip load")
if no_save: items.append("skip save")
if items:
print("WARNING: using fast-track mode {}".format(
' & '.join(items)))
self.locate_companion_shell()
if verbose:
print("Located companion in {}".format(self.companion))
if verbose:
print("Preparing tar of input shell scripts .. ", end="")
tarfile = self.prepare_tar(self.to_image)
if verbose:
print("Done in {}".format(tarfile))
keys = load_agent_keys()
if verbose:
print("We have found {} keys in the ssh agent".format(len(keys)))
#################### the 2 nodes we need to talk to
gateway_proxy = None
gwuser, gwname = self.user_host(self.gateway)
gateway_proxy = None if not gwuser else SshNode(
hostname=gwname,
username=gwuser,
keys=keys,
formatter=ColonFormatter(verbose=verbose),
)
# really not sure it makes sense to use username other than root
username, nodename = self.user_host(self.node)
node_proxy = SshNode(
gateway=gateway_proxy,
hostname=nodename,
username=username,
keys=keys,
formatter=ColonFormatter(verbose=verbose),
)
banner = 20 * '='
# now that node_proxy is initialized, we need to
# have a valid gateway_proxy for when we run all this from inside
# the gateway
if gateway_proxy is None:
print("WARNING: build-image is designed to be run on your laptop")
# best-effort, not even tested....
gateway_proxy = LocalNode()
#################### the little pieces
sequence = Sequence(
PrintJob("Checking for a valid lease"),
# bail out if we don't have a valid lease
SshJob(node = gateway_proxy,
command = "rhubarbe leases --check",
critical = True),
PrintJob("loading image {}".format(self.from_image)
if not no_load else "fast-track: skipping image load",
banner = banner,
# label = "welcome message",
),
SshJob(
node = gateway_proxy,
commands = [
Run("rhubarbe", "load", "-i", self.from_image, nodename) \
if not no_load else None,
Run("rhubarbe", "wait", "-v", "-t", "240", nodename),
],
# label = "load and wait image {}".format(self.from_image),
),
SshJob(
node = node_proxy,
commands = [
Run("rm", "-rf", "/etc/rhubarbe-history/{}".format(self.to_image)),
Run("mkdir", "-p", "/etc/rhubarbe-history"),
Push(localpaths = tarfile,
remotepath = "/etc/rhubarbe-history"),
RunScript(self.companion, nodename, self.from_image, self.to_image),
Pull(localpath = "{}/logs/".format(self.to_image),
remotepaths = "/etc/rhubarbe-history/{}/logs/".format(self.to_image),
recurse = True),
],
label = "set up and run scripts in /etc/rhubarbe-history/{}".format(self.to_image)),
)
# avoid creating an SshJob with void commands
if self.extra_logs:
sequence.append(
SshJob(
node=node_proxy,
label="collecting extra logs",
critical=False,
commands=[
Pull(localpath="{}/logs/".format(self.to_image),
remotepaths=extra_log,
recurse=True) for extra_log in self.extra_logs
],
))
# creating these as critical = True means the whole
# scenario will fail if these are not found
for binary in self.expected_binaries:
check_with = "ls" if os.path.isabs(binary) else ("type -p")
sequence.append(
Sequence(
PrintJob(
"Checking for expected binaries",
# label = "message about checking"
),
SshJob(
node=node_proxy,
command=[check_with, binary],
# label = "Checking for {}".format(binary)
)))
# xxx some flag
if no_save:
sequence.append(
PrintJob("fast-track: skipping image save", banner=banner))
else:
sequence.append(
Sequence(
PrintJob("saving image {} ...".format(self.to_image),
banner=banner),
# make sure we capture all the logs and all that
# mostly to test RunString
SshJob(
node=node_proxy,
command=RunString("sync ; sleep $1; sync; sleep $1",
1),
# label = 'sync',
),
SshJob(
node=gateway_proxy,
command=Run("rhubarbe", "save", "-o", self.to_image,
nodename),
# label = "save image {}".format(self.to_image),
),
SshJob(
node=gateway_proxy,
command="rhubarbe images -d",
# label = "list current images",
),
))
sched = Scheduler(sequence, verbose=verbose)
# sanitizing for the cases where some pieces are left out
sched.sanitize()
print(20 * '+', "before run")
sched.list(details=verbose)
print(20 * 'x')
if sched.orchestrate():
if verbose:
print(20 * '+', "after run")
sched.list()
print(20 * 'x')
print("image {} OK".format(self.to_image))
return True
else:
print("Something went wrong with image {}".format(self.to_image))
print(20 * '+', "after run - KO")
sched.debrief()
print(20 * 'x')
return False
def run(
*, # pylint: disable=r0912, r0914, r0915
# the pieces to use
slicename,
cn,
ran,
phones,
e3372_ues,
oai_ues,
gnuradios,
e3372_ue_xterms,
gnuradio_xterms,
# boolean flags
load_nodes,
reset_usb,
oscillo,
# the images to load
image_cn,
image_ran,
image_oai_ue,
image_e3372_ue,
image_gnuradio,
image_T_tracer,
# miscell
n_rb,
nodes_left_alone,
T_tracer,
verbose,
dry_run):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slicename : s.t like [email protected]
* cn : 7
* ran : 23
* phones: list of indices of phones to use
* e3372_ues : list of nodes to use as a UE using e3372
* oai_ues : list of nodes to use as a UE using OAI
* gnuradios : list of nodes to load with a gnuradio image
* T_tracer : list of nodes to load with a tracer image
* image_* : the name of the images to load on the various nodes
Plus
* load_nodes: whether to load images or not - in which case
image_cn, image_ran and image_*
are used to tell the image names
* reset_usb : the USRP board will be reset when this is set
"""
# what argparse knows as a slice actually is about the gateway (user + host)
gwuser, gwhost = r2lab_parse_slice(slicename)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
hostnames = [r2lab_hostname(x) for x in (cn, ran)]
cnnode, rannode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
scheduler = Scheduler(verbose=verbose, label="CORE EXP")
########## prepare the image-loading phase
# focus on the experiment, and use
# prepare_testbed_scheduler later on to prepare testbed
# all we need to do at this point is compute a mapping dict
# image -> list-of-nodes
images_to_load = defaultdict(list)
images_to_load[image_cn] += [cn]
images_to_load[image_ran] += [ran]
if e3372_ues:
images_to_load[image_e3372_ue] += e3372_ues
if e3372_ue_xterms:
images_to_load[image_e3372_ue] += e3372_ue_xterms
if oai_ues:
images_to_load[image_oai_ue] += oai_ues
if gnuradios:
images_to_load[image_gnuradio] += gnuradios
if gnuradio_xterms:
images_to_load[image_gnuradio] += gnuradio_xterms
if T_tracer:
images_to_load[image_T_tracer] += T_tracer
# start core network
job_start_cn = SshJob(
node=cnnode,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"),
"configure",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"),
"start",
includes=INCLUDES),
tcpdump_cn_service.start_command(),
],
label="start CN service",
scheduler=scheduler,
)
# prepare enodeb
reset_option = "-u" if reset_usb else ""
job_warm_ran = SshJob(
node=rannode,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"warm-up",
reset_option,
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"configure -b",
n_rb,
cn,
includes=INCLUDES),
],
label="Configure eNB",
scheduler=scheduler,
)
ran_requirements = [job_start_cn, job_warm_ran]
###
if oai_ues:
# prepare OAI UEs
for ue in oai_ues:
ue_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(ue),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
job_warm_ues = [
SshJob(node=ue_node,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(
find_local_embedded_script("mosaic-oai-ue.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(
find_local_embedded_script("mosaic-oai-ue.sh"),
"warm-up",
reset_option,
includes=INCLUDES),
RunScript(
find_local_embedded_script("mosaic-oai-ue.sh"),
"configure -b",
n_rb,
includes=INCLUDES),
],
label=f"Configure OAI UE on fit{ue}",
scheduler=scheduler)
]
ran_requirements.append(job_warm_ues)
###
if not load_nodes and phones:
job_turn_off_phones = SshJob(
node=gwnode,
commands=[
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{phone} phone-off") for phone in phones
],
scheduler=scheduler,
)
ran_requirements.append(job_turn_off_phones)
# wait for everything to be ready, and add an extra grace delay
grace = 5
grace_delay = PrintJob(
f"Allowing grace of {grace} seconds",
sleep=grace,
required=ran_requirements,
scheduler=scheduler,
label=f"settle for {grace}s",
)
# optionally start T_tracer
if T_tracer:
job_start_T_tracer = SshJob( # pylint: disable=w0612
node=SshNode(gateway=gwnode,
hostname=r2lab_hostname(T_tracer[0]),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose),
commands=[
Run(f"/root/trace {ran}", x11=True),
],
label="start T_tracer service",
required=ran_requirements,
scheduler=scheduler,
)
# ran_requirements.append(job_start_T_tracer)
# start services
graphical_option = "-x" if oscillo else ""
graphical_message = "graphical" if oscillo else "regular"
tracer_option = " -T" if T_tracer else ""
# we use a Python variable for consistency
# although it not used down the road
_job_service_ran = SshJob(
node=rannode,
commands=[
RunScript(
find_local_embedded_script("mosaic-ran.sh"),
"start",
graphical_option,
tracer_option,
includes=INCLUDES,
x11=oscillo,
),
],
label=f"start {graphical_message} softmodem on eNB",
required=grace_delay,
scheduler=scheduler,
)
########## run experiment per se
# Manage phone(s) and OAI UE(s)
# this starts at the same time as the eNB, but some
# headstart is needed so that eNB actually is ready to serve
sleeps = [20, 30]
phone_msgs = [
f"wait for {sleep}s for eNB to start up before waking up phone{id}"
for sleep, id in zip(sleeps, phones)
]
wait_commands = [
f"echo {msg}; sleep {sleep}" for msg, sleep in zip(phone_msgs, sleeps)
]
job_start_phones = [
SshJob(node=gwnode,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{id}",
"r2lab-embedded/shell/macphone.sh",
"phone-on",
includes=INCLUDES),
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{id}",
"r2lab-embedded/shell/macphone.sh",
"phone-start-app",
includes=INCLUDES),
],
label=f"turn off airplace mode on phone {id}",
required=grace_delay,
scheduler=scheduler)
for id, wait_command in zip(phones, wait_commands)
]
if oai_ues:
delay = 25
for ue in oai_ues:
msg = f"wait for {delay}s for eNB to start up before running UE on node fit{ue}"
wait_command = f"echo {msg}; sleep {delay}"
ue_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(ue),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
job_start_ues = [
SshJob(node=ue_node,
commands=[
Run(wait_command),
RunScript(
find_local_embedded_script("mosaic-oai-ue.sh"),
"start",
includes=INCLUDES),
],
label=f"Start OAI UE on fit{ue}",
required=grace_delay,
scheduler=scheduler)
]
delay += 20
for ue in oai_ues:
ue_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(ue),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
msg = f"Wait 60s and then ping faraday gateway from UE on fit{ue}"
_job_ping_gw_from_ue = [
SshJob(node=ue_node,
commands=[
Run(f"echo {msg}; sleep 60"),
Run(f"ping -c 5 -I oip1 faraday.inria.fr"),
],
label=f"ping faraday gateway from UE on fit{ue}",
critical=False,
required=job_start_ues,
scheduler=scheduler)
]
# ditto
_job_ping_phones_from_cn = [
SshJob(
node=cnnode,
commands=[
Run("sleep 20"),
Run(f"ping -c 100 -s 100 -i .05 172.16.0.{id+1} &> /root/ping-phone{id}"
),
],
label=f"ping phone {id} from core network",
critical=False,
required=job_start_phones,
scheduler=scheduler) for id in phones
]
########## xterm nodes
colors = ("wheat", "gray", "white", "darkolivegreen")
xterms = e3372_ue_xterms + gnuradio_xterms
for xterm, color in zip(xterms, cycle(colors)):
xterm_node = SshNode(gateway=gwnode,
hostname=r2lab_hostname(xterm),
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
SshJob(
node=xterm_node,
command=Run(f"xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*",
f" -bg {color} -geometry 90x10",
x11=True),
label=f"xterm on node {xterm_node.hostname}",
scheduler=scheduler,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
# remove dangling requirements - if any
# should not be needed but won't hurt either
scheduler.sanitize()
##########
print(10 * "*", "nodes usage summary")
if load_nodes:
for image, nodes in images_to_load.items():
for node in nodes:
print(f"node {node} : {image}")
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
print(10 * "*", "phones usage summary")
if phones:
for phone in phones:
print(f"Using phone{phone}")
else:
print("No phone involved")
if nodes_left_alone:
print(f"Ignore following fit nodes: {nodes_left_alone}")
# wrap scheduler into global scheduler that prepares the testbed
scheduler = prepare_testbed_scheduler(gwnode, load_nodes, scheduler,
images_to_load, nodes_left_alone)
scheduler.check_cycles()
# Update the .dot and .png file for illustration purposes
name = "mosaic-load" if load_nodes else "mosaic"
print(10 * '*', 'See main scheduler in', scheduler.export_as_pngfile(name))
if verbose:
scheduler.list()
if dry_run:
return True
if verbose:
input('OK ? - press control C to abort ? ')
if not scheduler.orchestrate():
print(f"RUN KO : {scheduler.why()}")
scheduler.debrief()
return False
print("RUN OK")
return True
def one_run(gwhost,
gwuser,
keys,
sendername,
receivername,
packets,
size,
period,
formatter,
verbose=False):
"""
gwhost, gwuser, keys: where to reach the testbed gateway
sendername, receivername : hostnames for the test nodes
packets, size, period : details of the traffic to send
formatter: how to report results
"""
# we keep all 'environment' data for one run in a dedicated subdir
# using this name scheme to store results locally
# xxx inherited from the NEPI version - unused for now
dataname = os.path.join("csi-{}-{}-{}-{}-{}".format(
receivername, sendername, packets, size, period))
# we have reused the shell script from the NEPI version as-is
auxiliary_script = "./angle-measure.sh"
# the proxy to enter faraday
r2lab_gateway = SshNode(
hostname=gwhost,
username=gwuser,
keys=keys,
formatter=formatter,
)
# the sender node
sender = SshNode(
# specifying the gateway attribute means this node will be reached
# through the ssh connection to the gateway
gateway=r2lab_gateway,
# hostname needs to make sense in the context of the gateway; so e.g. 'fit01' is fine
hostname=sendername,
# from the gateway we enter the R2lab nodes as root
username='******',
formatter=formatter,
)
# the receiver node - ditto
receiver = SshNode(
hostname=receivername,
username='******',
gateway=r2lab_gateway,
formatter=formatter,
)
# one initialization job per node
init_sender = SshJob(
# on what node to run the command
node=sender,
# the command to run; being a JobSshScript, the first item in this
# list is expected to be a **LOCAL** script that gets puhed remotely
# before being run
# a simple JobSsh is more suitable to issue standard Unix commands for instance
command=RunScript(auxiliary_script, "init-sender", 64, "HT20"),
# for convenience purposes
label="init-sender")
init_receiver = SshJob(node=receiver,
command=RunScript(auxiliary_script, "init-receiver",
64, "HT20"),
label="init-receiver")
# ditto for actually running the experiment
run_sender = SshJob(node=sender,
command=RunScript(auxiliary_script, "run-sender",
packets, size, period),
label="run-sender")
# run the sender only once both nodes are ready
run_sender.requires(init_sender, init_receiver)
run_receiver = SshJob(node=receiver,
commands=[
RunScript(auxiliary_script, "run-receiver",
packets, size, period),
Pull(remotepaths='rawdata', localpath=dataname),
],
label="run-receiver")
# ditto
run_receiver.requires(init_sender, init_receiver)
# print a one-liner for that receiver, sender couple
summary = "{} ==> {} - {} packets of {} bytes, each {}us"\
.format(sendername, receivername, packets, size, period)
print(10 * '-', summary)
# create an Scheduler object that will orchestrate this scenario
e = Scheduler(init_sender,
init_receiver,
run_sender,
run_receiver,
verbose=verbose)
print(20 * '*', "before run")
e.list(details=verbose)
print(20 * '*')
if e.orchestrate(timeout=3 * 60):
print("========== experiment OK")
else:
print("!!!!!!!!!! orchestration KO")
e.debrief()
# setting up the wireless interface on all nodes
#
# this is a python feature known as a list comprehension
# we just create as many SshJob instances as we have
# (id, SshNode) couples in node_index
# and gather them all in init_wireless_jobs
# they all depend on green_light
init_wireless_jobs = [
SshJob(
scheduler = scheduler,
required = green_light,
node = node,
verbose = verbose_jobs,
label = "init {}".format(id),
command = RunScript(
"B3-wireless.sh", "init-ad-hoc-network",
wireless_driver, "foobar", 2412,
))
for id, node in node_index.items() ]
########## let the wireless network settle
settle_wireless_job = PrintJob(
"Let the wireless network settle",
sleep = settle_delay,
scheduler = scheduler,
required = init_wireless_jobs,
label = "settling")
##########
# create all the ping jobs, i.e. max*(max-1)/2
# this again is a python list comprehension
def run_scenario(slicename=gateway_username, load_images=load_images,
node_ids=node_ids, verbose_mode=verbose_mode,
node_sender=node_sender):
"""
Performs L2BM experimentation
Arguments:
slicename: the Unix login name (slice name) to enter the gateway
load_images: a boolean specifying whether nodes should be re-imaged
first, else nodes will be reset to allow reconfiguration
node_ids: a list of node ids to run the scenario on; strings or ints
are OK;
node_sender: the sender node id, must be part of selected nodes
"""
if node_sender not in node_ids:
print("sender node {} must be part of selected fit nodes {}".format(node_sender, node_ids))
exit(1)
faraday = SshNode(hostname=default_gateway, username=slicename,
verbose=verbose_mode,
formatter=TimeColonFormatter())
node_ovs = SshNode(gateway=faraday, hostname=fitname(node_sender),
username="******",
verbose=verbose_mode,
formatter=TimeColonFormatter())
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id),
username="******",formatter=TimeColonFormatter(),
verbose=verbose_mode)
for id in node_ids
}
receiver_index = dict(node_index)
del receiver_index[node_sender]
fit_sender = fitname(node_sender)
ip_sender = "10.0.0.{}".format(node_sender)
# the global scheduler
scheduler = Scheduler(verbose=verbose_mode)
##########
check_lease = SshJob(
scheduler=scheduler,
node = faraday,
critical = True,
verbose=verbose_mode,
command = Run("rhubarbe leases --check"),
)
if load_images:
green_light = SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose_mode,
commands=[
Run("rhubarbe", "load", "-i", fit_image, *node_ids),
Run("rhubarbe", "wait", *node_ids)
]
)
else:
# reset nodes if images are already loaded
green_light = SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose_mode,
commands=[
Run("rhubarbe", "reset", *node_ids),
Run("rhubarbe", "wait", *node_ids)
]
)
##########
# setting up the wireless interface on all nodes
init_nodes = [
SshJob(
scheduler=scheduler,
required=green_light,
node=node,
critical=True,
verbose=verbose_mode,
label="init fit node {}".format(id),
command=RunScript(
"l2bm-setup.sh", "init-ad-hoc-network",
wireless_driver, ssid, frequency)
) for id, node in node_index.items()
]
# test Wi-Fi ad hoc connectivity between receivers and the sender
ping = [
SshJob(
scheduler=scheduler,
required=init_nodes,
node=node,
verbose=verbose_mode,
label="ping sender from receiver {}".format(id),
command=RunScript(
"l2bm-setup.sh", "my-ping", ip_sender, 20)
) for id, node in receiver_index.items()
]
# Setting up OVS and libfluid on the sender node
ovs_setup = SshJob(
scheduler=scheduler,
required=ping,
node=node_ovs,
critical=True,
verbose=verbose_mode,
command=RunScript("l2bm-setup.sh", "ovs-setup")
)
# we need to wait for OVS and libfluid controller setup
wait_ovs_job = PrintJob(
"Let the OVS and Libfluid settle",
scheduler=scheduler,
required=ping,
sleep=60,
label="settling ovs and libfluid"
)
iperf_sender = SshJob(
scheduler=scheduler,
required = wait_ovs_job,
node = node_ovs,
verbose=verbose_mode,
command = RunScript("l2bm-setup.sh", "iperf_sender")
)
# Run an iperf receiver at each receiving nodes
iperf_receivers = [
SshJob(
scheduler=scheduler,
required=wait_ovs_job,
node=node,
verbose=verbose_mode,
label="run iperf on receiver {}".format(id),
command = RunScript("l2bm-setup.sh", "iperf_receiver")
) for id, node in receiver_index.items()
]
##########
# orchestration scheduler jobs
ok = scheduler.orchestrate()
# give details if it failed
if not ok:
scheduler.debrief()
def main(argv):
if len(argv) == 3:
print("!! Unfinished routines !!")
else:
print("++ Using default settings ++")
###########################
## Local Variables
# platform='multiGPU'
platform='distributed'
gateway_user='******'
gateway_host='gw_host'
node_username='******'
#########################################################
## Distributed Requirements
num_ps = 1
num_workers = 2
#########################################################
gateway = SshNode(
gateway_host,
username=gateway_user
)
##########################################################
elif platform == 'distributed':
## Jetson-TX2 Cluster
hosts = [cluster_ip_host]
#########################################################
## Use the Server node for processing the first satge Data-mining
server = ResourceManager._set_Node(master_host, master_user, gateway,)
############################
# Push the launch file (run_splitpoint)
# With the Parameters Connfiguration on the server
# To execute the First Satege in this host
job_launch_S1 = SshJob(
node = server,
commands = [
## Run the script locate in the laptop
RunScript("run_dataspworkers_mlp.sh", platform, num_ps, num_workers),
Run("echo Split Data DONE"),
],
)
#############################
## A collection of the PS node
ps = []
[ps.append(ResourceManager._set_Node(hosts[i],
node_username, gateway,))
for i in range(num_ps)]
#############################
## A collection of the workers node
workers = []
[workers.append(ResourceManager._set_Node(hosts[num_ps+i],
node_username, gateway,))
for i in range(num_workers)]
#########################################################
## Setting Parameters for the First Stage
FEATURES_NAME = "FULL-W1_x1_x2_x3_x4_x5_x7_x8_Y1"
SANDBOX=str("/data_B/datasets/drg-PACA/healthData/sandbox-"+FEATURES_NAME)
YEAR=str(2008)
## Stage 1
# localdir = "/1_Mining-Stage/"
# SP_Dir_X = str(SANDBOX+localdir+"BPPR-"+FEATURES_NAME+"-"YEAR)
#############################
## Setting parameters for the Second Stage
S_PLOINT = str(3072) #1536)
#SP_ARGV = str(S_PLOINT+"-"+platform)
SP_ARGV = platform+"-"+str(num_workers)
SP2=str(SANDBOX+"/2_Split-Point-"+SP_ARGV+"/")
#############################
## BPPR Directories
dir_train = "/data_training/"
dir_valid = "/data_valid/"
dir_test = "/data_test/"
############################
## Worker data management
worker_healthData = "/opt/diagnosenet/healthData/"
worker_sandbox = str(worker_healthData+"/sandbox-"+FEATURES_NAME)
worker_splitpoint = str(worker_sandbox+"/2_Split-Point-"+SP_ARGV+"/")
worker_train = str(worker_splitpoint+dir_train)
worker_valid = str(worker_splitpoint+dir_valid)
worker_test = str(worker_splitpoint+dir_test)
############################
## Worker commands
mkd_worker_sandbox = str("mkdir"+" "+worker_sandbox)
mkd_worker_splitpoint = str("mkdir"+" "+worker_splitpoint)
mkd_worker_train = str("mkdir"+" "+worker_train)
mkd_worker_valid = str("mkdir"+" "+worker_valid)
mkd_worker_test = str("mkdir"+" "+worker_test)
#############################
## Create a JOB to build the sandbox for each Worker
job_build_sandbox = []
[ job_build_sandbox.append(SshJob(
node = workers[i],
commands = [
RunString(mkd_worker_sandbox),
RunString(mkd_worker_splitpoint),
RunString(mkd_worker_train),
RunString(mkd_worker_valid),
RunString(mkd_worker_test),
Run("echo SANDBOX ON WORKER DONE"), ],
)) for i in range(len(workers)) ]
#############################
## Create a command for transfer data
scp = "scp"
cmd_X_train_transfer = []
cmd_y_train_transfer = []
cmd_X_valid_transfer = []
cmd_y_valid_transfer = []
cmd_X_test_transfer = []
cmd_y_test_transfer = []
for i in range(num_workers):
worker_host = str(node_user+"@"+ hosts[num_ps+i] +":")
num_file = str(i+1)
## Commands to transfer Training dataset
X_train_splitted = str(SP2+dir_train+"X_training-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_X_train_transfer.append(str(scp+" "+X_train_splitted+" "+worker_host+worker_train))
y_train_splitted = str(SP2+dir_train+"y_training-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_y_train_transfer.append(str(scp+" "+y_train_splitted+" "+worker_host+worker_train))
## Commands to transfer Validation dataset
X_valid_splitted = str(SP2+dir_valid+"X_valid-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_X_valid_transfer.append(str(scp+" "+X_valid_splitted+" "+worker_host+worker_valid))
y_valid_splitted = str(SP2+dir_valid+"y_valid-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_y_valid_transfer.append(str(scp+" "+y_valid_splitted+" "+worker_host+worker_valid))
## Commands to transfer Test dataset
X_test_splitted = str(SP2+dir_test+"X_test-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_X_test_transfer.append(str(scp+" "+X_test_splitted+" "+worker_host+worker_test))
y_test_splitted = str(SP2+dir_test+"y_test-"+FEATURES_NAME+"-"+YEAR+"-"+num_file+".txt")
cmd_y_test_transfer.append(str(scp+" "+y_test_splitted+" "+worker_host+worker_test))
############################
## Build a JOB for transfering data to each worker sandbox
job_data_transfer = []
[job_data_transfer.append(SshJob(
node = server,
commands = [
RunString(cmd_X_train_transfer[i]),
RunString(cmd_y_train_transfer[i]),
Run("echo SENDER TRAINING DATA DONE"),
RunString(cmd_X_valid_transfer[i]),
RunString(cmd_y_valid_transfer[i]),
Run("echo SENDER VALID DATA DONE"),
RunString(cmd_X_test_transfer[i]),
RunString(cmd_y_test_transfer[i]),
Run("echo SENDER TEST DATA DONE"),
],)
) for i in range(len(workers))]
#########################################################
## Create a sequence orchestration scheduler instance upfront
worker_seq = []
## Add the Stage-1 JOB into Scheduler
worker_seq.append(Scheduler(Sequence(
job_launch_S1)))
## Add the worker JOBs into Scheduler
[worker_seq.append(Scheduler(Sequence(
job_build_sandbox[i],
job_data_transfer[i], ))
) for i in range(len(workers))]
#############################
## Old method
## Add the JOB PS Replicas into Scheduler
# worker_seq.append(Scheduler(Sequence(
# job_PS_replicas)))
#
# ## Add the JOB WORKER Replicas into Scheduler
# worker_seq.append(Scheduler(Sequence(
# job_WORKER_replicas)))
#############################
## Run the Sequence JOBS
# [seq.orchestrate() for seq in worker_seq]
#########################################################
#########################################################
## Push the launch file (run_secondstage_distributed)
## With the Distributed Parameters for each worker replicas
## To distributed training of Unsupervised Embedding
#############################
## Build a collection of TensorFlow Hosts for PS
tf_ps = []
[tf_ps.append(str(hosts[i]+":2222")) for i in range(num_ps)]
# print("+++ tf_ps: {}".format(tf_ps))
tf_ps=','.join(tf_ps)
#############################
## Build a collection of TensorFlow Hosts for workers
tf_workers = []
[tf_workers.append(str(hosts[num_ps+i]+":2222")) for i in range(num_workers)]
# print("+++ tf_workers: {}".format(tf_workers))
tf_workers=','.join(tf_workers)
job_PS_replicas = []
[job_PS_replicas.append(SshJob(
node = ps[i],
commands = [
## Launches local script to execute on cluster
# RunScript("run_secondstage_distributed.sh",
# platform, tf_ps, tf_workers,
# num_ps, num_workers, "ps", i),
RunScript("run_thirdstage_distributed_mlp.sh",
platform, tf_ps, tf_workers,
num_ps, num_workers, "ps", i),
Run("echo PS REPLICA DONE"),
],)
) for i in range(len(ps))]
job_WORKER_replicas = []
[job_WORKER_replicas.append(SshJob(
node = workers[i],
commands = [
## Launches local script to execute on cluster
# RunScript("run_secondstage_distributed.sh",
# platform, tf_ps, tf_workers,
# num_ps, num_workers, "worker", i),
RunScript("run_thirdstage_distributed_mlp.sh",
platform, tf_ps, tf_workers,
num_ps, num_workers, "worker", i),
Run("echo WORKER REPLICA DONE"),
], )
) for i in range(len(workers))]
#############################
### Simultaneous jobs
s_distraining = Scheduler()
[s_distraining.add(job_PS_replicas[i]) for i in range(len(ps))]
[s_distraining.add(job_WORKER_replicas[i]) for i in range(len(workers))]
s_distraining.run(jobs_window = int(num_ps+num_workers+1))
def prepare_testbed_scheduler( # pylint: disable=r0913, r0914
gateway: SshNode,
load_flag: bool,
experiment_scheduler: Scheduler,
images_mapping,
nodes_left_alone=None,
sdrs_left_alone=None,
phones_left_alone=None,
verbose_jobs=False):
"""
This function is designed as a standard way for experiments to warm up.
Experimenters only need to write a scheduler that defines the behaviour
of their core experiment, this function will add additional steps that
take care of a) checking for a valid lease, b) load images on nodes, and
c) turn off unused devices.
It is generally desirable to write an experiment script that has a
`--load/-l` boolean flag; typically, one would use the ``--load`` flag the
first time that an experiment is launched during a given timeslot, while
subsequent calls won't. That is the purpose of the ``load_flag`` below;
when set to False, only step a) is performed, otherwise the resulting
scheduler will go for the full monty.
Parameters:
gateway_sshnode: the ssh handle to the gateway
load_flag(bool): if not set, only the lease is checked
experiment_scheduler: core scheduler for the experiment
images_mapping: a dictionary that specifies images to be loaded on nodes;
see examples below
nodes_left_alone: a list of node numbers that should be left intact,
neither loaded nor turned off;
phones_left_alone: a list of node numbers that should be left intact,
i.e. not switched to airplane mode.
Return :
The overall scheduler where the input ``experiment_scheduler`` is embedded.
Examples:
Specify a mapping like the following::
images_mapping = { "ubuntu" : [1, 4, 5], "gnuradio": [16]}
Note that the format for ``images_mapping``, is flexible;
if only one node is to be loaded, the iterable level is optional;
also each node can be specified as an ``int``, a ``bytes``, a ``str``,
in which case non numeric characters are ignored. So this is a legitimate
requirement as well::
images_mapping = { 'openair-cn': 12 + 4,
'openair-enodeb': ('fit32',),
'ubuntu': {12, 'reboot1', '004',
'you-get-the-picture-34'}
}
"""
# handle default mutable args
nodes_left_alone = set(nodes_left_alone) if nodes_left_alone else set()
sdrs_left_alone = set(sdrs_left_alone) if sdrs_left_alone else set()
phones_left_alone = set(phones_left_alone) if phones_left_alone else set()
scheduler = Scheduler(label="Preparation")
check_lease = SshJob(
scheduler=scheduler,
node=gateway,
verbose=verbose_jobs,
label="Check lease {}".format(gateway.username),
command=Run("rhubarbe leases --check", label="rlease"),
)
# if no image loading is requested, we're done here
if not load_flag:
scheduler.add(experiment_scheduler)
experiment_scheduler.requires(check_lease)
return scheduler
# otherwise, we want to do in parallel
# (*) as many image-loading jobs as we have entries in images_mapping
# (*) one job to turn off phones, nodes and usrps
# as parallelizing brings no speed up at all
# todo ideally we could also probe the testbed to figure out which nodes
# are currently unavailable, and let them alone as well; but well.
# the jobs that we need to wait for before going on with the real stuff
octopus = []
loaded_nodes = set()
for image, nodes in images_mapping.items():
# let's be as flexible as possible
# (1) empty node list should be fine
if not nodes:
continue
# (2) atomic types should be allowed
if isinstance(nodes, (int, str, bytes)):
nodes = [nodes]
# (3) accept all forms of inputs
nodes = {r2lab_id(node) for node in nodes}
duplicates = loaded_nodes & nodes
if duplicates:
print("WARNING - nodes in {} have been assigned several images".
format(duplicates))
loaded_nodes.update(nodes)
# for there on we need strings
node_args = " ".join(str(node) for node in nodes)
octopus.append(
SshJob(
gateway,
scheduler=scheduler,
required=check_lease,
label=("loading {} on {}".format(image, node_args)),
commands=[
Run("rhubarbe load -i {} {}".format(image, node_args)),
Run("rhubarbe wait {}".format(node_args)),
],
verbose=verbose_jobs,
))
### turn off stuff
# nodes
dont_off_nodes = nodes_left_alone | loaded_nodes
# do turn off usrp device even on loaded nodes
dont_off_sdrs = sdrs_left_alone
# phones - there's no equivalent of --all ~ notation with phones
off_phones = set(range(1, PHONES+1)) \
- {r2lab_id(ph) for ph in phones_left_alone}
r2lab_includes = [
find_local_embedded_script(x) for x in ("faraday.sh", "r2labutils.sh")
]
if off_phones:
octopus.append(
SshJob(gateway,
scheduler=scheduler,
required=check_lease,
critical=False,
commands=[
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(phone),
"r2lab-embedded/shell/macphone.sh",
"phone-off",
label="turn off phone {}".format(phone),
includes=r2lab_includes)
for phone in off_phones
],
verbose=verbose_jobs))
octopus.append(
SshJob(
gateway,
scheduler=scheduler,
required=check_lease,
label="Turn off unused devices",
commands=[
Run(_rhubarbe_command(verb="off", left_alone=dont_off_nodes)),
Run(_rhubarbe_command(verb="usrpoff",
left_alone=dont_off_sdrs)),
],
verbose=verbose_jobs,
))
# embed experiment scheduler
experiment_scheduler.requires(octopus)
scheduler.add(experiment_scheduler)
return scheduler
def test_local_script_args(self):
self.run_one_job(
SshJob(node=self.gateway(),
command=RunScript("tests/script-with-args.sh", "foo", "bar",
"tutu"),
label='script'))
def collect(run_name, slicename, cn, ran, oai_ues, verbose, dry_run):
"""
retrieves all relevant logs under a common name
otherwise, same signature as run() for convenience
retrieved stuff will be made of
* one pcap file for the CN
* compressed tgz files, one per node, gathering logs and configs and datas
* for convenience the tgz files are unwrapped in run_name/id0
"""
# the local dir to store incoming raw files. mostly tar files
local_path = Path(f"{run_name}")
if not local_path.exists():
print(f"Creating directory {local_path}")
local_path.mkdir()
gwuser, gwhost = r2lab_parse_slice(slicename)
gwnode = SshNode(hostname=gwhost, username=gwuser,
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
functions = ["cn", "ran"]
hostnames = [r2lab_hostname(x) for x in (cn, ran)]
node_cn, node_ran = nodes = [
SshNode(gateway=gwnode, hostname=hostname, username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
for hostname in hostnames
]
if oai_ues:
hostnames_ue = [r2lab_hostname(x) for x in oai_ues]
nodes_ue = [
SshNode(gateway=gwnode, hostname=hostname, username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
for hostname in hostnames_ue]
# all nodes involved are managed in the same way
# node: a SshNode instance
# id: the fit number
# function, a string like 'cn' or 'ran' or 'oai-ue'
local_nodedirs_tars = []
scheduler = Scheduler(verbose=verbose)
for (node, id, function) in zip(
chain(nodes, nodes_ue),
chain( [cn, ran], oai_ues),
chain(functions, cycle(["oai-ue"]))):
# nodes on 2 digits
id0 = f"{id:02d}"
# node-dep collect dir
node_dir = local_path / id0
node_dir.exists() or node_dir.mkdir()
local_tar = f"{local_path}/{function}-{id0}.tgz"
SshJob(
node=node,
commands=[
# first run a 'capture-all' function remotely
# to gather all the relevant files and commands remotely
RunScript(
find_local_embedded_script(f"mosaic-{function}.sh"),
f"capture-all", f"{run_name}-{function}",
includes=INCLUDES),
# and retrieve it locally
Pull(
remotepaths=f"{run_name}-{function}.tgz",
localpath=local_tar),
],
scheduler=scheduler)
local_nodedirs_tars.append((node_dir, local_tar))
# retrieve tcpdump on CN
SshJob(
node=node_cn,
commands=[
tcpdump_cn_service.stop_command(),
Pull(remotepaths=[tcpdump_cn_pcap],
localpath=local_path),
],
scheduler=scheduler
)
print(10*'*', 'See collect scheduler in',
scheduler.export_as_pngfile("cefore-collect"))
if verbose:
scheduler.list()
if dry_run:
return
if not scheduler.run():
print("KO")
scheduler.debrief()
return
# unwrap
for node_dir, tar in local_nodedirs_tars:
print(f"Untaring {tar} in {node_dir}/")
os.system(f"tar -C {node_dir} -xzf {tar}")
def run(*, # pylint: disable=r0912, r0914, r0915
# the pieces to use
slicename, cn, ran, phones,
e3372_ues, oai_ues, gnuradios,
e3372_ue_xterms, gnuradio_xterms, ns3,
# boolean flags
load_nodes, reset_usb, oscillo, tcp_streaming,
# the images to load
image_cn, image_ran, image_oai_ue, image_e3372_ue, image_gnuradio, image_T_tracer, image_ns3,
# miscell
n_rb, nodes_left_alone, T_tracer, publisher_ip, verbose, dry_run):
"""
##########
# 3 methods to get nodes ready
# (*) load images
# (*) reset nodes that are known to have the right image
# (*) do nothing, proceed to experiment
expects e.g.
* slicename : s.t like [email protected]
* cn : 7
* ran : 23
* ns3 : 32
* phones: list of indices of phones to use
* e3372_ues : list of nodes to use as a UE using e3372
* oai_ues : list of nodes to use as a UE using OAI
* gnuradios : list of nodes to load with a gnuradio image
* T_tracer : list of nodes to load with a tracer image
* image_* : the name of the images to load on the various nodes
Plus
* load_nodes: whether to load images or not - in which case
image_cn, image_ran and image_*
are used to tell the image names
* reset_usb : the USRP board will be reset when this is set
* tcp_streaming : set up TCP streaming scenario
* publisher_ip : IP address of the publisher
"""
# what argparse knows as a slice actually is about the gateway (user + host)
gwuser, gwhost = r2lab_parse_slice(slicename)
gwnode = SshNode(hostname=gwhost, username=gwuser,
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
hostnames = [r2lab_hostname(x) for x in (cn, ran)]
cnnode, rannode = [
SshNode(gateway=gwnode, hostname=hostname, username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
for hostname in hostnames
]
scheduler = Scheduler(verbose=verbose, label="CORE EXP")
########## prepare the image-loading phase
# focus on the experiment, and use
# prepare_testbed_scheduler later on to prepare testbed
# all we need to do at this point is compute a mapping dict
# image -> list-of-nodes
images_to_load = defaultdict(list)
images_to_load[image_cn] += [cn]
images_to_load[image_ran] += [ran]
if e3372_ues:
images_to_load[image_e3372_ue] += e3372_ues
if e3372_ue_xterms:
images_to_load[image_e3372_ue] += e3372_ue_xterms
if oai_ues:
images_to_load[image_oai_ue] += oai_ues
if gnuradios:
images_to_load[image_gnuradio] += gnuradios
if gnuradio_xterms:
images_to_load[image_gnuradio] += gnuradio_xterms
if T_tracer:
images_to_load[image_T_tracer] += T_tracer
if ns3:
images_to_load[image_ns3] += [ns3]
# start core network
job_start_cn = SshJob(
node=cnnode,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"), "configure",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-cn.sh"), "start",
includes=INCLUDES),
tcpdump_cn_service.start_command(),
],
label="start CN service",
scheduler=scheduler,
)
# prepare enodeb
reset_option = "-u" if reset_usb else ""
job_warm_ran = SshJob(
node=rannode,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"warm-up", reset_option,
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"configure -b", n_rb, cn,
includes=INCLUDES),
],
label="Configure eNB",
scheduler=scheduler,
)
ran_requirements = [job_start_cn, job_warm_ran]
###
if oai_ues:
# prepare OAI UEs
for ue in oai_ues:
ue_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(ue), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
job_warm_ues = [
SshJob(
node=ue_node,
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"git-pull-r2lab",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-oai-ue.sh"),
"journal --vacuum-time=1s",
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-oai-ue.sh"),
"warm-up", reset_option,
includes=INCLUDES),
RunScript(find_local_embedded_script("mosaic-oai-ue.sh"),
"configure -b", n_rb,
includes=INCLUDES),
],
label=f"Configure OAI UE on fit{ue:02d}",
scheduler=scheduler)
]
ran_requirements.append(job_warm_ues)
###
if not load_nodes and phones:
job_turn_off_phones = SshJob(
node=gwnode,
commands=[
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{phone} phone-off")
for phone in phones],
scheduler=scheduler,
)
ran_requirements.append(job_turn_off_phones)
# wait for everything to be ready, and add an extra grace delay
grace = 5
grace_delay = PrintJob(
f"Allowing grace of {grace} seconds",
sleep=grace,
required=ran_requirements,
scheduler=scheduler,
label=f"settle for {grace}s",
)
# optionally start T_tracer
if T_tracer:
job_start_T_tracer = SshJob( # pylint: disable=w0612
node=SshNode(
gateway=gwnode, hostname=r2lab_hostname(T_tracer[0]), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose),
commands=[
Run(f"/root/trace {ran}",
x11=True),
],
label="start T_tracer service",
required=ran_requirements,
scheduler=scheduler,
)
# ran_requirements.append(job_start_T_tracer)
# start services
graphical_option = "-x" if oscillo else ""
graphical_message = "graphical" if oscillo else "regular"
tracer_option = " -T" if T_tracer else ""
# we use a Python variable for consistency
# although it not used down the road
_job_service_ran = SshJob(
node=rannode,
commands=[
RunScript(find_local_embedded_script("mosaic-ran.sh"),
"start", graphical_option, tracer_option,
includes=INCLUDES,
x11=oscillo,
),
],
label=f"start {graphical_message} softmodem on eNB",
required=grace_delay,
scheduler=scheduler,
)
########## run experiment per se
# Manage phone(s) and OAI UE(s)
# this starts at the same time as the eNB, but some
# headstart is needed so that eNB actually is ready to serve
sleeps = [20, 30]
phone_msgs = [f"wait for {sleep}s for eNB to start up before waking up phone{id}"
for sleep, id in zip(sleeps, phones)]
wait_commands = [f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone_msgs, sleeps)]
job_start_phones = [
SshJob(
node=gwnode,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-on",
includes=INCLUDES),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-start-app",
includes=INCLUDES),
],
label=f"turn off airplace mode on phone {id}",
required=grace_delay,
scheduler=scheduler)
for id, wait_command in zip(phones, wait_commands)]
if oai_ues:
delay = 25
for ue in oai_ues:
msg = f"wait for {delay}s for eNB to start up before running UE on node fit{ue:02d}"
wait_command = f"echo {msg}; sleep {delay}"
ue_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(ue), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
job_start_ues = [
SshJob(
node=ue_node,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("mosaic-oai-ue.sh"),
"start",
includes=INCLUDES),
],
label=f"Start OAI UE on fit{ue:02d}",
required=grace_delay,
scheduler=scheduler)
]
delay += 20
for ue in oai_ues:
environ = {'USER': '******'}
cefnet_ue_service = Service("cefnetd", service_id="cefnet", environ=environ)
ue_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(ue), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
msg = f"Wait 60s and then ping faraday gateway from UE on fit{ue:02d}"
ue_commands = f"echo {msg}; sleep 60; ping -c 5 -I oip1 faraday.inria.fr"
if tcp_streaming:
# TCP streaming scenario
if load_nodes:
ue_commands += "sysctl -w net.ipv4.ip_forward=1;"
ue_commands += "ip route add 10.1.1.0/24 via 192.168.2.32 dev data"
job_setup_ue = [
SshJob(
node=ue_node,
commands=[
Run(ue_commands,label="test UE link and set up routing for TCP streaming"),
],
label=f"ping faraday gateway from UE on fit{ue:02d} and set up routing for the TCP streaming scenario",
critical=True,
required=job_start_ues,
scheduler=scheduler)
]
else:
# Cefore streaming scenario
if load_nodes:
ue_commands += "sysctl -w net.ipv4.ip_forward=1;"
ue_commands += f"ip route add {publisher_ip}/32 dev oip1;"
ue_commands += "ip route add 10.1.1.0/24 via 192.168.2.32 dev data;"
ue_commands += "iptables -t nat -A POSTROUTING -s 10.1.1.2/32 -j SNAT --to-source 172.16.0.2;"
ue_commands += "iptables -t nat -A PREROUTING -d 172.16.0.2 -j DNAT --to-destination 10.1.1.2;"
ue_commands += "iptables -A FORWARD -d 10.1.1.2/32 -i oip1 -j ACCEPT;"
ue_commands += f"iptables -A FORWARD -d {publisher_ip}/32 -i data -j ACCEPT;"
ue_commands += "ip rule del from all to 172.16.0.2 lookup 201;"
ue_commands += "ip rule del from 172.16.0.2 lookup 201;"
ue_commands += "ip rule add from 10.1.1.2 lookup lte prio 32760;"
ue_commands += "ip rule add from all to 172.16.0.2 lookup lte prio 32761;"
ue_commands += "ip rule add from all fwmark 0x1 lookup lte prio 32762;"
ue_commands += "ip route add table lte 10.1.1.0/24 via 192.168.2.32 dev data;"
# ue_commands += "killall cefnetd || true"
job_setup_ue = [
SshJob(
node=ue_node,
commands=[
Run(ue_commands,label="test UE link and set up routing for Cefore streaming"),
cefnet_ue_service.start_command(),
],
label=f"ping faraday gateway from fit{ue:02d} UE and set up routing for the Cefore streaming scenario",
critical=True,#old cefnetd not killed when running new one...
required=job_start_ues,
scheduler=scheduler)
]
if ns3:
ns3_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(ns3), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
msg = f"Wait for the UE node to be ready before running the streaming scenario with ns-3 on fit{ns3}"
if load_nodes:
job_prepare_ns3_node = [
SshJob(
node=ns3_node,
commands=[
Run("turn-on-data"),
Run("ifconfig data promisc up"),
Run("ip route add default via 192.168.2.6 dev data || true"),
Run("sysctl -w net.ipv4.ip_forward=1"),
],
label=f"setup routing on ns-3 fit{ns3:02d} node",
# ip route may already be there so the ip route command may fail
critical=True,
required=job_setup_ue,
scheduler=scheduler)
]
ns3_requirements = job_prepare_ns3_node
else:
ns3_requirements = job_setup_ue
if not tcp_streaming:
environ = {'USER': '******'}
cefnet_ns3_service = Service("cefnetd", service_id="cefnet", environ=environ)
job_start_cefnet_on_cn = [
SshJob(
node=cnnode,
commands=[
Run(f"echo 'ccn:/streaming tcp {publisher_ip}:80' > /usr/local/cefore/cefnetd.conf"),
# Run("killall cefnetd || true"),# not done by default with service.start_command()
cefnet_ns3_service.start_command(),
],
label=f"Start Cefnet on EPC running at fit{cn:02d}",
critical=True,#old cefnetd not killed when running new one...
required=ns3_requirements,
scheduler=scheduler,
)
]
# ditto
_job_ping_phones_from_cn = [
SshJob(
node=cnnode,
commands=[
Run("sleep 20"),
Run(f"ping -c 100 -s 100 -i .05 172.16.0.{id+1} &> /root/ping-phone{id}"),
],
label=f"ping phone {id} from core network",
critical=False,
required=job_start_phones,
scheduler=scheduler)
for id in phones]
########## xterm nodes
colors = ("wheat", "gray", "white", "darkolivegreen")
xterms = e3372_ue_xterms + gnuradio_xterms
for xterm, color in zip(xterms, cycle(colors)):
xterm_node = SshNode(
gateway=gwnode, hostname=r2lab_hostname(xterm), username='******',
formatter=TimeColonFormatter(verbose=verbose), debug=verbose)
SshJob(
node=xterm_node,
command=Run(f"xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*",
f" -bg {color} -geometry 90x10",
x11=True),
label=f"xterm on node {xterm_node.hostname}",
scheduler=scheduler,
# don't set forever; if we do, then these xterms get killed
# when all other tasks have completed
# forever = True,
)
# remove dangling requirements - if any
# should not be needed but won't hurt either
scheduler.sanitize()
##########
print(10*"*", "nodes usage summary")
if load_nodes:
for image, nodes in images_to_load.items():
for node in nodes:
print(f"node fit{node:02d} : {image}")
else:
print("NODES ARE USED AS IS (no image loaded, no reset)")
print(10*"*", "phones usage summary")
if phones:
for phone in phones:
print(f"Using phone{phone}")
else:
print("No phone involved")
if nodes_left_alone:
print(f"Ignore following fit nodes: {nodes_left_alone}")
print(f"Publisher IP is {publisher_ip}")
if tcp_streaming:
print("Run streaming scenario with TCP")
else:
print("Run streaming scenario with Cefore")
# wrap scheduler into global scheduler that prepares the testbed
scheduler = prepare_testbed_scheduler(
gwnode, load_nodes, scheduler, images_to_load, nodes_left_alone)
scheduler.check_cycles()
# Update the .dot and .png file for illustration purposes
name = "cefore-load" if load_nodes else "cefore"
print(10*'*', 'See main scheduler in',
scheduler.export_as_pngfile(name))
if verbose:
scheduler.list()
if dry_run:
return True
if verbose:
input('OK ? - press control C to abort ? ')
if not scheduler.orchestrate():
print(f"RUN KO : {scheduler.why()}")
scheduler.debrief()
return False
print("RUN OK")
print(80*'*')
if tcp_streaming:
# TCP streaming scenario
print(f"Now it's time to run the ns-3 script on node fit{ns3:02d}")
print(f"root@fit{ns3:02d}:~# /root/NS3/source/ns-3-dce/waf --run dce-tcp-test")
print("Then, run iperf on the publisher host:")
print("yourlogin@publisher:~# iperf -s -P 1 -p 80")
print(f"Log file will be available on fit{ns3:02d} at:")
print(" /root/NS3/source/ns-3-dce/files-4/var/log/56884/stdout")
else:
# Cefore streaming scenario
print("Now, if not already done, copy cefnetd and cefputfile binaries on your publisher host")
print("login@your_host:r2lab-demos/cefore# scp bin/cefnetd yourlogin@publisher_node:/usr/local/sbin")
print("login@your_host:r2lab-demos/cefore# scp bin/cefputfile yourlogin@publisher_node:/user/local/bin")
print(f"After that, run on the ns-3 fit{ns3:02d} node the following command:")
print(f"root@fit{ns3:02d}:~# /root/NS3/source/ns-3-dce/waf --run dce-cefore-test ")
print("Then, run on the publisher host:")
print("yourlogin@publisher:~# cefnetd")
print("yourlogin@publisher:~# cefputfile ccn:/streaming/test -f ./[file-name] -r [1 <= streaming_rate <= 32 (Mbps)]")
print(f"Log file will be available on fit{ns3:02d} at:")
print(" /root/NS3/source/ns-3-dce/files-3/tmp/cefgetstream-thuputLog-126230400110000")
print(80*'*')
return True
def collect(run_name, slice, hss, epc, enb, verbose):
"""
retrieves all relevant logs under a common name
otherwise, same signature as run() for convenience
retrieved stuff will be 3 compressed tars named
<run_name>-(hss|epc|enb).tar.gz
xxx - todo - it would make sense to also unwrap them all
in a single place locally, like what "logs.sh unwrap" does
"""
gwuser, gwhost = r2lab_parse_slice(slice)
gwnode = SshNode(hostname=gwhost,
username=gwuser,
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose)
functions = "hss", "epc", "enb"
hostnames = hssname, epcname, enbname = [
r2lab_hostname(x) for x in (hss, epc, enb)
]
nodes = hssnode, epcnode, enbnode = [
SshNode(gateway=gwnode,
hostname=hostname,
username='******',
formatter=TimeColonFormatter(verbose=verbose),
debug=verbose) for hostname in hostnames
]
# first run a 'capture' function remotely to gather all the relevant
# info into a single tar named <run_name>.tgz
capturers = [
SshJob(
node=node,
command=RunScript(find_local_embedded_script("oai-common.sh"),
"capture-{}".format(function),
run_name,
includes=[
find_local_embedded_script(
"oai-{}.sh".format(function))
]),
label="capturer on {}".format(function),
# capture-enb will run oai-as-enb and thus requires oai-enb.sh
) for (node, function) in zip(nodes, functions)
]
collectors = [
SshJob(
node=node,
command=Pull(remotepaths=["{}-{}.tgz".format(run_name, function)],
localpath="."),
label="collector on {}".format(function),
required=capturers,
) for (node, function, capturer) in zip(nodes, functions, capturers)
]
sched = Scheduler(verbose=verbose)
sched.update(capturers)
sched.update(collectors)
if verbose:
sched.list()
if not sched.orchestrate():
print("KO")
sched.debrief()
return
print("OK")
if os.path.exists(run_name):
print("local directory {} already exists = NOT UNWRAPPED !".format(
run_name))
return
os.mkdir(run_name)
local_tars = [
"{run_name}-{ext}.tgz".format(run_name=run_name, ext=ext)
for ext in ['hss', 'epc', 'enb']
]
for tar in local_tars:
print("Untaring {} in {}".format(tar, run_name))
os.system("tar -C {} -xzf {}".format(run_name, tar))
def run(*, gateway, slicename, nodes, node_epc, node_enb, quectel_nodes,
phones, verbose, dry_run, load_images, epc_image, enb_image,
quectel_image):
"""
Launch latest OAICI EPC and eNB Docker images on R2lab
Arguments:
slicename: the Unix login name (slice name) to enter the gateway
quectel_nodes: list of indices of quectel UE nodes to use
phones: list of indices of phones to use
nodes: a list of node ids to run the scenario on; strings or ints
are OK;
node_epc: the node id on which to run the EPC
node_enb: the node id for the enb, which is connected to B210/eNB-duplexer
"""
quectel_ids = quectel_nodes[:]
quectel = len(quectel_ids) > 0
faraday = SshNode(hostname=default_gateway,
username=slicename,
verbose=verbose,
formatter=TimeColonFormatter())
epc = SshNode(gateway=faraday,
hostname=fitname(node_epc),
username="******",
verbose=verbose,
formatter=TimeColonFormatter())
node_index = {
id: SshNode(gateway=faraday,
hostname=fitname(id),
username="******",
formatter=TimeColonFormatter(),
verbose=verbose)
for id in nodes
}
nodes_quectel_index = {
id: SshNode(gateway=faraday,
hostname=fitname(id),
username="******",
formatter=TimeColonFormatter(),
verbose=verbose)
for id in quectel_nodes
}
allnodes = nodes + quectel_nodes
fit_epc = fitname(node_epc)
fit_enb = fitname(node_enb)
# the global scheduler
scheduler = Scheduler(verbose=verbose)
##########
check_lease = SshJob(
scheduler=scheduler,
node=faraday,
critical=True,
verbose=verbose,
command=Run("rhubarbe leases --check"),
)
green_light = check_lease
if load_images:
green_light = [
SshJob(scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label=f"Load image {epc_image} on {fit_epc}",
commands=[
Run(f"rhubarbe load {node_epc} -i {epc_image}"),
Run(f"rhubarbe wait {node_epc}"),
RunScript("oaici.sh", "init-epc", node_epc, node_enb),
]),
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label=f"Load image {enb_image} on {fit_enb}",
commands=[
Run(f"rhubarbe usrpoff {node_enb}"
), # if usrp is on, load could be problematic...
Run(f"rhubarbe load {node_enb} -i {enb_image}"),
Run(f"rhubarbe wait {node_enb}"),
Run(f"rhubarbe usrpon {node_enb}"
), # ensure a reset of the USRP on the enB node
RunScript("oaici.sh", "init-enb", node_enb, node_epc),
],
),
SshJob(scheduler=scheduler,
required=check_lease,
node=faraday,
critical=False,
verbose=verbose,
label="turning off unused nodes",
command=[
Run("rhubarbe bye --all " + "".join(f"~{x} "
for x in allnodes))
])
]
if quectel:
prepare_quectel = SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label=f"Load image {quectel_image} on quectel UE nodes",
commands=[
Run("rhubarbe", "usrpoff", *quectel_ids),
Run("rhubarbe", "load", *quectel_ids, "-i", quectel_image),
Run("rhubarbe", "wait", *quectel_ids),
Run("rhubarbe", "usrpon", *quectel_ids),
],
),
##########
# Prepare the Quectel UE nodes
if quectel:
# wait 30s for Quectel modules show up
wait_quectel_ready = PrintJob(
"Let Quectel modules show up",
scheduler=scheduler,
required=prepare_quectel,
sleep=30,
label="sleep 30s for the Quectel modules to show up")
# run the Quectel Connection Manager as a service on each Quectel UE node
quectelCM_service = Service(
command="quectel-CM -s oai.ipv4 -4",
service_id="QuectelCM",
verbose=verbose,
)
init_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectel_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Init Quectel UE on fit node {id}",
commands=[
RunScript(find_local_embedded_script("nodes.sh"),
"check-quectel-on",
includes=INCLUDES),
quectelCM_service.start_command(),
],
) for id, node in nodes_quectel_index.items()
]
# wait 20s for Quectel Connection Manager to start up
wait_quectelCM_ready = PrintJob(
"Let QuectelCM start up",
scheduler=scheduler,
required=init_quectel_nodes,
sleep=20,
label="Sleep 20s for the Quectel Connection Manager(s) to start up"
)
detach_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectelCM_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Detach Quectel UE on fit node {id}",
command=RunScript(find_local_embedded_script("nodes.sh"),
"quectel-detach",
includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Launch the EPC
start_epc = SshJob(
scheduler=scheduler,
required=green_light,
node=faraday,
critical=True,
verbose=verbose,
label=f"Launch EPC on {fit_epc}",
commands=[
RunScript("oaici.sh", "start-epc", node_epc),
],
)
# Launch the eNB
if quectel:
req = (start_epc, detach_quectel_nodes)
else:
req = start_epc
start_enb = SshJob(
scheduler=scheduler,
required=req,
node=faraday,
critical=True,
verbose=verbose,
label=f"Launch eNB on {fit_enb}",
commands=[
RunScript("oaici.sh", "start-enb", node_enb),
],
)
wait_ran_ready = PrintJob("Let the eNB start up",
scheduler=scheduler,
required=start_enb,
sleep=50,
label="sleep 50s for the eNB to start up")
########## Test phone(s) connectivity
sleeps_ran = (0, 10)
phone_msgs = [
f"wait again for {sleep}s before waking up phone{id}"
for sleep, id in zip(sleeps_ran, phones)
]
wait_commands = [
f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone_msgs, sleeps_ran)
]
sleeps_phone = (10, 10)
phone2_msgs = [
f"wait for {sleep}s for phone{id} before starting tests"
for sleep, id in zip(sleeps_phone, phones)
]
wait2_commands = [
f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone2_msgs, sleeps_phone)
]
job_start_phones = [
SshJob(node=faraday,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{id}",
"r2lab-embedded/shell/macphone.sh",
"phone-on",
includes=INCLUDES),
Run(wait2_command),
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{id}",
"r2lab-embedded/shell/macphone.sh",
"phone-check-cx",
includes=INCLUDES),
RunScript(find_local_embedded_script("faraday.sh"),
f"macphone{id}",
"r2lab-embedded/shell/macphone.sh",
"phone-start-app",
includes=INCLUDES),
],
label=f"turn off airplane mode on phone {id}",
required=wait_ran_ready,
scheduler=scheduler) for id, wait_command, wait2_command in zip(
phones, wait_commands, wait2_commands)
]
if quectel:
job_attach_quectel = [
SshJob(
scheduler=scheduler,
required=(job_start_phones, wait_ran_ready,
detach_quectel_nodes),
node=node,
critical=True,
verbose=verbose,
label=f"Attach Quectel UE on fit node {id}",
command=RunScript(find_local_embedded_script("nodes.sh"),
"quectel-attach",
includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
# wait 30s for Quectel connection to set up
wait_quectel_cx_ready = PrintJob(
"Let the Quectel connection(s) set up",
scheduler=scheduler,
required=job_attach_quectel,
sleep=30,
label="Sleep 30s for the Quectel connection(s) to set up")
test_quectel_cx = [
SshJob(
scheduler=scheduler,
required=wait_quectel_cx_ready,
node=node,
verbose=verbose,
label=f"Check the Quectel cx on fit node {id}",
command=RunScript(find_local_embedded_script("nodes.sh"),
"check-quectel-cx",
includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Update the .dot and .png file for illustration purposes
scheduler.check_cycles()
name = "deploy-oaici"
print(10 * '*', 'See main scheduler in', scheduler.export_as_pngfile(name))
# orchestration scheduler jobs
if verbose:
scheduler.list()
if dry_run:
return True
if not scheduler.orchestrate():
print(f"RUN KO : {scheduler.why()}")
scheduler.debrief()
return False
print(
f"RUN OK, you can log now on the EPC node {fit_epc} and the eNB node {fit_enb} to check the logs"
)
print(80 * '*')
node=faraday,
# this means that a failure in any of the commands
# will cause the scheduler to bail out immediately
critical=True,
command=Run("rhubarbe leases --check"),
scheduler=scheduler,
)
##########
# setting up the wireless interface on both fit01 and fit02
init_node_01 = SshJob(
node=node1,
command=RunScript(
"B3-wireless.sh",
"init-ad-hoc-network",
wireless_driver,
"foobar",
2412,
),
required=check_lease,
scheduler=scheduler,
)
init_node_02 = SshJob(
node=node2,
command=RunScript(
"B3-wireless.sh",
"init-ad-hoc-network",
wireless_driver,
"foobar",
2412,
label="ditto",
def main(nodename1, nodename2, *, verbose=True):
# show ssh outputs on stdout as they appear
# together with corresponding hostname
formatter = ColonFormatter(verbose=verbose)
########## declare the needed ssh connections
# our main ssh connection
gateway = SshNode(hostname=gwname, username=slice, formatter=formatter)
# the ssh connections to each of the 2 nodes
node1, node2 = [
SshNode(
hostname=nodename,
username="******",
# this is how we create a 2-hop
# ssh connection behind a gateway
gateway=gateway,
formatter=formatter,
debug=verbose) for nodename in (nodename1, nodename2)
]
##########
job_warmup = SshJob(
node=gateway,
# with just Run()
# you can run a command already available on the remote
command=[
Run("rhubarbe leases --check"),
Run("rhubarbe on", nodename1, nodename2),
Run("rhubarbe wait", nodename1, nodename2),
])
job_prep_send = SshJob(
node=node1,
command=[
# an example of a compound job
# with RunScript, we run a command whose source is local here
RunScript("demo.sh", "prepare-sender"),
Run("ip address show control"),
],
# run this only once this job is done
required=job_warmup,
)
job_prep_recv = SshJob(
node=node2,
command=RunScript("demo.sh", "prepare-receiver"),
required=job_warmup,
)
job_run_send = SshJob(
node=node1,
command=[
RunScript("demo.sh", "run-sender"),
Pull("PREP", "PREP-SEND"),
Pull("RUN", "RUN-SEND"),
],
# start when both nodes are ready
required=(job_prep_send, job_prep_recv),
)
job_run_recv = SshJob(
node=node2,
command=[
RunScript("demo.sh", "run-receiver"),
Pull("PREP", "PREP-RECV"),
Pull("RUN", "RUN-RECV"),
],
required=(job_prep_send, job_prep_recv),
)
scheduler = Scheduler(job_warmup,
job_prep_send,
job_prep_recv,
job_run_send,
job_run_recv,
verbose=verbose)
scheduler.export_as_dotfile('demo.dot')
print("# produce .png file with the following command")
print("# install dot with e.g. brew install graphviz on macos")
print("dot -Tpng demo.dot -o demo.png")
print(20 * '=')
ok = scheduler.orchestrate()
if not ok:
scheduler.debrief()
def run(*, gateway, slicename,
disaggregated_cn, operator_version, nodes, node_master, node_enb, quectel_nodes, phones, flexran,
drone, verbose, dry_run,
load_images, master_image, worker_image, quectel_image):
"""
Install K8S on R2lab
Arguments:
slicename: the Unix login name (slice name) to enter the gateway
quectel_nodes: list of indices of quectel UE nodes to use
phones: list of indices of phones to use
nodes: a list of node ids to run the scenario on; strings or ints
are OK;
node_master: the master node id, must be part of selected nodes
node_enb: the node id for the enb, which is connected to usrp/duplexer
disaggregated_cn: Boolean; True for the disaggregated CN scenario. False for all-in-one CN.
operator_version: str, either "none" or "v1" or "v2".
"""
if operator_version == "none":
only_kube5g = True
else:
only_kube5g = False
if node_master not in nodes:
print(f"master node {node_master} must be part of selected fit nodes {nodes}")
exit(1)
if node_enb not in nodes:
print(f"eNB worker node {node_enb} must be part of selected fit nodes {nodes}")
exit(1)
# Check if the browser can be automatically run to display the Drone app
if drone:
run_browser = True
if platform == "linux":
cmd_open = "xdg-open"
elif platform == "darwin":
cmd_open = "open"
else:
run_browser = False
if run_browser:
print(f"**** Will run the browser with command {cmd_open}")
else:
print(f"**** Will not be able to run the browser as platform is {platform}")
worker_ids = nodes[:]
worker_ids.remove(node_master)
quectel_ids = quectel_nodes[:]
quectel = len(quectel_ids) > 0
faraday = SshNode(hostname=default_gateway, username=slicename,
verbose=verbose,
formatter=TimeColonFormatter())
master = SshNode(gateway=faraday, hostname=fitname(node_master),
username="******",
verbose=verbose,
formatter=TimeColonFormatter())
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id),
username="******",formatter=TimeColonFormatter(),
verbose=verbose)
for id in nodes
}
nodes_quectel_index = {
id: SshNode(gateway=faraday, hostname=fitname(id),
username="******",formatter=TimeColonFormatter(),
verbose=verbose)
for id in quectel_nodes
}
worker_index = dict(node_index)
del worker_index[node_master]
fit_master = fitname(node_master)
fit_enb = fitname(node_enb)
# the global scheduler
scheduler = Scheduler(verbose=verbose)
##########
check_lease = SshJob(
scheduler=scheduler,
node = faraday,
critical = True,
verbose=verbose,
command = Run("rhubarbe leases --check"),
)
green_light = check_lease
if load_images:
green_light = [
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {master_image} on master {fit_master}",
commands=[
Run(f"rhubarbe load {node_master} -i {master_image}"),
Run(f"rhubarbe wait {node_master}"),
]
),
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {worker_image} on worker nodes",
commands=[
Run(f"rhubarbe usrpoff {node_enb}"), # if usrp is on, load could be problematic...
Run("rhubarbe", "load", *worker_ids, "-i", worker_image),
Run("rhubarbe", "wait", *worker_ids),
Run(f"rhubarbe usrpon {node_enb}"), # ensure a reset of the USRP on the enB node
],
),
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=False,
verbose=verbose,
label="turning off unused nodes",
command=[
Run("rhubarbe bye --all "
+ "".join(f"~{x} " for x in nodes))
]
)
]
if quectel:
prepare_quectel = SshJob(
scheduler=scheduler,
required=green_light,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {quectel_image} on quectel UE nodes",
commands=[
Run("rhubarbe", "usrpoff", *quectel_ids),
Run("rhubarbe", "load", *quectel_ids, "-i", quectel_image),
Run("rhubarbe", "wait", *quectel_ids),
Run("rhubarbe", "usrpon", *quectel_ids),
],
),
##########
if quectel:
# wait 30s for Quectel modules show up
wait_quectel_ready = PrintJob(
"Let Quectel modules show up",
scheduler=scheduler,
required=prepare_quectel,
sleep=30,
label="sleep 30s for the Quectel modules to show up"
)
# run the Quectel Connection Manager as a service on each Quectel UE node
quectelCM_service = Service(
command="quectel-CM -s oai.ipv4 -4",
service_id="QuectelCM",
verbose=verbose,
)
init_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectel_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Init Quectel UE on fit node {id}",
commands = [
RunScript(find_local_embedded_script("nodes.sh"), "check-quectel-on", includes=INCLUDES),
quectelCM_service.start_command(),
],
) for id, node in nodes_quectel_index.items()
]
# wait 20s for Quectel Connection Manager to start up
wait_quectelCM_ready = PrintJob(
"Let QuectelCM start up",
scheduler=scheduler,
required=init_quectel_nodes,
sleep=20,
label="Sleep 20s for the Quectel Connection Manager(s) to start up"
)
detach_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectelCM_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Detach Quectel UE on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"), "quectel-detach", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Initialize k8s on the master node
init_master = SshJob(
scheduler=scheduler,
required=green_light,
node=master,
critical=True,
verbose=verbose,
label = f"Install and launch k8s on the master {node_master}",
commands = [
Run("swapoff -a"),
Run("hostnamectl set-hostname master-node"),
Run("kubeadm init --pod-network-cidr=10.244.0.0/16 > /tmp/join_msg.txt"),
Run("tail -2 /tmp/join_msg.txt > /tmp/join_msg"),
Run("mkdir -p $HOME/.kube"),
Run("cp -i /etc/kubernetes/admin.conf $HOME/.kube/config"),
Run("chown $(id -u):$(id -g) $HOME/.kube/config"),
Run("kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"),
Run("kubectl get pods --all-namespaces"),
],
)
init_workers = [
SshJob(
scheduler=scheduler,
required=init_master,
node=node,
critical=True,
verbose=verbose,
label=f"Init k8s on fit node {id} and join the cluster",
commands = [
Run("swapoff -a"),
Run("increase-control-mtu"),
Run(f"scp -o 'StrictHostKeyChecking no' {fit_master}:/tmp/join_msg /tmp/join_msg"),
Run("chmod a+x /tmp/join_msg"),
Run("/tmp/join_msg"),
],
) for id, node in worker_index.items()
]
# wait 10s for K8S nodes setup
wait_k8nodes_ready = PrintJob(
"Let k8s set up",
scheduler=scheduler,
required=init_workers,
sleep=10,
label="sleep 10s for the k8s nodes to settle"
)
init_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8nodes_ready,
node = master,
verbose=verbose,
label = f"Add oai:ran label to oai-ran pod on {node_enb} and start 5GOperator pod",
commands = [
Run("kubectl get nodes"),
# add label to the eNB node to help k8s scheduler selects the right fit node
Run(f"kubectl label nodes fit{node_enb} oai=ran"),
Run("kubectl get nodes -Loai"),
## retrieve the kube5g operator
#Run("git clone -b develop [email protected]:mosaic5g/kube5g.git"),
# install a few dependencies
Run("apt install -y python3-pip"),
Run("pip3 install --upgrade pip"),
Run("pip3 install ruamel.yaml==0.16.12 colorlog==4.6.2"),
Run("sed -i 's/oairan:v1-1.0-1/oairan:v1-1.0-3/g' /root/kube5g/common/config-manager/conf_global_default.yaml"),
# specify the R2lab specific configuration
Run("cd /root/kube5g/common/config-manager; ./conf-manager.py -s conf_short_r2lab.yaml"),
# apply the R2lab CRD
Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh -n"),
# start the kube5g operator pod
Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh container start"),
Run("kubectl get pods"),
],
)
# wait 30s for K8S 5G Operator setup
wait_k8_5GOp_ready = PrintJob(
"Let 5G Operator set up",
scheduler=scheduler,
required=init_kube5g,
sleep=30,
label="wait 30s for the 5G Operator pod to settle"
)
if only_kube5g:
finish = SshJob(
scheduler=scheduler,
required = wait_k8_5GOp_ready,
node = master,
verbose=verbose,
label = f"showing nodes and pods before leaving",
commands = [
Run("kubectl get nodes -Loai"),
Run("kubectl get pods"),
],
)
else:
if disaggregated_cn:
cn_type="disaggregated-cn"
# setup_time = 120
setup_time = 200
else:
cn_type="all-in-one"
# setup_time = 60
setup_time = 140
if flexran:
flexran_opt="flexran"
else:
flexran_opt=""
run_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8_5GOp_ready,
node = master,
verbose=verbose,
label = f"deploy {operator_version} {cn_type} {flexran_opt} pods",
commands = [
Run("kubectl get nodes -Loai"),
Run(f"cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh deploy {operator_version} {cn_type} {flexran_opt}"),
Run("kubectl get pods"),
],
)
# Coffee Break -- wait 1 or 2mn for K8S 5G pods setup
wait_k8_5Gpods_ready = PrintJob(
"Let all 5G pods set up",
scheduler=scheduler,
required=run_kube5g,
sleep=setup_time,
label=f"waiting {setup_time}s for all 5G pods to settle"
)
check_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8_5Gpods_ready,
node = master,
verbose=verbose,
label = "Check which pods are deployed",
commands = [
Run("kubectl get nodes -Loai"),
Run("kubectl get pods"),
],
)
if drone:
# the place where runtime variables get stored
env = Variables()
#
# Define and run all the services to launch the Drone app locally on a firefox browser
#
drone_service = Service(
command=f"python /root/mosaic5g/store/sdk/frontend/drone/drone.py --port=8088 --tasks --address=192.168.3.{node_enb}",
service_id="drone_app",
verbose=verbose,
)
k8s_port9999_fwd_service = Service(
command=Deferred("kubectl port-forward {{flexran_pod}} 9999:9999 --address 0.0.0.0", env),
service_id="k8s-port9999-fwd",
verbose=verbose,
# somehow this is required for kubectl to run properly
environ={'KUBECONFIG': '/etc/kubernetes/admin.conf'}
)
# can't use a Service instance on the local box if it's not a Linux
# and we have macs...
local_port_fwd = (f"ssh -f -N -4"
f" -L9999:192.168.3.{node_master}:9999"
f" -L8088:192.168.3.{node_enb}:8088"
f" -o ExitOnForwardFailure=yes"
f" {slicename}@faraday.inria.fr")
browser_service = Service(
command=f"sleep 10; {cmd_open} http://127.0.0.1:8088/",
service_id="drone_browser",
verbose=verbose,
)
run_drone=SshJob(
scheduler=scheduler,
required=check_kube5g,
node=worker_index[node_enb],
verbose=verbose,
label=f"Run the drone app on worker node {node_enb} as a service",
commands=[
drone_service.start_command(),
],
)
get_flexran_podname=SshJob(
scheduler=scheduler,
required=check_kube5g,
node=master,
verbose=verbose,
label=f"Retrieve the name of the FlexRAN pod",
commands=[
# xxx here
Run("kubectl get --no-headers=true pods -l app=flexran -o custom-columns=:metadata.name",
capture=Capture('flexran_pod', env)),
],
)
run_k8s_port9999_fwd=SshJob(
scheduler=scheduler,
required=get_flexran_podname,
node=master,
verbose=verbose,
label=f"Run port forwarding on the master node as a service",
commands=[
k8s_port9999_fwd_service.start_command(),
],
)
# On the local machine, impossible to use Services as the latter uses systemd-run, only available on Linux
run_local_ports_fwd = SshJob(
scheduler=scheduler,
required = check_kube5g,
node = LocalNode(),
verbose=verbose,
label = f"Forward local ports 8088 and 9999",
command=Run(local_port_fwd + "&", ignore_outputs=True),
)
if run_browser:
run_local_browser = SshJob(
scheduler=scheduler,
required = (run_drone, run_k8s_port9999_fwd, run_local_ports_fwd),
node = LocalNode(),
verbose=verbose,
label = f"Run the browser on the local node in background",
command=browser_service.command+"&",
)
phones_requirements=run_local_browser
else:
phones_requirements=run_k8s_port9999_fwd
else:
phones_requirements=check_kube5g
########## Test phone(s) connectivity
sleeps_ran = (20, 25)
phone_msgs = [f"wait for {sleep}s for eNB to start up before waking up phone{id}"
for sleep, id in zip(sleeps_ran, phones)]
wait_commands = [f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone_msgs, sleeps_ran)]
sleeps_phone = (15, 20)
phone2_msgs = [f"wait for {sleep}s for phone{id} before starting tests"
for sleep, id in zip(sleeps_phone, phones)]
wait2_commands = [f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone2_msgs, sleeps_phone)]
job_start_phones = [
SshJob(
node=faraday,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-on",
includes=INCLUDES),
Run(wait2_command),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-check-cx",
includes=INCLUDES),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-start-app",
includes=INCLUDES),
],
label=f"turn off airplane mode on phone {id}",
required=phones_requirements,
scheduler=scheduler)
for id, wait_command, wait2_command in zip(phones, wait_commands, wait2_commands)
]
if quectel:
# wait 60s for Quectel connection(s) to set up
wait_before_attach_quectel = PrintJob(
"Wait again 30s before attaching Quectel device(s)",
scheduler=scheduler,
required=(job_start_phones,check_kube5g,detach_quectel_nodes),
sleep=30,
label="Sleep 30s before attaching Quectel device(s)"
)
job_attach_quectel = [
SshJob(
scheduler=scheduler,
required=wait_before_attach_quectel,
node=node,
critical=True,
verbose=verbose,
label=f"Attach Quectel UE on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"),
"quectel-attach", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
# wait 30s for Quectel connection(s) to set up
wait_quectel_cx_ready = PrintJob(
"Let the Quectel connection(s) set up",
scheduler=scheduler,
required=job_attach_quectel,
sleep=30,
label="Sleep 30s for the Quectel connection(s) to set up"
)
test_quectel_cx = [
SshJob(
scheduler=scheduler,
required=wait_quectel_cx_ready,
node=node,
critical=False,
verbose=verbose,
label=f"Check the Quectel cx on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"),
"check-quectel-cx", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Update the .dot and .png file for illustration purposes
scheduler.check_cycles()
name = "deploy-kube5g"
print(10*'*', 'See main scheduler in',
scheduler.export_as_pngfile(name))
# orchestration scheduler jobs
if verbose:
scheduler.list()
if dry_run:
return True
if not scheduler.orchestrate():
print(f"RUN KO : {scheduler.why()}")
scheduler.debrief()
return False
print(f"RUN OK, you can log now on master node {fit_master} to manually change the scenario")
print(80*'*')
def one_run(tx_power,
phy_rate,
antenna_mask,
channel,
interference,
protocol,
*,
run_name=default_run_name,
slicename=default_slicename,
load_images=False,
node_ids=None,
verbose_ssh=False,
verbose_jobs=False,
dry_run=False,
tshark=False,
map=False,
warmup=False,
exp=default_exp,
dest=default_node_ids,
ping_number=default_ping_number,
route_sampling=False):
"""
Performs data acquisition on all nodes with the following settings
Arguments:
tx_power: in dBm, a string like 5, 10 or 14. Correspond to the transmission power.
phy_rate: a string among 1, 54. Correspond to the wifi rate.
antenna_mask: a string among 1, 3, 7.
channel: a string like e.g. 1 or 40. Correspond to the channel.
protocol: a string among batman , olsr. Correspond to the protocol
interference : in dBm, a string like 60 or 50. Correspond to the power of the noise generated in the root.
run_name: the name for a subdirectory where all data will be kept
successive runs should use the same name for further visualization
slicename: the Unix login name (slice name) to enter the gateway
load_images: a boolean specifying whether nodes should be re-imaged first
node_ids: a list of node ids to run the scenario against; strings or ints are OK;
defaults to the nodes [1, 4, 5, 12, 19, 22,27 ,31, 33, 37]
tshark: a boolean specifying wether we should format/parse the .pcap.
map: a boolean specifying wether we should fetch/parse the route tables of the nodes.
warmup: a boolean specifying wether we should run a ping before the experiment to be certain of the stabilisation on the network.
exp: a list of nodes from which we will launch the ping from. strings or ints are OK.
default to the node [1]
ping_number : The number of pings that will be generated
"""
# set default for the nodes parameter
node_ids = [int(id) for id in node_ids
] if node_ids is not None else default_node_ids
exp_ids = [int(id) for id in exp] if exp is not None else default_exp
dest_ids = [int(id)
for id in dest] if dest is not None else default_node_ids
#
# dry-run mode
# just display a one-liner with parameters
#
if dry_run:
print("************************************")
print("\n")
run_root = naming_scheme(protocol,
run_name,
tx_power,
phy_rate,
antenna_mask,
channel,
interference,
autocreate=False)
load_msg = "" if not load_images else " LOAD"
nodes = " ".join(str(n) for n in node_ids)
exps = " ".join(str(n) for n in exp)
pingst = [
"PING{}-->{}".format(e, j) for e in exp_ids
# and on the destination
for j in node_ids if e != j #and not
#(j in exp_ids and j < e)
]
print(
"dry-run:{protocol} {run_name}{load_msg} -"
" t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} I{interference}-"
"nodes {nodes}"
" exp {exps}".format(**locals()))
print(
"\nNodes from which the experiment will be launched : \n{}\nList of pings generated:\n"
.format(exps))
print(pingst)
print("\n")
if warmup:
print("Will do warmup pings\n")
if tshark:
print(
"Will format data using tshark and will agregate the RSSI into one RSSI.txt file"
)
if map:
print(
"Will fetch the routing tables of the node (when stabilited) and will agregate the results\n"
)
if route_sampling:
print("Will launch route sampling services on nodes")
#print("Test creation of ROUTES files")
#post_processor= ProcessRoutes(run_root, exp_ids, node_ids)
#post_processor.run()
#print("\nList of tracepaths generated:\n{}".format(tracepathst))
# in dry-run mode we are done
###
# create the logs directory based on input parameters
run_root = naming_scheme(protocol,
run_name,
tx_power,
phy_rate,
antenna_mask,
channel,
interference,
autocreate=False)
if (run_root.is_dir()):
purgedir(run_root)
run_root = naming_scheme(protocol,
run_name,
tx_power,
phy_rate,
antenna_mask,
channel,
interference,
autocreate=True)
exp_info_file_name = run_root / "info.txt"
with exp_info_file_name.open("w") as info_file:
info_file.write("Selected nodes : \n")
for node in node_ids[:-1]:
info_file.write(f"{node} ")
info_file.write(f"{node_ids[-1]}")
info_file.write("\nSources : \n")
for src in exp_ids[:-1]:
info_file.write(f"{src} ")
info_file.write(f"{exp_ids[-1]}")
info_file.write("\nDestinations : \n")
for dest in dest_ids[:-1]:
info_file.write(f"{dest} ")
info_file.write(f"{dest_ids[-1]}" + "\n")
# the nodes involved
faraday = SshNode(hostname=default_gateway,
username=slicename,
formatter=TimeColonFormatter(),
verbose=verbose_ssh)
# this is a python dictionary that allows to retrieve a node object
# from an id
node_index = {
id: SshNode(gateway=faraday,
hostname=fitname(id),
username="******",
formatter=TimeColonFormatter(),
verbose=verbose_ssh)
for id in node_ids
}
if interference != "None":
node_scrambler = SshNode(gateway=faraday,
hostname=fitname(scrambler_id),
username="******",
formatter=TimeColonFormatter(),
verbose=verbose_ssh)
# the global scheduler
scheduler = Scheduler(verbose=verbose_jobs)
# if tshark:
#scheduler_monitoring = Scheduler(verbose=verbose_jobs)
#if interference != "None":
#scheduler_interferences = Scheduler(verbose=verbose_jobs)
##########
check_lease = SshJob(
scheduler=scheduler,
node=faraday,
verbose=verbose_jobs,
critical=True,
label="rhubarbe check lease",
command=Run("rhubarbe leases --check", label="rlease"),
#keep_connection = True
)
# load images if requested
green_light = check_lease
if load_images:
# the nodes that we **do not** use should be turned off
# so if we have selected e.g. nodes 10 12 and 15, we will do
# rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15
negated_node_ids = ["~{}".format(id) for id in node_ids]
#Add the id of the scrambler in the list and load the gnuradio image
negated_node_ids.append("~{}".format(scrambler_id))
load_ids = [int(id) for id in node_ids
] if node_ids is not None else default_node_ids
load_ids.append(scrambler_id)
# replace green_light in this case
#We use a modified image of gnuradio where uhd_siggen handle the signal SIGTERM in order to finish properly
green_light = SshJob(
node=faraday,
required=check_lease,
#critical=True,
scheduler=scheduler,
verbose=verbose_jobs,
label="rhubarbe load/wait on nodes {}".format(load_ids),
commands=[
Run("rhubarbe",
"off",
"-a",
*negated_node_ids,
label="roff {}".format(negated_node_ids)),
Run("rhubarbe",
"load",
*node_ids,
label="rload {}".format(node_ids)),
Run("rhubarbe",
"load",
"-i",
"gnuradio_batman",
scrambler_id,
label="load gnuradio batman on {}".format(scrambler_id)),
Run("rhubarbe", "wait", *load_ids, label="rwait")
],
#keep_connection = True
)
##########
# setting up the wireless interface on all nodes
#
# this is a python feature known as a list comprehension
# we just create as many SshJob instances as we have
# (id, SshNode) couples in node_index
# and gather them all in init_wireless_jobs
# they all depend on green_light
#
# provide node-utilities with the ranges/units it expects
frequency = channel_frequency[int(channel)]
# tx_power_in_mBm not in dBm
tx_power_driver = tx_power * 100
init_wireless_sshjobs = [
SshJob(
#scheduler=scheduler,
#required=green_light,
node=node,
verbose=verbose_jobs,
label="init {}".format(id),
command=RunScript("node-utilities.sh",
"init-ad-hoc-network-{}".format(wireless_driver),
wireless_driver,
"foobar",
frequency,
phy_rate,
antenna_mask,
tx_power_driver,
label="init add-hoc network"),
#keep_connection = True
) for id, node in node_index.items()
]
init_wireless_jobs = Scheduler(
*init_wireless_sshjobs,
scheduler=scheduler,
required=green_light,
#critical = True,
verbose=verbose_jobs,
label="Initialisation of wireless chips")
green_light_prot = init_wireless_jobs
if interference != "None":
#Run uhd_siggen with the chosen power
frequency_str = frequency / 1000
frequency_str = str(frequency_str) + "G"
init_scrambler_job = [
SshJob(
forever=True,
node=node_scrambler,
verbose=verbose_jobs,
label="init scrambler on node {}".format(scrambler_id),
command=RunScript("node-utilities.sh",
"init-scrambler",
interference,
frequency_str,
label="init scambler"),
#keep_connection = True
)
]
init_scrambler = Scheduler(
*init_scrambler_job,
scheduler=scheduler,
required=green_light,
#forever = True,
#critical = True,
verbose=verbose_jobs,
label="Running interference")
# then install and run batman on fit nodes
run_protocol_job = [
SshJob(
#scheduler=scheduler,
node=node,
#required=green_light_prot,
label="init and run {} on fit node {}".format(protocol, i),
verbose=verbose_jobs,
command=RunScript("node-utilities.sh",
"run-{}".format(protocol),
label="run {}".format(protocol)),
#keep_connection = True
) for i, node in node_index.items()
]
run_protocol = Scheduler(
*run_protocol_job,
scheduler=scheduler,
required=green_light_prot,
#critical = True,
verbose=verbose_jobs,
label="init and run routing protocols")
# after that, run tcpdump on fit nodes, this job never ends...
if tshark:
run_tcpdump_job = [
SshJob(
#scheduler=scheduler_monitoring,
node=node,
forever=True,
label="run tcpdump on fit node".format(i),
verbose=verbose_jobs,
commands=[
RunScript("node-utilities.sh",
"run-tcpdump",
wireless_driver,
i,
label="run tcpdump")
],
#keep_connection = True
) for i, node in node_index.items()
]
run_tcpdump = Scheduler(
*run_tcpdump_job,
scheduler=scheduler,
required=run_protocol,
forever=True,
#critical = True,
verbose=verbose_jobs,
label="Monitoring (tcpdum) Jobs")
# let the wireless network settle
settle_wireless_job = PrintJob(
"Let the wireless network settle",
sleep=settle_delay,
scheduler=scheduler,
required=run_protocol,
label="settling for {} sec".format(settle_delay))
green_light_experiment = settle_wireless_job
if warmup:
warmup_pings_job = [
SshJob(
node=nodei,
#required=green_light_experiment,
label="warmup ping {} -> {}".format(i, j),
verbose=verbose_jobs,
commands=[
Run("echo {} '->' {}".format(i, j),
label="ping {} '->' {}".format(i, j)),
RunScript("node-utilities.sh",
"my-ping",
"10.0.0.{}".format(j),
ping_timeout,
ping_interval,
ping_size,
ping_number,
label="")
],
#keep_connection = True
)
#for each selected experiment nodes
for e in exp_ids
# looping on the source (to get the correct sshnodes)
for i, nodei in node_index.items()
# and on the destination
for j, nodej in node_index.items()
# and keep only sources that are in the selected experiment nodes and remove destination that are themselves
# and remove the couples that have already be done
# print("i {index} exp {expe}".format(index = i, expe= exp))
if (i == e) and e != j and not (j in exp_ids and j < e)
]
warmup_pings = Scheduler(
Sequence(*warmup_pings_job),
scheduler=scheduler,
required=green_light_experiment,
#critical = True,
verbose=verbose_jobs,
label="Warmup ping")
settle_wireless_job2 = PrintJob(
"Let the wireless network settle",
sleep=settle_delay / 2,
scheduler=scheduler,
required=warmup_pings,
label="settling-warmup for {} sec".format(settle_delay / 2))
green_light_experiment = settle_wireless_job2
##########
# create all the tracepath jobs from the first node in the list
#
if map:
routes_job = [
SshJob(
node=nodei,
#scheduler=scheduler,
#required=green_light_experiment,
label="Generating ROUTE file for prot {} on node {}".format(
protocol, i),
verbose=verbose_jobs,
commands=[
RunScript("node-utilities.sh",
"route-{}".format(protocol),
">",
"ROUTE-TABLE-{:02d}".format(i),
label="get route table"),
Pull(remotepaths="ROUTE-TABLE-{:02d}".format(i),
localpath=str(run_root),
label="")
],
#keep_connection = True
) for i, nodei in node_index.items()
]
routes = Scheduler(
*routes_job,
scheduler=scheduler,
required=green_light_experiment,
#critical = True,
verbose=verbose_jobs,
label="Snapshoting route files")
green_light_experiment = routes
if route_sampling:
routes_sampling_job2 = [
SshJob(
node=nodei,
label="Route sampling service for prot {} on node {}".format(
protocol, i),
verbose=False,
#forever = True,
commands=[
Push(localpaths=["route_sample_service.sh"],
remotepath=".",
label=""),
Run("source",
"route_sample_service.sh;",
"route-sample",
"ROUTE-TABLE-{:02d}-SAMPLED".format(i),
"{}".format(protocol),
label="run route sampling service"),
],
#keep_connection = True
) for i, nodei in node_index.items()
]
routes_sampling_job = [
SshJob(
node=nodei,
label="Route sampling service for prot {} on node {}".format(
protocol, i),
verbose=False,
forever=True,
#critical = True,
#required = green_light_experiment,
#scheduler = scheduler,
commands=[
RunScript("route_sample_service.sh",
"route-sample",
"ROUTE-TABLE-{:02d}-SAMPLED".format(i),
"{}".format(protocol),
label="run route sampling service"),
],
#keep_connection = True
) for i, nodei in node_index.items()
]
routes_sampling = Scheduler(
*routes_sampling_job,
scheduler=scheduler,
verbose=False,
forever=True,
#critical = True,
label="Route Sampling services launch",
required=green_light_experiment)
##########
# create all the ping jobs, i.e. max*(max-1)/2
# this again is a python list comprehension
# see the 2 for instructions at the bottom
#
# notice that these SshJob instances are not yet added
# to the scheduler, we will add them later on
# depending on the sequential/parallel strategy
pings_job = [
SshJob(
node=nodei,
#required=green_light_experiment,
label="ping {} -> {}".format(i, j),
verbose=verbose_jobs,
commands=[
Run("echo {} '->' {}".format(i, j),
label="ping {}'->' {}".format(i, j)),
RunScript("node-utilities.sh",
"my-ping",
"10.0.0.{}".format(j),
ping_timeout,
ping_interval,
ping_size,
ping_number,
">",
"PING-{:02d}-{:02d}".format(i, j),
label=""),
Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j),
localpath=str(run_root),
label=""),
],
#keep_connection = True
)
#for each selected experiment nodes
for e in exp_ids
# looping on the source (to get the correct sshnodes)
for i, nodei in node_index.items()
# and on the destination
for j in dest_ids
# and keep only sources that are in the selected experiment nodes and remove destination that are themselves
# and remove the couples that have already be done
if (i == e) and e != j and not (j in exp_ids and j < e)
]
pings = Scheduler(
scheduler=scheduler,
label="PINGS",
#critical = True,
verbose=verbose_jobs,
required=green_light_experiment)
# retrieve all pcap files from fit nodes
stop_protocol_job = [
SshJob(
#scheduler=scheduler,
node=nodei,
#required=pings,
label="kill routing protocol on fit{:02d}".format(i),
verbose=verbose_jobs,
#critical = True,
commands=[
RunScript("node-utilities.sh",
"kill-{}".format(protocol),
label="kill-{}".format(protocol)),
],
#keep_connection = False
) for i, nodei in node_index.items()
]
stop_protocol = Scheduler(
*stop_protocol_job,
scheduler=scheduler,
required=pings,
#critical = True,
label="Stop routing protocols",
)
if tshark:
retrieve_tcpdump_job = [
SshJob(
#scheduler=scheduler,
node=nodei,
#required=pings,
label="retrieve pcap trace from fit{:02d}".format(i),
verbose=verbose_jobs,
#critical = True,
commands=[
# RunScript("node-utilities.sh", "kill-{}".format(protocol), label = "kill-{}".format(protocol)),
RunScript("node-utilities.sh",
"kill-tcpdump",
label="kill-tcpdump"),
#Run("sleep 1"),
Run("echo retrieving pcap trace and result-{i}.txt from fit{i:02d}"
.format(i=i),
label=""),
Pull(remotepaths=["/tmp/fit{}.pcap".format(i)],
localpath=str(run_root),
label=""),
],
#keep_connection = True
) for i, nodei in node_index.items()
]
retrieve_tcpdump = Scheduler(
*retrieve_tcpdump_job,
scheduler=scheduler,
required=pings,
#critical = True,
label="Retrieve tcpdump",
)
if route_sampling:
retrieve_sampling_job = [
SshJob(
#scheduler=scheduler,
node=nodei,
#required=pings,
label="retrieve sampling trace from fit{:02d}".format(i),
verbose=verbose_jobs,
#critical = True,
commands=[
#RunScript("node-utilities.sh", "kill-route-sample", protocol,
# label = "kill route sample"),
RunScript("route_sample_service.sh",
"kill-route-sample",
label="kill route sample"),
Run("echo retrieving sampling trace from fit{i:02d}".
format(i=i),
label=""),
Pull(remotepaths=["ROUTE-TABLE-{:02d}-SAMPLED".format(i)],
localpath=str(run_root),
label=""),
],
#keep_connection = True
) for i, nodei in node_index.items()
]
retrieve_sampling = Scheduler(
*retrieve_sampling_job,
scheduler=scheduler,
required=pings,
#critical=True,
verbose=verbose_jobs,
label="Retrieve & stopping route sampling",
)
if tshark:
parse_pcaps_job = [
SshJob(
#scheduler=scheduler,
node=LocalNode(),
#required=retrieve_tcpdump,
label="parse pcap trace {path}/fit{node}.pcap".format(
path=run_root, node=i),
verbose=verbose_jobs,
#commands = [RunScript("parsepcap.sh", run_root, i)]
commands=[
Run("tshark",
"-2",
"-r",
"{path}/fit{node}.pcap".format(path=run_root, node=i),
"-R",
"'(ip.dst==10.0.0.{node} && icmp) && radiotap.dbm_antsignal'"
.format(node=i),
"-Tfields",
"-e",
"'ip.src'", "-e"
"'ip.dst'",
"-e",
"'radiotap.dbm_antsignal'",
">",
"{path}/result-{node}.txt".format(path=run_root,
node=i),
label="parse pcap locally")
],
#keep_connection = True
) for i in node_ids
]
parse_pcaps = Scheduler(
*parse_pcaps_job,
scheduler=scheduler,
required=retrieve_tcpdump,
#critical=True,
label="Parse pcap",
)
#TODO: TURN OFF USRP
if interference != "None":
kill_uhd_siggen = SshJob(
scheduler=scheduler,
node=node_scrambler,
required=pings,
label="killing uhd_siggen on the scrambler node {}".format(
scrambler_id),
verbose=verbose_jobs,
#critical = True,
commands=[Run("pkill", "uhd_siggen")],
#keep_connection = True
)
kill_2_uhd_siggen = SshJob(
scheduler=scheduler,
node=faraday,
required=kill_uhd_siggen,
label="turning off usrp on the scrambler node {}".format(
scrambler_id),
verbose=verbose_jobs,
commands=[
Run("rhubarbe", "usrpoff", "fit{}".format(scrambler_id))
],
#keep_connection = True
)
#if map:
#scheduler.add(Sequence(*tracepaths, scheduler=scheduler))
#if warmup:
# scheduler.add(Sequence(*warmup_pings_job, scheduler=scheduler))
pings.add(Sequence(*pings_job))
# for running sequentially we impose no limit on the scheduler
# that will be limitied anyways by the very structure
# of the required graph
#jobs_window = None
if dry_run:
scheduler.export_as_pngfile(run_root / "experiment_graph")
return True
# if not in dry-run mode, let's proceed to the actual experiment
ok = scheduler.orchestrate() #jobs_window=jobs_window)
scheduler.shutdown()
dot_file = run_root / "experiment_graph"
if not dot_file.is_file():
scheduler.export_as_dotfile(dot_file)
#TODO : Is it necessary? if the user want to see it he can just do it?
#call(["dot", "-Tpng", dot_file, "-o", run_root / "experitment_graph.png"])
#ok=True
#ok = False
# give details if it failed
if not ok:
scheduler.debrief()
scheduler.export_as_dotfile("debug")
if ok and map:
print("Creation of ROUTES files")
post_processor = ProcessRoutes(run_root, exp_ids, node_ids)
post_processor.run()
if ok and route_sampling:
post_processor = ProcessRoutes(run_root, exp_ids, node_ids)
post_processor.run_sampled()
print("END of creation for ROUTES FILES")
# data acquisition is done, let's aggregate results
# i.e. compute averages
if ok and tshark:
post_processor = Aggregator(run_root, node_ids, antenna_mask)
post_processor.run()
return ok
def one_run(tx_power, phy_rate, antenna_mask, channel, *,
run_name=default_run_name, slicename=default_slicename,
load_images=False, node_ids=None,
parallel=None,
verbose_ssh=False, verbose_jobs=False, dry_run=False):
"""
Performs data acquisition on all nodes with the following settings
Arguments:
tx_power: in dBm, a string like 5, 10 or 14
phy_rate: a string among 1, 54
antenna_mask: a string among 1, 3, 7
channel: a string like e.g. 1 or 40
run_name: the name for a subdirectory where all data will be kept
successive runs should use the same name for further visualization
slicename: the Unix login name (slice name) to enter the gateway
load_images: a boolean specifying whether nodes should be re-imaged first
node_ids: a list of node ids to run the scenario on; strings or ints are OK;
defaults to the all 37 nodes i.e. the whole testbed
parallel: a number of simulataneous jobs to run
1 means all data acquisition is sequential (default)
0 means maximum parallel
"""
#
# dry-run mode
# just display a one-liner with parameters
#
if dry_run:
load_msg = "" if not load_images else " LOAD"
nodes = " ".join(str(n) for n in node_ids)
print("dry-run: {run_name}{load_msg} -"
" t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} -"
"nodes {nodes}"
.format(**locals()))
# in dry-run mode we are done
return True
# set default for the nodes parameter
node_ids = [int(id)
for id in node_ids] if node_ids is not None else default_node_ids
###
# create the logs directory based on input parameters
run_root = naming_scheme(run_name, tx_power, phy_rate,
antenna_mask, channel, autocreate=True)
# the nodes involved
faraday = SshNode(hostname=default_gateway, username=slicename,
formatter=TimeColonFormatter(), verbose=verbose_ssh)
# this is a python dictionary that allows to retrieve a node object
# from an id
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id), username="******",
formatter=TimeColonFormatter(), verbose=verbose_ssh)
for id in node_ids
}
# the global scheduler
scheduler = Scheduler(verbose=verbose_jobs)
##########
check_lease = SshJob(
scheduler=scheduler,
node=faraday,
verbose=verbose_jobs,
critical=True,
command=Run("rhubarbe leases --check"),
)
# load images if requested
green_light = check_lease
if load_images:
# the nodes that we **do not** use should be turned off
# so if we have selected e.g. nodes 10 12 and 15, we will do
# rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15
negated_node_ids = ["~{}".format(id) for id in node_ids]
# replace green_light in this case
green_light = SshJob(
node=faraday,
required=check_lease,
critical=True,
scheduler=scheduler,
verbose=verbose_jobs,
commands=[
Run("rhubarbe", "off", "-a", *negated_node_ids),
Run("rhubarbe", "load", "-i", "u16-ath-noreg", *node_ids),
Run("rhubarbe", "wait", *node_ids)
]
)
##########
# setting up the wireless interface on all nodes
#
# this is a python feature known as a list comprehension
# we just create as many SshJob instances as we have
# (id, SshNode) couples in node_index
# and gather them all in init_wireless_jobs
# they all depend on green_light
#
# provide node-utilities with the ranges/units it expects
frequency = channel_frequency[int(channel)]
# tx_power_in_mBm not in dBm
tx_power_driver = tx_power * 100
if load_images:
# The first init_wireless_jobs always has troubles... Do it twice the first time (nasty hack)
init_wireless_jobs = [
SshJob(
scheduler=scheduler,
required=green_light,
node=node,
verbose=verbose_jobs,
label="init {}".format(id),
commands=[
RunScript("node-utilities.sh", "init-ad-hoc-network",
wireless_driver, "foobar", frequency, phy_rate,
antenna_mask, tx_power_driver),
RunScript("node-utilities.sh", "init-ad-hoc-network",
wireless_driver, "foobar", frequency, phy_rate,
antenna_mask, tx_power_driver)
]
)
for id, node in node_index.items()]
else:
init_wireless_jobs = [
SshJob(
scheduler=scheduler,
required=green_light,
node=node,
verbose=verbose_jobs,
label="init {}".format(id),
command=RunScript("node-utilities.sh", "init-ad-hoc-network",
wireless_driver, "foobar", frequency, phy_rate,
antenna_mask, tx_power_driver)
)
for id, node in node_index.items()]
# then install and run olsr on fit nodes
run_olsr = [
SshJob(
scheduler=scheduler,
node=node,
required=init_wireless_jobs,
label="init and run olsr on fit nodes",
verbose=verbose_jobs,
command=RunScript("node-utilities.sh", "run-olsr")
)
for i, node in node_index.items()]
# after that, run tcpdump on fit nodes, this job never ends...
run_tcpdump = [
SshJob(
scheduler=scheduler,
node=node,
required=run_olsr,
label="run tcpdump on fit nodes",
verbose=verbose_jobs,
commands=[
Run("echo run tcpdump on fit{:02d}".format(i)),
Run("tcpdump -U -i moni-{} -y ieee802_11_radio -w /tmp/fit{}.pcap".format(wireless_driver, i))
]
)
for i, node in node_index.items()]
# let the wireless network settle
settle_wireless_job = PrintJob(
"Let the wireless network settle",
sleep=settle_delay,
scheduler=scheduler,
required=run_olsr,
label="settling")
##########
# create all the ping jobs, i.e. max*(max-1)/2
# this again is a python list comprehension
# see the 2 for instructions at the bottom
#
# notice that these SshJob instances are not yet added
# to the scheduler, we will add them later on
# depending on the sequential/parallel strategy
pings = [
SshJob(
node=nodei,
required=settle_wireless_job,
label="ping {} -> {}".format(i, j),
verbose=verbose_jobs,
commands=[
Run("echo {} '->' {}".format(i, j)),
RunScript("node-utilities.sh", "my-ping",
"10.0.0.{}".format(j), ping_timeout, ping_interval,
ping_size, ping_number,
">", "PING-{:02d}-{:02d}".format(i, j)),
Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j),
localpath=str(run_root)),
]
)
# looping on the source, now only fit01 is source
for i, nodei in node_index.items()
# and on the destination
for j, nodej in node_index.items()
# and keep only half of the couples
if (j > i) and (i==1)
]
# retrieve all pcap files from fit nodes
retrieve_tcpdump = [
SshJob(
scheduler=scheduler,
node=nodei,
required=pings,
label="retrieve pcap trace from fit{:02d}".format(i),
verbose=verbose_jobs,
commands=[
RunScript("node-utilities.sh", "kill-olsr"),
Run("sleep 1;pkill tcpdump; sleep 1"),
RunScript("node-utilities.sh", "process-pcap", i),
Run(
"echo retrieving pcap trace and result-{i}.txt from fit{i:02d}".format(i=i)),
Pull(remotepaths=["/tmp/fit{}.pcap".format(i),
"/tmp/result-{}.txt".format(i)],
localpath=str(run_root)),
]
)
for i, nodei in node_index.items()
]
# xxx this is a little fishy
# should we not just consider that the default is parallel=1 ?
if parallel is None:
# with the sequential strategy, we just need to
# create a Sequence out of the list of pings
# Sequence will add the required relationships
scheduler.add(Sequence(*pings, scheduler=scheduler))
# for running sequentially we impose no limit on the scheduler
# that will be limitied anyways by the very structure
# of the required graph
jobs_window = None
else:
# with the parallel strategy
# we just need to insert all the ping jobs
# as each already has its required OK
scheduler.update(pings)
# this time the value in parallel is the one
# to use as the jobs_limit; if 0 then inch'allah
jobs_window = parallel
# if not in dry-run mode, let's proceed to the actual experiment
ok = scheduler.orchestrate(jobs_window=jobs_window)
# give details if it failed
if not ok:
scheduler.debrief()
# data acquisition is done, let's aggregate results
# i.e. compute averages
if ok:
post_processor = Aggregator(run_root, node_ids, antenna_mask)
post_processor.run()
return ok
)
# the shell script has gone into B3-wireless.sh
####################
##########
# setting up the wireless interface on both fit01 and fit02
init_node_01 = SshJob(
node=node1,
# RunString is replaced with RunScript
command=RunScript(
# first argument is the local filename
# where to find the script to run remotely
"B3-wireless.sh",
# then its arguments
"init-ad-hoc-network",
wireless_driver,
"foobar",
2412,
# no need to set remote_name with RunScript
),
required=check_lease,
scheduler=scheduler,
)
init_node_02 = SshJob(
node=node2,
command=RunScript(
"B3-wireless.sh",
"init-ad-hoc-network",
wireless_driver,
"foobar",