def hop1(self, hostname='localhost', username=None,
*, c1, commands, s_command='echo hop1-{}-{}',
nested_sched=(0, 1)):
"""
create
* <c1> connections to one node 1 hop away
* and on each <commands> commands
check current number of connections
"""
if username is None:
username = localuser()
verbose(f"creating {c1} hop1-connections - "
f"{commands} commands per conn - "
f" to {username}@{hostname}")
scheduler = Scheduler()
nodes = []
jobs = []
for n in range(c1):
node1 = SshNode(hostname, username=username,
formatter=ColonFormatter(verbose=False))
nodes.append(node1)
for c in range(commands):
jobs.append(SshJob(node=node1,
command=s_command.format(n, c),
))
scheduler = self.populate_sched(scheduler, jobs,
nested=nested_sched[0],
pack_job=nested_sched[1])
expected = c1
# record base status
in0, out0 = in_out_connections()
verbose(f"INITIAL count in={in0} out={out0}")
scheduler.export_as_pngfile("debug")
topology_as_pngfile(scheduler, "topology")
scheduler.run()
in1, out1 = in_out_connections()
verbose(f"AFTER RUN in={in1} out={out1}")
self.assertEqual(in1-in0, expected)
self.assertEqual(out1-out0, expected)
arg = nodes
# cleanup
close_ssh_in_scheduler(scheduler)
in1, out1 = in_out_connections()
verbose(f"AFTER CLEANUP in={in1} out={out1}")
self.assertEqual(in1-in0, 0)
self.assertEqual(out1-out0, 0)
def test_hop_depth(self, hostname='localhost', username=None,
depth=4, commands=1):
# Do not use the close_nodes manually on this test, it does keep the
# Order of the declared nodes.
if username is None:
username = localuser()
verbose(f"creating hop{depth}-connections - "
f"{commands} commands per conn "
f" to {username}@{hostname}")
scheduler = Scheduler(timeout=7)
nodes = []
jobs = []
gateway = None
for n in range(depth):
node = SshNode(hostname, gateway=gateway, username=username,
formatter=ColonFormatter(verbose=False))
nodes.append(node)
gateway = node
for c in range(commands):
jobs.append(SshJob(node=node,
command=f"echo hop{n}-{c}",
scheduler=scheduler))
expected = depth
# record base status
in0, out0 = in_out_connections()
verbose(f"INITIAL count in={in0} out={out0}")
# try:
scheduler.run()
#except Exception:
# pass
in1, out1 = in_out_connections()
verbose(f"AFTER RUN in={in1} out={out1}")
self.assertEqual(in1-in0, expected)
self.assertEqual(out1-out0, expected)
# cleanup
close_ssh_in_scheduler(scheduler)
#Lets wait a little bit to count
time.sleep(1)
in1, out1 = in_out_connections()
verbose(f"AFTER CLEANUP in={in1} out={out1}")
self.assertEqual(in1-in0, 0)
self.assertEqual(out1-out0, 0)
def one_run(*, protocol, interference,
run_name=default_run_name, slicename=default_slicename,
tx_power, phy_rate, antenna_mask, channel,
load_images=False,
node_ids=DEFAULT_NODE_IDS,
src_ids=DEFAULT_SRC_IDS, dest_ids=DEFAULT_DEST_IDS,
scrambler_id=DEFAULT_SCRAMBLER_ID,
tshark=False, map=False, warmup=False,
route_sampling=False, iperf=False,
verbose_ssh=False, verbose_jobs=False, dry_run=False,
run_number=None):
"""
Performs data acquisition on all nodes with the following settings
Arguments:
tx_power: in dBm, a string like 5, 10 or 14.
Corresponds to the transmission power.
phy_rate: a string among 1, 54. Correspond to the wifi rate.
antenna_mask: a string among 1, 3, 7.
channel: a string like e.g. 1 or 40. Correspond to the channel.
protocol: a string among batman , olsr. Correspond to the protocol
interference : in amplitude percentage, a string like 15 or 20.
Correspond to the power of the noise generated in the spectrum.
Can be either None or "None" to mean no interference.
run_name: the name for a subdirectory where all data will be kept
successive runs should use the same name for further visualization
slicename: the Unix login name (slice name) to enter the gateway
load_images: a boolean specifying whether nodes should be re-imaged first
node_ids: a list of node ids to run the scenario against;
strings or ints are OK;
tshark: a boolean specifying wether we should format/parse the .pcap.
map: a boolean specifying wether we should fetch/parse
the route tables of the nodes.
warmup: a boolean specifying whether we should run a ping before
the experiment to be certain of the stabilisation on the network.
src_ids: a list of nodes from which we will launch the ping from.
strings or ints are OK.
ping_messages : the number of ping packets that will be generated
"""
# set default for the nodes parameter
node_ids = ([int(id) for id in node_ids]
if node_ids is not None else DEFAULT_NODE_IDS)
src_ids = ([int(id) for id in src_ids]
if src_ids is not None else DEFAULT_SRC_IDS)
dest_ids = ([int(id) for id in dest_ids]
if dest_ids is not None else DEFAULT_NODE_IDS)
# all nodes - i.e. including sources and destinations -
# need to run the protocol
node_ids = list(set(node_ids).union(set(src_ids).union(set(dest_ids))))
if interference == "None":
interference = None
# open result dir no matter what
run_root = naming_scheme(
run_name=run_name, protocol=protocol,
interference=interference, autocreate=True)
# fix me trace = run_root / f"trace-{%m-%d-%H-%M}"
ref_time = apssh_time()
trace = run_root / f"trace-{ref_time}"
try:
with trace.open('w') as feed:
def log_line(line):
time_line(line, file=feed)
load_msg = f"{'WITH' if load_images else 'NO'} image loading"
interference_msg = (f"interference={interference} "
f"from scrambler={scrambler_id}")
nodes = " ".join(str(n) for n in node_ids)
srcs = " ".join(str(n) for n in src_ids)
dests = " ".join(str(n) for n in dest_ids)
ping_labels = [
f"PING {s} ➡︎ {d}"
for s in src_ids
# and on the destination
for d in dest_ids
if d != s
]
log_line(f"output in {run_root}")
log_line(f"trace in {trace}")
log_line(f"protocol={protocol}")
log_line(f"{load_msg}")
log_line(f"{interference_msg}")
log_line("----")
log_line(f"Selected nodes : {nodes}")
log_line(f"Sources : {srcs}")
log_line(f"Destinations : {dests}")
for label in ping_labels:
log_line(f"{label}")
log_line("----")
for feature in ('warmup', 'tshark', 'map',
'route_sampling', 'iperf'):
log_line(f"Feature {feature}: {locals()[feature]}")
except Exception as exc:
print(f"Cannot write into {trace} - aborting this run")
print(f"Found exception {type(exc)} - {exc}")
return False
#
# dry-run mode
# just display a one-liner with parameters
#
prelude = "" if not dry_run else "dry_run:"
with trace.open() as feed:
print(f"**************** {ref_time} one_run #{run_number}:")
for line in feed:
print(prelude, line, sep='', end='')
if dry_run:
return True
# the nodes involved
faraday = SshNode(hostname=default_gateway, username=slicename,
formatter=TimeColonFormatter(), verbose=verbose_ssh)
# this is a python dictionary that allows to retrieve a node object
# from an id
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id), username="******",
formatter=TimeColonFormatter(), verbose=verbose_ssh)
for id in node_ids
}
# extracts for sources and destinations
src_index = {id:node for (id, node) in node_index.items()
if id in src_ids}
dest_index = {id:node for (id, node) in node_index.items()
if id in dest_ids}
if interference:
node_scrambler = SshNode(
gateway=faraday, hostname=fitname(scrambler_id), username="******",
formatter=TimeColonFormatter(), verbose=verbose_ssh)
# the global scheduler
scheduler = Scheduler(verbose=verbose_jobs)
##########
check_lease = SshJob(
scheduler=scheduler,
node=faraday,
verbose=verbose_jobs,
label="rhubarbe check lease",
command=Run("rhubarbe leases --check", label="rlease"),
)
# load images if requested
green_light = check_lease
# at some point we did not load the scrambler if interference was None
# and that was a way to run faster loads with no interference
# but now we always load the scrambler node with gnuradio
# this is because when we do runs.py -i None 15 30 ...
# then the first call to one_run is with interference being None
# but it is still important to load the scrambler
if load_images:
# copy node_ids
load_ids = node_ids[:]
load_ids.append(scrambler_id)
# the nodes that we **do not** use should be turned off
# so if we have selected e.g. nodes 10 12 and 15, we will do
# rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15
negated_node_ids = [f"~{id}" for id in load_ids]
# we can do these three things in parallel
ready_jobs = [
SshJob(node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
command=Run("rhubarbe", "off", "-a", *negated_node_ids,
label="turn off unused nodes")),
SshJob(node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
label="load batman image",
command=Run("rhubarbe", "load", "-i",
"batman-olsr",
*node_ids,
label=f"load ubuntu on {node_ids}")),
SshJob(
node=faraday, required=green_light,
scheduler=scheduler, verbose=verbose_jobs,
label="load gnuradio image",
command=Run("rhubarbe", "load", "-i",
"batman-olsr-gnuradio",
scrambler_id,
label=f"load gnuradio on {scrambler_id}")),
]
# replace green_light in this case
green_light = SshJob(
node=faraday, required=ready_jobs,
scheduler=scheduler, verbose=verbose_jobs,
label="wait for nodes to come up",
command=Run("rhubarbe", "wait", *load_ids))
##########
# setting up the wireless interface on all nodes
#
# provide node-utilities with the ranges/units it expects
frequency = channel_frequency[int(channel)]
# tx_power_in_mBm not in dBm
tx_power_driver = tx_power * 100
#just in case somme services failed in the previous experiment
reset_failed_services_job = [
SshJob(
node=node,
verbose=verbose_jobs,
label="reset failed services",
command=Run("systemctl reset-failed",
label="reset-failed services"))
for id, node in node_index.items()
]
reset_failed_services = Scheduler(
*reset_failed_services_job,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Reset failed services")
init_wireless_sshjobs = [
SshJob(
node=node,
verbose=verbose_jobs,
label=f"init {id}",
command=RunScript(
"node-utilities.sh",
f"init-ad-hoc-network-{WIRELESS_DRIVER}",
WIRELESS_DRIVER, "foobar", frequency, phy_rate,
antenna_mask, tx_power_driver,
label="init add-hoc network"),
)
for id, node in node_index.items()]
init_wireless_jobs = Scheduler(
*init_wireless_sshjobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Initialisation of wireless chips")
if interference:
# Run uhd_siggen with the chosen power
init_scrambler_job = SshJob(
scheduler=scheduler,
required=green_light,
forever=True,
node=node_scrambler,
verbose=verbose_jobs,
#TODO : If exit-signal patch is done add exit-signal=["TERM"]
# to this run object and call uhd_siggen directly
commands=[RunScript("node-utilities.sh",
"init-scrambler",
label="init scrambler"),
Run(f"systemd-run --unit=uhd_siggen -t ",
f"uhd_siggen -a usrp -f {frequency}M",
f"--sine --amplitude 0.{interference}",
label="systemctl start uhd_siggen")
]
)
green_light = [init_wireless_jobs, reset_failed_services]
# then install and run batman on fit nodes
run_protocol_job = [
SshJob(
# scheduler=scheduler,
node=node,
label=f"init and run {protocol} on fit node {id}",
verbose=verbose_jobs,
# CAREFUL : These ones use sytemd-run
# with the ----service-type=forking option!
command=RunScript("node-utilities.sh",
f"run-{protocol}",
label=f"run {protocol}"),
)
for id, node in node_index.items()]
run_protocol = Scheduler(
*run_protocol_job,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="init and run routing protocols")
green_light = run_protocol
# after that, run tcpdump on fit nodes, this job never ends...
if tshark:
run_tcpdump_job = [
SshJob(
# scheduler=scheduler_monitoring,
node=node,
forever=True,
label=f"run tcpdump on fit node {id}",
verbose=verbose_jobs,
command=[
Run("systemd-run -t --unit=tcpdump",
f"tcpdump -U -i moni-{WIRELESS_DRIVER}",
f"-y ieee802_11_radio -w /tmp/fit{id}.pcap",
label=f"tcpdump {id}")
]
)
for id, node in node_index.items()
]
run_tcpdump = Scheduler(
*run_tcpdump_job,
scheduler=scheduler,
required=green_light,
forever=True,
verbose=verbose_jobs,
label="Monitoring - tcpdumps")
# let the wireless network settle
settle_scheduler = Scheduler(
scheduler=scheduler,
required=green_light,
)
if warmup:
# warmup pings don't need to be sequential, so let's
# do all the nodes at the same time
# on a given node though, we'll ping the other ends sequentially
# see the graph for more
warmup_jobs = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
RunScript("node-utilities.sh",
"my-ping", f"10.0.0.{d}",
warmup_ping_timeout,
warmup_ping_interval,
warmup_ping_size,
warmup_ping_messages,
f"warmup {s} ➡︎ {d}",
label=f"warmup {s} ➡︎ {d}")
for d in dest_index.keys()
if s != d
]
)
# for each selected experiment nodes
for s, node_s in src_index.items()
]
warmup_scheduler = Scheduler(
*warmup_jobs,
scheduler=settle_scheduler,
verbose=verbose_jobs,
label="Warmup pings")
settle_wireless_job2 = PrintJob(
"Let the wireless network settle after warmup",
sleep=settle_delay_shorter,
scheduler=settle_scheduler,
required=warmup_scheduler,
label=f"settling-warmup for {settle_delay_shorter} sec")
# this is a little cheating; could have gone before the bloc above
# but produces a nicer graphical output
# we might want to help asynciojobs if it offered a means
# to specify entry and exit jobs in a scheduler
settle_wireless_job = PrintJob(
"Let the wireless network settle",
sleep=settle_delay_long,
scheduler=settle_scheduler,
label=f"settling for {settle_delay_long} sec")
green_light = settle_scheduler
if iperf:
iperf_service_jobs = [
SshJob(
node=node_d,
verbose=verbose_jobs,
forever=True,
commands=[
Run("systemd-run -t --unit=iperf",
"iperf -s -p 1234 -u",
label=f"iperf serv on {d}"),
],
)
for d, node_d in dest_index.items()
]
iperf_serv_sched = Scheduler(
*iperf_service_jobs,
verbose=verbose_jobs,
label="Iperf Servers",
# for a nicer graphical output
# otherwise the exit arrow
# from scheduler 'iperf mode'
# to job 'settling for 60s'
# gets to start from this box
forever=True,
)
iperf_cli = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
Run("sleep 7", label=""),
Run(f"iperf",
f"-c 10.0.0.{d} -p 1234",
f"-u -b {phy_rate}M -t 60",
f"-l 1024 > IPERF-{s:02d}-{d:02d}",
label=f"run iperf {s} ➡︎ {d}")
]
)
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
iperf_cli_sched = Scheduler(
Sequence(*iperf_cli),
verbose=verbose_jobs,
label="Iperf Clients")
iperf_stop = [
SshJob(node=node_d,
verbose=verbose_jobs,
label=f"Stop iperf on {d}",
command=Run("systemctl stop iperf"))
for d, node_d in dest_index.items()
]
iperf_stop_sched = Scheduler(
*iperf_stop,
required=iperf_cli_sched,
verbose=verbose_jobs,
label="Iperf server stop")
iperf_fetch = [
SshJob(node=node_s,
verbose=verbose_jobs,
command=Pull(
remotepaths=[f"IPERF-{s:02d}-{d:02d}"],
localpath=str(run_root),
label="fetch iperf {s} ➡︎ {d}")
)
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
iperf_fetch_sched = Scheduler(
*iperf_fetch,
required=iperf_stop_sched,
verbose=verbose_jobs,
label="Iperf fetch report")
iperf_jobs = [iperf_serv_sched, iperf_cli_sched,
iperf_stop_sched, iperf_fetch_sched]
iperf_sched = Scheduler(
*iperf_jobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Iperf Module")
settle_wireless_job_iperf = PrintJob(
"Let the wireless network settle",
sleep=settle_delay_shorter,
scheduler=scheduler,
required=iperf_sched,
label=f"settling-iperf for {settle_delay_shorter} sec")
green_light = settle_wireless_job_iperf
# create all the tracepath jobs from the first node in the list
if map:
map_jobs = [
SshJob(
node=node,
label=f"Generating ROUTE file for proto {protocol} on node {id}",
verbose=verbose_jobs,
commands=[
RunScript(f"node-utilities.sh",
f"route-{protocol}",
f"> ROUTE-TABLE-{id:02d}",
label="get route table"),
Pull(remotepaths=[f"ROUTE-TABLE-{id:02d}"],
localpath=str(run_root),
label="")
],
)
for id, node in node_index.items()
]
map_scheduler = Scheduler(
*map_jobs,
scheduler=scheduler,
required=green_light,
verbose=verbose_jobs,
label="Snapshoting route files")
green_light = map_scheduler
if route_sampling:
route_sampling_jobs = [
SshJob(
node=node,
label=f"Route sampling service for proto {protocol} on node {id}",
verbose=False,
forever=True,
commands=[
Push(localpaths=["route-sample-service.sh"],
remotepath=".", label=""),
Run("chmod +x route-sample-service.sh", label=""),
Run("systemd-run -t --unit=route-sample",
"/root/route-sample-service.sh",
"route-sample",
f"ROUTE-TABLE-{id:02d}-SAMPLED",
protocol,
label="start route-sampling"),
],
)
for id, node in node_index.items()
]
route_sampling_scheduler = Scheduler(
*route_sampling_jobs,
scheduler=scheduler,
verbose=False,
forever=True,
label="Route Sampling services launch",
required=green_light)
##########
# create all the ping jobs, i.e. max*(max-1)/2
# this again is a python list comprehension
# see the 2 for instructions at the bottom
#
# notice that these SshJob instances are not yet added
# to the scheduler, we will add them later on
# depending on the sequential/parallel strategy
pings_job = [
SshJob(
node=node_s,
verbose=verbose_jobs,
commands=[
Run(f"echo actual ping {s} ➡︎ {d} using {protocol}",
label=f"ping {s} ➡︎ {d}"),
RunScript("node-utilities.sh", "my-ping",
f"10.0.0.{d}",
ping_timeout, ping_interval,
ping_size, ping_messages,
f"actual {s} ➡︎ {d}",
">", f"PING-{s:02d}-{d:02d}",
label=""),
Pull(remotepaths=[f"PING-{s:02d}-{d:02d}"],
localpath=str(run_root),
label=""),
],
)
# for each selected experiment nodes
for s, node_s in src_index.items()
for d, node_d in dest_index.items()
if s != d
]
pings = Scheduler(
scheduler=scheduler,
label="PINGS",
verbose=verbose_jobs,
required=green_light)
# retrieve all pcap files from fit nodes
stop_protocol_job = [
SshJob(
# scheduler=scheduler,
node=node,
# required=pings,
label=f"kill routing protocol on {id}",
verbose=verbose_jobs,
command=RunScript(f"node-utilities.sh",
f"kill-{protocol}",
label=f"kill-{protocol}"),
)
for id, node in node_index.items()
]
stop_protocol = Scheduler(
*stop_protocol_job,
scheduler=scheduler,
required=pings,
label="Stop routing protocols",
)
if tshark:
retrieve_tcpdump_job = [
SshJob(
# scheduler=scheduler,
node=nodei,
# required=pings,
label=f"retrieve pcap trace from fit{i:02d}",
verbose=verbose_jobs,
commands=[
Run("systemctl stop tcpdump",
label="stop tcpdump"),
#Run("systemctl reset-failed tcpdump"),
#RunScript("node-utilities.sh", "kill-tcpdump",
# label="kill-tcpdump"),
Run(
f"echo retrieving pcap trace and result-{i}.txt from fit{i:02d}",
label=""),
Pull(remotepaths=[f"/tmp/fit{i}.pcap"],
localpath=str(run_root), label=""),
],
)
for i, nodei in node_index.items()
]
retrieve_tcpdump = Scheduler(
*retrieve_tcpdump_job,
scheduler=scheduler,
required=pings,
label="Retrieve tcpdump",
)
if route_sampling:
retrieve_sampling_job = [
SshJob(
# scheduler=scheduler,
node=nodei,
# required=pings,
label=f"retrieve sampling trace from fit{i:02d}",
verbose=verbose_jobs,
commands=[
# RunScript("node-utilities.sh", "kill-route-sample", protocol,
# label = "kill route sample"),
#RunScript("route-sample-service.sh", "kill-route-sample",
# label="kill route sample"),
Run("systemctl stop route-sample",
label="stop route-sample"),
Run(
f"echo retrieving sampling trace from fit{i:02d}",
label=""),
Pull(remotepaths=[f"ROUTE-TABLE-{i:02d}-SAMPLED"],
localpath=str(run_root), label=""),
],
)
for i, nodei in node_index.items()
]
retrieve_sampling = Scheduler(
*retrieve_sampling_job,
scheduler=scheduler,
required=pings,
verbose=verbose_jobs,
label="Stop & retrieve route sampling",
)
if tshark:
parse_pcaps_job = [
SshJob(
# scheduler=scheduler,
node=LocalNode(),
# required=retrieve_tcpdump,
label=f"parse pcap trace {run_root}/fit{i}.pcap",
verbose=verbose_jobs,
#commands = [RunScript("parsepcap.sh", run_root, i)]
command=Run("tshark", "-2", "-r",
f"{run_root}/fit{i}.pcap",
"-R",
f"'(ip.dst==10.0.0.{i} && icmp) && radiotap.dbm_antsignal'",
"-Tfields",
"-e", "'ip.src'",
"-e" "'ip.dst'",
"-e", "'radiotap.dbm_antsignal'",
">", f"{run_root}/result-{i}.txt",
label=f"parsing pcap from {i}"),
)
for i in node_ids
]
parse_pcaps = Scheduler(
*parse_pcaps_job,
scheduler=scheduler,
required=retrieve_tcpdump,
label="Parse pcap",
)
if interference:
kill_uhd_siggen = SshJob(
scheduler=scheduler,
node=node_scrambler,
required=pings,
label=f"killing uhd_siggen on the scrambler node {scrambler_id}",
verbose=verbose_jobs,
commands=[Run("systemctl", "stop", "uhd_siggen"),
#Run("systemctl reset-failed tcpdump"),
],
)
kill_2_uhd_siggen = SshJob(
scheduler=scheduler,
node=faraday,
required=kill_uhd_siggen,
label=f"turning off usrp on the scrambler node {scrambler_id}",
verbose=verbose_jobs,
command=Run("rhubarbe", "usrpoff", scrambler_id),
)
pings.add(Sequence(*pings_job))
# for running sequentially we impose no limit on the scheduler
# that will be limitied anyways by the very structure
# of the required graph
# safety check
scheduler.export_as_pngfile(run_root / "experiment-graph")
if dry_run:
scheduler.list()
return True
# if not in dry-run mode, let's proceed to the actual experiment
ok = scheduler.run() # jobs_window=jobs_window)
# close all ssh connections
close_ssh_in_scheduler(scheduler)
# give details if it failed
if not ok:
scheduler.debrief()
scheduler.export_as_pngfile("debug")
if ok and map:
time_line("Creation of MAP files")
post_processor = ProcessRoutes(run_root, src_ids, node_ids)
post_processor.run()
if ok and route_sampling:
time_line("Creation of ROUTE SAMPLING files")
post_processor = ProcessRoutes(run_root, src_ids, node_ids)
post_processor.run_sampled()
# data acquisition is done, let's aggregate results
# i.e. compute averages
#if ok and tshark:
#post_processor = Aggregator(run_root, node_ids, antenna_mask)
#post_processor.run()
time_line("one_run done")
return ok
def hop2(self, hostname='localhost', username=None,
*, c1=1, c2=1, commands=1, s_command='echo hop2-{}-{}-{}',
nested_sched=(0, 1)):
"""
create
* <c1> connections to one node 1 hop away
* on each one, <c2> connections one hop behind
* and on each <commands> commands
check current number of connections
"""
if username is None:
username = localuser()
verbose(f"creating {c1}x{c2} hop2-connections - "
f"{commands} commands per conn "
f" to {username}@{hostname}")
scheduler = Scheduler(timeout=7)
nodes = []
#nodes2 = []
jobs = []
for n in range(c1):
node1 = SshNode(hostname, username=username,
formatter=ColonFormatter(verbose=False))
nodes.append(node1)
for m in range(c2):
node2 = SshNode(hostname, username=username,
gateway=node1,
formatter=ColonFormatter(verbose=False))
nodes.append(node2)
for c in range(commands):
jobs.append(SshJob(node=node2,
command=s_command.format(n, m, c),
))
scheduler = self.populate_sched(scheduler, jobs,
nested=nested_sched[0],
pack_job=nested_sched[1])
# for each hop1 conn, there are 1 hop1 + c2 hop2 connections alive
expected = c1 * (c2+1)
scheduler.export_as_pngfile("debug")
topology_as_pngfile(scheduler, "topology")
# record base status
in0, out0 = in_out_connections()
verbose(f"INITIAL count in={in0} out={out0}")
# try:
scheduler.run()
#except Exception:
# pass
in1, out1 = in_out_connections()
verbose(f"AFTER RUN in={in1} out={out1}")
self.assertEqual(in1-in0, expected)
self.assertEqual(out1-out0, expected)
# cleanup
close_ssh_in_scheduler(scheduler)
#Lets wait a little bit to count
time.sleep(1)
in1, out1 = in_out_connections()
verbose(f"AFTER CLEANUP in={in1} out={out1}")
self.assertEqual(in1-in0, 0)
self.assertEqual(out1-out0, 0)
def _simple(self, forever):
storage = f"/root/TCPDUMP-{forever}.pcap"
status = f"/root/TCPDUMP-{forever}.status"
tcpdump = Service(f"tcpdump -i lo -w {storage}",
service_id='tcpdump',
verbose=True)
monitor = ProcessMonitor()
scheduler = Scheduler()
node = SshNode("localhost")
SshJob(node,
scheduler=scheduler,
command=tcpdump.start_command(),
forever=forever)
Sequence(
SshJob(node, command="sleep 1"),
SshJob(node, command=tcpdump.status_command(output=status)),
SshJob(node, command="sleep 1"),
SshJob(node, command=tcpdump.stop_command()),
# could use a pull to retrive both files but that's not required
# since we run on localhost, so keep tests simple
scheduler=scheduler,
)
# cleanup before we run
paths = (Path(x) for x in (storage, status))
for path in paths:
if path.exists():
path.unlink()
self.assertFalse(path.exists())
produce_png(scheduler, f"service-{forever}")
self.assertTrue(scheduler.run())
scheduler.list()
for path in paths:
self.assertTrue(path.exists())
with Path(status).open() as feed:
contents = feed.read()
for needle in ('Loaded: loaded', 'Active: active'):
self.assertTrue(contents.find(needle) >= 0)
close_ssh_in_scheduler(scheduler)
# let it settle for a short while, and check the process space
import time
time.sleep(0.5)
monitor.difference()
news = monitor.news
if news:
print(f"we have {len(news)} new processes, {news}")
ps_command = "ps " + "".join(str(pid) for pid in news)
import os
os.system(ps_command)
self.assertEqual(len(news), 0)