def start_server(conn,
redis_addr,
outf,
shards=1,
ebpf=False,
server_batch="none",
stack_frag=False):
conn.run("sudo pkill -9 kvserver-ebpf")
conn.run("sudo pkill -9 kvserver-noebpf")
conn.run("sudo pkill -INT iokerneld")
write_shenango_config(conn)
conn.run("./iokerneld",
wd="~/burrito/shenango-chunnel/caladan",
sudo=True,
background=True)
time.sleep(2)
with_ebpf = "ebpf" if ebpf else "noebpf"
ok = conn.run(
f"RUST_LOG=info,kvstore=debug,bertha=debug ./target/release/kvserver-{with_ebpf} --ip-addr {conn.addr} --port 4242 --num-shards {shards} --redis-addr={redis_addr} -s host.config --batch-mode={server_batch} {'--fragment-stack' if stack_frag else ''} --log --trace-time={outf}.trace",
wd="~/burrito",
sudo=True,
background=True,
stdout=f"{outf}.out",
stderr=f"{outf}.err",
)
check(ok, "spawn server", conn.addr)
agenda.subtask("wait for kvserver check")
time.sleep(8)
conn.check_proc(f"kvserver-{with_ebpf}", f"{outf}.err")
def setup_machine(conn, outdir):
ok = conn.run(f"mkdir -p ~/burrito/{outdir}")
check(ok, "mk outdir", conn.addr)
agenda.subtask(f"building burrito on {conn.addr}")
ok = conn.run("make sharding", wd="~/burrito")
check(ok, "build", conn.addr)
return conn
def make_cloudlab_topology(config, headless=False):
agenda.section("Setup Cloudlab topology")
agenda.subtask(f"headless: {headless}")
driver = init_driver(
config['topology']['cloudlab']['username'],
config['topology']['cloudlab']['password'],
headless=headless,
)
machines = check_existing_experiment(driver)
if machines is None:
machines = launch(driver)
machines = [
cloudlab_conn_rgx.match(m).groupdict() for m in machines
if 'cloudlab.us' in m
]
for (name, host_id) in zip(['sender', 'inbox', 'outbox', 'receiver'],
[0, 1, 2, 2]):
overrides = config['topology'][name]
config['topology'][name] = machines[host_id]
for (k, v) in overrides.items():
config['topology'][name][k] = v
return config
def put(self, local_file, remote=None, preserve_mode=True):
if remote and remote[0] == "~":
remote = remote[2:]
agenda.subtask("[{}] scp localhost:{} -> {}:{}".format(
self.addr, local_file, self.addr, remote))
return super().put(local_file, remote, preserve_mode)
def start_server(self, config, node, execute):
if int(self.port) == 8900 or int(self.port) == 8901:
agenda.subtask("Skipping server!")
return
agenda.subtask("Start iperf server ({})".format(self))
iperf_out = os.path.join(config['iteration_dir'],
"iperf_server_{}.log".format(self.port))
expect(
node.run(
"{path} -s -p {port} --reverse -i {report_interval} -t {length} -P {num_flows} -Z {alg}"
.format(
path=os.path.join(config['structure']['bundler_root'],
'iperf/src/iperf'),
port=self.port,
report_interval=self.report_interval,
length=self.length,
num_flows=self.num_flows,
alg=self.alg,
),
background=True,
stdout=iperf_out,
stderr=iperf_out),
"Failed to start iperf server on {}".format(node.addr))
if not config['args'].dry_run:
time.sleep(1)
node.check_file('Server listening on TCP port', iperf_out)
config['iteration_outputs'].append((node, iperf_out))
return iperf_out
def parse_etg_logs(dirname, replot):
agenda.subtask("etg logs")
outf = os.path.join(dirname, "fcts.data")
if not replot and os.path.isfile(outf):
return
if replot and os.path.isfile(outf):
os.remove(outf)
g = glob.glob(dirname + "/**/*reqs.out", recursive=True)
some = None
cross_traffic_pattern = "0:60=empty1,60:120=iperfc1,120:150=empty2,150:210=cbr32,210:250=empty3"
print_head = True
for exp in g:
print(exp)
some = True
exp_root = "/".join(exp.split("/")[:-1])
exp_root = os.path.dirname(exp)
exp_root = exp_root.split(dirname)[-1].split("/")
try:
_, setup, alg, traffic, seed = exp_root
except Exception as e:
print(exp_root)
raise e
alg_sp = alg.split(".")
if not alg_sp:
alg = alg_sp[0]
sch, bw, rtt = setup.split("_")
subprocess.check_output(f"awk '{{print \"sch:{sch}, bw:{bw}, rtt:{rtt}, alg:{alg}, traffic:{traffic}, seed:{seed} \"$0}}' {exp} | python3 columnize.py \"{cross_traffic_pattern}\" {print_head}>> tmp", shell=True)
print_head = False
if some:
subprocess.call(f"mv tmp {outf}", shell=True)
def create_ssh_connections(config):
agenda.task("Creating SSH connections")
conns = {}
machines = {}
args = config['args']
for (role, details) in [(r, d) for r, d in config['topology'].items()
if r in ("sender", "inbox", "outbox", "receiver")]:
hostname = details['name']
is_self = 'self' in details and details['self']
if is_self:
agenda.subtask(hostname)
conns[hostname] = ConnectionWrapper('localhost',
nickname=role,
dry=args.dry_run,
verbose=args.verbose,
interact=args.interact)
config['self'] = conns[hostname]
elif not hostname in conns:
agenda.subtask(hostname)
user = None
port = None
if 'user' in details:
user = details['user']
if 'port' in details:
port = details['port']
conns[hostname] = ConnectionWrapper(hostname,
nickname=role,
user=user,
port=port,
dry=args.dry_run,
verbose=args.verbose,
interact=args.interact)
machines[role] = conns[hostname]
return (conns, machines)
def start_ccp(config, inbox, alg):
if config['args'].verbose:
agenda.subtask("Starting ccp")
ccp_binary = get_ccp_binary_path(config, alg['name'])
ccp_binary_name = ccp_binary.split('/')[-1]
ccp_out = os.path.join(config['iteration_dir'], "ccp.log")
alg_name = alg['name']
args = list(config['ccp'][alg_name]['args'].items())
args += [(k, alg[k]) for k in alg]
alg_args = [
f"--{arg}={val}" for arg, val in args
if val != "false" and arg != "name"
]
expect(
inbox.run(
"{} --ipc=unix {}".format(ccp_binary, " ".join(alg_args)),
sudo=True,
background=True,
stdout=ccp_out,
stderr=ccp_out,
), "Failed to start ccp")
if not config['args'].dry_run:
time.sleep(1)
inbox.check_proc(ccp_binary_name, ccp_out)
inbox.check_file('starting CCP', ccp_out)
config['iteration_outputs'].append((inbox, ccp_out))
return ccp_out
def start_client(self, config, node, in_bundler, execute):
agenda.subtask("Start iperf client ({})".format(self))
iperf_out = os.path.join(config['iteration_dir'],
"iperf_client_{}.log".format(self.port))
check_bundler_port(in_bundler, self, config)
cmd = "sleep {delay} && {path} -c {ip} -p {port} --reverse -i {report_interval} -t {length} -P {num_flows} -Z {alg}".format(
path=os.path.join(config['structure']['bundler_root'],
'iperf/src/iperf'),
ip=config['topology']['sender']['ifaces'][0]['addr']
if in_bundler else '$MAHIMAHI_BASE',
port=self.port,
report_interval=self.report_interval,
length=self.length,
num_flows=self.num_flows,
alg=self.alg,
delay=self.start_delay)
config['iteration_outputs'].append((node, iperf_out))
if execute:
expect(
node.run(cmd,
background=True,
stdout=iperf_out,
stderr=iperf_out),
"Failed to start iperf client on {}".format(node.addr))
else:
return cmd + " > {}".format(iperf_out)
def start_server_unix(neg_arg):
agenda.task("local rpcbench-server unix")
neg = '' if neg_arg != 'off' else '--no-negotiation'
cmd = f"./scripts/start-rpcbench-unix-server.sh ./target/release {neg} &"
agenda.subtask("running launch script")
sh.run(cmd, shell=True)
agenda.subtask("launched")
def start_inbox(self, qtype, q_buffer_size):
config = self.config
inbox = self.machines['inbox']
agenda.subtask("Starting inbox")
inbox_out = os.path.join(config['iteration_dir'], "inbox.log")
res = inbox.run(
"{path} --iface={iface} --port={port} --sample_rate={sample} --qtype={qtype} --buffer={buf}"
.format(path=get_inbox_binary(config),
iface=get_iface(config, 'inbox')['dev'],
port=config['topology']['inbox']['listen_port'],
sample=config['parameters']['initial_sample_rate'],
qtype=qtype,
buf=q_buffer_size),
sudo=True,
background=True,
stdout=inbox_out,
stderr=inbox_out,
)
if not config['args'].dry_run:
time.sleep(10)
inbox.check_proc('inbox', inbox_out)
inbox.check_file('Wait for CCP to install datapath program', inbox_out)
config['iteration_outputs'].append((inbox, inbox_out))
return inbox_out
def get_files(num):
fn = c.get
if c.local:
agenda.subtask(f"Use get_local: {c.host}")
fn = get_local
agenda.subtask(f"getting {outf}{num}-{c.addr}.err")
fn(
f"burrito/{outf}{num}.err",
local=f"{outf}{num}-{c.addr}.err",
preserve_mode=False,
)
agenda.subtask(f"getting {outf}{num}-{c.addr}.out")
fn(
f"burrito/{outf}{num}.out",
local=f"{outf}{num}-{c.addr}.out",
preserve_mode=False,
)
agenda.subtask(f"getting {outf}{num}-{c.addr}.data")
fn(
f"burrito/{outf}{num}.data",
local=f"{outf}{num}-{c.addr}.data",
preserve_mode=False,
)
agenda.subtask(f"getting {outf}{num}-{c.addr}.trace")
fn(
f"burrito/{outf}{num}.trace",
local=f"{outf}{num}-{c.addr}.trace",
preserve_mode=False,
)
def run(self,
cmd,
*args,
stdin=None,
stdout=None,
stderr=None,
ignore_out=False,
wd=None,
sudo=False,
background=False,
quiet=False,
pty=True,
**kwargs):
self.verbose = True
# Prepare command string
pre = ""
if wd:
pre += f"cd {wd} && "
if background:
pre += "screen -d -m "
#escape the strings
cmd = cmd.replace("\"", "\\\"")
if sudo:
pre += "sudo "
pre += "bash -c \""
if ignore_out:
stdin = "/dev/null"
stdout = "/dev/null"
stderr = "/dev/null"
if background:
stdin = "/dev/null"
full_cmd = f"{pre}{cmd}"
if stdout is not None:
full_cmd += f" > {stdout} "
if stderr is not None:
full_cmd += f" 2> {stderr} "
if stdin is not None:
full_cmd += f" < {stdin} "
full_cmd += "\""
# Prepare arguments for invoke/fabric
if background:
pty = False
# Print command if necessary
if not quiet:
agenda.subtask("[{}]{} {}".format(
self.addr.ljust(10), " (bg) " if background else " ",
full_cmd))
# Finally actually run it
return super().run(full_cmd,
*args,
hide=True,
warn=True,
pty=pty,
**kwargs)
def get_machines_from_experiment(driver):
driver.find_element_by_id("show_listview_tab").click()
time.sleep(1)
machines = [m.text for m in driver.find_elements_by_name("sshurl")]
agenda.subtask("Got machines")
for m in machines:
agenda.subtask(m)
return machines
def get(self, remote_file, local=None, preserve_mode=True):
if local is None:
local = remote_file
agenda.subtask("[{}] scp {}:{} -> localhost:{}".format(
self.addr, self.addr, remote_file, local))
return super().get(remote_file,
local=local,
preserve_mode=preserve_mode)
def check_ccp_alg(config, node):
for (alg, details) in config['ccp'].items():
agenda.task(alg)
alg_dir = get_ccp_alg_dir(config, alg)
if not node.file_exists(alg_dir):
expect(
node.run("git clone {} {}".format(details['repo'], alg_dir)),
"node failed to clone {}".format(alg))
branch = node.run("git -C {} rev-parse --abbrev-ref HEAD".format(
alg_dir)).stdout.strip()
if branch != details['branch']:
expect(
node.run("git -C {} checkout {}".format(
alg_dir, details['branch'])),
"node failed to checkout branch {} of {}".format(
details['branch'], alg))
commit = node.run(
"git -C {} rev-parse HEAD".format(alg_dir)).stdout.strip()
should_recompile = False
if not details['commit'] in commit:
pull = expect(node.run("git -C {} pull".format(alg_dir)),
"node failed to pull latest code for {}".format(
alg)).stdout.strip()
if details['commit'] == 'latest':
if not 'Already up-to-date.' in pull:
should_recompile = True
else:
expect(
node.run("git -C {} checkout {}".format(
alg_dir, details['commit'])),
"node failed to checkout commit {} of {}".format(
details['commit'], alg))
should_recompile = True
if details['language'] == 'rust':
ccp_binary = get_ccp_binary_path(config, alg)
if not node.file_exists(ccp_binary):
print("could not find ccp binary")
should_recompile = True
if should_recompile:
new_commit = node.run(
"git -C {} rev-parse HEAD".format(alg_dir)).stdout.strip()
if commit.strip() != new_commit.strip():
print("updated {}: {} -> {}".format(
alg, commit[:6], new_commit[:6]))
agenda.subtask("compiling ccp algorithm")
expect(
node.run("~/.cargo/bin/cargo build {}".format(
'--release' if 'release' in ccp_binary else ''),
wd=alg_dir),
"node failed to build {}".format(alg))
def exp_unix(args, n):
neg = f'--negotiation={n}'
outfile_arg = f"local-mode:rel-ux-msgs:{args.reqs}-perconn:{args.perconn}-neg:{n}"
cmd = f"./scripts/run-rpcbench-unix-client.sh \
{args.outdir} \
-i={args.reqs} \
--reqs-per-iter={args.perconn} \
{outfile_arg} \
{neg}"
agenda.subtask(f"outfile: {outfile_arg}")
sh.run(cmd, shell=True)
def update_sysctl(machines, config):
if 'sysctl' in config:
agenda.task("Updating sysctl")
for (name, conn) in set((m, machines[m]) for m in machines if m in ("sender", "inbox", "outbox", "receiver")):
agenda.subtask(f"{name}")
for k in config['sysctl']:
v = config['sysctl'][k]
expect(
conn.run(f"sysctl -w {k}=\"{v}\"", sudo=True),
f"Failed to set {k} on {conn.addr}"
)
def start_redis(machine, use_sudo=False):
machine.run("docker rm -f burrito-shard-redis", sudo=True)
agenda.task("Starting redis")
ok = machine.run("docker run \
--name burrito-shard-redis \
-d -p 6379:6379 redis:6",
sudo=True,
wd="~/burrito")
check(ok, "start redis", machine.addr)
ok = machine.run("docker ps | grep burrito-shard-redis", sudo=True)
check(ok, "start redis", machine.addr)
agenda.subtask(f"Started redis on {machine.host}")
return f"{machine.alt}:6379"
def start_tcpdump(config, machines):
agenda.subtask("Start tcpdump")
inbox = machines['inbox']
outbox = machines['outbox']
inbox_pcap = os.path.join(config['iteration_dir'], 'inbox.pcap')
outbox_pcap = os.path.join(config['iteration_dir'], 'outbox.pcap')
inbox.run(f"tcpdump -i {config['topology']['inbox']['ifaces'][1]['dev']} -n -s128 -w {inbox_pcap} \"src portrange {config['parameters']['bg_port_start']}-{config['parameters']['bg_port_end']}\"", sudo=True, background=True)
outbox.run(f"tcpdump -i {config['topology']['outbox']['ifaces'][0]['dev']} -n -s128 -w {outbox_pcap} \"src portrange {config['parameters']['bg_port_start']}-{config['parameters']['bg_port_end']}\"", sudo=True, background=True)
config['iteration_outputs'].append((inbox, inbox_pcap))
config['iteration_outputs'].append((outbox, outbox_pcap))
return config
def disable_tcp_offloads(config, machines):
agenda.task("Turn off TSO, GSO, and GRO")
for (name, conn) in set((m, machines[m]) for m in machines if m in ("sender", "inbox", "outbox", "receiver")):
agenda.subtask(name)
for iface in config['topology'][name]['ifaces']:
expect(
conn.run(
"ethtool -K {} tso off gso off gro off".format(
iface['dev']
),
sudo=True
),
"Failed to turn off optimizations"
)
def fetch_build_logs(self, config):
if self.machines is None or 'local_experiment_dir' not in config:
raise Exception("Tried to fetch build logs without connecting")
for m in self.machines:
agenda.subtask(f"fetch from {m}")
root = config['structure']['bundler_root']
if root.startswith("~/"):
root = root[2:]
self.machines[m].get(
f"{root}/{m}.out.mk",
f"{config['local_experiment_dir']}/{m}.out.mk")
self.machines[m].get(
f"{root}/{m}.err.mk",
f"{config['local_experiment_dir']}/{m}.err.mk")
def all_paths(m_id):
agenda.subtask(f"get measurement {m_id}")
# get all the paths from a given measurement
res = requests.get(f"https://atlas.ripe.net/api/v2/measurements/{m_id}/results/?format=json").text
res = json.loads(res)
if type(res) == dict:
res = [res]
for p in res:
path = read_path(p)
if path is None:
continue
if path['src_asn'] is None or path['dst_asn'] is None:
assert(False)
if len(path['path']) > 0:
yield path
def start_client(self, config, node, in_bundler, execute):
agenda.subtask("Create ETG config file")
if self.start_port < config['parameters'][
'bg_port_start'] or self.start_port + self.num_conns > config[
'parameters']['bg_port_end']:
fatal_warn(
"Requested poisson traffic would be outside of outbox portrange ({}-{})"
.format(config['parameters']['bg_port_start'],
config['parameters']['bg_port_end']))
i = 1
etg_config_path = os.path.join(config['iteration_dir'],
f"etgConfig{i}")
while node.file_exists(etg_config_path) and not config['args'].dry_run:
i += 1
etg_config_path = os.path.join(config['iteration_dir'],
f"etgConfig{i}")
with io.StringIO() as etg_config:
self.create_etg_config(node, config, etg_config)
node.put(etg_config,
remote=os.path.join(config['iteration_dir'],
f"etgConfig{i}"))
etg_out = os.path.join(config['iteration_dir'], "{}".format(i))
# NOTE: using cd + relative paths instead of absolute because etg has a buffer size of 80 for filenames
cmd = "sleep {delay} && cd {wd} && {path} -c {config} -l {out_prefix} -s {seed}".format(
wd=config['iteration_dir'],
path=config['etg_client_path'],
config=os.path.basename(etg_config_path),
out_prefix=str(i),
seed=self.seed,
delay=self.start_delay)
config['iteration_outputs'].append((node, etg_out + "_flows.out"))
config['iteration_outputs'].append((node, etg_out + "_reqs.out"))
if execute:
expect(
node.run(
cmd,
background=True,
stdout=etg_out,
stderr=etg_out,
), "Failed to start poisson client on {}".format(node.addr))
else:
return cmd
def start_server(self, config, node, execute):
agenda.subtask("Start cbr server ({})".format(self))
iperf_out = os.path.join(config['iteration_dir'],
"cbr_server_{}.log".format(self.port))
ccp_binary = get_ccp_binary_path(config, 'const')
#ccp_binary_name = ccp_binary.split('/')[-1]
ccp_out = os.path.join(config['iteration_dir'], "ccp_const.log")
agenda.subtask("Starting CBR CCP agent")
expect(
node.run(
"{ccp_path} --ipc=netlink --rate={rate} --cwnd_cap={cwnd_cap}".
format(
ccp_path=ccp_binary,
rate=self.rate,
cwnd_cap=self.cwnd_cap,
),
sudo=True,
background=True,
stdout=ccp_out,
stderr=ccp_out,
), "Failed to start ccp_const agent")
expect(
node.run(
"{path} -s -p {port} --reverse -i {report_interval} -t {length} -Z ccp"
.format(
path=os.path.join(config['structure']['bundler_root'],
'iperf/src/iperf'),
port=self.port,
report_interval=self.report_interval,
length=self.length,
),
background=True,
stdout=iperf_out,
stderr=iperf_out),
"Failed to start iperf server on {}".format(node.addr))
if not config['args'].dry_run:
time.sleep(2)
node.check_file('Server listening on TCP port', iperf_out)
node.check_proc("ccp_const", ccp_out)
node.check_file('starting CCP', ccp_out)
config['iteration_outputs'].append((node, iperf_out))
config['iteration_outputs'].append((node, ccp_out))
return iperf_out
def run_client(conn, server, redis_addr, interarrival, poisson_arrivals, batch,
shardtype, stack_frag, outf, wrkfile):
conn.run("sudo pkill -INT iokerneld")
timeout = get_timeout(wrkfile, interarrival)
write_shenango_config(conn)
if shardtype == 'client':
shard_arg = '--use-clientsharding'
elif shardtype == 'server':
shard_arg = ''
elif shardtype == 'basicclient':
shard_arg = '--use-basicclient'
else:
raise f"unknown shardtype {shardtype}"
batch_arg = f'--max-send-batching={batch}' if batch != 0 else ''
poisson_arg = "--poisson-arrivals" if poisson_arrivals else ''
conn.run("./iokerneld",
wd="~/burrito/shenango-chunnel/caladan",
sudo=True,
background=True)
time.sleep(2)
agenda.subtask(f"client starting, timeout {timeout} -> {outf}0.out")
ok = conn.run(
f"RUST_LOG=info,ycsb=debug,bertha=debug ./target/release/ycsb \
--addr {server}:4242 \
--redis-addr={redis_addr} \
-i {interarrival} \
--accesses {wrkfile} \
--out-file={outf}0.data \
-s host.config \
{batch_arg} \
{poisson_arg} \
{shard_arg} \
{'--fragment-stack' if stack_frag else ''} \
--logging --tracing --skip-loads",
wd="~/burrito",
stdout=f"{outf}0.out",
stderr=f"{outf}0.err",
timeout=timeout,
)
check(ok, "client", conn.addr)
conn.run("sudo pkill -INT iokerneld")
agenda.subtask("client done")
def parse_ccp_logs(dirname, sample_rate, replot):
agenda.subtask("ccp logs")
fields = [9,17,19,27,29,35,13]
log_header = "elapsed,rtt,zt,rout,rin,curr_rate,curr_q,elasticity2"
pattern = re.compile(r'(?P<sch>[a-z]+)_(?P<bw>[\d]+)_(?P<delay>[\d]+)/(?P<alg>[a-z_]+).(?P<args>[a-z_]+=[a-zA-Z_0-9].+)?/b=(?P<bg>[^_]*)_c=(?P<cross>[^/]*)/(?P<seed>[\d]+)/ccp.log')
g = glob.glob(dirname + "/**/ccp.log", recursive=True)
global_out_fname = os.path.join(dirname, 'ccp.parsed')
if not replot and os.path.isfile(global_out_fname):
return global_out_fname, len(g)
for exp in g:
exp_root = "/".join(exp.split("/")[:-1])
old_header = None
matches = pattern.search(exp)
if matches is not None and 'nimbus' in exp:
print(exp)
with open(exp) as f, open(os.path.join(exp_root, "ccp.parsed"), 'w') as out, open(os.path.join(exp_root, "ccp_switch.parsed"), 'w') as out_switch:
sch, bw, delay, args, bg, cross, seed, alg = matches.group('sch', 'bw', 'delay', 'args', 'bg', 'cross', 'seed', 'alg')
args = [a.split("=") for a in args.split(".")] if args else []
exp_header = f"sch,alg,rate,rtt,{','.join(a[0] for a in args)},bundle,cross,seed"
header = exp_header + "," + log_header
if old_header and old_header != header:
exit("headers do not align")
old_header = header
out.write(header + "\n")
bg = bg if bg != '' else 'None'
cross = cross if cross != '' else 'None'
prepend = f"{sch},{alg},{bw},{delay},{','.join(a[1] for a in args)},{bg},{cross},{seed}"
parse_nimbus_log(f, out, out_switch, header, prepend, fields, sample_rate)
else:
print(f"skipping {exp}, no regex match")
global_out_fname = os.path.join(dirname, 'ccp.parsed')
subprocess.call(f"rm -f {global_out_fname}", shell=True)
g = glob.glob(dirname + "/**/ccp.parsed", recursive=True)
tail = 1
for exp in g:
if exp != global_out_fname:
subprocess.call(f"tail -n +{tail} {exp} >> {global_out_fname}", shell=True)
if tail == 1:
tail = 2
return global_out_fname, len(g)
def parse_mahimahi_logs(dirname, sample_rate, replot, bundler_root):
agenda.subtask("mahimahi logs")
pattern = re.compile(r'(?P<sch>[a-z]+)_(?P<bw>[\d]+)_(?P<delay>[\d]+)/(?P<alg>[a-zA-Z]+).(?P<args>[a-z_]+=[a-zA-Z_0-9].+)?/b=(?P<bg>[^_]*)_c=(?P<cross>[^/]*)/(?P<seed>[\d]+)/downlink.log')
g = glob.glob(dirname + "/**/downlink.log", recursive=True)
for exp in g:
matches = pattern.search(exp)
if matches is not None:
delay = int(matches.group('delay'))
rtt = int(delay*2)
print(rtt,exp)
exp_root = "/".join(exp.split("/")[:-1])
exp_root = os.path.dirname(exp)
if not replot and os.path.isfile(os.path.join(exp_root, 'mm-graph.tmp')):
continue
subprocess.check_output(f"{bundler_root}/mahimahi/scripts/mm-graph {exp} {rtt} --fake --plot-direction ingress --agg \"5000:6000=bundle,8000:9000=cross\"", shell=True, executable="/bin/bash")
subprocess.check_output("mv /tmp/mm-graph.tmp {}".format(exp_root), shell=True)
else:
print(f"skipping {exp}, no regex match")
def check_existing_experiment(driver):
agenda.task("Check for existing experiment")
driver.get("https://www.cloudlab.us/user-dashboard.php#experiments")
table = None
try:
table = driver.find_element_by_id("experiments_table")
except:
agenda.subfailure("No existing experiment found")
return None
elements = [
e.text.split()[0]
for e in table.find_elements_by_xpath("//table/tbody")
if len(e.text.split()) > 0
]
agenda.subtask("Existing experiment found")
driver.find_element_by_link_text(elements[0]).click()
time.sleep(6)
return get_machines_from_experiment(driver)
def start_server(self, config, node, execute):
agenda.subtask(f"Start ETG server ({self}) on {node.addr}")
i = 1
etg_out = os.path.join(config['iteration_dir'],
"etg_server{}.log".format(i))
while node.file_exists(etg_out) and not config['args'].dry_run:
i += 1
etg_out = os.path.join(config['iteration_dir'],
"etg_server{}.log".format(i))
expect(
node.run(
"{sh} {start} {conns} {alg}".format(
sh=config['etg_server_path'],
start=self.start_port,
conns=self.num_conns,
alg=self.congalg),
wd=os.path.join(config['structure']['bundler_root'],
'empirical-traffic-gen'),
stdout=etg_out,
stderr=etg_out,
background=True,
), "Failed to start poisson servers on {}".format(node.addr))
if not config['args'].dry_run:
time.sleep(1)
num_servers_running = int(
node.run("pgrep -c etgServer").stdout.strip())
else:
num_servers_running = self.num_conns
if num_servers_running != self.num_conns:
fatal_warn(
"self pattern requested {} servers, but only {} are running properly."
.format(self.num_conns, num_servers_running),
exit=False)
with io.BytesIO() as f:
node.get(node.local_path(etg_out), local=f)
print(f.getvalue().decode("utf-8"))
sys.exit(1)
config['iteration_outputs'].append((node, etg_out))
return etg_out
#!/usr/bin/env python3
import agenda
agenda.section("Set up network")
agenda.task("Create Virtual Private Cloud")
agenda.task("Attach internet gateway")
agenda.task("Allocate subnet #1")
agenda.subtask("Hook in internet-enabled route table")
agenda.task("Allocate subnet #2")
agenda.task("Generate VPC key-pair")
agenda.subfailure("Could not create key-pair")
agenda.subtask("Attempting to delete old key-pair")
agenda.subtask("Attempting to generate new key-pair")
agenda.section("Launch instances")
agenda.task("Launch instances in cluster #1")
agenda.task("Launch instances in cluster #2")
agenda.task("Wait for HQ to start running")
agenda.subtask("Still in 'pending' state")
agenda.subtask("Still in 'pending' state")
agenda.task("Wait for workers to reach 'running' state")
agenda.task("Wait for HQ to become pingable")
print("54.84.179.156 | UNREACHABLE!")
print("54.84.179.156 | UNREACHABLE!")
print('54.84.179.156 | SUCCESS => {"changed": false, "ping": "pong"}')
agenda.task("Wait for workers to become pingable")
print('10.0.1.237 | SUCCESS => {"changed": false, "ping": "pong"}')
agenda.section("Deploy application")
print("""\
PLAY [ansible-playbook]
