def _run_rtt_monitor(self, program='nping'):
ping_source_ip = self.server.ip_wan
#self.server.ifname_remote #self.server.ip_lan
ping_dest_ip = self.client.ip_wan
ping_ssh_ip = self.server.ip_wan
ping_ssh_username = self.server.username
ping_ssh_key_filename = self.server.key_filename
# want out of band ping so try using basic ping
#start_ping_cmd = 'ping -i 5 -I {} {}'.format(ping_source_ip,
# ping_dest_ip)
if program == 'nping':
cmd = ('nping --count 0 --delay 1s {} > {} &').format(
ping_dest_ip,
self.logs['ping_log'])
elif program == 'ping':
cmd = 'ping -i 1 {} > {} &'.format(ping_dest_ip,
self.logs['ping_log'])
logging.info('Running cmd: {}'.format(cmd))
pid = None
try:
os.system(cmd)
if program == 'nping':
pid = run_local_command('pgrep -f "nping --count 0 --delay 1s"')
elif program == 'ping':
pid = run_local_command('pgrep -f "ping -i 1"')
assert(pid)
pid = int(pid)
logging.info('PID={}'.format(pid))
yield pid
finally:
logging.info('Cleaning up cmd: {}'.format(cmd))
run_local_command('kill {}'.format(pid))
def _show_bess_pipeline(self):
cmd = '/opt/bess/bessctl/bessctl show pipeline'
logging.info('\n'+run_local_command(cmd))
cmd = '/opt/bess/bessctl/bessctl command module queue0 get_status EmptyArg'
logging.info('\n'+run_local_command(cmd))
cmd = '/opt/bess/bessctl/bessctl command module queue_delay0 get_status EmptyArg'
logging.info('\n'+run_local_command(cmd))
def __init__(self, name, btlbw, queue_size,
flows, server, client, config_filename, server_nat_ip=None,
loss_rate=None):
self.exp_time = (datetime.now().isoformat()
.replace(':','').replace('-','').split('.')[0])
self.name = name
self.btlbw = btlbw
self.queue_size = queue_size
self.server = server
self.client = client
# add ability to specify loss rate for controlled experiments
self.loss_rate = loss_rate
# store what version of this code we are running -- could be useful later
self.cctestbed_git_commit = run_local_command('git rev-parse HEAD').strip()
self.bess_git_commit = run_local_command('git --git-dir=/opt/bess/.git '
'--work-tree=/opt/bess '
'rev-parse HEAD').strip()
self.config_filename = config_filename
self.logs = {
'server_tcpdump_log': '/tmp/server-tcpdump-{}-{}.pcap'.format(
self.name, self.exp_time),
'queue_log': '/tmp/queue-{}-{}.txt'.format(self.name, self.exp_time),
'client_tcpdump_log': '/tmp/client-tcpdump-{}-{}.pcap'.format(
self.name, self.exp_time),
'description_log': '/tmp/{}-{}.json'.format(self.name, self.exp_time),
'tcpprobe_log': '/tmp/tcpprobe-{}-{}.txt'.format(self.name, self.exp_time),
'dig_log': '/tmp/dig-{}-{}.txt'.format(self.name, self.exp_time),
'rtt_log': '/tmp/rtt-{}-{}.txt'.format(self.name, self.exp_time),
'capinfos_log': '/tmp/capinfos-{}-{}.txt'.format(self.name, self.exp_time),
'ping_log': '/tmp/ping-{}-{}.txt'.format(self.name, self.exp_time),
'website_log': '/tmp/website-{}-{}.json'.format(self.name, self.exp_time),
'http_log': '/tmp/http-{}-{}.pcap'.format(self.name, self.exp_time)
}
self.tar_filename = '/tmp/{}-{}.tar.gz'.format(self.name, self.exp_time)
# update flow objects with log filenames
self.flows = []
for flow in flows:
client_log = '/tmp/client-{}-{}-{}.iperf'.format(
flow.client_port, self.name, self.exp_time)
server_log = '/tmp/server-{}-{}-{}.iperf'.format(
flow.server_port, self.name, self.exp_time)
# tuples are immutable so we must create new ones to update log attributes
new_flow = flow._replace(client_log=client_log,
server_log=server_log)
self.flows.append(new_flow)
# deal with having server behind a NAT
# TODO: FIX THIS STRAWMAN SOLUTION
if server_nat_ip is None:
self.server_nat_ip = self.server.ip_lan
self.dig = False
else:
self.server_nat_ip = server_nat_ip
# do a dig to figure out if using CDN
self.dig = True
# will store measured rtt
self.rtt_measured = None
def start_bess(experiment, bess_config_name='active-middlebox-pmd'):
cmd = '/opt/bess/bessctl/bessctl daemon start'
try:
run_local_command(cmd, timeout=120)
except Exception as e:
pass
cmd = ("/opt/bess/bessctl/bessctl run {} "
"\"CCTESTBED_EXPERIMENT_DESCRIPTION='{}'\"").format(
bess_config_name, experiment.logs['description_log'])
run_local_command(cmd)
def start_bess():
cmd = '/opt/bess/bessctl/bessctl daemon start'
try:
run_local_command(cmd, timeout=120)
except Exception as e:
pass
cmd = ("/opt/bess/bessctl/bessctl run {} "
"\"CCTESTBED_EXPERIMENT_DESCRIPTION='{}'\"").format(
"active-middlebox-pmd", "/tmp/webrtc-tmp.json")
run_local_command(cmd)
def _run_bess_monitor(self):
# have to use system here; subprocess just hangs for some reason
with open(self.logs['queue_log'], 'w') as f:
f.write(
'dequeued,time,src,seq,datalen,size,dropped,queued,batch\n')
# only log "good" lines, those that start with 0 or 1
cmd = 'tail -n1 -f /tmp/bessd.INFO > {} &'.format(
self.logs['queue_log'])
logging.info('Running cmd: {}'.format(cmd))
pid = None
try:
os.system(cmd)
pid = run_local_command('pgrep -f "tail -n1 -f /tmp/bessd.INFO"')
assert (pid)
pid = int(pid)
logging.info('PID={}'.format(pid))
yield pid
finally:
logging.info('Cleaning up cmd: {}'.format(cmd))
run_local_command('kill {}'.format(pid))
run_local_command(
'grep -e "^0" -e "^1" {} > /tmp/queue-log-tmp.txt'.format(
self.logs['queue_log']),
shell=True)
run_local_command('mv /tmp/queue-log-tmp.txt {}'.format(
self.logs['queue_log']))
def cleanup_last_experiment(self, cleanup_tail=True):
logging.info('Cleaning up last experiment just in case...')
if cleanup_tail:
run_local_command('pkill tail')
cleanup_cmds = [('sudo pkill -9 tcpdump', ('server', 'client')),
('sudo pkill -9 cat', ('client')),
('sudo pkill iperf', ('server','client')),
('sudo rmmod tcp_probe_ray', ('client'))]
for cmd, machines in cleanup_cmds:
if 'client' in machines:
with get_ssh_client(self.client.ip_wan,
username=self.client.username,
key_filename=self.client.key_filename) as ssh_client:
exec_command(ssh_client, self.client.ip_wan, cmd)
if 'server' in machines:
with get_ssh_client(self.server.ip_wan,
username=self.server.username,
key_filename=self.server.key_filename) as ssh_client:
exec_command(ssh_client, self.server.ip_wan, cmd)
def connect_dpdk(server, client, dpdk_driver='igb_uio'):
# check if DPDK already connected
cmd = '/opt/bess/bin/dpdk-devbind.py --status | grep drv={}'.format('dpdk_driver')
proc = subprocess.run(cmd, check=False, shell=True, stdout=subprocess.PIPE)
if proc.returncode == 0:
logging.info('Interfaces already connected to DPDK')
return server.pci, client.pci
# make sure we can ssh into jicama from taro (ASSUMES POTATO IS CLIENT)
if server.ifname_local == 'enp11s0f0' or server.ifname_local == 'enp11s0f0':
check_cmd = 'route | grep 192.0.0.1'
proc = subprocess.run(check_cmd, check=False, shell=True, stdout=subprocess.PIPE)
if proc.returncode != 0:
route_cmd = 'sudo ip route add 192.0.0.1 dev ens3f0'
subprocess.run(route_cmd, check=True, shell=True, stdout=subprocess.PIPE)
subprocess.run(check_cmd, check=True, shell=True, stdout=subprocess.PIPE)
# get pcis
expected_server_pci = get_interface_pci(server.ifname_local)
expected_client_pci = get_interface_pci(client.ifname_local)
if expected_server_pci is None or expected_server_pci.strip()=='':
return server.pci, client.pci
assert(expected_server_pci == server.pci)
assert(expected_client_pci == client.pci)
# get ips on this machine
server_if_ip, server_ip_mask = get_interface_ip(server.ifname_local)
client_if_ip, client_ip_mask = get_interface_ip(client.ifname_local)
logging.info('Server: ifname = {}, '
'pci = {}, if_ip = {}/{}'.format(
server.ifname_local, server.pci, server_if_ip, server_ip_mask))
logging.info('Client: ifname = {}, '
'pci = {}, if_ip = {}/{}'.format(
client.ifname_local, client.pci, client_if_ip, client_ip_mask))
# make sure hugepages is started
cmd = 'sudo sysctl vm.nr_hugepages=1024'
run_local_command(cmd)
# connect NIC interfaces to DPDK
if dpdk_driver == 'igb_uio':
cmd = 'sudo modprobe uio'
run_local_command(cmd)
cmd = 'sudo insmod /opt/bess/deps/dpdk-17.11/build/kmod/igb_uio.ko'
run_local_command(cmd)
else:
cmd = 'sudo modprobe {}'.format(dpdk_driver)
run_local_command(cmd)
cmd = 'sudo ifconfig {} down'
run_local_command(cmd.format(server.ifname_local))
run_local_command(cmd.format(client.ifname_local))
cmd = 'sudo /opt/bess/bin/dpdk-devbind.py -b {} {}'
run_local_command(cmd.format(dpdk_driver, server.pci))
run_local_command(cmd.format(dpdk_driver, client.pci))
return server.pci, client.pci
def stop_bess():
cmd = '/opt/bess/bessctl/bessctl daemon stop'
try:
run_local_command(cmd, timeout=60)
except Exception as e:
logging.error('Couldnt shutdown bess')
def _run_dig(self):
run_local_command('dig -x {} > {}'.format(
self.client.ip_wan,
self.logs['dig_log']), shell=True)
return
