def get_host_info(htype='1', netint='1', netmac='1'):
"Populate the host info caches"
if htype == '1':
get_type_cached(env.host_string)
if netint == '1':
get_netint_cached(env.host_string, int_no=-1)
get_netint_windump_cached(env.host_string, int_no=-1)
get_netint_cached(env.host_string, int_no=-1, internal_int='0')
get_netint_windump_cached(env.host_string, int_no=-1, internal_int='0')
if netmac == '1':
get_netmac_cached(env.host_string)
def get_host_info(htype='1', netint='1', netmac='1'):
"Populate the host info caches"
if htype == '1':
get_type_cached(env.host_string)
if netint == '1':
get_netint_cached(env.host_string, int_no=-1)
get_netint_windump_cached(env.host_string, int_no=-1)
get_netint_cached(env.host_string, int_no=-1, internal_int='0')
get_netint_windump_cached(env.host_string, int_no=-1,
internal_int='0')
if netmac == '1':
get_netmac_cached(env.host_string)
def init_router():
"Initialize router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(init_dummynet)
elif htype == 'Linux':
interfaces = get_netint_cached(env.host_string, int_no=-1)
interfaces[0] = 'eno1' # swapna
interfaces[1] = 'enx000000000f92' # swapna
#interfaces = ['eno1']
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('sudo ethtool -K %s tso off' % interface)
run('sudo ethtool -K %s gso off' % interface)
run('sudo ethtool -K %s lro off' % interface)
run('sudo ethtool -K %s gro off' % interface)
run('sudo ethtool -K %s ufo off' % interface)
execute(init_tc)
else:
abort("Router must be running FreeBSD or Linux")
def show_tc_setup():
interfaces = get_netint_cached(env.host_string, int_no=-1)
run('tc -d -s qdisc show')
cnt = 0
for interface in interfaces:
run('tc -d -s class show dev %s' % interface)
run('tc -d -s filter show dev %s' % interface)
pseudo_interface = 'ifb' + str(cnt)
run('tc -d -s class show dev %s' % pseudo_interface)
run('tc -d -s filter show dev %s' % pseudo_interface)
cnt += 1
run('iptables -t mangle -vL')
def show_tc_setup():
interfaces = get_netint_cached(env.host_string, int_no=-1)
run('sudo tc -d -s qdisc show')
cnt = 0
for interface in interfaces:
run('sudo tc -d -s class show dev %s' % interface)
run('sudo tc -d -s filter show dev %s' % interface)
pseudo_interface = 'ifb' + str(cnt)
run('sudo tc -d -s class show dev %s' % pseudo_interface)
run('sudo tc -d -s filter show dev %s' % pseudo_interface)
cnt += 1
run('sudo iptables -t mangle -vL')
def log_queue_stats(file_prefix='', remote_dir='', local_dir='.'):
"Get queue statistics after experiment"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_queue_stats.log"
if htype == 'FreeBSD':
run('echo ipfw pipe show > %s' % file_name)
run('ipfw pipe show >> %s' % file_name)
run('echo ipfw show >> %s' % file_name)
run('ipfw show >> %s' % file_name)
elif htype == 'Linux':
run('echo tc -d -s qdisc show > %s' % file_name)
run('tc -d -s qdisc show >> %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
cnt = 0
for interface in interfaces:
run('echo >> %s' % file_name)
run('echo tc -s class show dev %s >> %s' % (interface, file_name))
run('tc -s class show dev %s >> %s' % (interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -s filter show dev %s >> %s' % (interface, file_name))
run('tc -s filter show dev %s >> %s' % (interface, file_name))
pseudo_interface = 'ifb' + str(cnt)
run('echo >> %s' % file_name)
run('echo tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
cnt += 1
run('echo iptables -t mangle -vL >> %s' % file_name)
run('iptables -t mangle -vL >> %s' % file_name)
getfile(file_name, local_dir)
def log_queue_stats(file_prefix='', remote_dir='', local_dir='.'):
"Get queue statistics after experiment"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_queue_stats.log"
if htype == 'FreeBSD':
run('echo ipfw pipe show > %s' % file_name)
run('ipfw pipe show >> %s' % file_name)
run('echo ipfw show >> %s' % file_name)
run('ipfw show >> %s' % file_name)
elif htype == 'Linux':
run('echo tc -d -s qdisc show > %s' % file_name)
run('tc -d -s qdisc show >> %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
cnt = 0
for interface in interfaces:
run('echo >> %s' % file_name)
run('echo tc -s class show dev %s >> %s' % (interface, file_name))
run('tc -s class show dev %s >> %s' % (interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -s filter show dev %s >> %s' % (interface, file_name))
run('tc -s filter show dev %s >> %s' % (interface, file_name))
pseudo_interface = 'ifb' + str(cnt)
run('echo >> %s' % file_name)
run('echo tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
cnt += 1
run('echo iptables -t mangle -vL >> %s' % file_name)
run('iptables -t mangle -vL >> %s' % file_name)
getfile(file_name, local_dir)
def init_tc():
# load pseudo interface mdoule
run('sudo modprobe ifb')
# get all interfaces
interfaces = get_netint_cached(env.host_string, int_no=-1)
if env.host_string == '172.16.10.2':
interfaces[0] = 'eno1'
print str(interfaces)
print str(env.host_string)
# delete all rules
for interface in interfaces:
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('sudo tc qdisc del dev %s root' % interface)
# set root qdisc
run('sudo tc qdisc add dev %s root handle 1 htb' % interface)
# bring up pseudo ifb interfaces (for netem)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
run('sudo ifconfig %s down' % pseudo_interface)
run('sudo ifconfig %s up' % pseudo_interface)
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('sudo tc qdisc del dev %s root' % pseudo_interface)
# set root qdisc
run('sudo tc qdisc add dev %s root handle 1 htb' % pseudo_interface)
cnt += 1
run('sudo iptables -t mangle -F')
# this is just for counting all packets
run('sudo iptables -t mangle -A POSTROUTING -j MARK --set-mark 0')
def init_tc():
# get all interfaces
interfaces = get_netint_cached(env.host_string, int_no=-1)
# load pseudo interface module
# need to remove it first in case it is already loaded with diff. numifbs
run('rmmod ifb; modprobe ifb numifbs=%d' % len(interfaces))
# delete all rules
for interface in interfaces:
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('tc qdisc del dev %s root' % interface)
# set root qdisc
run('tc qdisc add dev %s root handle 1 htb' % interface)
# bring up pseudo ifb interfaces (for netem)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
run('ifconfig %s down' % pseudo_interface)
run('ifconfig %s up' % pseudo_interface)
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('tc qdisc del dev %s root' % pseudo_interface)
# set root qdisc
run('tc qdisc add dev %s root handle 1 htb' % pseudo_interface)
cnt += 1
run('iptables -t mangle -F')
# this is just for counting all packets
run('iptables -t mangle -A POSTROUTING -j MARK --set-mark 0')
def init_tc():
# load pseudo interface mdoule
run('modprobe ifb')
# get all interfaces
interfaces = get_netint_cached(env.host_string, int_no=-1)
# delete all rules
for interface in interfaces:
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('tc qdisc del dev %s root' % interface)
# set root qdisc
run('tc qdisc add dev %s root handle 1 htb' % interface)
# bring up pseudo ifb interfaces (for netem)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
run('ifconfig %s down' % pseudo_interface)
run('ifconfig %s up' % pseudo_interface)
with settings(warn_only=True):
# run with warn_only since it will return error if no tc commands
# exist
run('tc qdisc del dev %s root' % pseudo_interface)
# set root qdisc
run('tc qdisc add dev %s root handle 1 htb' % pseudo_interface)
cnt += 1
run('iptables -t mangle -F')
# this is just for counting all packets
run('iptables -t mangle -A POSTROUTING -j MARK --set-mark 0')
def init_router():
"Initialize router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(init_dummynet)
elif htype == 'Linux':
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
execute(init_tc)
else:
abort("Router must be running FreeBSD or Linux")
def init_tc_pipe(counter='1',
source='',
dest='',
rate='',
delay='',
rtt='',
loss='',
queue_size='',
queue_size_mult='1.0',
queue_disc='',
queue_disc_params='',
bidir='0',
attach_to_queue=''):
# compatibility with FreeBSD
if queue_disc == 'fifo':
# pfifo is the default for HTB classes
queue_disc = 'pfifo'
queue_size = str(queue_size)
if queue_size.lower() == 'bdp':
_rate = rate.replace('kbit', '000')
_rate = _rate.replace('mbit', '000000')
if rtt == '':
rtt = str(2 * int(delay))
if queue_disc == 'pfifo' or queue_disc == 'codel' or \
queue_disc == 'fq_codel' or queue_disc == 'pie':
# queue size in packets
avg_packet = 600 # average packet size
queue_size = int(
float(_rate) * (float(rtt) / 1000.0) / 8 / avg_packet)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 1:
queue_size = 1 # minimum 1 packet
queue_size = str(queue_size)
elif queue_disc == 'bfifo' or queue_disc == 'red':
# queue size in bytes
queue_size = int(float(_rate) * (float(rtt) / 1000.0) / 8)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 2048:
queue_size = 2048 # minimum 2kB
queue_size = str(queue_size)
else:
abort('Can\'t specify \'bdp\' for queuing discipline %s' %
queue_disc)
# class/handle numbers
class_no = str(int(counter) + 0)
if attach_to_queue == '':
queue_class_no = class_no
else:
# if attach_to_queue is set we attach this to existing (previously
# configured pipe). this means packets will go through an existing htb
# and leaf qdisc, but a separate netem.
# so we can have different flows going through the same bottleneck
# queue, but with different emulated delays or loss rates
queue_class_no = attach_to_queue
netem_class_no = class_no
qdisc_no = str(int(counter) + 1000)
netem_no = str(int(counter) + 1000)
# disciplines: fq_codel, codel, red, choke, pfifo, pfifo_fast (standard
# magic), pie (only as patch), ...
if queue_disc == '':
queue_disc = 'pfifo'
# for pie we need to make sure the kernel module is loaded (for kernel pre
# 3.14 only, for new kernels it happens automatically via tc use!)
if queue_disc == 'pie':
with settings(warn_only=True):
run('sudo modprobe pie')
if rate == '':
rate = '1000mbit'
if queue_size == '':
# set default queue size to 1000 packet (massive but default for e.g.
# codel)
queue_size = '1000'
if loss != '':
# convert to percentage
loss = str(float(loss) * 100)
interfaces = get_netint_cached(env.host_string, int_no=-1)
interfaces[0] = 'eno1' # swapna
interfaces[1] = 'enx000000000f92' # swapna
# our approach works as follows:
# - shaping, aqm and delay/loss emulation is done on egress interface
# (as usual)
# - use htb qdisc for rate limiting with the aqm qdisc (e.g. pfifo, codel)
# as leave node
# - after shaping and aqm, emulate loss and delay with netem
# - for each "pipe" we setup a new class on all (two) interfaces
# - if pipes are unidirectional a class is only used on one of the two ifaces;
# otherwise it is used on both interfaces (XXX could optimise the
# unidirectional case and omit unused pipes)
# - traffic flow is as follows:
# 1. packets are marked by iptables in mangle table POSTROUTING hook
# depending on defined source/dest (unique mark for each pipe)
# 2. marked packets are classified into appropriate class (1-1 mapping
# between marks and classes) and redirected to pseudo interface
# 3. pseudo interface does the shaping with htb and aqm (leaf qdisc)
# 4. packets go back to actual interface
# 5. actual interface does network emulation (delay/loss), here htb is set to
# max rate (1Gbps) and pfifo is used (effectively no shaping or aqm here)
# note that according to my information the htb has a build-in buffer of 1
# packet as well (cannot be changed)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
# config rate limiting on pseudo interface
config_tc_cmd = 'sudo tc class add dev %s parent 1: classid 1:%s htb rate %s ceil %s' % \
(pseudo_interface, queue_class_no, rate, rate)
if attach_to_queue == '':
run(config_tc_cmd)
# config queuing discipline and buffer limit on pseudo interface
config_tc_cmd = 'sudo tc qdisc add dev %s parent 1:%s handle %s: %s limit %s %s' % \
(pseudo_interface,
queue_class_no,
qdisc_no,
queue_disc,
queue_size,
queue_disc_params)
if attach_to_queue == '':
run(config_tc_cmd)
# configure filter to classify traffic based on mark on pseudo device
config_tc_cmd = 'sudo tc filter add dev %s protocol ip parent 1: ' \
'handle %s fw flowid 1:%s' % (
pseudo_interface, class_no, queue_class_no)
run(config_tc_cmd)
# configure class for actual interface with max rate
config_tc_cmd = 'sudo tc class add dev %s parent 1: classid 1:%s ' \
'htb rate 1000mbit ceil 1000mbit' % \
(interface, netem_class_no)
run(config_tc_cmd)
# config netem on actual interface
config_tc_cmd = 'sudo tc qdisc add dev %s parent 1:%s handle %s: ' \
'netem limit 1000' % (
interface, netem_class_no, netem_no)
if delay != "":
config_tc_cmd += " delay %sms" % delay
if loss != "":
config_tc_cmd += " loss %s%%" % loss
run(config_tc_cmd)
# configure filter to redirect traffic to pseudo device first and also
# classify traffic based on mark after leaving the pseudo interface traffic
# will go back to actual interface
config_tc_cmd = 'sudo tc filter add dev %s protocol ip parent 1: handle %s ' \
'fw flowid 1:%s action mirred egress redirect dev %s' % \
(interface, class_no, netem_class_no, pseudo_interface)
run(config_tc_cmd)
cnt += 1
# filter on specific ips
config_it_cmd = 'sudo iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(source, dest, class_no)
run(config_it_cmd)
if bidir == '1':
config_it_cmd = 'sudo iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(dest, source, class_no)
run(config_it_cmd)
def init_host():
"Perform host initialization"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# record the number of reassembly queue overflows
run('sysctl net.inet.tcp.reass.overflows')
# disable auto-tuning of receive buffer
run('sysctl net.inet.tcp.recvbuf_auto=0')
# disable tso
run('sysctl net.inet.tcp.tso=0')
# send and receiver buffer max (2MB by default on FreeBSD 9.2 anyway)
run('sysctl net.inet.tcp.sendbuf_max=2097152')
run('sysctl net.inet.tcp.recvbuf_max=2097152')
# clear host cache quickly, otherwise successive TCP connections will
# start with ssthresh and cwnd from the end of most recent tcp
# connections to the same host
run('sysctl net.inet.tcp.hostcache.expire=1')
run('sysctl net.inet.tcp.hostcache.prune=5')
run('sysctl net.inet.tcp.hostcache.purge=1')
elif htype == 'Linux':
# disable host cache
run('sysctl net.ipv4.tcp_no_metrics_save=1')
# disable auto-tuning of receive buffer
run('sysctl net.ipv4.tcp_moderate_rcvbuf=0')
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
# send and recv buffer max (set max to 2MB)
run('sysctl net.core.rmem_max=2097152')
run('sysctl net.core.wmem_max=2097152')
# tcp recv buffer max (min 4kB, default 87kB, max 6MB; this is standard
# on kernel 3.7)
run('sysctl net.ipv4.tcp_rmem=\'4096 87380 6291456\'')
# tcp send buffer max (min 4kB, default 64kB, max 4MB; 4x the default
# otherwise standard values from kernel 3.7)
run('sysctl net.ipv4.tcp_wmem=\'4096 65535 4194304\'')
elif htype == 'Darwin':
# disable tso
run('sysctl -w net.inet.tcp.tso=0')
# diable lro (off by default anyway)
run('sysctl -w net.inet.tcp.lro=0')
# disable auto tuning of buffers
run('sysctl -w net.inet.tcp.doautorcvbuf=0')
run('sysctl -w net.inet.tcp.doautosndbuf=0')
# send and receive buffer max (2MB). kern.ipc.maxsockbuf max be the sum
# (but is 4MB by default anyway)
run('sysctl -w kern.ipc.maxsockbuf=4194304')
run('sysctl -w net.inet.tcp.sendspace=2097152')
run('sysctl -w net.inet.tcp.recvspace=2097152')
# set the auto receive/send buffer max to 2MB as well just in case
run('sysctl -w net.inet.tcp.autorcvbufmax=2097152')
run('sysctl -w net.inet.tcp.autosndbufmax=2097152')
elif htype == 'CYGWIN':
# disable ip/tcp/udp offload processing
run('netsh int tcp set global chimney=disabled', pty=False)
run('netsh int ip set global taskoffload=disabled', pty=False)
# enable tcp timestamps
run('netsh int tcp set global timestamps=enabled', pty=False)
# disable tcp window scaling heuristics, enforce user-set auto-tuning
# level
run('netsh int tcp set heuristics disabled', pty=False)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
# stop and restart interface to make the changes
run('netsh int set int "Local Area Connection %s" disabled' %
interface,
pty=False)
run('netsh int set int "Local Area Connection %s" enabled' %
interface,
pty=False)
def check_host():
"Check that needed tools are installed on hosts"
# get type of current host
htype = get_type_cached(env.host_string)
# run checks
if env.host_string in config.TPCONF_router:
if htype == 'FreeBSD':
run('sudo which ipfw')
if htype == "Linux":
run('sudo which tc')
run('sudo which iptables')
# XXX check that kernel tick rate is high (>= 1000)
else:
if htype == 'FreeBSD':
run('sudo which md5')
run('sudo which tcpdump')
elif htype == 'Darwin':
run('sudo which md5')
run('sudo which tcpdump')
run('sudo which dsiftr-osx-teacup.d')
elif htype == 'Linux':
run('sudo which ethtool')
run('sudo which md5sum')
run('sudo which tcpdump')
#run('sudo which web10g-listconns')
#run('sudo which web10g-readvars')
#updated for ttprobe support
try:
linux_tcp_logger = config.TPCONF_linux_tcp_logger
except AttributeError:
linux_tcp_logger = 'web10g'
if linux_tcp_logger == 'ttprobe' or linux_tcp_logger == 'both':
#checking the availability of ttprobe.ko kernel module
run('sudo ls /lib/modules/$(uname -r)/extra/ttprobe.ko')
if linux_tcp_logger == 'web10g' or linux_tcp_logger == 'both':
run('sudo which web10g-logger')
elif htype == 'CYGWIN':
run('sudo which WinDump', pty=False)
run('sudo which win-estats-logger', pty=False)
# if we don't have proper ntp installed then
# start time service if not started and force resync
with settings(warn_only=True):
ret = run(
'sudo ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code != 0:
run('sudo net start w32time', pty=False)
run('sudo w32tm /resync', pty=False)
# try to enable any test network interfaces that are (accidently)
# disabled after reboot
with settings(warn_only=True):
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('sudo netsh int set int "Local Area Connection %s" enabled'
% interface,
pty=False)
run('sudo which killall', pty=False)
run('sudo which pkill', pty=False)
run('sudo which ps', pty=False)
run('sudo which gzip', pty=False)
run('sudo which dd', pty=False)
# check for traffic sender/receiver tools
run('sudo which iperf', pty=False)
run('sudo which ping', pty=False)
run('sudo which httperf', pty=False)
run('sudo which lighttpd', pty=False)
run('sudo which nttcp', pty=False)
put(config.TPCONF_script_path + '/runbg_wrapper.sh',
'/usr/bin',
use_sudo=True)
run('sudo chmod a+x /usr/bin/runbg_wrapper.sh', pty=False)
run('sudo which runbg_wrapper.sh', pty=False)
put(config.TPCONF_script_path + '/kill_iperf.sh',
'/usr/bin',
use_sudo=True)
run('sudo chmod a+x /usr/bin/kill_iperf.sh', pty=False)
run('sudo which kill_iperf.sh', pty=False)
put(config.TPCONF_script_path + '/pktgen.sh', '/usr/bin', use_sudo=True)
run('sudo chmod a+x /usr/bin/pktgen.sh', pty=False)
run('sudo which pktgen.sh', pty=False)
def init_host():
"Perform host initialization"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# record the number of reassembly queue overflows
run('sysctl net.inet.tcp.reass.overflows')
# disable tso
run('sysctl net.inet.tcp.tso=0')
# send and receiver buffer max (2MB by default on FreeBSD 9.2 anyway)
run('sysctl net.inet.tcp.sendbuf_max=2097152')
run('sysctl net.inet.tcp.recvbuf_max=2097152')
# clear host cache quickly, otherwise successive TCP connections will
# start with ssthresh and cwnd from the end of most recent tcp
# connections to the same host
run('sysctl net.inet.tcp.hostcache.expire=1')
run('sysctl net.inet.tcp.hostcache.prune=5')
run('sysctl net.inet.tcp.hostcache.purge=1')
elif htype == 'Linux':
# disable host cache
run('sysctl net.ipv4.tcp_no_metrics_save=1')
# disable auto-tuning of receive buffer
run('sysctl net.ipv4.tcp_moderate_rcvbuf=0')
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
# send and recv buffer max (set max to 2MB)
run('sysctl net.core.rmem_max=2097152')
run('sysctl net.core.wmem_max=2097152')
# tcp recv buffer max (min 4kB, default 87kB, max 6MB; this is standard
# on kernel 3.7)
run('sysctl net.ipv4.tcp_rmem=\'4096 87380 6291456\'')
# tcp send buffer max (min 4kB, default 32kB, max 6MB; doubled default
# otherwise standard on kernel 3.7)
run('sysctl net.ipv4.tcp_wmem=\'4096 65535 4194304\'')
elif htype == 'Darwin':
# disable tso
run('sysctl -w net.inet.tcp.tso=0')
# diable lro (off by default anyway)
run('sysctl -w net.inet.tcp.lro=0')
# disable auto tuning of buffers
run('sysctl -w net.inet.tcp.doautorcvbuf=0')
run('sysctl -w net.inet.tcp.doautosndbuf=0')
# send and receive buffer max (2MB). kern.ipc.maxsockbuf max be the sum
# (but is 4MB by default anyway)
run('sysctl -w kern.ipc.maxsockbuf=4194304')
run('sysctl -w net.inet.tcp.sendspace=2097152')
run('sysctl -w net.inet.tcp.recvspace=2097152')
# set the auto receive/send buffer max to 2MB as well just in case
run('sysctl -w net.inet.tcp.autorcvbufmax=2097152')
run('sysctl -w net.inet.tcp.autosndbufmax=2097152')
elif htype == 'CYGWIN':
# disable ip/tcp/udp offload processing
run('netsh int tcp set global chimney=disabled', pty=False)
run('netsh int ip set global taskoffload=disabled', pty=False)
# enable tcp timestamps
run('netsh int tcp set global timestamps=enabled', pty=False)
# disable tcp window scaling heuristics, enforce user-set auto-tuning
# level
run('netsh int tcp set heuristics disabled', pty=False)
if interfaces == '':
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
# stop and restart interface to make the changes
run('netsh int set int "Local Area Connection %s" disabled' %
interface, pty=False)
run('netsh int set int "Local Area Connection %s" enabled' %
interface, pty=False)
def log_sysdata(file_prefix='', remote_dir='', local_dir='.'):
"Log various information for each system"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_uname.log"
run('uname -a > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_netstat.log"
run('netstat -nr > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_sysctl.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('sysctl -a > %s' % file_name)
else:
run('echo "netsh int show int" > %s' % file_name, pty=False)
run('netsh int show int >> %s' % file_name, pty=False)
run('echo "netsh int tcp show global" >> %s' % file_name, pty=False)
run('netsh int tcp show global >> %s' % file_name, pty=False)
run('echo "netsh int tcp show heuristics" >> %s' %
file_name, pty=False)
run('netsh int tcp show heuristics >> %s' % file_name, pty=False)
run('echo "netsh int tcp show security" >> %s' % file_name, pty=False)
run('netsh int tcp show security >> %s' % file_name, pty=False)
run('echo "netsh int tcp show chimneystats" >> %s' %
file_name, pty=False)
run('netsh int tcp show chimneystats >> %s' % file_name, pty=False)
run('echo "netsh int ip show offload" >> %s' % file_name, pty=False)
run('netsh int ip show offload >> %s' % file_name, pty=False)
run('echo "netsh int ip show global" >> %s' % file_name, pty=False)
run('netsh int ip show global >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ifconfig.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ifconfig -a > %s' % file_name)
else:
run('ipconfig > %s' % file_name, pty=False)
# log interface speeds
run('echo "wmic NIC where NetEnabled=true get Name, Speed" >> %s' % file_name, pty=False)
run('wmic NIC where NetEnabled=true get Name, Speed >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_procs.log"
if htype == 'FreeBSD' or htype == 'Linux':
run('ps -axu > %s' % file_name)
elif htype == 'Darwin':
run('ps -axu root > %s' % file_name)
else:
run('ps -alW > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ntp.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ntpq -4p > %s' % file_name)
else:
with settings(warn_only=True):
# if we have ntp installed then use ntpq, otherwise use w32tm
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code == 0:
run('"/cygdrive/c/Program Files (x86)/NTP/bin/ntpq" -4p > %s' %
file_name, pty=False)
else:
run('w32tm /query /status > %s' % file_name, pty=False)
getfile(file_name, local_dir)
# log tcp module parameters (Linux only)
if htype == 'Linux':
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_tcpmod.log"
run("find /sys/module/tcp* -type f -exec grep -sH '' '{}' \; | "
"grep -v Binary > %s" % file_name)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ethtool.log"
run('touch %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('ethtool -k %s >> %s' % (interface, file_name))
getfile(file_name, local_dir)
def start_tcpdump(
file_prefix='', remote_dir='', local_dir='.', snap_len='80',
tcpdump_filter='', internal_int='1'):
"Start tcpdump instance on host"
# get host type
htype = get_type_cached(env.host_string)
if env.host_string in config.TPCONF_router:
interfaces = get_netint_cached(env.host_string, int_no=-1,
internal_int=internal_int)
else:
if htype == 'CYGWIN':
interfaces = get_netint_windump_cached(env.host_string,
int_no=0,
internal_int=internal_int)
else:
interfaces = get_netint_cached(env.host_string,
int_no=0,
internal_int=internal_int)
if len(interfaces) < 1:
abort('Internal interface not specified')
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
for interface in interfaces:
if env.host_string in config.TPCONF_router:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + \
'_' + interface + '_router.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
else:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
tcpdump_cmd = 'tcpdump -n -s %s -i %s -w %s \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
else:
# CYGWIN
tcpdump_cmd = 'WinDump -n -s %s -i %s -w ' \
'"$(cygpath -aw "%s")" \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
pid = runbg(tcpdump_cmd)
name = 'tcpdump-' + interface
#bgproc.register_proc(env.host_string, name, '0', pid, file_name)
bgproc.register_proc_later(
env.host_string,
local_dir,
name,
'0',
pid,
file_name)
def log_sysdata(file_prefix='', remote_dir='', local_dir='.'):
"Log various information for each system"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_uname.log"
run('uname -a > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_netstat.log"
run('netstat -nr > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_sysctl.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('sysctl -a > %s' % file_name)
else:
run('echo "netsh int show int" > %s' % file_name, pty=False)
run('netsh int show int >> %s' % file_name, pty=False)
run('echo "netsh int tcp show global" >> %s' % file_name, pty=False)
run('netsh int tcp show global >> %s' % file_name, pty=False)
run('echo "netsh int tcp show heuristics" >> %s' %
file_name, pty=False)
run('netsh int tcp show heuristics >> %s' % file_name, pty=False)
run('echo "netsh int tcp show security" >> %s' % file_name, pty=False)
run('netsh int tcp show security >> %s' % file_name, pty=False)
run('echo "netsh int tcp show chimneystats" >> %s' %
file_name, pty=False)
run('netsh int tcp show chimneystats >> %s' % file_name, pty=False)
run('echo "netsh int ip show offload" >> %s' % file_name, pty=False)
run('netsh int ip show offload >> %s' % file_name, pty=False)
run('echo "netsh int ip show global" >> %s' % file_name, pty=False)
run('netsh int ip show global >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ifconfig.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ifconfig -a > %s' % file_name)
else:
run('ipconfig > %s' % file_name, pty=False)
# log interface speeds
run('echo "wmic NIC where NetEnabled=true get Name, Speed" >> %s' % file_name, pty=False)
run('wmic NIC where NetEnabled=true get Name, Speed >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_procs.log"
if htype == 'FreeBSD' or htype == 'Linux':
run('ps -axu > %s' % file_name)
elif htype == 'Darwin':
run('ps -axu root > %s' % file_name)
else:
run('ps -alW > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ntp.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ntpq -4p > %s' % file_name)
else:
with settings(warn_only=True):
# if we have ntp installed then use ntpq, otherwise use w32tm
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code == 0:
run('"/cygdrive/c/Program Files (x86)/NTP/bin/ntpq" -4p > %s' %
file_name, pty=False)
else:
run('w32tm /query /status > %s' % file_name, pty=False)
getfile(file_name, local_dir)
# log tcp module parameters (Linux only)
if htype == 'Linux':
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_tcpmod.log"
run("find /sys/module/tcp* -type f -exec grep -sH '' '{}' \; | "
"grep -v Binary > %s" % file_name)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ethtool.log"
run('touch %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('ethtool -k %s >> %s' % (interface, file_name))
getfile(file_name, local_dir)
def check_host():
"Check that needed tools are installed on hosts"
# get type of current host
htype = get_type_cached(env.host_string)
# run checks
if env.host_string in config.TPCONF_router:
if htype == 'FreeBSD':
run('which ipfw')
if htype == "Linux":
run('which tc')
run('which iptables')
# XXX check that kernel tick rate is high (>= 1000)
else:
if htype == 'FreeBSD':
run('which md5')
run('which tcpdump')
elif htype == 'Darwin':
run('which md5')
run('which tcpdump')
run('which dsiftr-osx-teacup.d')
elif htype == 'Linux':
run('which ethtool')
run('which md5sum')
run('which tcpdump')
#run('which web10g-listconns')
#run('which web10g-readvars')
run('which web10g-logger')
elif htype == 'CYGWIN':
run('which WinDump', pty=False)
run('which win-estats-logger', pty=False)
# if we don't have proper ntp installed then
# start time service if not started and force resync
with settings(warn_only=True):
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code != 0:
run('net start w32time', pty=False)
run('w32tm /resync', pty=False)
# try to enable any test network interfaces that are (accidently)
# disabled after reboot
with settings(warn_only=True):
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('netsh int set int "Local Area Connection %s" enabled' %
interface, pty=False)
run('which killall', pty=False)
run('which pkill', pty=False)
run('which ps', pty=False)
run('which gzip', pty=False)
run('which dd', pty=False)
# check for traffic sender/receiver tools
run('which iperf', pty=False)
run('which ping', pty=False)
run('which httperf', pty=False)
run('which lighttpd', pty=False)
run('which nttcp', pty=False)
put(config.TPCONF_script_path + '/runbg_wrapper.sh', '/usr/bin')
run('chmod a+x /usr/bin/runbg_wrapper.sh', pty=False)
run('which runbg_wrapper.sh', pty=False)
put(config.TPCONF_script_path + '/kill_iperf.sh', '/usr/bin')
run('chmod a+x /usr/bin/kill_iperf.sh', pty=False)
run('which kill_iperf.sh', pty=False)
put(config.TPCONF_script_path + '/pktgen.sh', '/usr/bin')
run('chmod a+x /usr/bin/pktgen.sh', pty=False)
run('which pktgen.sh', pty=False)
def start_tcpdump(
file_prefix='', remote_dir='', local_dir='.', snap_len='80',
tcpdump_filter='', internal_int='1'):
"Start tcpdump instance on host"
# get host type
htype = get_type_cached(env.host_string)
if env.host_string in config.TPCONF_router:
interfaces = get_netint_cached(env.host_string, int_no=-1,
internal_int=internal_int)
else:
if htype == 'CYGWIN':
interfaces = get_netint_windump_cached(env.host_string,
int_no=0,
internal_int=internal_int)
else:
interfaces = get_netint_cached(env.host_string,
int_no=0,
internal_int=internal_int)
if len(interfaces) < 1:
abort('Internal interface not specified')
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
for interface in interfaces:
if env.host_string in config.TPCONF_router:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + \
'_' + interface + '_router.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
else:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
tcpdump_cmd = 'tcpdump -n -s %s -i %s -w %s \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
else:
# CYGWIN
tcpdump_cmd = 'WinDump -n -s %s -i %s -w ' \
'"$(cygpath -aw "%s")" \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
pid = runbg(tcpdump_cmd)
name = 'tcpdump-' + interface
#bgproc.register_proc(env.host_string, name, '0', pid, file_name)
bgproc.register_proc_later(
env.host_string,
local_dir,
name,
'0',
pid,
file_name)
def init_tc_pipe(counter='1', source='', dest='', rate='', delay='', rtt='', loss='',
queue_size='', queue_size_mult='1.0', queue_disc='',
queue_disc_params='', bidir='0', attach_to_queue=''):
# compatibility with FreeBSD
if queue_disc == 'fifo':
# pfifo is the default for HTB classes
queue_disc = 'pfifo'
queue_size = str(queue_size)
if queue_size.lower() == 'bdp':
_rate = rate.replace('kbit', '000')
_rate = _rate.replace('mbit', '000000')
if rtt == '':
rtt = str(2 * int(delay))
if queue_disc == 'pfifo' or queue_disc == 'codel' or \
queue_disc == 'fq_codel' or queue_disc == 'pie':
# queue size in packets
avg_packet = 600 # average packet size
queue_size = int(
float(_rate) * (float(rtt) / 1000.0) / 8 / avg_packet)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 1:
queue_size = 1 # minimum 1 packet
queue_size = str(queue_size)
elif queue_disc == 'bfifo' or queue_disc == 'red':
# queue size in bytes
queue_size = int(float(_rate) * (float(rtt) / 1000.0) / 8)
if queue_size_mult != '1.0':
queue_size = int(float(queue_size) * float(queue_size_mult))
if queue_size < 2048:
queue_size = 2048 # minimum 2kB
queue_size = str(queue_size)
else:
abort(
'Can\'t specify \'bdp\' for queuing discipline %s' %
queue_disc)
# class/handle numbers
class_no = str(int(counter) + 0)
if attach_to_queue == '':
queue_class_no = class_no
else:
# if attach_to_queue is set we attach this to existing (previously
# configured pipe). this means packets will go through an existing htb
# and leaf qdisc, but a separate netem.
# so we can have different flows going through the same bottleneck
# queue, but with different emulated delays or loss rates
queue_class_no = attach_to_queue
netem_class_no = class_no
qdisc_no = str(int(counter) + 1000)
netem_no = str(int(counter) + 1000)
# disciplines: fq_codel, codel, red, choke, pfifo, pfifo_fast (standard
# magic), pie (only as patch), ...
if queue_disc == '':
queue_disc = 'pfifo'
# for pie we need to make sure the kernel module is loaded (for kernel pre
# 3.14 only, for new kernels it happens automatically via tc use!)
if queue_disc == 'pie':
with settings(warn_only=True):
run('modprobe pie')
if rate == '':
rate = '1000mbit'
if queue_size == '':
# set default queue size to 1000 packet (massive but default for e.g.
# codel)
queue_size = '1000'
if loss != '':
# convert to percentage
loss = str(float(loss) * 100)
interfaces = get_netint_cached(env.host_string, int_no=-1)
# our approach works as follows:
# - shaping, aqm and delay/loss emulation is done on egress interface
# (as usual)
# - use htb qdisc for rate limiting with the aqm qdisc (e.g. pfifo, codel)
# as leave node
# - after shaping and aqm, emulate loss and delay with netem
# - for each "pipe" we setup a new class on all (two) interfaces
# - if pipes are unidirectional a class is only used on one of the two ifaces;
# otherwise it is used on both interfaces (XXX could optimise the
# unidirectional case and omit unused pipes)
# - traffic flow is as follows:
# 1. packets are marked by iptables in mangle table POSTROUTING hook
# depending on defined source/dest (unique mark for each pipe)
# 2. marked packets are classified into appropriate class (1-1 mapping
# between marks and classes) and redirected to pseudo interface
# 3. pseudo interface does the shaping with htb and aqm (leaf qdisc)
# 4. packets go back to actual interface
# 5. actual interface does network emulation (delay/loss), here htb is set to
# max rate (1Gbps) and pfifo is used (effectively no shaping or aqm here)
# note that according to my information the htb has a build-in buffer of 1
# packet as well (cannot be changed)
cnt = 0
for interface in interfaces:
pseudo_interface = 'ifb' + str(cnt)
# config rate limiting on pseudo interface
config_tc_cmd = 'tc class add dev %s parent 1: classid 1:%s htb rate %s ceil %s' % \
(pseudo_interface, queue_class_no, rate, rate)
if attach_to_queue == '':
run(config_tc_cmd)
# config queuing discipline and buffer limit on pseudo interface
config_tc_cmd = 'tc qdisc add dev %s parent 1:%s handle %s: %s limit %s %s' % \
(pseudo_interface,
queue_class_no,
qdisc_no,
queue_disc,
queue_size,
queue_disc_params)
if attach_to_queue == '':
run(config_tc_cmd)
# configure filter to classify traffic based on mark on pseudo device
config_tc_cmd = 'tc filter add dev %s protocol ip parent 1: ' \
'handle %s fw flowid 1:%s' % (
pseudo_interface, class_no, queue_class_no)
run(config_tc_cmd)
# configure class for actual interface with max rate
config_tc_cmd = 'tc class add dev %s parent 1: classid 1:%s ' \
'htb rate 1000mbit ceil 1000mbit' % \
(interface, netem_class_no)
run(config_tc_cmd)
# config netem on actual interface
config_tc_cmd = 'tc qdisc add dev %s parent 1:%s handle %s: ' \
'netem limit 1000' % (
interface, netem_class_no, netem_no)
if delay != "":
config_tc_cmd += " delay %sms" % delay
if loss != "":
config_tc_cmd += " loss %s%%" % loss
run(config_tc_cmd)
# configure filter to redirect traffic to pseudo device first and also
# classify traffic based on mark after leaving the pseudo interface traffic
# will go back to actual interface
config_tc_cmd = 'tc filter add dev %s protocol ip parent 1: handle %s ' \
'fw flowid 1:%s action mirred egress redirect dev %s' % \
(interface, class_no, netem_class_no, pseudo_interface)
run(config_tc_cmd)
cnt += 1
# filter on specific ips
config_it_cmd = 'iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(source, dest, class_no)
run(config_it_cmd)
if bidir == '1':
config_it_cmd = 'iptables -t mangle -A POSTROUTING -s %s -d %s -j MARK --set-mark %s' % \
(dest, source, class_no)
run(config_it_cmd)