def get_host_info(htype='1', netint='1', netmac='1'):
"Populate the host info caches"
if htype == '1':
get_type_cached(env.host_string)
if netint == '1':
get_netint_cached(env.host_string, int_no=-1)
get_netint_windump_cached(env.host_string, int_no=-1)
get_netint_cached(env.host_string, int_no=-1, internal_int='0')
get_netint_windump_cached(env.host_string, int_no=-1, internal_int='0')
if netmac == '1':
get_netmac_cached(env.host_string)
def get_host_info(htype='1', netint='1', netmac='1'):
"Populate the host info caches"
if htype == '1':
get_type_cached(env.host_string)
if netint == '1':
get_netint_cached(env.host_string, int_no=-1)
get_netint_windump_cached(env.host_string, int_no=-1)
get_netint_cached(env.host_string, int_no=-1, internal_int='0')
get_netint_windump_cached(env.host_string, int_no=-1,
internal_int='0')
if netmac == '1':
get_netmac_cached(env.host_string)
def runbg(command, wait='0.0', out_file="/dev/null", shell=False, pty=True):
# get type of current host
htype = get_type_cached(env.host_string)
# on Linux with pty set to true, we don't get tool output in log,
# but that doesn't matter so much, since we are starting most things
# delayed with runbg_wrapper.sh. problem with pty=false is that
# with small sleep values processes may not get started on Linux (on
# slow systems such as VMs)
if htype == 'Linux' or htype == 'FreeBSD' or htype == 'Darwin':
result = run(
'nohup runbg_wrapper.sh %s %s >%s & sleep 0.1 ; echo "[1] $!" ' %
(wait, command, out_file), shell, pty)
else:
result = run(
'nohup runbg_wrapper.sh %s %s >%s & sleep 0.1 ; echo "[1] $!" ' %
(wait, command, out_file),
shell,
pty=False)
# get pid from output
result = result.replace('\r', '')
out_array = result.split('\n')
for line in out_array:
if line.find('[1] ') > -1:
pid = line.split(" ")[-1]
break
# check it is actually running (XXX slightly delay this?)
# if command executes very fast, this will cause task to fail, but quick
# commands should not be run with runbg()
run('kill -0 %s' % pid, pty=False)
return pid
def check_connectivity():
"Check connectivity between each pair of hosts with ping"
# get type of current host
htype = get_type_cached(env.host_string)
# get host test IP
test_ip = config.TPCONF_host_internal_ip[env.host_string][0]
# get list of reachable /24 subnets
# reachable does not store /24 subnets, but the actual IPs, since we always
# ignore the last octet in comparisons anyway
reachable = [test_ip]
for r in config.TPCONF_router:
for r_ip in config.TPCONF_host_internal_ip[r]:
if in_subnets(r_ip, reachable):
# if we are connected to a subnet of the router, then
# add the other subnet(s) router is connected to the
# reachable list
for x_ip in config.TPCONF_host_internal_ip[r]:
if x_ip != r_ip and x_ip not in reachable:
reachable.append(x_ip)
break # continue with next router
all_hosts = config.TPCONF_router + config.TPCONF_hosts
for host in all_hosts:
for ihost in config.TPCONF_host_internal_ip[host]:
if in_subnets(ihost, reachable):
if htype == "CYGWIN":
run('sudo ping -n 2 %s' % ihost, pty=False)
else:
run('sudo ping -c 2 %s' % ihost, pty=False)
def init_pipe(counter='1',
source='',
dest='',
rate='',
delay='',
rtt='',
loss='',
queue_size='',
queue_size_mult='1.0',
queue_disc='',
queue_disc_params='',
bidir='0',
attach_to_queue=''):
"Configure pipe on router, including rate shaping, AQM, loss/delay emulation"
# get internal addresses
dummy, source_internal = get_address_pair(source)
dummy, dest_internal = get_address_pair(dest)
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(init_dummynet_pipe, counter, source_internal, dest_internal,
rate, delay, rtt, loss, queue_size, queue_size_mult,
queue_disc, queue_disc_params, bidir)
elif htype == 'Linux':
execute(init_tc_pipe, counter, source_internal, dest_internal, rate,
delay, rtt, loss, queue_size, queue_size_mult, queue_disc,
queue_disc_params, bidir, attach_to_queue)
else:
abort("Router must be running FreeBSD or Linux")
def init_router():
"Initialize router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(init_dummynet)
elif htype == 'Linux':
interfaces = get_netint_cached(env.host_string, int_no=-1)
interfaces[0] = 'eno1' # swapna
interfaces[1] = 'enx000000000f92' # swapna
#interfaces = ['eno1']
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('sudo ethtool -K %s tso off' % interface)
run('sudo ethtool -K %s gso off' % interface)
run('sudo ethtool -K %s lro off' % interface)
run('sudo ethtool -K %s gro off' % interface)
run('sudo ethtool -K %s ufo off' % interface)
execute(init_tc)
else:
abort("Router must be running FreeBSD or Linux")
def show_pipes():
"Show pipe setup on router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(show_dummynet_pipes)
elif htype == 'Linux':
execute(show_tc_setup)
else:
abort("Router must be running FreeBSD or Linux")
def show_pipes():
"Show pipe setup on router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(show_dummynet_pipes)
elif htype == 'Linux':
execute(show_tc_setup)
else:
abort("Router must be running FreeBSD or Linux")
def check_connectivity():
"Check connectivity between each pair of hosts with ping"
# get type of current host
htype = get_type_cached(env.host_string)
all_hosts = config.TPCONF_router + config.TPCONF_hosts
for host in all_hosts:
for ihost in config.TPCONF_host_internal_ip[host]:
if htype == "CYGWIN":
run('ping -n 2 %s' % ihost, pty=False)
else:
run('ping -c 2 %s' % ihost, pty=False)
def log_queue_stats(file_prefix='', remote_dir='', local_dir='.'):
"Get queue statistics after experiment"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_queue_stats.log"
if htype == 'FreeBSD':
run('sudo echo ipfw pipe show > %s' % file_name)
run('sudo ipfw pipe show >> %s' % file_name)
run('sudo echo ipfw show >> %s' % file_name)
run('sudo ipfw show >> %s' % file_name)
elif htype == 'Linux':
run('sudo echo tc -d -s qdisc show > %s' % file_name)
run('sudo tc -d -s qdisc show >> %s' % file_name)
# interfaces = get_netint_cached(env.host_string, int_no=-1)
# interfaces[0] = 'eno1' # swapna
# interfaces[1] = 'enx000000000f92' # swapna
cnt = 0
for interface in interfaces:
run('sudo echo >> %s' % file_name)
run('sudo echo tc -s class show dev %s >> %s' %
(interface, file_name))
run('sudo tc -s class show dev %s >> %s' % (interface, file_name))
run('sudo echo >> %s' % file_name)
run('sudo echo tc -s filter show dev %s >> %s' %
(interface, file_name))
run('sudo tc -s filter show dev %s >> %s' % (interface, file_name))
pseudo_interface = 'ifb' + str(cnt)
run('sudo echo >> %s' % file_name)
run('sudo echo tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('sudo tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('sudo echo >> %s' % file_name)
run('sudo echo tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
run('sudo tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
cnt += 1
run('sudo echo iptables -t mangle -vL >> %s' % file_name)
run('sudo iptables -t mangle -vL >> %s' % file_name)
getfile(file_name, local_dir)
def stop_tcp_logger(file_prefix='', remote_dir='', local_dir='.'):
"Stop TCP logger (e.g. siftr on FreeBSD)"
# get host type
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
run('sysctl net.inet.siftr.enabled=0')
run('kldunload siftr')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_siftr.log'
elif htype == 'Linux':
#run('killall web100_logger')
run('killall web100-logger')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_web10g.log'
elif htype == 'Darwin':
pass
elif htype == 'CYGWIN':
run('killall win-estats-logger')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_web10g.log'
if logfile == '':
if remote_dir != '':
logfile = remote_dir + '/' + logfile
if file_prefix != '' or remote_dir != '':
file_name = logfile
else:
file_name = bgproc.get_proc_log(env.host_string, 'tcplogger', '00')
# add a small delay to allow logger to write data to disk completely
time.sleep(0.5)
# commented out: I think it may be confusing if the stats not match etc.
# if htype == 'FreeBSD':
# filter out control traffic from siftr log but
# stats and flow list in last line of log is left unchanged
#host = env.host_string.split(':')[0]
#tmp_file = local('mktemp "tmp.XXXXXXXXXX"', capture=True)
# run('cat %s | grep -v ",%s," > %s && mv %s %s' % \
# (file_name, host, tmp_file, tmp_file, file_name))
getfile(file_name, local_dir)
bgproc.remove_proc(env.host_string, 'tcplogger', '00')
def log_queue_stats(file_prefix='', remote_dir='', local_dir='.'):
"Get queue statistics after experiment"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_queue_stats.log"
if htype == 'FreeBSD':
run('echo ipfw pipe show > %s' % file_name)
run('ipfw pipe show >> %s' % file_name)
run('echo ipfw show >> %s' % file_name)
run('ipfw show >> %s' % file_name)
elif htype == 'Linux':
run('echo tc -d -s qdisc show > %s' % file_name)
run('tc -d -s qdisc show >> %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
cnt = 0
for interface in interfaces:
run('echo >> %s' % file_name)
run('echo tc -s class show dev %s >> %s' % (interface, file_name))
run('tc -s class show dev %s >> %s' % (interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -s filter show dev %s >> %s' % (interface, file_name))
run('tc -s filter show dev %s >> %s' % (interface, file_name))
pseudo_interface = 'ifb' + str(cnt)
run('echo >> %s' % file_name)
run('echo tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s class show dev %s >> %s' %
(pseudo_interface, file_name))
run('echo >> %s' % file_name)
run('echo tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
run('tc -d -s filter show dev %s >> %s' %
(pseudo_interface, file_name))
cnt += 1
run('echo iptables -t mangle -vL >> %s' % file_name)
run('iptables -t mangle -vL >> %s' % file_name)
getfile(file_name, local_dir)
def _get_md5val(file_name='', for_local='0'):
"Get MD5 hash for file depending on OS"
# get type of current host
htype = get_type_cached(env.host_string, for_local)
if htype == 'FreeBSD' or htype == 'Darwin':
md5_command = "md5 %s | awk '{ print $NF }'" % file_name
elif htype == 'Linux' or htype == 'CYGWIN':
md5_command = "md5sum %s | awk '{ print $1 }'" % file_name
else:
md5_command = ''
if for_local == '1':
local(md5_command, capture=True)
else:
run(md5_command, pty=False, shell=False)
def init_pipe(counter='1', source='', dest='', rate='', delay='', rtt='', loss='',
queue_size='', queue_size_mult='1.0', queue_disc='',
queue_disc_params='', bidir='0', attach_to_queue=''):
"Configure pipe on router, including rate shaping, AQM, loss/delay emulation"
# get internal addresses
dummy, source_internal = get_address_pair(source)
dummy, dest_internal = get_address_pair(dest)
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(
init_dummynet_pipe,
counter,
source_internal,
dest_internal,
rate,
delay,
rtt,
loss,
queue_size,
queue_size_mult,
queue_disc,
queue_disc_params,
bidir)
elif htype == 'Linux':
execute(
init_tc_pipe,
counter,
source_internal,
dest_internal,
rate,
delay,
rtt,
loss,
queue_size,
queue_size_mult,
queue_disc,
queue_disc_params,
bidir,
attach_to_queue)
else:
abort("Router must be running FreeBSD or Linux")
def get_netint_windump_cached(host='', int_no=0, internal_int='1'):
global host_internal_windump_int
global host_external_windump_int
# get type of current host
htype = get_type_cached(env.host_string)
if internal_int == '1':
if htype == 'CYGWIN' and host not in host_internal_windump_int:
host_internal_windump_int.update({host: []})
for i in range(len(config.TPCONF_host_internal_ip[host])):
host_internal_windump_int[host].append(
execute(
get_netint,
int_no=i,
windump='1',
internal_int='1',
hosts=host)[host])
res = host_internal_windump_int.get(host, '')
if int_no == -1:
return res
else:
if len(res) > int_no:
return [ res[int_no] ]
else:
return ['']
else:
if htype == 'CYGWIN' and host not in host_external_windump_int:
host_external_windump_int.update({host: []})
host_external_windump_int[host].append(
execute(
get_netint,
int_no=0,
windump='1',
internal_int='0',
hosts=host)[host])
res = host_external_windump_int.get(host, '')
return res
def _get_netmac(host=''):
htype = get_type_cached(env.host_string)
# resolve to ip if name
host = socket.gethostbyname(host)
# populate arp table
run('ping -c 1 %s' % host)
# get mac address
if htype == 'FreeBSD':
mac = run("arp %s | cut -d' ' -f 4 | head -1" % host)
elif htype == 'Linux':
mac = run("arp -a %s | cut -d' ' -f 4 | head -1" % host)
else:
abort("Can't determine MAC address for OS %s" % htype)
return mac
def _get_netmac(host=''):
htype = get_type_cached(env.host_string)
# resolve to ip if name
host = socket.gethostbyname(host)
# populate arp table
run('ping -c 1 %s' % host)
# get mac address
if htype == 'FreeBSD':
mac = run("arp %s | cut -d' ' -f 4 | head -1" % host)
elif htype == 'Linux':
mac = run("arp -a %s | cut -d' ' -f 4 | head -1" % host)
else:
abort("Can't determine MAC address for OS %s" % htype)
return mac
def get_netint_windump_cached(host='', int_no=0, internal_int='1'):
global host_internal_windump_int
global host_external_windump_int
# get type of current host
htype = get_type_cached(env.host_string)
if internal_int == '1':
if htype == 'CYGWIN' and host not in host_internal_windump_int:
host_internal_windump_int.update({host: []})
for i in range(len(config.TPCONF_host_internal_ip[host])):
host_internal_windump_int[host].append(
execute(
get_netint,
int_no=i,
windump='1',
internal_int='1',
hosts=host)[host])
res = host_internal_windump_int.get(host, '')
if int_no == -1:
return res
else:
if len(res) > int_no:
return [ res[int_no] ]
else:
return ['']
else:
if htype == 'CYGWIN' and host not in host_external_windump_int:
host_external_windump_int.update({host: []})
host_external_windump_int[host].append(
execute(
get_netint,
int_no=0,
windump='1',
internal_int='0',
hosts=host)[host])
res = host_external_windump_int.get(host, '')
return res
def kill_old_processes():
"Kill old logging or traffic generation processes still running"
# get type of current host
htype = get_type_cached(env.host_string)
with settings(warn_only=True):
if htype == 'FreeBSD':
run('killall tcpdump', pty=False)
elif htype == 'Linux':
run('killall tcpdump', pty=False)
# updated for ttprobe support
# Kill ttprobe user space process and unload kernel module
# run('pkill -f "cat /proc/net/ttprobe"')
with settings(warn_only=True):
run('rmmod ttprobe')
#run('killall web10g_logger.sh')
run('killall qdisc_logger.sh')
run('killall web10g-logger')
elif htype == 'Darwin':
run('killall tcpdump', pty=False)
run('killall dsiftr-osx-teacup.d', pty=False)
elif htype == 'CYGWIN':
run('killall WinDump', pty=False)
run('killall win-estats-logger', pty=False)
if htype == 'CYGWIN':
# on new cygwin does stop anymore on sigterm
run('killall -9 iperf', pty=False)
else:
run('killall iperf', pty=False)
run('killall ping', pty=False)
run('killall httperf', pty=False)
run('killall lighttpd', pty=False)
# delete old lighttp pid files (XXX would be better to delete after
# experiment)
run('rm -f /var/run/*lighttpd.pid', pty=False)
run('killall runbg_wrapper.sh', pty=False)
run('killall nttcp')
run('killall pktgen.sh ; killall python')
# remove old log stuff in /tmp
run('rm -f /tmp/*.log', pty=False)
def check_time_sync():
"Check time synchronisation between control host and testbed host clocks"
allowed_time_diff = 1
try:
allowed_time_diff = config.TPCONF_max_time_diff
except AttributeError:
pass
# get type of current host
htype = get_type_cached(env.host_string)
# get timestamps in unix time to avoid having to do time format conversions
# XXX should get timestamps in milliseconds, cause now we have huge quantisation
# error, but how to do this in a portable way?
t1 = datetime.datetime.now()
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
rdate = run('date +\'%s\'')
elif htype == 'CYGWIN':
rdate = run('date +\'%s\'', pty=False)
ldate = local('date +\'%s\'', capture=True)
t2 = datetime.datetime.now()
dt_diff = t2 - t1
sec_diff = (dt_diff.days * 24 * 3600 + dt_diff.seconds) + \
(dt_diff.microseconds / 1000000.0)
puts(
'Local time: %s, remote time: %s, proc delay: %s' %
(ldate, rdate, str(sec_diff)))
diff = abs(int(ldate) - int(rdate) - sec_diff)
if diff > allowed_time_diff:
abort(
'Host %s time synchronisation error (difference > %s seconds)' %
(env.host_string, str(allowed_time_diff)))
def getfile(file_name='', local_dir='.'):
"Get file from remote and check that file is not corrupt"
if file_name == '':
abort('Must specify file name')
if file_name[0] != '/':
# get type of current host
htype = get_type_cached(env.host_string)
# need to guess the path
if env.user == 'root' and not htype == 'CYGWIN':
remote_dir = '/root'
else:
remote_dir = '/home/' + env.user
file_name = remote_dir + '/' + file_name
else:
remote_dir = os.path.dirname(file_name)
# gzip and download (XXX could use bzip2 instead, slower but better
# compression)
run('gzip -f %s' % file_name, pty=False)
file_name += '.gz'
local_file_name = get(file_name, local_dir)[0]
# get MD5 on remote
md5_val = _get_md5val(file_name, '0')
if md5_val != '':
# get MD5 for downloaded file
local_md5_val = _get_md5val(local_file_name, '1')
# check if MD5 is correct
if md5_val != local_md5_val:
abort('Failed MD5 check')
else:
puts('MD5 OK')
run('rm -f %s' % file_name, pty=False)
def getfile(file_name='', local_dir='.'):
"Get file from remote and check that file is not corrupt"
if file_name == '':
abort('Must specify file name')
if file_name[0] != '/':
# get type of current host
htype = get_type_cached(env.host_string)
# need to guess the path
if env.user == 'root' and not htype == 'CYGWIN':
remote_dir = '/root'
else:
remote_dir = '/home/' + env.user
file_name = remote_dir + '/' + file_name
else:
remote_dir = os.path.dirname(file_name)
# gzip and download (XXX could use bzip2 instead, slower but better
# compression)
run('gzip -f %s' % file_name, pty=False)
file_name += '.gz'
local_file_name = get(file_name, local_dir)[0]
# get MD5 on remote
md5_val = _get_md5val(file_name, '0')
if md5_val != '':
# get MD5 for downloaded file
local_md5_val = _get_md5val(local_file_name, '1')
# check if MD5 is correct
if md5_val != local_md5_val:
abort('Failed MD5 check')
else:
puts('MD5 OK')
run('rm -f %s' % file_name, pty=False)
def init_router():
"Initialize router"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
execute(init_dummynet)
elif htype == 'Linux':
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
execute(init_tc)
else:
abort("Router must be running FreeBSD or Linux")
def stop_tcpdump(file_prefix='', remote_dir='', local_dir='.'):
"Stop tcpdump instance on host"
pid = bgproc.get_proc_pid(env.host_string, 'tcpdump', '0')
with settings(warn_only=True):
if pid != "":
run('kill %s' % pid, pty=False)
else:
# get host type
htype = get_type_cached(env.host_string)
if htype == "FreeBSD" or htype == "Linux" or htype == 'Darwin':
run('killall tcpdump')
else:
run('killall WinDump', pty=False)
if file_prefix != "" or remote_dir != "":
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + ".dmp"
else:
file_name = bgproc.get_proc_log(env.host_string, 'tcpdump', '0')
getfile(file_name, local_dir)
bgproc.remove_proc(env.host_string, 'tcpdump', '0')
def stop_tcpdump(file_prefix='', remote_dir='', local_dir='.'):
"Stop tcpdump instance on host"
pid = bgproc.get_proc_pid(env.host_string, 'tcpdump', '0')
with settings(warn_only=True):
if pid != "":
run('kill %s' % pid, pty=False)
else:
# get host type
htype = get_type_cached(env.host_string)
if htype == "FreeBSD" or htype == "Linux" or htype == 'Darwin':
run('killall tcpdump')
else:
run('killall WinDump', pty=False)
if file_prefix != "" or remote_dir != "":
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + ".dmp"
else:
file_name = bgproc.get_proc_log(env.host_string, 'tcpdump', '0')
getfile(file_name, local_dir)
bgproc.remove_proc(env.host_string, 'tcpdump', '0')
def kill_old_processes():
"Kill old logging or traffic generation processes still running"
# get type of current host
htype = get_type_cached(env.host_string)
with settings(warn_only=True):
if htype == 'FreeBSD':
run('killall tcpdump', pty=False)
elif htype == 'Linux':
run('killall tcpdump', pty=False)
#run('killall web10g_logger.sh')
run('killall web10g-logger')
elif htype == 'Darwin':
run('killall tcpdump', pty=False)
run('killall dsiftr-osx-teacup.d', pty=False)
elif htype == 'CYGWIN':
run('killall WinDump', pty=False)
run('killall win-estats-logger', pty=False)
if htype == 'CYGWIN':
# on new cygwin does stop anymore on sigterm
run('killall -9 iperf', pty=False)
else:
run('killall iperf', pty=False)
run('killall ping', pty=False)
run('killall httperf', pty=False)
run('killall lighttpd', pty=False)
# delete old lighttp pid files (XXX would be better to delete after
# experiment)
run('rm -f /var/run/*lighttpd.pid', pty=False)
run('killall runbg_wrapper.sh', pty=False)
run('killall nttcp')
run('killall pktgen.sh ; killall python')
# remove old log stuff in /tmp
run('rm -f /tmp/*.log', pty=False)
def init_ecn(ecn='0'):
"Initialize whether ECN is used or not"
if ecn != '0' and ecn != '1':
abort("Parameter ecn must be set to '0' or '1'")
# get type of current host
htype = get_type_cached(env.host_string)
# enable/disable RED
if htype == 'FreeBSD':
run('sysctl net.inet.tcp.ecn.enable=%s' % ecn)
elif htype == 'Linux':
run('sysctl net.ipv4.tcp_ecn=%s' % ecn)
elif htype == 'Darwin':
run('sysctl -w net.inet.tcp.ecn_initiate_out=%s' % ecn)
run('sysctl -w net.inet.tcp.ecn_negotiate_in=%s' % ecn)
elif htype == 'CYGWIN':
if ecn == '1':
run('netsh int tcp set global ecncapability=enabled', pty=False)
else:
run('netsh int tcp set global ecncapability=disabled', pty=False)
else:
abort("Can't enable/disable ECN for OS '%s'" % htype)
def init_ecn(ecn='0'):
"Initialize whether ECN is used or not"
if ecn != '0' and ecn != '1':
abort("Parameter ecn must be set to '0' or '1'")
# get type of current host
htype = get_type_cached(env.host_string)
# enable/disable RED
if htype == 'FreeBSD':
run('sysctl net.inet.tcp.ecn.enable=%s' % ecn)
elif htype == 'Linux':
run('sysctl net.ipv4.tcp_ecn=%s' % ecn)
elif htype == 'Darwin':
run('sysctl -w net.inet.tcp.ecn_initiate_out=%s' % ecn)
run('sysctl -w net.inet.tcp.ecn_negotiate_in=%s' % ecn)
elif htype == 'CYGWIN':
if ecn == '1':
run('netsh int tcp set global ecncapability=enabled', pty=False)
else:
run('netsh int tcp set global ecncapability=disabled', pty=False)
else:
abort("Can't enable/disable ECN for OS '%s'" % htype)
def init_os(file_prefix='', os_list='', force_reboot='0', do_power_cycle='0',
boot_timeout='100', local_dir='.',
linux_kern_router='3.10.18-vanilla-10000hz',
linux_kern_hosts='3.9.8-desktop-web10g',
tftp_server='10.1.1.11:8080',
mac_list=''):
"Boot host with selected operating system"
_boot_timeout = int(boot_timeout)
if _boot_timeout < 60:
warn('Boot timeout value too small, using 60 seconds')
_boot_timeout = '60'
host_os_vals = os_list.split(',')
if len(env.all_hosts) < len(host_os_vals):
abort('Number of OSs specified must be the same as number of hosts')
# duplicate last one until we reach correct length
while len(host_os_vals) < len(env.all_hosts):
host_os_vals.append(host_os_vals[-1])
host_mac = {}
if mac_list != '':
mac_vals = mac_list.split(',')
if len(env.all_hosts) != len(mac_vals):
abort('Must specify one MAC address for each host')
# create a dictionary
host_mac = dict(zip(env.all_hosts, mac_vals))
# get type of current host if possible
# XXX try to suppress Fabric exception traceback in case host is not
# accessible, but doesn't seem to work properly
with settings(hide('debug', 'warnings'), warn_only=True):
htype = get_type_cached(env.host_string)
if type(htype) == NetworkError:
# host not accessible, set htype to unknown
htype = '?'
# get dictionary from host and OS lists
host_os = dict(zip(env.all_hosts, host_os_vals))
# os we want
target_os = host_os.get(env.host_string, '')
kern = ''
target_kern = ''
if target_os == 'Linux':
if env.host_string in config.TPCONF_router:
target_kern = linux_kern_router
else:
target_kern = linux_kern_hosts
if htype == 'Linux':
kern = run('uname -r')
else:
kern = '?'
if target_kern == 'running' or target_kern == 'current':
if htype == 'Linux':
target_kern = kern
else:
warn('Host not running Linux, ignoring "running" or "current"')
if target_os != '' and (
force_reboot == '1' or target_os != htype or target_kern != kern):
# write pxe config file
pxe_template = config.TPCONF_script_path + \
'/conf-macaddr_xx\:xx\:xx\:xx\:xx\:xx.ipxe.in'
# if we have a mac address specified use it, otherwise try to automatically
# get the mac address
if env.host_string in host_mac:
mac = host_mac[env.host_string]
else:
mac = get_netmac_cached(env.host_string)
file_name = 'conf-macaddr_' + mac + '.ipxe'
hdd_partition = ''
if target_os == 'CYGWIN':
hdd_partition = '(hd0,0)'
elif target_os == 'Linux':
hdd_partition = '(hd0,1)'
elif target_os == 'FreeBSD':
hdd_partition = '(hd0,2)'
try:
hdd_partition = config.TPCONF_os_partition[target_os]
except AttributeError:
pass
if target_os == 'Linux':
# could remove the if, but might need in future if we specify
# different options for router and hosts
if env.host_string in config.TPCONF_router:
config_str = 'root ' + hdd_partition + '; kernel \/boot\/vmlinuz-' + \
target_kern + ' splash=0 quiet showopts; ' + \
'initrd \/boot\/initrd-' + target_kern
else:
config_str = 'root ' + hdd_partition + '; kernel \/boot\/vmlinuz-' + \
target_kern + ' splash=0 quiet showopts; ' + \
'initrd \/boot\/initrd-' + target_kern
elif target_os == 'CYGWIN':
if env.host_string in config.TPCONF_router:
abort('Router has no Windows')
config_str = 'root ' + hdd_partition + '; chainloader +1'
elif target_os == 'FreeBSD':
config_str = 'root ' + hdd_partition + '; chainloader +1'
elif target_os == 'Darwin':
pass
else:
abort('Unknown OS %s' % target_os)
if force_reboot == '1':
puts('Forced reboot (TPCONF_force_reboot = \'1\')')
puts(
'Switching %s from OS %s %s to OS %s %s' %
(env.host_string, htype, kern, target_os, target_kern))
if htype != 'Darwin':
# no PXE booting for Macs
local(
'cat %s | sed -e "s/@CONFIG@/%s/" | sed -e "s/@TFTPSERVER@/%s/" > %s' %
(pxe_template, config_str, tftp_server, file_name))
# make backup of current file if not exists yet
full_file_name = config.TPCONF_tftpboot_dir + '/' + file_name
full_file_name_backup = config.TPCONF_tftpboot_dir + \
'/' + file_name + '.bak'
with settings(warn_only=True):
local('mv -f %s %s' % (full_file_name, full_file_name_backup))
local('rm -f %s' % full_file_name)
# XXX should combine the next two into one shell command to make it
# more atomic
local('cp %s %s' % (file_name, config.TPCONF_tftpboot_dir))
local('chmod a+rw %s' % full_file_name)
if file_prefix != '':
file_name2 = local_dir + '/' + file_prefix + '_' + file_name
local('mv %s %s' % (file_name, file_name2))
# reboot
with settings(warn_only=True):
if htype == '?':
# we cannot login to issue shutdown command, so power cycle and hope
# for the best
execute(power_cycle)
elif htype == 'Linux' or htype == 'FreeBSD' or htype == 'Darwin':
run('shutdown -r now', pty=False)
elif htype == 'CYGWIN':
run('shutdown -r -t 0', pty=False)
# give some time to shutdown
puts('Waiting for reboot...')
time.sleep(60)
# wait until up
t = 60
while t <= _boot_timeout:
# ret = local('ping -c 1 %s' % env.host_string) # ping works before
# ssh and can't ping from inside jail
with settings(warn_only=True):
try:
ret = run(
'echo waiting for OS %s to start' %
target_os,
timeout=2,
pty=False)
if ret.return_code == 0:
break
except:
pass
time.sleep(8)
t += 10
if t > _boot_timeout and do_power_cycle == '1':
# host still not up, may be hanging so power cycle it
puts('Power cycling host...')
execute(power_cycle)
puts('Waiting for reboot...')
time.sleep(60)
# wait until up
t = 60
while t <= _boot_timeout:
# ret = local('ping -c 1 %s' % env.host_string) # ping works
# before ssh and can't ping from inside jail
with settings(warn_only=True):
try:
ret = run(
'echo waiting for OS %s to start' %
target_os,
timeout=2,
pty=False)
if ret.return_code == 0:
break
except:
pass
time.sleep(8)
t += 10
# finally check if host is up again with desired OS
# XXX if the following fails because we can't connect to host Fabric
# will crash with weird error (not super important since we would abort
# anyway but annoying)
htype = execute(get_type, hosts=[env.host_string])[env.host_string]
if target_os == 'Linux':
kern = run('uname -r')
if htype == target_os and kern == target_kern:
puts(
'Host %s running OS %s %s' %
(env.host_string, target_os, target_kern))
else:
abort(
'Error switching %s to OS %s %s' %
(env.host_string, target_os, target_kern))
else:
if target_os == '':
target_os = htype
puts(
'Leaving %s as OS %s %s' %
(env.host_string, target_os, target_kern))
def start_tcpdump(
file_prefix='', remote_dir='', local_dir='.', snap_len='80',
tcpdump_filter='', internal_int='1'):
"Start tcpdump instance on host"
# get host type
htype = get_type_cached(env.host_string)
if env.host_string in config.TPCONF_router:
interfaces = get_netint_cached(env.host_string, int_no=-1,
internal_int=internal_int)
else:
if htype == 'CYGWIN':
interfaces = get_netint_windump_cached(env.host_string,
int_no=0,
internal_int=internal_int)
else:
interfaces = get_netint_cached(env.host_string,
int_no=0,
internal_int=internal_int)
if len(interfaces) < 1:
abort('Internal interface not specified')
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
for interface in interfaces:
if env.host_string in config.TPCONF_router:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + \
'_' + interface + '_router.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
else:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
tcpdump_cmd = 'tcpdump -n -s %s -i %s -w %s \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
else:
# CYGWIN
tcpdump_cmd = 'WinDump -n -s %s -i %s -w ' \
'"$(cygpath -aw "%s")" \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
pid = runbg(tcpdump_cmd)
name = 'tcpdump-' + interface
#bgproc.register_proc(env.host_string, name, '0', pid, file_name)
bgproc.register_proc_later(
env.host_string,
local_dir,
name,
'0',
pid,
file_name)
def get_netint(int_no=0, windump='0', internal_int='1'):
"Get network interface name"
# need to convert if we run task from command line
int_no = int(int_no)
# check int_no paramter
if int_no < 0:
int_no = 0
if int_no >= len(config.TPCONF_host_internal_ip[env.host_string]):
int_no = len(config.TPCONF_host_internal_ip[env.host_string]) - 1
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
# get ip and set last octet to 0
if internal_int == '1':
iip = config.TPCONF_host_internal_ip[env.host_string][int_no]
else:
iip = socket.gethostbyname(env.host_string.split(':')[0])
a = iip.split('.')
del a[3]
iip = '.'.join(a)
int_name = ''
field_idx = -1
lines = run('netstat -nr', shell=False)
for line in lines.split('\n'):
if line != '':
fields = line.split()
if len(fields) > 0 and fields[0] == 'Destination' and \
int_name == '' :
for i in range(len(fields)) :
if fields[i] == 'Netif' :
field_idx = i
if len(fields) > 0 and (fields[0].split('/')[0] == iip + '.0' or
fields[0].split('/')[0] == iip) :
int_name = fields[field_idx]
#puts('Interface: %s' % int_name)
return int_name
elif htype == "CYGWIN":
# on windows we have two numbers
# windows numbering of interfaces
# numbering used by windump
if windump == '0':
# get interface IPs and numbers
output = run(
'ipconfig | egrep "Local Area|IPv4" | grep -v "Tunnel"',
pty=False)
lines = output.split("\n")
for i in range(0, len(lines), 2):
int_num = lines[i].replace(":", "").split(" ")[-1]
if int_num == "": # XXX not sure what we are doing here
int_num = "1"
int_ip = lines[i + 1].split(":")[1].strip()
if internal_int == '1' and int_ip == config.TPCONF_host_internal_ip[
env.host_string][int_no] or \
internal_int == '0' and int_ip == socket.gethostbyname(
env.host_string.split(':')[0]):
puts('Interface: %s' % int_num)
return int_num
else:
# get list of interface numbers and interface IDs
output = run(
'winDUmp -D | sed "s/\([0-9]\)\.[^{]*{\([^}]*\).*/\\1 \\2/"',
pty=False)
# get list of interface macs and interface IDs
output2 = run(
'getmac | '
'grep "^[0-9]" | sed "s/^\([0-9A-Fa-f-]*\)[^{]*{\([^}]*\).*/\\1 \\2/"',
pty=False)
# get mac of the internal/external interface
mac = execute(
get_netmac,
internal_int=internal_int,
hosts=[env.host_string]).values()[0]
# find interface ID
int_id = ''
lines = output2.split("\n")
for line in lines:
_int_mac, _int_id = line.split(' ')
# get mac print with '-' instead of ':'
_int_mac = _int_mac.replace('-', ':').lower()
if _int_mac == mac:
int_id = _int_id
break
# get interface number (could use ID but it's a bit long)
lines = output.split("\n")
for line in lines:
_int_num, _int_id = line.split(' ')
if _int_id == int_id:
puts('Interface: %s' % _int_num)
return _int_num
else:
abort('Cannot determine network interface for OS %s' % htype)
def init_cc_algo(algo='default', *args, **kwargs):
"Initialize TCP congestion control algorithm"
if algo[0:4] == 'host':
arr = algo.split('t')
if len(arr) == 2 and arr[1].isdigit():
num = int(arr[1])
else:
abort('If you specify host<N>, the <N> must be an integer number')
algo_list = config.TPCONF_host_TCP_algos.get(env.host_string, [])
if len(algo_list) == 0:
abort(
'No TCP congestion control algos defined for host %s' %
env.host_string)
if num > len(algo_list) - 1:
warn(
'No TCP congestion control algo specified for <N> = %d, ' \
'setting <N> = 0' % num)
num = 0
algo = algo_list[num]
puts('Selecting TCP congestion control algorithm: %s' % algo)
if algo != 'default' and algo != 'newreno' and algo != 'cubic' and \
algo != 'cdg' and algo != 'htcp' and \
algo != 'compound' and algo != 'hd' and algo != 'vegas':
abort(
'Available TCP algorithms: ' +
'default, newreno, cubic, cdg, hd, htcp, compound, vegas')
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
if algo == 'newreno' or algo == 'default':
algo = 'newreno'
elif algo == 'cubic':
with settings(warn_only=True):
ret = run('kldstat | grep cc_cubic')
if ret.return_code != 0:
run('kldload cc_cubic')
elif algo == 'hd':
with settings(warn_only=True):
ret = run('kldstat | grep cc_hd')
if ret.return_code != 0:
run('kldload cc_hd')
elif algo == 'htcp':
with settings(warn_only=True):
ret = run('kldstat | grep cc_htcp')
if ret.return_code != 0:
run('kldload cc_htcp')
elif algo == 'cdg':
with settings(warn_only=True):
ret = run('kldstat | grep cc_cdg')
if ret.return_code != 0:
# cdg is only available by default since FreeBSD 9.2
run('kldload cc_cdg')
elif algo == 'vegas':
with settings(warn_only=True):
ret = run('kldstat | grep cc_vegas')
if ret.return_code != 0:
run('kldload cc_vegas')
else:
abort("Congestion algorithm '%s' not supported by FreeBSD" % algo)
run('sysctl net.inet.tcp.cc.algorithm=%s' % algo)
elif htype == 'Linux':
if algo == 'newreno':
# should be there by default
algo = 'reno'
elif algo == 'cubic' or algo == 'default':
# should also be there by default
algo = 'cubic'
elif algo == 'htcp':
run('modprobe tcp_htcp')
elif algo == 'vegas':
run('modprobe tcp_vegas')
else:
abort("Congestion algorithm '%s' not supported by Linux" % algo)
run('sysctl net.ipv4.tcp_congestion_control=%s' % algo)
elif htype == 'Darwin':
if algo == 'newreno' or algo == 'default':
algo = 'newreno'
else:
abort("Congestion algorithm '%s' not supported by MacOS" % algo)
elif htype == 'CYGWIN':
if algo == 'newreno' or algo == 'default':
run('netsh int tcp set global congestionprovider=none', pty=False)
elif algo == 'compound':
run('netsh int tcp set global congestionprovider=ctcp', pty=False)
else:
abort("Congestion algorithm '%s' not supported by Windows" % algo)
else:
abort("Can't set TCP congestion control algo for OS '%s'" % htype)
# now set the cc algorithm parameters
execute(init_cc_algo_params, algo=algo, *args, **kwargs)
def get_netmac(internal_int='0'):
"Get MAC address for external/ctrl network interface"
# not so easy to get local mac address for all different OS
# use following approach:
# get all non-router macs via the router using arp -a
# get router mac using ifconfig
# if host is a vm which we access via localhost then use ifconfig not arp
# method
# XXX windows: getmac, ipconfig /all
host_string = env.host_string
# if we have a port then strip off port
if host_string.find(':') > -1:
host_string = host_string.split(':')[0]
if host_string == 'localhost':
host_string = '127.0.0.1'
if host_string in config.TPCONF_router or host_string == '127.0.0.1':
# get MAC of router
htype = get_type_cached(env.host_string)
# complicated awk code to get pairs of ip and mac addresses from
# ifconfig
if htype == 'FreeBSD':
macips = run(
'ifconfig | awk \'/ether / { printf("%s ", $0); next } 1\' | '
+ 'grep ether | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $2, $4) }\'',
shell=False)
elif htype == 'Linux':
macips = run(
'ifconfig | awk \'/HWaddr / { printf("%s ", $0); next } 1\' | '
+ 'grep HWaddr | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $5, $7) }\' | sed -e "s/addr://"')
else:
abort("Can't determine MAC address for OS %s" % htype)
ip_mac_map = {}
for line in macips.split('\n'):
a = line.split(' ')
ip_mac_map.update({a[1].strip(): a[0].strip()})
# print(ip_mac_map)
# check if it looks like a name
if not re.match('[0-9.]+', host_string):
# try dns and assume we get an answer
ip = socket.gethostbyname(host_string)
else:
ip = host_string
if ip != '127.0.0.1':
mac = ip_mac_map.get(ip)
else:
# guess it's the first NIC
# XXX should return MAC based on router IP, not simply the first
mac = ip_mac_map.get(ip_mac_map.keys()[0])
else:
# get MAC of non-router
if internal_int == '0':
host_string = env.host_string
else:
host_string = config.TPCONF_host_internal_ip.get(
env.host_string, '')[0]
mac = execute(_get_netmac,
host_string.split(':')[0],
hosts=[config.TPCONF_router[0]])[config.TPCONF_router[0]]
return mac.lower()
def stop_tcp_logger(file_prefix='', remote_dir='', local_dir='.'):
"Stop TCP logger (e.g. siftr on FreeBSD)"
# get host type
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
run('sysctl net.inet.siftr.enabled=0')
run('kldunload siftr')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_siftr.log'
elif htype == 'Linux':
# In fact, stop_tcp_logger is called just when Linux
# and ttprobe are used (not with web10g)
# but we check linux_tcp_logger type just in case
# run('killall web100-logger')
try:
linux_tcp_logger = config.TPCONF_linux_tcp_logger
except AttributeError:
linux_tcp_logger = 'web10g'
if linux_tcp_logger == 'ttprobe' or linux_tcp_logger == 'both':
# flush ttprobe module buffer
run('echo flush > /proc/net/ttprobe')
time.sleep(0.5)
run('echo finish > /proc/net/ttprobe')
#run('pkill -f "cat /proc/net/ttprobe"')
run('rmmod ttprobe')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ttprobe.log'
# complete other tasks and exit from this function because ttprobe has differnt bgproce
# name
if file_prefix != '' or remote_dir != '':
file_name = logfile
else:
file_name = bgproc.get_proc_log(env.host_string, 'tcploggerprobe', '00')
getfile(file_name, local_dir)
bgproc.remove_proc(env.host_string, 'tcploggerprobe', '00')
return
elif htype == 'Darwin':
pass
elif htype == 'CYGWIN':
run('killall win-estats-logger')
logfile = file_prefix + '_' + \
env.host_string.replace(':', '_') + '_web10g.log'
if logfile == '':
if remote_dir != '':
logfile = remote_dir + '/' + logfile
if file_prefix != '' or remote_dir != '':
file_name = logfile
else:
file_name = bgproc.get_proc_log(env.host_string, 'tcplogger', '00')
# add a small delay to allow logger to write data to disk completely
time.sleep(0.5)
# commented out: I think it may be confusing if the stats not match etc.
# if htype == 'FreeBSD':
# filter out control traffic from siftr log but
# stats and flow list in last line of log is left unchanged
#host = env.host_string.split(':')[0]
#tmp_file = local('mktemp "tmp.XXXXXXXXXX"', capture=True)
# run('cat %s | grep -v ",%s," > %s && mv %s %s' % \
# (file_name, host, tmp_file, tmp_file, file_name))
getfile(file_name, local_dir)
bgproc.remove_proc(env.host_string, 'tcplogger', '00')
def log_sysdata(file_prefix='', remote_dir='', local_dir='.'):
"Log various information for each system"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_uname.log"
run('uname -a > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_netstat.log"
run('netstat -nr > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_sysctl.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('sysctl -a > %s' % file_name)
else:
run('echo "netsh int show int" > %s' % file_name, pty=False)
run('netsh int show int >> %s' % file_name, pty=False)
run('echo "netsh int tcp show global" >> %s' % file_name, pty=False)
run('netsh int tcp show global >> %s' % file_name, pty=False)
run('echo "netsh int tcp show heuristics" >> %s' %
file_name, pty=False)
run('netsh int tcp show heuristics >> %s' % file_name, pty=False)
run('echo "netsh int tcp show security" >> %s' % file_name, pty=False)
run('netsh int tcp show security >> %s' % file_name, pty=False)
run('echo "netsh int tcp show chimneystats" >> %s' %
file_name, pty=False)
run('netsh int tcp show chimneystats >> %s' % file_name, pty=False)
run('echo "netsh int ip show offload" >> %s' % file_name, pty=False)
run('netsh int ip show offload >> %s' % file_name, pty=False)
run('echo "netsh int ip show global" >> %s' % file_name, pty=False)
run('netsh int ip show global >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ifconfig.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ifconfig -a > %s' % file_name)
else:
run('ipconfig > %s' % file_name, pty=False)
# log interface speeds
run('echo "wmic NIC where NetEnabled=true get Name, Speed" >> %s' % file_name, pty=False)
run('wmic NIC where NetEnabled=true get Name, Speed >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_procs.log"
if htype == 'FreeBSD' or htype == 'Linux':
run('ps -axu > %s' % file_name)
elif htype == 'Darwin':
run('ps -axu root > %s' % file_name)
else:
run('ps -alW > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ntp.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ntpq -4p > %s' % file_name)
else:
with settings(warn_only=True):
# if we have ntp installed then use ntpq, otherwise use w32tm
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code == 0:
run('"/cygdrive/c/Program Files (x86)/NTP/bin/ntpq" -4p > %s' %
file_name, pty=False)
else:
run('w32tm /query /status > %s' % file_name, pty=False)
getfile(file_name, local_dir)
# log tcp module parameters (Linux only)
if htype == 'Linux':
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_tcpmod.log"
run("find /sys/module/tcp* -type f -exec grep -sH '' '{}' \; | "
"grep -v Binary > %s" % file_name)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ethtool.log"
run('touch %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('ethtool -k %s >> %s' % (interface, file_name))
getfile(file_name, local_dir)
def init_os(file_prefix='',
os_list='',
force_reboot='0',
do_power_cycle='0',
boot_timeout='100',
local_dir='.',
linux_kern_router='3.10.18-vanilla-10000hz',
linux_kern_hosts='3.9.8-desktop-web10g',
tftp_server='10.1.1.11:8080',
mac_list=''):
"Boot host with selected operating system"
_boot_timeout = int(boot_timeout)
if _boot_timeout < 60:
warn('Boot timeout value too small, using 60 seconds')
_boot_timeout = '60'
host_os_vals = os_list.split(',')
if len(env.all_hosts) < len(host_os_vals):
abort('Number of OSs specified must be the same as number of hosts')
# duplicate last one until we reach correct length
while len(host_os_vals) < len(env.all_hosts):
host_os_vals.append(host_os_vals[-1])
host_mac = {}
if mac_list != '':
mac_vals = mac_list.split(',')
if len(env.all_hosts) != len(mac_vals):
abort('Must specify one MAC address for each host')
# create a dictionary
host_mac = dict(zip(env.all_hosts, mac_vals))
# get type of current host if possible
# XXX try to suppress Fabric exception traceback in case host is not
# accessible, but doesn't seem to work properly
with settings(hide('debug', 'warnings'), warn_only=True):
htype = get_type_cached(env.host_string)
if type(htype) == NetworkError:
# host not accessible, set htype to unknown
htype = '?'
# get dictionary from host and OS lists
host_os = dict(zip(env.all_hosts, host_os_vals))
# os we want
target_os = host_os.get(env.host_string, '')
kern = ''
target_kern = ''
if target_os == 'Linux':
if env.host_string in config.TPCONF_router:
target_kern = linux_kern_router
else:
target_kern = linux_kern_hosts
if htype == 'Linux':
kern = run('uname -r')
else:
kern = '?'
if target_kern == 'running' or target_kern == 'current':
if htype == 'Linux':
target_kern = kern
else:
warn('Host not running Linux, ignoring "running" or "current"')
if target_os != '' and (force_reboot == '1' or target_os != htype
or target_kern != kern):
# write pxe config file
pxe_template = config.TPCONF_script_path + \
'/conf-macaddr_xx\:xx\:xx\:xx\:xx\:xx.ipxe.in'
# if we have a mac address specified use it, otherwise try to automatically
# get the mac address
if env.host_string in host_mac:
mac = host_mac[env.host_string]
else:
mac = get_netmac_cached(env.host_string)
file_name = 'conf-macaddr_' + mac + '.ipxe'
hdd_partition = ''
if target_os == 'CYGWIN':
hdd_partition = '(hd0,0)'
elif target_os == 'Linux':
hdd_partition = '(hd0,1)'
elif target_os == 'FreeBSD':
hdd_partition = '(hd0,2)'
try:
hdd_partition = config.TPCONF_os_partition[target_os]
except AttributeError:
pass
if target_os == 'Linux':
# could remove the if, but might need in future if we specify
# different options for router and hosts
if env.host_string in config.TPCONF_router:
config_str = 'root ' + hdd_partition + '; kernel \/boot\/vmlinuz-' + \
target_kern + ' splash=0 quiet showopts; ' + \
'initrd \/boot\/initrd-' + target_kern
else:
config_str = 'root ' + hdd_partition + '; kernel \/boot\/vmlinuz-' + \
target_kern + ' splash=0 quiet showopts; ' + \
'initrd \/boot\/initrd-' + target_kern
elif target_os == 'CYGWIN':
if env.host_string in config.TPCONF_router:
abort('Router has no Windows')
config_str = 'root ' + hdd_partition + '; chainloader +1'
elif target_os == 'FreeBSD':
config_str = 'root ' + hdd_partition + '; chainloader +1'
elif target_os == 'Darwin':
pass
else:
abort('Unknown OS %s' % target_os)
if force_reboot == '1':
puts('Forced reboot (TPCONF_force_reboot = \'1\')')
puts('Switching %s from OS %s %s to OS %s %s' %
(env.host_string, htype, kern, target_os, target_kern))
if htype != 'Darwin':
# no PXE booting for Macs
local(
'cat %s | sed -e "s/@CONFIG@/%s/" | sed -e "s/@TFTPSERVER@/%s/" > %s'
% (pxe_template, config_str, tftp_server, file_name))
# make backup of current file if not exists yet
full_file_name = config.TPCONF_tftpboot_dir + '/' + file_name
full_file_name_backup = config.TPCONF_tftpboot_dir + \
'/' + file_name + '.bak'
with settings(warn_only=True):
local('mv -f %s %s' % (full_file_name, full_file_name_backup))
local('rm -f %s' % full_file_name)
# XXX should combine the next two into one shell command to make it
# more atomic
local('cp %s %s' % (file_name, config.TPCONF_tftpboot_dir))
local('chmod a+rw %s' % full_file_name)
if file_prefix != '':
file_name2 = local_dir + '/' + file_prefix + '_' + file_name
local('mv %s %s' % (file_name, file_name2))
# reboot
with settings(warn_only=True):
if htype == '?':
# we cannot login to issue shutdown command, so power cycle and hope
# for the best
execute(power_cycle)
elif htype == 'Linux' or htype == 'FreeBSD' or htype == 'Darwin':
run('shutdown -r now', pty=False)
elif htype == 'CYGWIN':
run('shutdown -r -t 0', pty=False)
# give some time to shutdown
puts('Waiting for reboot...')
time.sleep(60)
# wait until up
t = 60
while t <= _boot_timeout:
# ret = local('ping -c 1 %s' % env.host_string) # ping works before
# ssh and can't ping from inside jail
with settings(warn_only=True):
try:
ret = run('echo waiting for OS %s to start' % target_os,
timeout=2,
pty=False)
if ret.return_code == 0:
break
except:
pass
time.sleep(8)
t += 10
if t > _boot_timeout and do_power_cycle == '1':
# host still not up, may be hanging so power cycle it
puts('Power cycling host...')
execute(power_cycle)
puts('Waiting for reboot...')
time.sleep(60)
# wait until up
t = 60
while t <= _boot_timeout:
# ret = local('ping -c 1 %s' % env.host_string) # ping works
# before ssh and can't ping from inside jail
with settings(warn_only=True):
try:
ret = run('echo waiting for OS %s to start' %
target_os,
timeout=2,
pty=False)
if ret.return_code == 0:
break
except:
pass
time.sleep(8)
t += 10
# finally check if host is up again with desired OS
# XXX if the following fails because we can't connect to host Fabric
# will crash with weird error (not super important since we would abort
# anyway but annoying)
htype = execute(get_type, hosts=[env.host_string])[env.host_string]
if target_os == 'Linux':
kern = run('uname -r')
if htype == target_os and kern == target_kern:
puts('Host %s running OS %s %s' %
(env.host_string, target_os, target_kern))
else:
abort('Error switching %s to OS %s %s' %
(env.host_string, target_os, target_kern))
else:
if target_os == '':
target_os = htype
puts('Leaving %s as OS %s %s' %
(env.host_string, target_os, target_kern))
def init_host():
"Perform host initialization"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# record the number of reassembly queue overflows
run('sysctl net.inet.tcp.reass.overflows')
# disable auto-tuning of receive buffer
run('sysctl net.inet.tcp.recvbuf_auto=0')
# disable tso
run('sysctl net.inet.tcp.tso=0')
# send and receiver buffer max (2MB by default on FreeBSD 9.2 anyway)
run('sysctl net.inet.tcp.sendbuf_max=2097152')
run('sysctl net.inet.tcp.recvbuf_max=2097152')
# clear host cache quickly, otherwise successive TCP connections will
# start with ssthresh and cwnd from the end of most recent tcp
# connections to the same host
run('sysctl net.inet.tcp.hostcache.expire=1')
run('sysctl net.inet.tcp.hostcache.prune=5')
run('sysctl net.inet.tcp.hostcache.purge=1')
elif htype == 'Linux':
# disable host cache
run('sysctl net.ipv4.tcp_no_metrics_save=1')
# disable auto-tuning of receive buffer
run('sysctl net.ipv4.tcp_moderate_rcvbuf=0')
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
# send and recv buffer max (set max to 2MB)
run('sysctl net.core.rmem_max=2097152')
run('sysctl net.core.wmem_max=2097152')
# tcp recv buffer max (min 4kB, default 87kB, max 6MB; this is standard
# on kernel 3.7)
run('sysctl net.ipv4.tcp_rmem=\'4096 87380 6291456\'')
# tcp send buffer max (min 4kB, default 64kB, max 4MB; 4x the default
# otherwise standard values from kernel 3.7)
run('sysctl net.ipv4.tcp_wmem=\'4096 65535 4194304\'')
elif htype == 'Darwin':
# disable tso
run('sysctl -w net.inet.tcp.tso=0')
# diable lro (off by default anyway)
run('sysctl -w net.inet.tcp.lro=0')
# disable auto tuning of buffers
run('sysctl -w net.inet.tcp.doautorcvbuf=0')
run('sysctl -w net.inet.tcp.doautosndbuf=0')
# send and receive buffer max (2MB). kern.ipc.maxsockbuf max be the sum
# (but is 4MB by default anyway)
run('sysctl -w kern.ipc.maxsockbuf=4194304')
run('sysctl -w net.inet.tcp.sendspace=2097152')
run('sysctl -w net.inet.tcp.recvspace=2097152')
# set the auto receive/send buffer max to 2MB as well just in case
run('sysctl -w net.inet.tcp.autorcvbufmax=2097152')
run('sysctl -w net.inet.tcp.autosndbufmax=2097152')
elif htype == 'CYGWIN':
# disable ip/tcp/udp offload processing
run('netsh int tcp set global chimney=disabled', pty=False)
run('netsh int ip set global taskoffload=disabled', pty=False)
# enable tcp timestamps
run('netsh int tcp set global timestamps=enabled', pty=False)
# disable tcp window scaling heuristics, enforce user-set auto-tuning
# level
run('netsh int tcp set heuristics disabled', pty=False)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
# stop and restart interface to make the changes
run('netsh int set int "Local Area Connection %s" disabled' %
interface,
pty=False)
run('netsh int set int "Local Area Connection %s" enabled' %
interface,
pty=False)
def init_topology_host():
"Topology setup host"
if env.host_string not in config.TPCONF_hosts:
abort('Host %s not found in TPCONF_hosts' % env.host_string)
# get test ip
try:
test_ip = config.TPCONF_host_internal_ip[env.host_string][0]
except AttributeError:
abort('No entry for host %s in TPCONF_host_internal_ip' %
env.host_string)
# get interface speed setting if defined
link_speed = get_link_speed(env.host_string)
#
# set network interface
#
# find the router we are connected to
conn_router = ''
for r in config.TPCONF_router:
for r_ip in config.TPCONF_host_internal_ip[r]:
if same_subnet(test_ip, r_ip):
conn_router = r
break
if conn_router != '':
break
if conn_router == '':
abort('Cant find router host %s is connected to' % env.host_string)
a = test_ip.split('.')
del a[3]
test_subnet = '.'.join(a)
subnet1 = config.TPCONF_host_internal_ip[conn_router][0]
a = subnet1.split('.')
del a[3]
subnet1 = '.'.join(a)
subnet2 = config.TPCONF_host_internal_ip[conn_router][1]
a = subnet2.split('.')
del a[3]
subnet2 = '.'.join(a)
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'Linux':
# set link speed via ethtool options
ethtool_options = ''
if link_speed == 'auto':
ethtool_options = 'autoneg on duplex full'
else:
if link_speed == '10':
ethtool_options = 'autoneg off speed %s duplex full' % link_speed
else:
ethtool_options = 'autoneg on speed %s duplex full' % link_speed
test_if_config = "BOOTPROTO='static'\n" + \
"BROADCAST=''\n" + \
"ETHTOOL_OPTIONS='" + ethtool_options + "'\n" + \
"IPADDR='" + test_ip + "/24'\n" + \
"MTU=''\n" + \
"NAME='Test IF'\n" + \
"NETWORK=''\n" + \
"REMOTE_IPADDR=''\n" + \
"STARTMODE='auto'\n" + \
"USERCONTROL='no'"
fname = env.host_string + '_test_if_config'
with open(fname, 'w') as f:
f.write(test_if_config)
#interface = 'enp2s0'
interface = 'eth1'
put(fname, '/etc/sysconfig/network/ifcfg-%s' % interface)
os.remove(fname)
if test_subnet == subnet1:
route = subnet2 + '.0 ' + subnet1 + '.1 255.255.255.0 ' + interface
else:
route = subnet1 + '.0 ' + subnet2 + '.1 255.255.255.0 ' + interface
run('echo %s > /etc/sysconfig/network/routes' % route)
#run('/etc/rc.d/network restart')
run('systemctl restart network.service')
elif htype == 'FreeBSD':
ctl_interface = 'em0'
interface = 'em1'
run('cp -a /etc/rc.conf /etc/rc.conf.bak')
run('cat /etc/rc.conf | egrep -v ^static_routes | ' \
'egrep -v route_ | egrep -v ^ifconfig_%s > __tmp' % interface)
run('mv __tmp /etc/rc.conf')
media_settings = ''
if link_speed == '10':
media_settings = ' media 10baseT mediaopt full-duplex'
else:
# setting mediaopt to full-duplex causes the link establishment to fail (with switch set to
# auto-neg). despite the fact that it is listed in 'man em', it did not work on FreeBSD 10.1.
# also we need to set type to auto, if switch uses auto-net, otherwise we get no carrier. also
# if we just run ifconfig with media <type>, it seems we can reliable kill the interface (no carrier).
# however, with netif restart it works.
#media_settings = ' media auto mediaopt full-duplex'
media_settings = ' media auto'
# the most insane quoting ever :). Since fabric will quote command in double quotes and root has
# csh we cannot echo a double quote with /". We must terminate Fabrics double quotes and put the
# echo string in single quotes. We must use raw strings to be able to pass \" to shell. We must
# also tell Fabric to not escape doubel quotes with \.
run('echo "\'%s\' >> /etc/rc.conf"' %
(r'ifconfig_' + interface + r'=\"' + test_ip +
r' netmask 255.255.255.0' + media_settings + r'\"'),
shell_escape=False)
if test_subnet == subnet1:
route1 = r'static_routes=\"internalnet2\"'
route2 = r'route_internalnet2=\"-net ' + subnet2 + r'.0/24 ' + subnet1 + r'.1\"'
else:
route1 = r'static_routes=\"internalnet1\"'
route2 = r'route_internalnet1=\"-net ' + subnet1 + r'.0/24 ' + subnet2 + r'.1\"'
run('echo "\'%s\' >> /etc/rc.conf"' % route1, shell_escape=False)
run('echo "\'%s\' >> /etc/rc.conf"' % route2, shell_escape=False)
# restart network
with settings(warn_only=True):
run('/etc/rc.d/netif restart %s' % interface)
with settings(warn_only=True):
run('/etc/rc.d/routing restart')
# routing restart actually "looses" the default router if DHCP is used, so need to restart our
# control interface as well, which will restore the default route
with settings(warn_only=True):
run('/etc/rc.d/netif restart %s' % ctl_interface)
elif htype == 'CYGWIN':
# remove all testbed routes
run('route delete %s.0 -p' % subnet1)
run('route delete %s.0 -p' % subnet2)
# search for right interface based on start of MAC
interface = ''
interfaces_all = run('ipconfig /all')
for line in interfaces_all.splitlines():
if line.find('Ethernet adapter') > -1:
interface = line.replace('Ethernet adapter ',
'').replace(':', '').rstrip()
if line.find('68-05-CA-') > -1:
break
# interface config
cmd = r'netsh interface ip set address \"%s\" static %s 255.255.255.0' % (
interface, test_ip)
run('"\'%s\'"' % cmd, pty=False, shell_escape=False)
time.sleep(5)
# set static route
# first need to find interface id for routing purposes based on MAC
interface = ''
interfaces_all = run('route print')
for line in interfaces_all.splitlines():
if line.find('68 05 ca') > -1:
interface = line.lstrip()[0:2]
interface = interface.replace('.', '')
break
if test_subnet == subnet1:
route = 'route add ' + subnet2 + '.0 mask 255.255.255.0 ' + subnet1 + '.1 if %s -p' % interface
else:
route = 'route add ' + subnet1 + '.0 mask 255.255.255.0 ' + subnet2 + '.1 if %s -p' % interface
run(route, pty=False)
# there seems to be no command line tools on Windows that can set link speed, cause link speed setting
# is implemented in nic driver and can be configured via driver GUI. possible command line solution is
# to manipulate the registry value that store the link speed value for the testbed nic. however, the
# implementation would be specific to the supported nic, as the registry entries are nic specific.
# by default autonegotiation is enabled though, so the switch will force the host to 100, 100,
# show interface speeds
run('wmic NIC where NetEnabled=true get Name, Speed')
elif htype == 'Darwin':
# remove all testbed routes
run('route -n delete %s.0/24' % subnet1)
run('route -n delete %s.0/24' % subnet2)
# setup interface
run('networksetup -setmanual "Ethernet" %s 255.255.255.0' % test_ip)
# set static route
if test_subnet == subnet1:
par1 = subnet2
par2 = subnet1
else:
par1 = subnet1
par2 = subnet2
interface = 'en0'
run('route -n add %s.0/24 -interface %s' % (par2, interface))
run('cat /Library/StartupItems/AddRoutes/AddRoutes | sed "s/route add .*$/route add %s.0\/24 %s.1/" > __tmp' \
' && mv __tmp /Library/StartupItems/AddRoutes/AddRoutes' %
(par1, par2))
run('chmod a+x /Library/StartupItems/AddRoutes/AddRoutes')
run('/Library/StartupItems/AddRoutes/AddRoutes start')
# XXX for Mac the link speed setting is not permanent for now. need to add new script under StartupItems
# to make this permanent (plus accompanying .plist file), similar to the AddRoutes approach
if link_speed == '10':
run('ifconfig %s media 10baseT/UTP mediaopt full-duplex' %
interface)
elif link_speed == '100':
run('ifconfig %s media 100baseTX mediaopt full-duplex' % interface)
else:
run('ifconfig %s media 1000baseT mediaopt full-duplex' % interface)
def init_cc_algo(algo='default', *args, **kwargs):
"Initialize TCP congestion control algorithm"
if algo[0:4] == 'host':
arr = algo.split('t')
if len(arr) == 2 and arr[1].isdigit():
num = int(arr[1])
else:
abort('If you specify host<N>, the <N> must be an integer number')
algo_list = config.TPCONF_host_TCP_algos.get(env.host_string, [])
if len(algo_list) == 0:
abort('No TCP congestion control algos defined for host %s' %
env.host_string)
if num > len(algo_list) - 1:
warn(
'No TCP congestion control algo specified for <N> = %d, ' \
'setting <N> = 0' % num)
num = 0
algo = algo_list[num]
puts('Selecting TCP congestion control algorithm: %s' % algo)
if algo != 'default' and algo != 'reno' and algo != 'newreno' and \
algo != 'cubic' and algo != 'cdg' and algo != 'htcp' and \
algo != 'compound' and algo != 'hd' and algo != 'vegas' and \
algo != 'dctcp' and algo != 'lgc' and algo != 'abc':
abort(
'Available TCP algorithms: ' +
'default, (new)reno, cubic, cdg, hd, htcp, compound, vegas, dctcp, lgc, abc'
)
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
if algo == 'newreno' or algo == 'default':
algo = 'newreno'
elif algo == 'cubic':
with settings(warn_only=True):
ret = run('kldstat | grep cc_cubic')
if ret.return_code != 0:
run('kldload cc_cubic')
elif algo == 'hd':
with settings(warn_only=True):
ret = run('kldstat | grep cc_hd')
if ret.return_code != 0:
run('kldload cc_hd')
elif algo == 'htcp':
with settings(warn_only=True):
ret = run('kldstat | grep cc_htcp')
if ret.return_code != 0:
run('kldload cc_htcp')
elif algo == 'cdg':
with settings(warn_only=True):
ret = run('kldstat | grep cc_cdg')
if ret.return_code != 0:
# cdg is only available by default since FreeBSD 9.2
run('kldload cc_cdg')
elif algo == 'vegas':
with settings(warn_only=True):
ret = run('kldstat | grep cc_vegas')
if ret.return_code != 0:
run('kldload cc_vegas')
else:
abort("Congestion algorithm '%s' not supported by FreeBSD" % algo)
run('sysctl net.inet.tcp.cc.algorithm=%s' % algo)
elif htype == 'Linux':
if algo == 'newreno' or algo == 'reno':
# should be there by default
algo = 'reno'
elif algo == 'cubic' or algo == 'default':
# should also be there by default
algo = 'cubic'
elif algo == 'htcp':
run('modprobe tcp_htcp')
elif algo == 'vegas':
run('modprobe tcp_vegas')
elif algo == 'dctcp' or algo == 'lgc' or algo == 'abc':
puts('TCP congestion control algorithm %s is supported only in ' \
'Linux' % algo)
else:
abort("Congestion algorithm '%s' not supported by Linux" % algo)
run('sysctl net.ipv4.tcp_congestion_control=%s' % algo)
elif htype == 'Darwin':
if algo == 'newreno' or algo == 'default':
algo = 'newreno'
else:
abort("Congestion algorithm '%s' not supported by MacOS" % algo)
elif htype == 'CYGWIN':
if algo == 'newreno' or algo == 'default':
run('netsh int tcp set global congestionprovider=none', pty=False)
elif algo == 'compound':
run('netsh int tcp set global congestionprovider=ctcp', pty=False)
else:
abort("Congestion algorithm '%s' not supported by Windows" % algo)
else:
abort("Can't set TCP congestion control algo for OS '%s'" % htype)
# now set the cc algorithm parameters
execute(init_cc_algo_params, algo=algo, *args, **kwargs)
def init_topology_host():
"Topology setup host"
if env.host_string not in config.TPCONF_hosts:
abort('Host %s not found in TPCONF_hosts' % env.host_string)
# get test ip
try:
test_ip = config.TPCONF_host_internal_ip[env.host_string][0]
except AttributeError:
abort('No entry for host %s in TPCONF_host_internal_ip' %
env.host_string)
# get interface speed setting if defined
link_speed = get_link_speed(env.host_string)
#
# set network interface
#
a = test_ip.split('.')
del a[3]
test_subnet = '.'.join(a)
subnet1 = config.TPCONF_host_internal_ip[config.TPCONF_router[0]][0]
a = subnet1.split('.')
del a[3]
subnet1 = '.'.join(a)
subnet2 = config.TPCONF_host_internal_ip[config.TPCONF_router[0]][1]
a = subnet2.split('.')
del a[3]
subnet2 = '.'.join(a)
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'Linux':
# set link speed via ethtool options
ethtool_options = ''
if link_speed == 'auto':
ethtool_options = 'autoneg on duplex full'
else:
if link_speed == '10':
ethtool_options = 'autoneg off speed %s duplex full' % link_speed
else:
ethtool_options = 'autoneg on speed %s duplex full' % link_speed
test_if_config = "BOOTPROTO='static'\n" + \
"BROADCAST=''\n" + \
"ETHTOOL_OPTIONS='" + ethtool_options + "'\n" + \
"IPADDR='" + test_ip + "/24'\n" + \
"MTU=''\n" + \
"NAME='Test IF'\n" + \
"NETWORK=''\n" + \
"REMOTE_IPADDR=''\n" + \
"STARTMODE='auto'\n" + \
"USERCONTROL='no'"
fname = env.host_string + '_test_if_config'
with open(fname, 'w') as f:
f.write(test_if_config)
#interface = 'enp2s0'
interface = 'eth1'
put(fname, '/etc/sysconfig/network/ifcfg-%s' % interface)
os.remove(fname)
if test_subnet == subnet1:
route = subnet2 + '.0 ' + subnet1 + '.1 255.255.255.0 ' + interface
else:
route = subnet1 + '.0 ' + subnet2 + '.1 255.255.255.0 ' + interface
run('echo %s > /etc/sysconfig/network/routes' % route)
#run('/etc/rc.d/network restart')
run('systemctl restart network.service')
elif htype == 'FreeBSD':
interface = 'em1'
run('cp -a /etc/rc.conf /etc/rc.conf.bak')
run('cat /etc/rc.conf | egrep -v ^static_routes | ' \
'egrep -v route_ | egrep -v ^ifconfig_%s > __tmp' % interface)
run('mv __tmp /etc/rc.conf')
media_settings = ''
if link_speed == '10':
media_settings = ' media 10baseT mediaopt full-duplex'
else:
# setting mediaopt to full-duplex causes the link establishment to fail (with switch set to
# auto-neg). despite the fact that it is listed in 'man em', it did not work on FreeBSD 10.1.
# also we need to set type to auto, if switch uses auto-net, otherwise we get no carrier. also
# if we just run ifconfig with media <type>, it seems we can reliable kill the interface (no carrier).
# however, with netif restart it works.
#media_settings = ' media auto mediaopt full-duplex'
media_settings = ' media auto'
# the most insane quoting ever :). Since fabric will quote command in double quotes and root has
# csh we cannot echo a double quote with /". We must terminate Fabrics double quotes and put the
# echo string in single quotes. We must use raw strings to be able to pass \" to shell. We must
# also tell Fabric to not escape doubel quotes with \.
run('echo "\'%s\' >> /etc/rc.conf"' %
(r'ifconfig_' + interface + r'=\"' + test_ip + r' netmask 255.255.255.0' + media_settings + r'\"'),
shell_escape=False)
if test_subnet == subnet1 :
route1 = r'static_routes=\"internalnet2\"'
route2 = r'route_internalnet2=\"-net ' + subnet2 + r'.0/24 ' + subnet1 + r'.1\"'
else:
route1 = r'static_routes=\"internalnet1\"'
route2 = r'route_internalnet1=\"-net ' + subnet1 + r'.0/24 ' + subnet2 + r'.1\"'
run('echo "\'%s\' >> /etc/rc.conf"' % route1, shell_escape=False)
run('echo "\'%s\' >> /etc/rc.conf"' % route2, shell_escape=False)
# restart network
run('/etc/rc.d/netif restart')
time.sleep(1)
with settings(warn_only=True):
run('/etc/rc.d/routing restart')
elif htype == 'CYGWIN':
# remove all testbed routes
run('route delete %s.0 -p' % subnet1)
run('route delete %s.0 -p' % subnet2)
# search for right interface based on start of MAC
interface = ''
interfaces_all = run('ipconfig /all')
for line in interfaces_all.splitlines():
if line.find('Ethernet adapter') > -1:
interface = line.replace('Ethernet adapter ', '').replace(':','').rstrip()
if line.find('68-05-CA-') > -1 :
break
# interface config
cmd = r'netsh interface ip set address \"%s\" static %s 255.255.255.0' % (interface, test_ip)
run('"\'%s\'"' % cmd, pty=False, shell_escape=False)
time.sleep(5)
# set static route
# first need to find interface id for routing purposes based on MAC
interface = ''
interfaces_all = run('route print')
for line in interfaces_all.splitlines():
if line.find('68 05 ca') > -1 :
interface = line.lstrip()[0:2]
interface = interface.replace('.', '')
break
if test_subnet == subnet1 :
route = 'route add ' + subnet2 + '.0 mask 255.255.255.0 ' + subnet1 + '.1 if %s -p' % interface
else:
route = 'route add ' + subnet1 + '.0 mask 255.255.255.0 ' + subnet2 + '.1 if %s -p' % interface
run(route, pty=False)
# there seems to be no command line tools on Windows that can set link speed, cause link speed setting
# is implemented in nic driver and can be configured via driver GUI. possible command line solution is
# to manipulate the registry value that store the link speed value for the testbed nic. however, the
# implementation would be specific to the supported nic, as the registry entries are nic specific.
# by default autonegotiation is enabled though, so the switch will force the host to 100, 100,
# show interface speeds
run('wmic NIC where NetEnabled=true get Name, Speed')
elif htype == 'Darwin':
# remove all testbed routes
run('route -n delete %s.0/24' % subnet1)
run('route -n delete %s.0/24' % subnet2)
# setup interface
run('networksetup -setmanual "Ethernet" %s 255.255.255.0' % test_ip)
# set static route
if test_subnet == subnet1 :
par1 = subnet2
par2 = subnet1
else :
par1 = subnet1
par2 = subnet2
interface = 'en0'
run('route -n add %s.0/24 -interface %s' % (par2, interface))
run('cat /Library/StartupItems/AddRoutes/AddRoutes | sed "s/route add .*$/route add %s.0\/24 %s.1/" > __tmp' \
' && mv __tmp /Library/StartupItems/AddRoutes/AddRoutes' %
(par1, par2))
run('chmod a+x /Library/StartupItems/AddRoutes/AddRoutes')
run('/Library/StartupItems/AddRoutes/AddRoutes start')
# XXX for Mac the link speed setting is not permanent for now. need to add new script under StartupItems
# to make this permanent (plus accompanying .plist file), similar to the AddRoutes approach
if link_speed == '10':
run('ifconfig %s media 10baseT/UTP mediaopt full-duplex' % interface)
elif link_speed == '100':
run('ifconfig %s media 100baseTX mediaopt full-duplex' % interface)
else:
run('ifconfig %s media 1000baseT mediaopt full-duplex' % interface)
def log_sysdata(file_prefix='', remote_dir='', local_dir='.'):
"Log various information for each system"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_uname.log"
run('uname -a > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_netstat.log"
run('netstat -nr > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_sysctl.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('sysctl -a > %s' % file_name)
else:
run('echo "netsh int show int" > %s' % file_name, pty=False)
run('netsh int show int >> %s' % file_name, pty=False)
run('echo "netsh int tcp show global" >> %s' % file_name, pty=False)
run('netsh int tcp show global >> %s' % file_name, pty=False)
run('echo "netsh int tcp show heuristics" >> %s' %
file_name, pty=False)
run('netsh int tcp show heuristics >> %s' % file_name, pty=False)
run('echo "netsh int tcp show security" >> %s' % file_name, pty=False)
run('netsh int tcp show security >> %s' % file_name, pty=False)
run('echo "netsh int tcp show chimneystats" >> %s' %
file_name, pty=False)
run('netsh int tcp show chimneystats >> %s' % file_name, pty=False)
run('echo "netsh int ip show offload" >> %s' % file_name, pty=False)
run('netsh int ip show offload >> %s' % file_name, pty=False)
run('echo "netsh int ip show global" >> %s' % file_name, pty=False)
run('netsh int ip show global >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ifconfig.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ifconfig -a > %s' % file_name)
else:
run('ipconfig > %s' % file_name, pty=False)
# log interface speeds
run('echo "wmic NIC where NetEnabled=true get Name, Speed" >> %s' % file_name, pty=False)
run('wmic NIC where NetEnabled=true get Name, Speed >> %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_procs.log"
if htype == 'FreeBSD' or htype == 'Linux':
run('ps -axu > %s' % file_name)
elif htype == 'Darwin':
run('ps -axu root > %s' % file_name)
else:
run('ps -alW > %s' % file_name, pty=False)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ntp.log"
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
run('ntpq -4p > %s' % file_name)
else:
with settings(warn_only=True):
# if we have ntp installed then use ntpq, otherwise use w32tm
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code == 0:
run('"/cygdrive/c/Program Files (x86)/NTP/bin/ntpq" -4p > %s' %
file_name, pty=False)
else:
run('w32tm /query /status > %s' % file_name, pty=False)
getfile(file_name, local_dir)
# log tcp module parameters (Linux only)
if htype == 'Linux':
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_tcpmod.log"
run("find /sys/module/tcp* -type f -exec grep -sH '' '{}' \; | "
"grep -v Binary > %s" % file_name)
getfile(file_name, local_dir)
file_name = remote_dir + file_prefix + "_" + \
env.host_string.replace(":", "_") + "_ethtool.log"
run('touch %s' % file_name)
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('ethtool -k %s >> %s' % (interface, file_name))
getfile(file_name, local_dir)
def check_host():
"Check that needed tools are installed on hosts"
# get type of current host
htype = get_type_cached(env.host_string)
# run checks
if env.host_string in config.TPCONF_router:
if htype == 'FreeBSD':
run('which ipfw')
if htype == "Linux":
run('which tc')
run('which iptables')
# XXX check that kernel tick rate is high (>= 1000)
else:
if htype == 'FreeBSD':
run('which md5')
run('which tcpdump')
elif htype == 'Darwin':
run('which md5')
run('which tcpdump')
run('which dsiftr-osx-teacup.d')
elif htype == 'Linux':
run('which ethtool')
run('which md5sum')
run('which tcpdump')
#run('which web10g-listconns')
#run('which web10g-readvars')
run('which web10g-logger')
elif htype == 'CYGWIN':
run('which WinDump', pty=False)
run('which win-estats-logger', pty=False)
# if we don't have proper ntp installed then
# start time service if not started and force resync
with settings(warn_only=True):
ret = run('ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code != 0:
run('net start w32time', pty=False)
run('w32tm /resync', pty=False)
# try to enable any test network interfaces that are (accidently)
# disabled after reboot
with settings(warn_only=True):
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('netsh int set int "Local Area Connection %s" enabled' %
interface, pty=False)
run('which killall', pty=False)
run('which pkill', pty=False)
run('which ps', pty=False)
run('which gzip', pty=False)
run('which dd', pty=False)
# check for traffic sender/receiver tools
run('which iperf', pty=False)
run('which ping', pty=False)
run('which httperf', pty=False)
run('which lighttpd', pty=False)
run('which nttcp', pty=False)
put(config.TPCONF_script_path + '/runbg_wrapper.sh', '/usr/bin')
run('chmod a+x /usr/bin/runbg_wrapper.sh', pty=False)
run('which runbg_wrapper.sh', pty=False)
put(config.TPCONF_script_path + '/kill_iperf.sh', '/usr/bin')
run('chmod a+x /usr/bin/kill_iperf.sh', pty=False)
run('which kill_iperf.sh', pty=False)
put(config.TPCONF_script_path + '/pktgen.sh', '/usr/bin')
run('chmod a+x /usr/bin/pktgen.sh', pty=False)
run('which pktgen.sh', pty=False)
def start_tcpdump(
file_prefix='', remote_dir='', local_dir='.', snap_len='80',
tcpdump_filter='', internal_int='1'):
"Start tcpdump instance on host"
# get host type
htype = get_type_cached(env.host_string)
if env.host_string in config.TPCONF_router:
interfaces = get_netint_cached(env.host_string, int_no=-1,
internal_int=internal_int)
else:
if htype == 'CYGWIN':
interfaces = get_netint_windump_cached(env.host_string,
int_no=0,
internal_int=internal_int)
else:
interfaces = get_netint_cached(env.host_string,
int_no=0,
internal_int=internal_int)
if len(interfaces) < 1:
abort('Internal interface not specified')
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
for interface in interfaces:
if env.host_string in config.TPCONF_router:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + \
'_' + interface + '_router.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
else:
if internal_int == '1':
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '.dmp'
else:
file_name = remote_dir + file_prefix + '_' + \
env.host_string.replace(':', '_') + '_ctl.dmp'
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
tcpdump_cmd = 'tcpdump -n -s %s -i %s -w %s \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
else:
# CYGWIN
tcpdump_cmd = 'WinDump -n -s %s -i %s -w ' \
'"$(cygpath -aw "%s")" \'%s\'' % (
snap_len, interface, file_name, tcpdump_filter)
pid = runbg(tcpdump_cmd)
name = 'tcpdump-' + interface
#bgproc.register_proc(env.host_string, name, '0', pid, file_name)
bgproc.register_proc_later(
env.host_string,
local_dir,
name,
'0',
pid,
file_name)
def get_netint(int_no=0, windump='0', internal_int='1'):
"Get network interface name"
# need to convert if we run task from command line
int_no = int(int_no)
# check int_no paramter
if int_no < 0:
int_no = 0
if int_no >= len(config.TPCONF_host_internal_ip[env.host_string]):
int_no = len(config.TPCONF_host_internal_ip[env.host_string]) - 1
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD' or htype == 'Linux' or htype == 'Darwin':
# get ip and set last octet to 0
if internal_int == '1':
iip = config.TPCONF_host_internal_ip[env.host_string][int_no]
else:
iip = socket.gethostbyname(env.host_string.split(':')[0])
a = iip.split('.')
del a[3]
iip = '.'.join(a)
int_name = ''
field_idx = -1
lines = run('netstat -nr', shell=False)
for line in lines.split('\n'):
if line != '':
fields = line.split()
if len(fields) > 0 and fields[0] == 'Destination' and \
int_name == '' :
for i in range(len(fields)) :
if fields[i] == 'Netif' :
field_idx = i
if len(fields) > 0 and (fields[0].split('/')[0] == iip + '.0' or
fields[0].split('/')[0] == iip) :
int_name = fields[field_idx]
#puts('Interface: %s' % int_name)
return int_name
elif htype == "CYGWIN":
# on windows we have two numbers
# windows numbering of interfaces
# numbering used by windump
if windump == '0':
# get interface IPs and numbers
output = run(
'ipconfig | egrep "Local Area|IPv4" | grep -v "Tunnel"',
pty=False)
lines = output.split("\n")
for i in range(0, len(lines), 2):
int_num = lines[i].replace(":", "").split(" ")[-1]
if int_num == "": # XXX not sure what we are doing here
int_num = "1"
int_ip = lines[i + 1].split(":")[1].strip()
if internal_int == '1' and int_ip == config.TPCONF_host_internal_ip[
env.host_string][int_no] or \
internal_int == '0' and int_ip == socket.gethostbyname(
env.host_string.split(':')[0]):
puts('Interface: %s' % int_num)
return int_num
else:
# get list of interface numbers and interface IDs
output = run(
'winDUmp -D | sed "s/\([0-9]\)\.[^{]*{\([^}]*\).*/\\1 \\2/"',
pty=False)
# get list of interface macs and interface IDs
output2 = run(
'getmac | '
'grep "^[0-9]" | sed "s/^\([0-9A-Fa-f-]*\)[^{]*{\([^}]*\).*/\\1 \\2/"',
pty=False)
# get mac of the internal/external interface
mac = execute(
get_netmac,
internal_int=internal_int,
hosts=[env.host_string]).values()[0]
# find interface ID
int_id = ''
lines = output2.split("\n")
for line in lines:
_int_mac, _int_id = line.split(' ')
# get mac print with '-' instead of ':'
_int_mac = _int_mac.replace('-', ':').lower()
if _int_mac == mac:
int_id = _int_id
break
# get interface number (could use ID but it's a bit long)
lines = output.split("\n")
for line in lines:
_int_num, _int_id = line.split(' ')
if _int_id == int_id:
puts('Interface: %s' % _int_num)
return _int_num
else:
abort('Cannot determine network interface for OS %s' % htype)
def start_tcp_logger(file_prefix='', remote_dir='', local_dir='.'):
"Start TCP information logger (e.g. siftr on FreeBSD)"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# load kernel module
with settings(warn_only=True):
run('kldunload siftr')
# run('kldunload h_ertt') # do not attempt to unload, can cause
# kernel panic
# if h_ertt is loaded siftr outputs unsmoothed rtt as well???
# h_ertt appears to have a bug that makes it impossible to unload,
# so don't try to unload. test if present and only try to load if
# not present
with settings(warn_only=True):
ret = run('kldstat | grep h_ertt')
if ret.return_code != 0:
run('kldload h_ertt')
run('kldload siftr')
# we need an absolute path
if remote_dir == '':
remote_dir = '/tmp/'
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_siftr.log"
run('sysctl net.inet.siftr.logfile=%s' % logfile)
run('sysctl net.inet.siftr.ppl=1')
run('sysctl net.inet.siftr.genhashes=1')
run('sysctl net.inet.siftr.enabled=1')
#bgproc.register_proc(env.host_string, 'tcplogger', '00', '0', logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
'0',
logfile)
elif htype == 'Linux':
try:
linux_tcp_logger = config.TPCONF_linux_tcp_logger
except AttributeError:
linux_tcp_logger = 'web10g'
if linux_tcp_logger == 'ttprobe' or linux_tcp_logger == 'both':
ttprobe_output_mode = config.TPCONF_ttprobe_output_mode
ttprobe_direction = config.TPCONF_ttprobe_direction
if ttprobe_output_mode == 'binary':
ttprobe_output_mode = '1'
else:
ttprobe_output_mode = '0'
if ttprobe_direction == 'o':
ttprobe_direction = '0'
elif ttprobe_direction == 'i':
ttprobe_direction = '1'
else:
ttprobe_direction = '2'
# ttprobe - start_tcp_logger code
with settings(warn_only=True):
run('rmmod ttprobe')
# start new_tcp_probe kernel module
run('modprobe ttprobe port=0 full=1 bufsize=16384 omode=%s direction=%s' %
(ttprobe_output_mode, ttprobe_direction))
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_ttprobe.log"
host = env.host_string.split(':')[0]
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg('cat /proc/net/ttprobe', out_file=logfile)
#pid = runbg('cat /proc/kmsg', out_file=logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcploggerprobe',
'00',
pid,
logfile)
if linux_tcp_logger == 'web10g' or linux_tcp_logger == 'both':
# set default sample interval to roughly 10ms
sample_interval = 0.01
try:
sample_interval = float(config.TPCONF_web10g_poll_interval) / 1000.0
except AttributeError:
pass
# with new web10g we use the provided web10g-logger which needs interval
# in milliseconds, so convert here
with settings(warn_only=True):
out = run('web10g-logger -h | grep "poll interval given in seconds"')
if out == '':
sample_interval *= 1000
# turn off logging and remove kernel module
with settings(warn_only=True):
run('echo 0 > /proc/sys/net/ipv4/tcp_estats') # version 2.0.8+
run('rmmod tcp_estats_nl')
# start kernel module
run('modprobe tcp_estats_nl')
# for new web10g (version 2.08 and higher) we need to turn it on, note
# that /proc/sys/net/ipv4/tcp_estats is not just 0/1 but we need to
# specify a bit mask to turn on/off different features.
# we turn on all features to be compatible to previous version
# (see /usr/src/<linux>/include/net/tcp_estats.h for bit flags,
# the TCP_ESTATS_TABLEMASK_* constants)
with settings(warn_only=True):
run('echo 95 > /proc/sys/net/ipv4/tcp_estats')
# OLD:
# check userland code is there
#run('which web10g-listconns')
#run('which web10g-readvars')
# make sure script is there
#put(config.TPCONF_script_path + '/web10g_logger.sh', '/usr/bin')
#run('chmod a+x /usr/bin/web10g_logger.sh')
# check userland code is there
run('which web10g-logger')
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_web10g.log"
host = env.host_string.split(':')[0]
# OLD:
#pid = runbg('web10g_logger.sh %f %s %s' % (sample_interval, logfile, host))
# run with high priority
#run('renice -n -20 -p %s' % pid)
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'web10g-logger -e %s -i %f' %
(host_ip, sample_interval), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
elif htype == 'Darwin':
# start dtrace based logging tool
run('which dsiftr-osx-teacup.d')
# since the Mac tool produces the same output as SIFTR
# we give it the same _siftr.log extension
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_siftr.log"
host = env.host_string.split(':')[0]
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'dsiftr-osx-teacup.d "%s"' %
(host_ip), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
elif htype == 'CYGWIN':
# set sample interval to roughly 10ms
sample_interval = 0.01
try:
sample_interval = float(config.TPCONF_web10g_poll_interval) / 1000.0
except AttributeError:
pass
# check userland code is there
run('which win-estats-logger')
# since the windows tool produces the same output as the web10g logger
# we give it the same _web10g.log extension
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_web10g.log"
host = env.host_string.split(':')[0]
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'win-estats-logger -q -e %s -i %f' %
(host_ip, sample_interval), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
else:
warn("No TCP logger available on OS '%s'" % htype)
def kill_old_processes(*args, **kwargs):
"Kill old logging or traffic generation processes still running"
# get type of current host
htype = get_type_cached(env.host_string)
try:
custom_loggers = config.TPCONF_custom_loggers
with settings(warn_only=True):
for c, v in config.TPCONF_custom_loggers:
v = re.sub("(V_[a-zA-Z0-9_-]*)", "_param('\\1', kwargs)", v)
logger_conf = re.search(r',(.*)|$', v).group(1)
_nargs, _kwargs = eval('_args(%s)' % logger_conf)
logger_name = _kwargs.get('name', '')
run('killall %s' % logger_name, pty=False)
except AttributeError:
pass
with settings(warn_only=True):
if htype == 'FreeBSD':
run('killall tcpdump', pty=False)
elif htype == 'Linux':
run('killall tcpdump', pty=False)
# updated for ttprobe support
# Kill ttprobe user space process and unload kernel module
# run('pkill -f "cat /proc/net/ttprobe"')
with settings(warn_only=True):
run('rmmod ttprobe')
#run('killall web10g_logger.sh')
run('killall web10g-logger')
elif htype == 'Darwin':
run('killall tcpdump', pty=False)
run('killall dsiftr-osx-teacup.d', pty=False)
elif htype == 'CYGWIN':
run('killall WinDump', pty=False)
run('killall win-estats-logger', pty=False)
if htype == 'CYGWIN':
# on new cygwin does stop anymore on sigterm
run('killall -9 iperf', pty=False)
else:
run('killall iperf', pty=False)
for t, c, v in config.TPCONF_traffic_gens:
v = re.sub("(V_[a-zA-Z0-9_-]*)", "_param('\\1', kwargs)", v)
traffic_conf = re.search(r',(.*)|$', v).group(1)
_nargs, _kwargs = eval('_args(%s)' % traffic_conf)
traffic_name = _kwargs.get('name', '')
if traffic_name != '':
run('killall %s' % traffic_name, pty=False)
#str_list=v.split(',')
#run('killall %s' % str_list[0], pty=False)
run('killall ping', pty=False)
run('killall httperf', pty=False)
run('killall lighttpd', pty=False)
# delete old lighttp pid files (XXX would be better to delete after
# experiment)
run('rm -f /var/run/*lighttpd.pid', pty=False)
run('killall runbg_wrapper.sh', pty=False)
run('killall nttcp')
run('killall pktgen.sh ; killall python')
# remove old log stuff in /tmp
run('rm -f /tmp/*.log', pty=False)
def get_netmac(internal_int='0'):
"Get MAC address for external/ctrl network interface"
# not so easy to get local mac address for all different OS
# use following approach:
# get all non-router macs via the router using arp -a
# get router mac using ifconfig
# if host is a vm which we access via localhost then use ifconfig not arp
# method
# XXX windows: getmac, ipconfig /all
host_string = env.host_string
# if we have a port then strip off port
if host_string.find(':') > -1:
host_string = host_string.split(':')[0]
if host_string == 'localhost':
host_string = '127.0.0.1'
if host_string in config.TPCONF_router or host_string == '127.0.0.1':
# get MAC of router
htype = get_type_cached(env.host_string)
# complicated awk code to get pairs of ip and mac addresses from
# ifconfig
if htype == 'FreeBSD':
macips = run(
'ifconfig | awk \'/ether / { printf("%s ", $0); next } 1\' | ' +
'grep ether | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $2, $4) }\'',
shell=False)
elif htype == 'Linux':
macips = run(
'ifconfig | awk \'/HWaddr / { printf("%s ", $0); next } 1\' | ' +
'grep HWaddr | grep "inet " | ' +
'awk \'{ printf("%s %s\\n", $5, $7) }\' | sed -e "s/addr://"')
else:
abort("Can't determine MAC address for OS %s" % htype)
ip_mac_map = {}
for line in macips.split('\n'):
a = line.split(' ')
ip_mac_map.update({a[1].strip(): a[0].strip()})
# print(ip_mac_map)
# check if it looks like a name
if not re.match('[0-9.]+', host_string):
# try dns and assume we get an answer
ip = socket.gethostbyname(host_string)
else:
ip = host_string
if ip != '127.0.0.1':
mac = ip_mac_map.get(ip)
else:
# guess it's the first NIC
# XXX should return MAC based on router IP, not simply the first
mac = ip_mac_map.get(ip_mac_map.keys()[0])
else:
# get MAC of non-router
if internal_int == '0':
host_string = env.host_string
else:
host_string = config.TPCONF_host_internal_ip.get(
env.host_string,
'')[0]
mac = execute(_get_netmac, host_string.split(':')[0],
hosts = [ config.TPCONF_router[0] ])[config.TPCONF_router[0]]
return mac.lower()
def start_tcp_logger(file_prefix='', remote_dir='', local_dir='.'):
"Start TCP information logger (e.g. siftr on FreeBSD)"
if remote_dir != '' and remote_dir[-1] != '/':
remote_dir += '/'
# get host type
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# load kernel module
with settings(warn_only=True):
run('kldunload siftr')
# run('kldunload h_ertt') # do not attempt to unload, can cause
# kernel panic
# if h_ertt is loaded siftr outputs unsmoothed rtt as well???
# h_ertt appears to have a bug that makes it impossible to unload,
# so don't try to unload. test if present and only try to load if
# not present
with settings(warn_only=True):
ret = run('kldstat | grep h_ertt')
if ret.return_code != 0:
run('kldload h_ertt')
run('kldload siftr')
# we need an absolute path
if remote_dir == '':
remote_dir = '/tmp/'
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_siftr.log"
run('sysctl net.inet.siftr.logfile=%s' % logfile)
run('sysctl net.inet.siftr.ppl=1')
run('sysctl net.inet.siftr.genhashes=1')
run('sysctl net.inet.siftr.enabled=1')
#bgproc.register_proc(env.host_string, 'tcplogger', '00', '0', logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
'0',
logfile)
elif htype == 'Linux':
# set default sample interval to roughly 10ms
sample_interval = 0.01
try:
sample_interval = float(config.TPCONF_web10g_poll_interval) / 1000.0
except AttributeError:
pass
# with new web10g we use the provided web10g-logger which needs interval
# in milliseconds, so convert here
with settings(warn_only=True):
out = run('web10g-logger -h | grep "poll interval given in seconds"')
if out == '':
sample_interval *= 1000
# turn off logging and remove kernel module
with settings(warn_only=True):
run('echo 0 > /proc/sys/net/ipv4/tcp_estats') # version 2.0.8+
run('rmmod tcp_estats_nl')
# start kernel module
run('modprobe tcp_estats_nl')
# for new web10g (version 2.08 and higher) we need to turn it on, note
# that /proc/sys/net/ipv4/tcp_estats is not just 0/1 but we need to
# specify a bit mask to turn on/off different features.
# we turn on all features to be compatible to previous version
# (see /usr/src/<linux>/include/net/tcp_estats.h for bit flags,
# the TCP_ESTATS_TABLEMASK_* constants)
with settings(warn_only=True):
run('echo 95 > /proc/sys/net/ipv4/tcp_estats')
# OLD:
# check userland code is there
#run('which web10g-listconns')
#run('which web10g-readvars')
# make sure script is there
#put(config.TPCONF_script_path + '/web10g_logger.sh', '/usr/bin')
#run('chmod a+x /usr/bin/web10g_logger.sh')
# check userland code is there
run('which web10g-logger')
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_web10g.log"
host = env.host_string.split(':')[0]
# OLD:
#pid = runbg('web10g_logger.sh %f %s %s' % (sample_interval, logfile, host))
# run with high priority
#run('renice -n -20 -p %s' % pid)
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'web10g-logger -e %s -i %f' %
(host_ip, sample_interval), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
elif htype == 'Darwin':
# start dtrace based logging tool
run('which dsiftr-osx-teacup.d')
# since the Mac tool produces the same output as SIFTR
# we give it the same _siftr.log extension
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_siftr.log"
host = env.host_string.split(':')[0]
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'dsiftr-osx-teacup.d "%s"' %
(host_ip), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
elif htype == 'CYGWIN':
# set sample interval to roughly 10ms
sample_interval = 0.01
try:
sample_interval = float(config.TPCONF_web10g_poll_interval) / 1000.0
except AttributeError:
pass
# check userland code is there
run('which win-estats-logger')
# since the windows tool produces the same output as the web10g logger
# we give it the same _web10g.log extension
logfile = remote_dir + file_prefix + '_' + \
env.host_string.replace(":", "_") + "_web10g.log"
host = env.host_string.split(':')[0]
# resolve to IP to make sure we filter on right address
host_ip = socket.gethostbyname(host)
if host_ip == '':
host_ip = host
pid = runbg(
'win-estats-logger -q -e %s -i %f' %
(host_ip, sample_interval), out_file=logfile)
#bgproc.register_proc(env.host_string, 'tcplogger', '00', pid, logfile)
bgproc.register_proc_later(
env.host_string,
local_dir,
'tcplogger',
'00',
pid,
logfile)
else:
warn("No TCP logger available on OS '%s'" % htype)
def init_host():
"Perform host initialization"
# get type of current host
htype = get_type_cached(env.host_string)
if htype == 'FreeBSD':
# record the number of reassembly queue overflows
run('sysctl net.inet.tcp.reass.overflows')
# disable tso
run('sysctl net.inet.tcp.tso=0')
# send and receiver buffer max (2MB by default on FreeBSD 9.2 anyway)
run('sysctl net.inet.tcp.sendbuf_max=2097152')
run('sysctl net.inet.tcp.recvbuf_max=2097152')
# clear host cache quickly, otherwise successive TCP connections will
# start with ssthresh and cwnd from the end of most recent tcp
# connections to the same host
run('sysctl net.inet.tcp.hostcache.expire=1')
run('sysctl net.inet.tcp.hostcache.prune=5')
run('sysctl net.inet.tcp.hostcache.purge=1')
elif htype == 'Linux':
# disable host cache
run('sysctl net.ipv4.tcp_no_metrics_save=1')
# disable auto-tuning of receive buffer
run('sysctl net.ipv4.tcp_moderate_rcvbuf=0')
interfaces = get_netint_cached(env.host_string, int_no=-1)
# disable all offloading, e.g. tso = tcp segment offloading
for interface in interfaces:
run('ethtool -K %s tso off' % interface)
run('ethtool -K %s gso off' % interface)
run('ethtool -K %s lro off' % interface)
run('ethtool -K %s gro off' % interface)
run('ethtool -K %s ufo off' % interface)
# send and recv buffer max (set max to 2MB)
run('sysctl net.core.rmem_max=2097152')
run('sysctl net.core.wmem_max=2097152')
# tcp recv buffer max (min 4kB, default 87kB, max 6MB; this is standard
# on kernel 3.7)
run('sysctl net.ipv4.tcp_rmem=\'4096 87380 6291456\'')
# tcp send buffer max (min 4kB, default 32kB, max 6MB; doubled default
# otherwise standard on kernel 3.7)
run('sysctl net.ipv4.tcp_wmem=\'4096 65535 4194304\'')
elif htype == 'Darwin':
# disable tso
run('sysctl -w net.inet.tcp.tso=0')
# diable lro (off by default anyway)
run('sysctl -w net.inet.tcp.lro=0')
# disable auto tuning of buffers
run('sysctl -w net.inet.tcp.doautorcvbuf=0')
run('sysctl -w net.inet.tcp.doautosndbuf=0')
# send and receive buffer max (2MB). kern.ipc.maxsockbuf max be the sum
# (but is 4MB by default anyway)
run('sysctl -w kern.ipc.maxsockbuf=4194304')
run('sysctl -w net.inet.tcp.sendspace=2097152')
run('sysctl -w net.inet.tcp.recvspace=2097152')
# set the auto receive/send buffer max to 2MB as well just in case
run('sysctl -w net.inet.tcp.autorcvbufmax=2097152')
run('sysctl -w net.inet.tcp.autosndbufmax=2097152')
elif htype == 'CYGWIN':
# disable ip/tcp/udp offload processing
run('netsh int tcp set global chimney=disabled', pty=False)
run('netsh int ip set global taskoffload=disabled', pty=False)
# enable tcp timestamps
run('netsh int tcp set global timestamps=enabled', pty=False)
# disable tcp window scaling heuristics, enforce user-set auto-tuning
# level
run('netsh int tcp set heuristics disabled', pty=False)
if interfaces == '':
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
# stop and restart interface to make the changes
run('netsh int set int "Local Area Connection %s" disabled' %
interface, pty=False)
run('netsh int set int "Local Area Connection %s" enabled' %
interface, pty=False)
def check_host():
"Check that needed tools are installed on hosts"
# get type of current host
htype = get_type_cached(env.host_string)
# run checks
if env.host_string in config.TPCONF_router:
if htype == 'FreeBSD':
run('sudo which ipfw')
if htype == "Linux":
run('sudo which tc')
run('sudo which iptables')
# XXX check that kernel tick rate is high (>= 1000)
else:
if htype == 'FreeBSD':
run('sudo which md5')
run('sudo which tcpdump')
elif htype == 'Darwin':
run('sudo which md5')
run('sudo which tcpdump')
run('sudo which dsiftr-osx-teacup.d')
elif htype == 'Linux':
run('sudo which ethtool')
run('sudo which md5sum')
run('sudo which tcpdump')
#run('sudo which web10g-listconns')
#run('sudo which web10g-readvars')
#updated for ttprobe support
try:
linux_tcp_logger = config.TPCONF_linux_tcp_logger
except AttributeError:
linux_tcp_logger = 'web10g'
if linux_tcp_logger == 'ttprobe' or linux_tcp_logger == 'both':
#checking the availability of ttprobe.ko kernel module
run('sudo ls /lib/modules/$(uname -r)/extra/ttprobe.ko')
if linux_tcp_logger == 'web10g' or linux_tcp_logger == 'both':
run('sudo which web10g-logger')
elif htype == 'CYGWIN':
run('sudo which WinDump', pty=False)
run('sudo which win-estats-logger', pty=False)
# if we don't have proper ntp installed then
# start time service if not started and force resync
with settings(warn_only=True):
ret = run(
'sudo ls "/cygdrive/c/Program Files (x86)/NTP/bin/ntpq"')
if ret.return_code != 0:
run('sudo net start w32time', pty=False)
run('sudo w32tm /resync', pty=False)
# try to enable any test network interfaces that are (accidently)
# disabled after reboot
with settings(warn_only=True):
interfaces = get_netint_cached(env.host_string, int_no=-1)
for interface in interfaces:
run('sudo netsh int set int "Local Area Connection %s" enabled'
% interface,
pty=False)
run('sudo which killall', pty=False)
run('sudo which pkill', pty=False)
run('sudo which ps', pty=False)
run('sudo which gzip', pty=False)
run('sudo which dd', pty=False)
# check for traffic sender/receiver tools
run('sudo which iperf', pty=False)
run('sudo which ping', pty=False)
run('sudo which httperf', pty=False)
run('sudo which lighttpd', pty=False)
run('sudo which nttcp', pty=False)
put(config.TPCONF_script_path + '/runbg_wrapper.sh',
'/usr/bin',
use_sudo=True)
run('sudo chmod a+x /usr/bin/runbg_wrapper.sh', pty=False)
run('sudo which runbg_wrapper.sh', pty=False)
put(config.TPCONF_script_path + '/kill_iperf.sh',
'/usr/bin',
use_sudo=True)
run('sudo chmod a+x /usr/bin/kill_iperf.sh', pty=False)
run('sudo which kill_iperf.sh', pty=False)
put(config.TPCONF_script_path + '/pktgen.sh', '/usr/bin', use_sudo=True)
run('sudo chmod a+x /usr/bin/pktgen.sh', pty=False)
run('sudo which pktgen.sh', pty=False)
