def waitListening( client=None, server='127.0.0.1', port=80, timeout=None ):
"""Wait until server is listening on port.
returns True if server is listening"""
runCmd = ( client.cmd if client else
partial( quietRun, shell=True ) )
if not runCmd( 'which telnet' ):
raise Exception('Could not find telnet' )
# pylint: disable=maybe-no-member
serverIP = server if isinstance( server, BaseString ) else server.IP()
cmd = ( 'echo A | telnet -e A %s %s' % ( serverIP, port ) )
time = 0
result = runCmd( cmd )
while 'Connected' not in result:
if 'No route' in result:
rtable = runCmd( 'route' )
error( 'no route to %s:\n%s' % ( server, rtable ) )
return False
if timeout and time >= timeout:
error( 'could not connect to %s on port %d\n' % ( server, port ) )
return False
debug( 'waiting for', server, 'to listen on port', port, '\n' )
info( '.' )
sleep( .5 )
time += .5
result = runCmd( cmd )
return True
def moduleDeps( subtract=None, add=None ):
"""Handle module dependencies.
subtract: string or list of module names to remove, if already loaded
add: string or list of module names to add, if not already loaded"""
subtract = subtract if subtract is not None else []
add = add if add is not None else []
if isinstance( subtract, BaseString ):
subtract = [ subtract ]
if isinstance( add, BaseString ):
add = [ add ]
for mod in subtract:
if mod in lsmod():
info( '*** Removing ' + mod + '\n' )
rmmodOutput = rmmod( mod )
if rmmodOutput:
error( 'Error removing ' + mod + ': "%s">\n' % rmmodOutput )
exit( 1 )
if mod in lsmod():
error( 'Failed to remove ' + mod + '; still there!\n' )
exit( 1 )
for mod in add:
if mod not in lsmod():
info( '*** Loading ' + mod + '\n' )
modprobeOutput = modprobe( mod )
if modprobeOutput:
error( 'Error inserting ' + mod +
' - is it installed and available via modprobe?\n' +
'Error was: "%s"\n' % modprobeOutput )
if mod not in lsmod():
error( 'Failed to insert ' + mod + ' - quitting.\n' )
exit( 1 )
else:
debug( '*** ' + mod + ' already loaded\n' )
def config(self, **params):
"""Configure the NAT and iptables"""
super(NAT, self).config(**params)
if not self.localIntf:
self.localIntf = self.defaultIntf()
if self.flush:
self.cmd('sysctl net.ipv4.ip_forward=0')
self.cmd('iptables -F')
self.cmd('iptables -t nat -F')
# Create default entries for unmatched traffic
self.cmd('iptables -P INPUT ACCEPT')
self.cmd('iptables -P OUTPUT ACCEPT')
self.cmd('iptables -P FORWARD DROP')
# Install NAT rules
self.cmd('iptables -I FORWARD', '-i', self.localIntf, '-d',
self.subnet, '-j DROP')
self.cmd('iptables -A FORWARD', '-i', self.localIntf, '-s',
self.subnet, '-j ACCEPT')
self.cmd('iptables -A FORWARD', '-o', self.localIntf, '-d',
self.subnet, '-j ACCEPT')
self.cmd('iptables -t nat -A POSTROUTING', '-s', self.subnet, "'!'",
'-d', self.subnet, '-j MASQUERADE')
# Instruct the kernel to perform forwarding
self.cmd('sysctl net.ipv4.ip_forward=1')
# Prevent network-manager from messing with our interface
# by specifying manual configuration in /etc/network/interfaces
intf = self.localIntf
cfile = '/etc/network/interfaces'
line = '\niface %s inet manual\n' % intf
config = open(cfile).read()
if (line) not in config:
info('*** Adding "' + line.strip() + '" to ' + cfile + '\n')
with open(cfile, 'a') as f:
f.write(line)
# Probably need to restart network-manager to be safe -
# hopefully this won't disconnect you
self.cmd('service network-manager restart')
def run(self):
"Run our cmdloop(), catching KeyboardInterrupt"
while True:
try:
# Make sure no nodes are still waiting
for node in self.mn.values():
while node.waiting:
info('stopping', node, '\n')
node.sendInt()
node.waitOutput()
if self.isatty():
quietRun('stty echo sane intr ^C')
self.cmdloop()
break
except KeyboardInterrupt:
# Output a message - unless it's also interrupted
# pylint: disable=broad-except
try:
output('\nInterrupt\n')
except Exception:
pass
def __init__(self, mininet, stdin=sys.stdin, script=None):
"""Start and run interactive or batch mode CLI
mininet: Mininet network object
stdin: standard input for CLI
script: script to run in batch mode"""
self.mn = mininet
# Local variable bindings for py command
self.locals = {'net': mininet}
# Attempt to handle input
self.stdin = stdin
self.inPoller = poll()
self.inPoller.register(stdin)
self.inputFile = script
RCmd.__init__(self)
# Containernet allows '.' in host identifiers to build human readable hierarchical name spaces:
self.identchars = string.ascii_letters + string.digits + '_' + '.'
info('*** Starting CLI:\n')
if self.inputFile:
self.do_source(self.inputFile)
return
self.initReadline()
self.run()
def cleanup( cls):
"""Clean up junk which might be left over from old runs;
do fast stuff before slow dp and link removal!"""
info( "*** Removing excess controllers/ofprotocols/ofdatapaths/"
"pings/noxes\n" )
zombies = ( 'controller ofprotocol ofdatapath ping nox_core'
'lt-nox_core ovs-openflowd ovs-controller'
'ovs-testcontroller udpbwtest mnexec ivs ryu-manager' )
# Note: real zombie processes can't actually be killed, since they
# are already (un)dead. Then again,
# you can't connect to them either, so they're mostly harmless.
# Send SIGTERM first to give processes a chance to shutdown cleanly.
sh( 'killall ' + zombies + ' 2> /dev/null' )
time.sleep( 1 )
sh( 'killall -9 ' + zombies + ' 2> /dev/null' )
# And kill off sudo mnexec
sh( 'pkill -9 -f "sudo mnexec"')
info( "*** Removing junk from /tmp\n" )
sh( 'rm -f /tmp/vconn* /tmp/vlogs* /tmp/*.out /tmp/*.log' )
info( "*** Removing old X11 tunnels\n" )
cleanUpScreens()
info( "*** Removing excess kernel datapaths\n" )
dps = sh( "ps ax | egrep -o 'dp[0-9]+' | sed 's/dp/nl:/'"
).splitlines()
for dp in dps:
if dp:
sh( 'dpctl deldp ' + dp )
info( "*** Removing OVS datapaths\n" )
dps = sh("ovs-vsctl --timeout=1 list-br").strip().splitlines()
if dps:
sh( "ovs-vsctl " + " -- ".join( "--if-exists del-br " + dp
for dp in dps if dp ) )
# And in case the above didn't work...
dps = sh( "ovs-vsctl --timeout=1 list-br" ).strip().splitlines()
for dp in dps:
sh( 'ovs-vsctl del-br ' + dp )
info( "*** Removing all links of the pattern foo-ethX\n" )
links = sh( "ip link show | "
"egrep -o '([-_.[:alnum:]]+-eth[[:digit:]]+)'"
).splitlines()
# Delete blocks of links
n = 1000 # chunk size
for i in range( 0, len( links ), n ):
cmd = ';'.join( 'ip link del %s' % link
for link in links[ i : i + n ] )
sh( '( %s ) 2> /dev/null' % cmd )
if 'tap9' in sh( 'ip link show' ):
info( "*** Removing tap9 - assuming it's from cluster edition\n" )
sh( 'ip link del tap9' )
info( "*** Killing stale mininet node processes\n" )
killprocs( 'mininet:' )
info( "*** Shutting down stale tunnels\n" )
killprocs( 'Tunnel=Ethernet' )
killprocs( '.ssh/mn')
sh( 'rm -f ~/.ssh/mn/*' )
# Call any additional cleanup code if necessary
for callback in cls.callbacks:
callback()
# Containernet should also cleanup pending Docker
cmd = "docker rm -f $( docker ps --filter 'label=com.containernet' -a -q)"
call(cmd, shell=True, stdout=open(os.devnull, 'wb'), stderr=open(os.devnull, 'wb'))
# cleanup any remaining iptables rules from external SAPs with NAT
# we use iptc module to iterate through the loops, but due to a bug, we cannot use iptc to delete the rules
# we rely on iptables CLI to delete the found rules
info("*** Removing SAP NAT rules\n")
table = iptc.Table(iptc.Table.NAT)
chain = iptc.Chain(table, 'POSTROUTING')
for rule in chain.rules:
if SAP_PREFIX in str(rule.out_interface):
src_CIDR = str(ipaddress.IPv4Network(u'{}'.format(str(rule.src))))
rule0_ = "iptables -t nat -D POSTROUTING ! -o {0} -s {1} -j MASQUERADE".\
format(rule.out_interface.strip('!'), src_CIDR)
p = Popen(shlex.split(rule0_))
p.communicate()
info("delete NAT rule from SAP: {1} - {0} - {2}\n".format(rule.out_interface, rule.in_interface, src_CIDR))
table = iptc.Table(iptc.Table.FILTER)
chain = iptc.Chain(table, 'FORWARD')
for rule in chain.rules:
src_CIDR = str(ipaddress.IPv4Network(u'{}'.format(str(rule.src))))
if SAP_PREFIX in str(rule.out_interface):
rule1_ = "iptables -D FORWARD -o {0} -j ACCEPT".format(rule.out_interface)
p = Popen(shlex.split(rule1_))
p.communicate()
info("delete FORWARD rule from SAP: {1} - {0} - {2}\n".format(rule.out_interface, rule.in_interface,
src_CIDR))
if SAP_PREFIX in str(rule.in_interface):
rule2_ = "iptables -D FORWARD -i {0} -j ACCEPT".format(rule.in_interface)
p = Popen(shlex.split(rule2_))
p.communicate()
info("delete FORWARD rule from SAP: {1} - {0} - {2}\n".format(rule.out_interface, rule.in_interface,
src_CIDR))
info( "*** Cleanup complete.\n" )
def sh( cmd ):
"Print a command and send it to the shell"
info( cmd + '\n' )
result = Popen( [ '/bin/sh', '-c', cmd ], stdout=PIPE ).communicate()[ 0 ]
return decode( result )
def errRun( *cmd, **kwargs ):
"""Run a command and return stdout, stderr and return code
cmd: string or list of command and args
stderr: STDOUT to merge stderr with stdout
shell: run command using shell
echo: monitor output to console"""
# By default we separate stderr, don't run in a shell, and don't echo
stderr = kwargs.get( 'stderr', PIPE )
shell = kwargs.get( 'shell', False )
echo = kwargs.get( 'echo', False )
if echo:
# cmd goes to stderr, output goes to stdout
info( cmd, '\n' )
if len( cmd ) == 1:
cmd = cmd[ 0 ]
# Allow passing in a list or a string
if isinstance( cmd, BaseString ) and not shell:
cmd = cmd.split( ' ' )
cmd = [ str( arg ) for arg in cmd ]
elif isinstance( cmd, list ) and shell:
cmd = " ".join( arg for arg in cmd )
debug( '*** errRun:', cmd, '\n' )
popen = Popen( cmd, stdout=PIPE, stderr=stderr, shell=shell )
# We use poll() because select() doesn't work with large fd numbers,
# and thus communicate() doesn't work either
out, err = '', ''
poller = poll()
poller.register( popen.stdout, POLLIN )
fdtofile = { popen.stdout.fileno(): popen.stdout }
outDone, errDone = False, True
if popen.stderr:
fdtofile[ popen.stderr.fileno() ] = popen.stderr
poller.register( popen.stderr, POLLIN )
errDone = False
while not outDone or not errDone:
readable = poller.poll()
for fd, event in readable:
f = fdtofile[ fd ]
if event & POLLIN:
data = f.read( 1024 )
if Python3:
data = data.decode( Encoding )
if echo:
output( data )
if f == popen.stdout:
out += data
if data == '':
outDone = True
elif f == popen.stderr:
err += data
if data == '':
errDone = True
else: # POLLHUP or something unexpected
if f == popen.stdout:
outDone = True
elif f == popen.stderr:
errDone = True
poller.unregister( fd )
returncode = popen.wait()
# Python 3 complains if we don't explicitly close these
popen.stdout.close()
if stderr == PIPE:
popen.stderr.close()
debug( out, err, returncode )
return out, err, returncode
def config(self,
bw=None,
delay=None,
jitter=None,
loss=None,
gro=False,
txo=True,
rxo=True,
speedup=0,
use_hfsc=False,
use_tbf=False,
latency_ms=None,
enable_ecn=False,
enable_red=False,
max_queue_size=None,
**params):
"""Configure the port and set its properties.
bw: bandwidth in b/s (e.g. '10m')
delay: transmit delay (e.g. '1ms' )
jitter: jitter (e.g. '1ms')
loss: loss (e.g. '1%' )
gro: enable GRO (False)
txo: enable transmit checksum offload (True)
rxo: enable receive checksum offload (True)
speedup: experimental switch-side bw option
use_hfsc: use HFSC scheduling
use_tbf: use TBF scheduling
latency_ms: TBF latency parameter
enable_ecn: enable ECN (False)
enable_red: enable RED (False)
max_queue_size: queue limit parameter for netem"""
# Support old names for parameters
gro = not params.pop('disable_gro', not gro)
result = Intf.config(self, **params)
def on(isOn):
"Helper method: bool -> 'on'/'off'"
return 'on' if isOn else 'off'
# Set offload parameters with ethool
self.cmd('ethtool -K', self, 'gro', on(gro), 'tx', on(txo), 'rx',
on(rxo))
# Optimization: return if nothing else to configure
# Question: what happens if we want to reset things?
if (bw is None and not delay and not loss and max_queue_size is None):
return
# Clear existing configuration
tcoutput = self.tc('%s qdisc show dev %s')
if "priomap" not in tcoutput and "noqueue" not in tcoutput:
cmds = ['%s qdisc del dev %s root']
else:
cmds = []
# Bandwidth limits via various methods
bwcmds, parent = self.bwCmds(bw=bw,
speedup=speedup,
use_hfsc=use_hfsc,
use_tbf=use_tbf,
latency_ms=latency_ms,
enable_ecn=enable_ecn,
enable_red=enable_red)
cmds += bwcmds
# Delay/jitter/loss/max_queue_size using netem
delaycmds, parent = self.delayCmds(delay=delay,
jitter=jitter,
loss=loss,
max_queue_size=max_queue_size,
parent=parent)
cmds += delaycmds
# Ugly but functional: display configuration info
stuff = ((['%.2fMbit' % bw] if bw is not None else []) +
(['%s delay' % delay] if delay is not None else []) +
(['%s jitter' % jitter] if jitter is not None else []) +
(['%.5f%% loss' % loss] if loss is not None else []) +
(['ECN'] if enable_ecn else ['RED'] if enable_red else []))
info('(' + ' '.join(stuff) + ') ')
# Execute all the commands in our node
debug("at map stage w/cmds: %s\n" % cmds)
tcoutputs = [self.tc(cmd) for cmd in cmds]
for output in tcoutputs:
if output != '' and output != 'RTNETLINK answers: No such file or directory\r\n':
error("*** Error: %s" % output)
debug("cmds:", cmds, '\n')
debug("outputs:", tcoutputs, '\n')
result['tcoutputs'] = tcoutputs
result['parent'] = parent
return result