def precheck( self ):
"""Pre-check to make sure connection works and that
we can call sudo without a password"""
result = 0
info( '*** Checking servers\n' )
for server in self.servers:
ip = self.serverIP[ server ]
if not server or server == 'localhost':
continue
info( server, '' )
dest = '%s@%s' % ( self.user, ip )
cmd = [ 'sudo', '-E', '-u', self.user ]
cmd += self.sshcmd + [ '-n', dest, 'sudo true' ]
debug( ' '.join( cmd ), '\n' )
_out, _err, code = errRun( cmd )
if code != 0:
error( '\nstartConnection: server connection check failed '
'to %s using command:\n%s\n'
% ( server, ' '.join( cmd ) ) )
result |= code
if result:
error( '*** Server precheck failed.\n'
'*** Make sure that the above ssh command works'
' correctly.\n'
'*** You may also need to run mn -c on all nodes, and/or\n'
'*** use sudo -E.\n' )
sys.exit( 1 )
info( '\n' )
def _popen( self, cmd, sudo=True, tt=True, **params):
"""Spawn a process on a remote node
cmd: remote command to run (list)
**params: parameters to Popen()
returns: Popen() object"""
if type( cmd ) is str:
cmd = cmd.split()
if self.isRemote:
if sudo:
cmd = [ 'sudo', '-E' ] + cmd
if tt:
cmd = self.sshcmd + cmd
else:
# Hack: remove -tt
sshcmd = list( self.sshcmd )
sshcmd.remove( '-tt' )
cmd = sshcmd + cmd
else:
if self.user and not sudo:
# Drop privileges
cmd = [ 'sudo', '-E', '-u', self.user ] + cmd
params.update( preexec_fn=self._ignoreSignal )
debug( '_popen', cmd, '\n' )
popen = super( RemoteMixin, self )._popen( cmd, **params )
return popen
def errRun(*cmd, **kwargs):
"""Run a command and return stdout, stderr and return code
cmd: string or list of command and args
stderr: STDOUT to merge stderr with stdout
shell: run command using shell
echo: monitor output to console"""
# By default we separate stderr, don't run in a shell, and don't echo
stderr = kwargs.get('stderr', PIPE)
shell = kwargs.get('shell', False)
echo = kwargs.get('echo', False)
if echo:
# cmd goes to stderr, output goes to stdout
info(cmd, '\n')
if len(cmd) == 1:
cmd = cmd[0]
# Allow passing in a list or a string
if isinstance(cmd, str) and not shell:
cmd = cmd.split(' ')
cmd = [str(arg) for arg in cmd]
elif isinstance(cmd, list) and shell:
cmd = " ".join(arg for arg in cmd)
debug('*** errRun:', cmd, '\n')
popen = Popen(cmd, stdout=PIPE, stderr=stderr, shell=shell)
# We use poll() because select() doesn't work with large fd numbers,
# and thus communicate() doesn't work either
out, err = '', ''
poller = poll()
poller.register(popen.stdout, POLLIN)
fdtofile = {popen.stdout.fileno(): popen.stdout}
outDone, errDone = False, True
if popen.stderr:
fdtofile[popen.stderr.fileno()] = popen.stderr
poller.register(popen.stderr, POLLIN)
errDone = False
while not outDone or not errDone:
readable = poller.poll()
for fd, event in readable:
f = fdtofile[fd]
if event & POLLIN:
data = f.read(1024)
if echo:
output(data)
if f == popen.stdout:
out += data
if data == '':
outDone = True
elif f == popen.stderr:
err += data
if data == '':
errDone = True
else: # POLLHUP or something unexpected
if f == popen.stdout:
outDone = True
elif f == popen.stderr:
errDone = True
poller.unregister(fd)
returncode = popen.wait()
debug(out, err, returncode)
return out, err, returncode
def waitListening(client=None, server='127.0.0.1', port=80, timeout=None):
"""Wait until server is listening on port.
returns True if server is listening"""
runCmd = (client.cmd if client else partial(quietRun, shell=True))
if not runCmd('which telnet'):
raise Exception('Could not find telnet')
# pylint: disable=maybe-no-member
serverIP = server if isinstance(server, basestring) else server.IP()
cmd = ('echo A | telnet -e A %s %s' % (serverIP, port))
time = 0
result = runCmd(cmd)
while 'Connected' not in result:
if 'No route' in result:
rtable = runCmd('route')
error('no route to %s:\n%s' % (server, rtable))
return False
if timeout and time >= timeout:
error('could not connect to %s on port %d\n' % (server, port))
return False
debug('waiting for', server, 'to listen on port', port, '\n')
info('.')
sleep(.5)
time += .5
result = runCmd(cmd)
return True
def fixLimits():
"Fix ridiculously small resource limits."
debug("*** Setting resource limits\n")
try:
rlimitTestAndSet(RLIMIT_NPROC, 8192)
rlimitTestAndSet(RLIMIT_NOFILE, 16384)
#Increase open file limit
sysctlTestAndSet('fs.file-max', 10000)
#Increase network buffer space
sysctlTestAndSet('net.core.wmem_max', 16777216)
sysctlTestAndSet('net.core.rmem_max', 16777216)
sysctlTestAndSet('net.ipv4.tcp_rmem', '10240 87380 16777216')
sysctlTestAndSet('net.ipv4.tcp_wmem', '10240 87380 16777216')
sysctlTestAndSet('net.core.netdev_max_backlog', 5000)
#Increase arp cache size
sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh1', 4096)
sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh2', 8192)
sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh3', 16384)
#Increase routing table size
sysctlTestAndSet('net.ipv4.route.max_size', 32768)
#Increase number of PTYs for nodes
sysctlTestAndSet('kernel.pty.max', 20000)
# pylint: disable=broad-except
except Exception:
warn("*** Warning: setting resource limits. "
"Mininet's performance may be affected.\n")
def moduleDeps(subtract=None, add=None):
"""Handle module dependencies.
subtract: string or list of module names to remove, if already loaded
add: string or list of module names to add, if not already loaded"""
subtract = subtract if subtract is not None else []
add = add if add is not None else []
if isinstance(subtract, basestring):
subtract = [subtract]
if isinstance(add, basestring):
add = [add]
for mod in subtract:
if mod in lsmod():
info('*** Removing ' + mod + '\n')
rmmodOutput = rmmod(mod)
if rmmodOutput:
error('Error removing ' + mod + ': "%s">\n' % rmmodOutput)
exit(1)
if mod in lsmod():
error('Failed to remove ' + mod + '; still there!\n')
exit(1)
for mod in add:
if mod not in lsmod():
info('*** Loading ' + mod + '\n')
modprobeOutput = modprobe(mod)
if modprobeOutput:
error('Error inserting ' + mod +
' - is it installed and available via modprobe?\n' +
'Error was: "%s"\n' % modprobeOutput)
if mod not in lsmod():
error('Failed to insert ' + mod + ' - quitting.\n')
exit(1)
else:
debug('*** ' + mod + ' already loaded\n')
def iperf(self, hosts=None, l4Type='TCP', udpBw='10M', fmt=None,
seconds=5, port=5001):
"""Run iperf between two hosts.
hosts: list of hosts; if None, uses first and last hosts
l4Type: string, one of [ TCP, UDP ]
udpBw: bandwidth target for UDP test
fmt: iperf format argument if any
seconds: iperf time to transmit
port: iperf port
returns: two-element array of [ server, client ] speeds
note: send() is buffered, so client rate can be much higher than
the actual transmission rate; on an unloaded system, server
rate should be much closer to the actual receive rate"""
hosts = hosts or [self.hosts[0], self.hosts[-1]]
assert len(hosts) == 2
client, server = hosts
output('*** Iperf: testing', l4Type, 'bandwidth between',
client, 'and', server, '\n')
server.cmd('killall -9 iperf')
iperfArgs = 'iperf -p %d ' % port
bwArgs = ''
if l4Type == 'UDP':
iperfArgs += '-u '
bwArgs = '-b ' + udpBw + ' '
elif l4Type != 'TCP':
raise Exception('Unexpected l4 type: %s' % l4Type)
if fmt:
iperfArgs += '-f %s ' % fmt
server.sendCmd(iperfArgs + '-s')
if l4Type == 'TCP':
if not waitListening(client, server.IP(), port):
raise Exception('Could not connect to iperf on port %d'
% port)
cliout = client.cmd(iperfArgs + '-t %d -c ' % seconds +
server.IP() + ' ' + bwArgs)
debug('Client output: %s\n' % cliout)
servout = ''
# We want the last *b/sec from the iperf server output
# for TCP, there are two fo them because of waitListening
count = 2 if l4Type == 'TCP' else 1
while len(re.findall('/sec', servout)) < count:
servout += server.monitor(timeoutms=5000)
server.sendInt()
servout += server.waitOutput()
debug('Server output: %s\n' % servout)
result = [self._parseIperf(servout), self._parseIperf(cliout)]
if l4Type == 'UDP':
result.insert(0, udpBw)
output('*** Results: %s\n' % result)
return result
def makeTunnel( self, node1, node2, intfname1, intfname2,
addr1=None, addr2=None ):
"Make a tunnel across switches on different servers"
# We should never try to create a tunnel to ourselves!
assert node1.server != 'localhost' or node2.server != 'localhost'
# And we can't ssh into this server remotely as 'localhost',
# so try again swappping node1 and node2
if node2.server == 'localhost':
return self.makeTunnel( node2, node1, intfname2, intfname1,
addr2, addr1 )
# 1. Create tap interfaces
for node in node1, node2:
# For now we are hard-wiring tap9, which we will rename
cmd = 'ip tuntap add dev tap9 mode tap user ' + node.user
result = node.rcmd( cmd )
if result:
raise Exception( 'error creating tap9 on %s: %s' %
( node, result ) )
# 2. Create ssh tunnel between tap interfaces
# -n: close stdin
dest = '%s@%s' % ( node2.user, node2.serverIP )
cmd = [ 'ssh', '-n', '-o', 'Tunnel=Ethernet', '-w', '9:9',
dest, 'echo @' ]
self.cmd = cmd
tunnel = node1.rpopen( cmd, sudo=False )
# When we receive the character '@', it means that our
# tunnel should be set up
debug( 'Waiting for tunnel to come up...\n' )
ch = tunnel.stdout.read( 1 )
if ch != '@':
raise Exception( 'makeTunnel:\n',
'Tunnel setup failed for',
'%s:%s' % ( node1, node1.dest ), 'to',
'%s:%s\n' % ( node2, node2.dest ),
'command was:', cmd, '\n' )
# 3. Move interfaces if necessary
for node in node1, node2:
if not self.moveIntf( 'tap9', node ):
raise Exception( 'interface move failed on node %s' % node )
# 4. Rename tap interfaces to desired names
for node, intf, addr in ( ( node1, intfname1, addr1 ),
( node2, intfname2, addr2 ) ):
if not addr:
result = node.cmd( 'ip link set tap9 name', intf )
else:
result = node.cmd( 'ip link set tap9 name', intf,
'address', addr )
if result:
raise Exception( 'error renaming %s: %s' % ( intf, result ) )
return tunnel
def __init__(self, label, topo=None, resources_table=None):
# self.name = "vi%d" % VirtualInstance._COUNTER
self.name = label
VirtualInstance._COUNTER += 1
self.topo = topo
self.label = label
self.res_table = resources_table
self.containers = {}
self.allocated_resources = {}
self.resource_model = None
self.switch = self.topo.addSwitch("%s.s1" % self.name)
debug("created data center switch: %s\n" % str(self.switch))
def removeHost(self, name, **params):
"""
Remove a host from the network at runtime.
"""
if not isinstance(name, basestring) and name is not None:
name = name.name # if we get a host object
try:
h = self.get(name)
except:
error("Host: %s not found. Cannot remove it.\n" % name)
return False
if h is not None:
if h in self.hosts:
self.hosts.remove(h)
if name in self.nameToNode:
del self.nameToNode[name]
h.stop(deleteIntfs=True)
debug("Removed: %s\n" % name)
return True
return False
def config(self, bw=None, delay=None, jitter=None, loss=None,
disable_gro=True, speedup=0, use_hfsc=False, use_tbf=False,
latency_ms=None, enable_ecn=False, enable_red=False,
max_queue_size=None, **params):
"Configure the port and set its properties."
result = Intf.config(self, **params)
# Disable GRO
if disable_gro:
self.cmd('ethtool -K %s gro off' % self)
# Optimization: return if nothing else to configure
# Question: what happens if we want to reset things?
if (bw is None and not delay and not loss
and max_queue_size is None):
return
# Clear existing configuration
tcoutput = self.tc('%s qdisc show dev %s')
if "priomap" not in tcoutput:
cmds = ['%s qdisc del dev %s root']
else:
cmds = []
# Bandwidth limits via various methods
bwcmds, parent = self.bwCmds(bw=bw, speedup=speedup,
use_hfsc=use_hfsc, use_tbf=use_tbf,
latency_ms=latency_ms,
enable_ecn=enable_ecn,
enable_red=enable_red)
cmds += bwcmds
# Delay/jitter/loss/max_queue_size using netem
delaycmds, parent = self.delayCmds(delay=delay, jitter=jitter,
loss=loss,
max_queue_size=max_queue_size,
parent=parent)
cmds += delaycmds
# Ugly but functional: display configuration info
stuff = ((['%.2fMbit' % bw] if bw is not None else []) +
(['%s delay' % delay] if delay is not None else []) +
(['%s jitter' % jitter] if jitter is not None else []) +
(['%d%% loss' % loss] if loss is not None else []) +
(['ECN'] if enable_ecn else ['RED']
if enable_red else []))
info('(' + ' '.join(stuff) + ') ')
# Execute all the commands in our node
debug("at map stage w/cmds: %s\n" % cmds)
tcoutputs = [self.tc(cmd) for cmd in cmds]
for output in tcoutputs:
if output != '' and output != 'RTNETLINK answers: No such file or directory\r\n':
error("*** Error: %s" % output)
debug("cmds:", cmds, '\n')
debug("outputs:", tcoutputs, '\n')
result['tcoutputs'] = tcoutputs
result['parent'] = parent
return result
def tc(self, cmd, tc='tc'):
"Execute tc command for our interface"
c = cmd % (tc, self) # Add in tc command and our name
debug(" *** executing command: %s\n" % c)
return self.cmd(c)
