def precheck( self ): """Pre-check to make sure connection works and that we can call sudo without a password""" result = 0 info( '*** Checking servers\n' ) for server in self.servers: ip = self.serverIP[ server ] if not server or server == 'localhost': continue info( server, '' ) dest = '%s@%s' % ( self.user, ip ) cmd = [ 'sudo', '-E', '-u', self.user ] cmd += self.sshcmd + [ '-n', dest, 'sudo true' ] debug( ' '.join( cmd ), '\n' ) _out, _err, code = errRun( cmd ) if code != 0: error( '\nstartConnection: server connection check failed ' 'to %s using command:\n%s\n' % ( server, ' '.join( cmd ) ) ) result |= code if result: error( '*** Server precheck failed.\n' '*** Make sure that the above ssh command works' ' correctly.\n' '*** You may also need to run mn -c on all nodes, and/or\n' '*** use sudo -E.\n' ) sys.exit( 1 ) info( '\n' )
def _popen( self, cmd, sudo=True, tt=True, **params): """Spawn a process on a remote node cmd: remote command to run (list) **params: parameters to Popen() returns: Popen() object""" if type( cmd ) is str: cmd = cmd.split() if self.isRemote: if sudo: cmd = [ 'sudo', '-E' ] + cmd if tt: cmd = self.sshcmd + cmd else: # Hack: remove -tt sshcmd = list( self.sshcmd ) sshcmd.remove( '-tt' ) cmd = sshcmd + cmd else: if self.user and not sudo: # Drop privileges cmd = [ 'sudo', '-E', '-u', self.user ] + cmd params.update( preexec_fn=self._ignoreSignal ) debug( '_popen', cmd, '\n' ) popen = super( RemoteMixin, self )._popen( cmd, **params ) return popen
def errRun(*cmd, **kwargs): """Run a command and return stdout, stderr and return code cmd: string or list of command and args stderr: STDOUT to merge stderr with stdout shell: run command using shell echo: monitor output to console""" # By default we separate stderr, don't run in a shell, and don't echo stderr = kwargs.get('stderr', PIPE) shell = kwargs.get('shell', False) echo = kwargs.get('echo', False) if echo: # cmd goes to stderr, output goes to stdout info(cmd, '\n') if len(cmd) == 1: cmd = cmd[0] # Allow passing in a list or a string if isinstance(cmd, str) and not shell: cmd = cmd.split(' ') cmd = [str(arg) for arg in cmd] elif isinstance(cmd, list) and shell: cmd = " ".join(arg for arg in cmd) debug('*** errRun:', cmd, '\n') popen = Popen(cmd, stdout=PIPE, stderr=stderr, shell=shell) # We use poll() because select() doesn't work with large fd numbers, # and thus communicate() doesn't work either out, err = '', '' poller = poll() poller.register(popen.stdout, POLLIN) fdtofile = {popen.stdout.fileno(): popen.stdout} outDone, errDone = False, True if popen.stderr: fdtofile[popen.stderr.fileno()] = popen.stderr poller.register(popen.stderr, POLLIN) errDone = False while not outDone or not errDone: readable = poller.poll() for fd, event in readable: f = fdtofile[fd] if event & POLLIN: data = f.read(1024) if echo: output(data) if f == popen.stdout: out += data if data == '': outDone = True elif f == popen.stderr: err += data if data == '': errDone = True else: # POLLHUP or something unexpected if f == popen.stdout: outDone = True elif f == popen.stderr: errDone = True poller.unregister(fd) returncode = popen.wait() debug(out, err, returncode) return out, err, returncode
def waitListening(client=None, server='127.0.0.1', port=80, timeout=None): """Wait until server is listening on port. returns True if server is listening""" runCmd = (client.cmd if client else partial(quietRun, shell=True)) if not runCmd('which telnet'): raise Exception('Could not find telnet') # pylint: disable=maybe-no-member serverIP = server if isinstance(server, basestring) else server.IP() cmd = ('echo A | telnet -e A %s %s' % (serverIP, port)) time = 0 result = runCmd(cmd) while 'Connected' not in result: if 'No route' in result: rtable = runCmd('route') error('no route to %s:\n%s' % (server, rtable)) return False if timeout and time >= timeout: error('could not connect to %s on port %d\n' % (server, port)) return False debug('waiting for', server, 'to listen on port', port, '\n') info('.') sleep(.5) time += .5 result = runCmd(cmd) return True
def fixLimits(): "Fix ridiculously small resource limits." debug("*** Setting resource limits\n") try: rlimitTestAndSet(RLIMIT_NPROC, 8192) rlimitTestAndSet(RLIMIT_NOFILE, 16384) #Increase open file limit sysctlTestAndSet('fs.file-max', 10000) #Increase network buffer space sysctlTestAndSet('net.core.wmem_max', 16777216) sysctlTestAndSet('net.core.rmem_max', 16777216) sysctlTestAndSet('net.ipv4.tcp_rmem', '10240 87380 16777216') sysctlTestAndSet('net.ipv4.tcp_wmem', '10240 87380 16777216') sysctlTestAndSet('net.core.netdev_max_backlog', 5000) #Increase arp cache size sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh1', 4096) sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh2', 8192) sysctlTestAndSet('net.ipv4.neigh.default.gc_thresh3', 16384) #Increase routing table size sysctlTestAndSet('net.ipv4.route.max_size', 32768) #Increase number of PTYs for nodes sysctlTestAndSet('kernel.pty.max', 20000) # pylint: disable=broad-except except Exception: warn("*** Warning: setting resource limits. " "Mininet's performance may be affected.\n")
def moduleDeps(subtract=None, add=None): """Handle module dependencies. subtract: string or list of module names to remove, if already loaded add: string or list of module names to add, if not already loaded""" subtract = subtract if subtract is not None else [] add = add if add is not None else [] if isinstance(subtract, basestring): subtract = [subtract] if isinstance(add, basestring): add = [add] for mod in subtract: if mod in lsmod(): info('*** Removing ' + mod + '\n') rmmodOutput = rmmod(mod) if rmmodOutput: error('Error removing ' + mod + ': "%s">\n' % rmmodOutput) exit(1) if mod in lsmod(): error('Failed to remove ' + mod + '; still there!\n') exit(1) for mod in add: if mod not in lsmod(): info('*** Loading ' + mod + '\n') modprobeOutput = modprobe(mod) if modprobeOutput: error('Error inserting ' + mod + ' - is it installed and available via modprobe?\n' + 'Error was: "%s"\n' % modprobeOutput) if mod not in lsmod(): error('Failed to insert ' + mod + ' - quitting.\n') exit(1) else: debug('*** ' + mod + ' already loaded\n')
def iperf(self, hosts=None, l4Type='TCP', udpBw='10M', fmt=None, seconds=5, port=5001): """Run iperf between two hosts. hosts: list of hosts; if None, uses first and last hosts l4Type: string, one of [ TCP, UDP ] udpBw: bandwidth target for UDP test fmt: iperf format argument if any seconds: iperf time to transmit port: iperf port returns: two-element array of [ server, client ] speeds note: send() is buffered, so client rate can be much higher than the actual transmission rate; on an unloaded system, server rate should be much closer to the actual receive rate""" hosts = hosts or [self.hosts[0], self.hosts[-1]] assert len(hosts) == 2 client, server = hosts output('*** Iperf: testing', l4Type, 'bandwidth between', client, 'and', server, '\n') server.cmd('killall -9 iperf') iperfArgs = 'iperf -p %d ' % port bwArgs = '' if l4Type == 'UDP': iperfArgs += '-u ' bwArgs = '-b ' + udpBw + ' ' elif l4Type != 'TCP': raise Exception('Unexpected l4 type: %s' % l4Type) if fmt: iperfArgs += '-f %s ' % fmt server.sendCmd(iperfArgs + '-s') if l4Type == 'TCP': if not waitListening(client, server.IP(), port): raise Exception('Could not connect to iperf on port %d' % port) cliout = client.cmd(iperfArgs + '-t %d -c ' % seconds + server.IP() + ' ' + bwArgs) debug('Client output: %s\n' % cliout) servout = '' # We want the last *b/sec from the iperf server output # for TCP, there are two fo them because of waitListening count = 2 if l4Type == 'TCP' else 1 while len(re.findall('/sec', servout)) < count: servout += server.monitor(timeoutms=5000) server.sendInt() servout += server.waitOutput() debug('Server output: %s\n' % servout) result = [self._parseIperf(servout), self._parseIperf(cliout)] if l4Type == 'UDP': result.insert(0, udpBw) output('*** Results: %s\n' % result) return result
def makeTunnel( self, node1, node2, intfname1, intfname2, addr1=None, addr2=None ): "Make a tunnel across switches on different servers" # We should never try to create a tunnel to ourselves! assert node1.server != 'localhost' or node2.server != 'localhost' # And we can't ssh into this server remotely as 'localhost', # so try again swappping node1 and node2 if node2.server == 'localhost': return self.makeTunnel( node2, node1, intfname2, intfname1, addr2, addr1 ) # 1. Create tap interfaces for node in node1, node2: # For now we are hard-wiring tap9, which we will rename cmd = 'ip tuntap add dev tap9 mode tap user ' + node.user result = node.rcmd( cmd ) if result: raise Exception( 'error creating tap9 on %s: %s' % ( node, result ) ) # 2. Create ssh tunnel between tap interfaces # -n: close stdin dest = '%s@%s' % ( node2.user, node2.serverIP ) cmd = [ 'ssh', '-n', '-o', 'Tunnel=Ethernet', '-w', '9:9', dest, 'echo @' ] self.cmd = cmd tunnel = node1.rpopen( cmd, sudo=False ) # When we receive the character '@', it means that our # tunnel should be set up debug( 'Waiting for tunnel to come up...\n' ) ch = tunnel.stdout.read( 1 ) if ch != '@': raise Exception( 'makeTunnel:\n', 'Tunnel setup failed for', '%s:%s' % ( node1, node1.dest ), 'to', '%s:%s\n' % ( node2, node2.dest ), 'command was:', cmd, '\n' ) # 3. Move interfaces if necessary for node in node1, node2: if not self.moveIntf( 'tap9', node ): raise Exception( 'interface move failed on node %s' % node ) # 4. Rename tap interfaces to desired names for node, intf, addr in ( ( node1, intfname1, addr1 ), ( node2, intfname2, addr2 ) ): if not addr: result = node.cmd( 'ip link set tap9 name', intf ) else: result = node.cmd( 'ip link set tap9 name', intf, 'address', addr ) if result: raise Exception( 'error renaming %s: %s' % ( intf, result ) ) return tunnel
def __init__(self, label, topo=None, resources_table=None): # self.name = "vi%d" % VirtualInstance._COUNTER self.name = label VirtualInstance._COUNTER += 1 self.topo = topo self.label = label self.res_table = resources_table self.containers = {} self.allocated_resources = {} self.resource_model = None self.switch = self.topo.addSwitch("%s.s1" % self.name) debug("created data center switch: %s\n" % str(self.switch))
def removeHost(self, name, **params): """ Remove a host from the network at runtime. """ if not isinstance(name, basestring) and name is not None: name = name.name # if we get a host object try: h = self.get(name) except: error("Host: %s not found. Cannot remove it.\n" % name) return False if h is not None: if h in self.hosts: self.hosts.remove(h) if name in self.nameToNode: del self.nameToNode[name] h.stop(deleteIntfs=True) debug("Removed: %s\n" % name) return True return False
def config(self, bw=None, delay=None, jitter=None, loss=None, disable_gro=True, speedup=0, use_hfsc=False, use_tbf=False, latency_ms=None, enable_ecn=False, enable_red=False, max_queue_size=None, **params): "Configure the port and set its properties." result = Intf.config(self, **params) # Disable GRO if disable_gro: self.cmd('ethtool -K %s gro off' % self) # Optimization: return if nothing else to configure # Question: what happens if we want to reset things? if (bw is None and not delay and not loss and max_queue_size is None): return # Clear existing configuration tcoutput = self.tc('%s qdisc show dev %s') if "priomap" not in tcoutput: cmds = ['%s qdisc del dev %s root'] else: cmds = [] # Bandwidth limits via various methods bwcmds, parent = self.bwCmds(bw=bw, speedup=speedup, use_hfsc=use_hfsc, use_tbf=use_tbf, latency_ms=latency_ms, enable_ecn=enable_ecn, enable_red=enable_red) cmds += bwcmds # Delay/jitter/loss/max_queue_size using netem delaycmds, parent = self.delayCmds(delay=delay, jitter=jitter, loss=loss, max_queue_size=max_queue_size, parent=parent) cmds += delaycmds # Ugly but functional: display configuration info stuff = ((['%.2fMbit' % bw] if bw is not None else []) + (['%s delay' % delay] if delay is not None else []) + (['%s jitter' % jitter] if jitter is not None else []) + (['%d%% loss' % loss] if loss is not None else []) + (['ECN'] if enable_ecn else ['RED'] if enable_red else [])) info('(' + ' '.join(stuff) + ') ') # Execute all the commands in our node debug("at map stage w/cmds: %s\n" % cmds) tcoutputs = [self.tc(cmd) for cmd in cmds] for output in tcoutputs: if output != '' and output != 'RTNETLINK answers: No such file or directory\r\n': error("*** Error: %s" % output) debug("cmds:", cmds, '\n') debug("outputs:", tcoutputs, '\n') result['tcoutputs'] = tcoutputs result['parent'] = parent return result
def tc(self, cmd, tc='tc'): "Execute tc command for our interface" c = cmd % (tc, self) # Add in tc command and our name debug(" *** executing command: %s\n" % c) return self.cmd(c)