def do_ipall(self, line): "Print ip of each host" for node in self.mn.hosts: output("%s -> %s\n" % (node.name, node.IP())) if not self.mn.hosts: output("No host found.\n")
def run(self): "Run our cmdloop(), catching KeyboardInterrupt" while True: try: # Make sure no nodes are still waiting for node in self.mn.values(): while node.waiting: info('stopping', node, '\n') node.sendInt() node.waitOutput() if self.isatty(): quietRun('stty echo sane intr ^C') self.cmdloop() break except KeyboardInterrupt: # Output a message - unless it's also interrupted # pylint: disable=broad-except try: output('\nInterrupt\n') except Exception: pass # pylint: enable=broad-except except Exception as e: info("Exiting cmd loop due to error.\n") self.mn.stop() raise
def do_ip(self, line): "Print ip of host" node = self.mn.get(line) if node is None: output("Error: Node '%s' does not exist\n" % line) else: output("%s\n" % node.IP())
def errRun(*cmd, **kwargs): """Run a command and return stdout, stderr and return code cmd: string or list of command and args stderr: STDOUT to merge stderr with stdout shell: run command using shell echo: monitor output to console""" # By default we separate stderr, don't run in a shell, and don't echo stderr = kwargs.get('stderr', PIPE) shell = kwargs.get('shell', False) echo = kwargs.get('echo', False) if echo: # cmd goes to stderr, output goes to stdout info(cmd, '\n') if len(cmd) == 1: cmd = cmd[0] # Allow passing in a list or a string if isinstance(cmd, str) and not shell: cmd = cmd.split(' ') cmd = [str(arg) for arg in cmd] elif isinstance(cmd, list) and shell: cmd = " ".join(arg for arg in cmd) debug('*** errRun:', cmd, '\n') popen = Popen(cmd, stdout=PIPE, stderr=stderr, shell=shell) # We use poll() because select() doesn't work with large fd numbers, # and thus communicate() doesn't work either out, err = '', '' poller = poll() poller.register(popen.stdout, POLLIN) fdtofile = {popen.stdout.fileno(): popen.stdout} outDone, errDone = False, True if popen.stderr: fdtofile[popen.stderr.fileno()] = popen.stderr poller.register(popen.stderr, POLLIN) errDone = False while not outDone or not errDone: readable = poller.poll() for fd, event in readable: f = fdtofile[fd] if event & POLLIN: data = f.read(1024) if echo: output(data) if f == popen.stdout: out += data if data == '': outDone = True elif f == popen.stderr: err += data if data == '': errDone = True else: # POLLHUP or something unexpected if f == popen.stdout: outDone = True elif f == popen.stderr: errDone = True poller.unregister(fd) returncode = popen.wait() debug(out, err, returncode) return out, err, returncode
def do_px(self, line): """Execute a Python statement. Node names may be used, e.g.: px print h1.cmd('ls')""" try: exec(line, globals(), self.getLocals()) except Exception, e: output(str(e) + '\n')
def do_placement( self, _line ): "Describe node placement" mn = self.mn nodes = mn.hosts + mn.switches + mn.controllers for server in mn.servers: names = [ n.name for n in nodes if hasattr( n, 'server' ) and n.server == server ] output( '%s: %s\n' % ( server, ' '.join( names ) ) )
def do_dpctl(self, line): """Run dpctl (or ovs-ofctl) command on all switches. Usage: dpctl command [arg1] [arg2] ...""" args = line.split() if len(args) < 1: error('usage: dpctl command [arg1] [arg2] ...\n') return for sw in self.mn.switches: output('*** ' + sw.name + ' ' + ('-' * 72) + '\n') output(sw.dpctl(*args))
def do_status( self, _line ): "Report on node shell status" nodes = self.mn.hosts + self.mn.switches for node in nodes: node.shell.poll() exited = [ node for node in nodes if node.shell.returncode is not None ] if exited: for node in exited: output( '%s has exited with code %d\n' % ( node, node.shell.returncode ) ) else: output( 'All nodes are still running.\n' )
def iperf(self, hosts=None, l4Type='TCP', udpBw='10M', fmt=None, seconds=5, port=5001): """Run iperf between two hosts. hosts: list of hosts; if None, uses first and last hosts l4Type: string, one of [ TCP, UDP ] udpBw: bandwidth target for UDP test fmt: iperf format argument if any seconds: iperf time to transmit port: iperf port returns: two-element array of [ server, client ] speeds note: send() is buffered, so client rate can be much higher than the actual transmission rate; on an unloaded system, server rate should be much closer to the actual receive rate""" hosts = hosts or [self.hosts[0], self.hosts[-1]] assert len(hosts) == 2 client, server = hosts output('*** Iperf: testing', l4Type, 'bandwidth between', client, 'and', server, '\n') server.cmd('killall -9 iperf') iperfArgs = 'iperf -p %d ' % port bwArgs = '' if l4Type == 'UDP': iperfArgs += '-u ' bwArgs = '-b ' + udpBw + ' ' elif l4Type != 'TCP': raise Exception('Unexpected l4 type: %s' % l4Type) if fmt: iperfArgs += '-f %s ' % fmt server.sendCmd(iperfArgs + '-s') if l4Type == 'TCP': if not waitListening(client, server.IP(), port): raise Exception('Could not connect to iperf on port %d' % port) cliout = client.cmd(iperfArgs + '-t %d -c ' % seconds + server.IP() + ' ' + bwArgs) debug('Client output: %s\n' % cliout) servout = '' # We want the last *b/sec from the iperf server output # for TCP, there are two fo them because of waitListening count = 2 if l4Type == 'TCP' else 1 while len(re.findall('/sec', servout)) < count: servout += server.monitor(timeoutms=5000) server.sendInt() servout += server.waitOutput() debug('Server output: %s\n' % servout) result = [self._parseIperf(servout), self._parseIperf(cliout)] if l4Type == 'UDP': result.insert(0, udpBw) output('*** Results: %s\n' % result) return result
def runCpuLimitTest(self, cpu, duration=5): """run CPU limit test with 'while true' processes. cpu: desired CPU fraction of each host duration: test duration in seconds (integer) returns a single list of measured CPU fractions as floats. """ cores = int(quietRun('nproc')) pct = cpu * 100 info('*** Testing CPU %.0f%% bandwidth limit\n' % pct) hosts = self.hosts cores = int(quietRun('nproc')) # number of processes to run a while loop on per host num_procs = int(ceil(cores * cpu)) pids = {} for h in hosts: pids[h] = [] for _core in range(num_procs): h.cmd('while true; do a=1; done &') pids[h].append(h.cmd('echo $!').strip()) outputs = {} time = {} # get the initial cpu time for each host for host in hosts: outputs[host] = [] with open('/sys/fs/cgroup/cpuacct/%s/cpuacct.usage' % host, 'r') as f: time[host] = float(f.read()) for _ in range(duration): sleep(1) for host in hosts: with open('/sys/fs/cgroup/cpuacct/%s/cpuacct.usage' % host, 'r') as f: readTime = float(f.read()) outputs[host].append(((readTime - time[host]) / 1000000000) / cores * 100) time[host] = readTime for h, pids in pids.items(): for pid in pids: h.cmd('kill -9 %s' % pid) cpu_fractions = [] for _host, outputs in outputs.items(): for pct in outputs: cpu_fractions.append(pct) output('*** Results: %s\n' % cpu_fractions) return cpu_fractions
def dumpNodeConnections(nodes): "Dump connections to/from nodes." def dumpConnections(node): "Helper function: dump connections to node" for intf in node.intfList(): output(' %s:' % intf) if intf.link: intfs = [intf.link.intf1, intf.link.intf2] intfs.remove(intf) output(intfs[0]) else: output(' ') for node in nodes: output(node.name) dumpConnections(node) output('\n')
def dumpPorts(switches): "dump interface to openflow port mappings for each switch" for switch in switches: output('%s ' % switch.name) for intf in switch.intfList(): port = switch.ports[intf] output('%s:%d ' % (intf, port)) output('\n')
def waitForNode(self, node): "Wait for a node to finish, and print its output." # Pollers nodePoller = poll() nodePoller.register(node.stdout) bothPoller = poll() bothPoller.register(self.stdin, POLLIN) bothPoller.register(node.stdout, POLLIN) if self.isatty(): # Buffer by character, so that interactive # commands sort of work quietRun('stty -isig -icanon min 1') while node.shell: try: bothPoller.poll() # XXX BL: this doesn't quite do what we want. if False and self.inputFile: key = self.inputFile.read(1) if key is not '': node.write(key) else: self.inputFile = None if isReadable(self.inPoller): key = self.stdin.read(1) node.write(key) if isReadable(nodePoller): data = node.monitor() output(data) if not node.waiting: quietRun('stty isig') break except KeyboardInterrupt: # There is an at least one race condition here, since # it's possible to interrupt ourselves after we've # read data but before it has been printed. node.sendInt()
def dumpConnections(node): "Helper function: dump connections to node" for intf in node.intfList(): output(' %s:' % intf) if intf.link: intfs = [intf.link.intf1, intf.link.intf2] intfs.remove(intf) output(intfs[0]) else: output(' ')
def printConnections(switches): "Compactly print connected nodes to each switch" for sw in switches: output('%s: ' % sw) for intf in sw.intfList(): link = intf.link if link: intf1, intf2 = link.intf1, link.intf2 remote = intf1 if intf1.node != sw else intf2 output('%s(%s) ' % (remote.node, sw.ports[intf])) output('\n')
def do_py(self, line): """Evaluate a Python expression. Node names may be used, e.g.: py h1.cmd('ls')""" try: result = eval(line, globals(), self.getLocals()) if not result: return elif isinstance(result, str): output(result + '\n') else: output(repr(result) + '\n') except Exception, e: output(str(e) + '\n')
def do_EOF(self, line): "Exit" output('\n') return self.do_exit(line)
def do_help(self, line): "Describe available CLI commands." Cmd.do_help(self, line) if line is '': output(self.helpStr)
def pingFull(self, hosts=None, timeout=None, manualdestip=None): """Ping between all specified hosts and return all data. hosts: list of hosts timeout: time to wait for a response, as string returns: all ping data; see function body.""" # should we check if running? # Each value is a tuple: (src, dsd, [all ping outputs]) all_outputs = [] if not hosts: hosts = self.hosts output('*** Ping: testing ping reachability\n') for node in hosts: output('%s -> ' % node.name) if manualdestip is not None: opts = '' if timeout: opts = '-W %s' % timeout result = node.cmd('ping -c1 %s %s' % (opts, manualdestip)) outputs = self._parsePingFull(result) sent, received, rttmin, rttavg, rttmax, rttdev = outputs all_outputs.append((node, manualdestip, outputs)) output(('%s ' % manualdestip) if received else 'X ') output('\n') else: for dest in hosts: if node != dest: opts = '' if timeout: opts = '-W %s' % timeout result = node.cmd('ping -c1 %s %s' % (opts, dest.IP())) outputs = self._parsePingFull(result) sent, received, rttmin, rttavg, rttmax, rttdev = outputs all_outputs.append((node, dest, outputs)) output(('%s ' % dest.name) if received else 'X ') output('\n') output("*** Results: \n") for outputs in all_outputs: src, dest, ping_outputs = outputs sent, received, rttmin, rttavg, rttmax, rttdev = ping_outputs output(" %s->%s: %s/%s, " % (src, dest, sent, received)) output("rtt min/avg/max/mdev %0.3f/%0.3f/%0.3f/%0.3f ms\n" % (rttmin, rttavg, rttmax, rttdev)) return all_outputs
def ping(self, hosts=None, timeout=None, manualdestip=None): """Ping between all specified hosts. hosts: list of hosts timeout: time to wait for a response, as string manualdestip: sends pings from each h in hosts to manualdestip returns: ploss packet loss percentage""" # should we check if running? packets = 0 lost = 0 ploss = None if not hosts: hosts = self.hosts output('*** Ping: testing ping reachability\n') for node in hosts: output('%s -> ' % node.name) if manualdestip is not None: opts = '' if timeout: opts = '-W %s' % timeout result = node.cmd('ping -c1 %s %s' % (opts, manualdestip)) sent, received = self._parsePing(result) packets += sent if received > sent: error('*** Error: received too many packets') error('%s' % result) node.cmdPrint('route') exit(1) lost += sent - received output(('%s ' % manualdestip) if received else 'X ') else: for dest in hosts: if node != dest: opts = '' if timeout: opts = '-W %s' % timeout if dest.intfs: result = node.cmd('ping -c1 %s %s' % (opts, dest.IP())) sent, received = self._parsePing(result) else: sent, received = 0, 0 packets += sent if received > sent: error('*** Error: received too many packets') error('%s' % result) node.cmdPrint('route') exit(1) lost += sent - received output(('%s ' % dest.name) if received else 'X ') output('\n') if packets > 0: ploss = 100.0 * lost / packets received = packets - lost output("*** Results: %i%% dropped (%d/%d received)\n" % (ploss, received, packets)) else: ploss = 0 output("*** Warning: No packets sent\n") return ploss
def do_nodes(self, _line): "List all nodes." nodes = ' '.join(sorted(self.mn)) output('available nodes are: \n%s\n' % nodes)
def do_traffic(self, line): switch_map = {} intf_map = {} for switch in self.mn.switches: switch_map.update(switch.traffic_by_interface()) for link in self.mn.links: intf_map[link.intf1.name] = link.intf2.name for node in self.mn.hosts: output(node.name + '\n') for intf in node.nameToIntf.keys(): output(" " + intf + '\n') output(" received:\n") map_rx = switch_map[intf_map[intf]]['tx'] for attr in map_rx.keys(): output(" " + attr + ":" + str(map_rx[attr]) + '\n') output(" transmitted:\n") map_tx = switch_map[intf_map[intf]]['rx'] for attr in map_tx.keys(): output(" " + attr + ":" + str(map_tx[attr]) + '\n')
def do_print(self, line): "print input line" output('%s\n' % line)
def do_dump(self, _line): "Dump node info." for node in self.mn.values(): output('%s\n' % repr(node))
def do_intfs(self, _line): "List interfaces." for node in self.mn.values(): output('%s: %s\n' % (node.name, ','.join(node.intfNames())))