def do_ipall(self, line):
"Print ip of each host"
for node in self.mn.hosts:
output("%s -> %s\n" % (node.name, node.IP()))
if not self.mn.hosts:
output("No host found.\n")
def run(self):
"Run our cmdloop(), catching KeyboardInterrupt"
while True:
try:
# Make sure no nodes are still waiting
for node in self.mn.values():
while node.waiting:
info('stopping', node, '\n')
node.sendInt()
node.waitOutput()
if self.isatty():
quietRun('stty echo sane intr ^C')
self.cmdloop()
break
except KeyboardInterrupt:
# Output a message - unless it's also interrupted
# pylint: disable=broad-except
try:
output('\nInterrupt\n')
except Exception:
pass
# pylint: enable=broad-except
except Exception as e:
info("Exiting cmd loop due to error.\n")
self.mn.stop()
raise
def do_ip(self, line):
"Print ip of host"
node = self.mn.get(line)
if node is None:
output("Error: Node '%s' does not exist\n" % line)
else:
output("%s\n" % node.IP())
def errRun(*cmd, **kwargs):
"""Run a command and return stdout, stderr and return code
cmd: string or list of command and args
stderr: STDOUT to merge stderr with stdout
shell: run command using shell
echo: monitor output to console"""
# By default we separate stderr, don't run in a shell, and don't echo
stderr = kwargs.get('stderr', PIPE)
shell = kwargs.get('shell', False)
echo = kwargs.get('echo', False)
if echo:
# cmd goes to stderr, output goes to stdout
info(cmd, '\n')
if len(cmd) == 1:
cmd = cmd[0]
# Allow passing in a list or a string
if isinstance(cmd, str) and not shell:
cmd = cmd.split(' ')
cmd = [str(arg) for arg in cmd]
elif isinstance(cmd, list) and shell:
cmd = " ".join(arg for arg in cmd)
debug('*** errRun:', cmd, '\n')
popen = Popen(cmd, stdout=PIPE, stderr=stderr, shell=shell)
# We use poll() because select() doesn't work with large fd numbers,
# and thus communicate() doesn't work either
out, err = '', ''
poller = poll()
poller.register(popen.stdout, POLLIN)
fdtofile = {popen.stdout.fileno(): popen.stdout}
outDone, errDone = False, True
if popen.stderr:
fdtofile[popen.stderr.fileno()] = popen.stderr
poller.register(popen.stderr, POLLIN)
errDone = False
while not outDone or not errDone:
readable = poller.poll()
for fd, event in readable:
f = fdtofile[fd]
if event & POLLIN:
data = f.read(1024)
if echo:
output(data)
if f == popen.stdout:
out += data
if data == '':
outDone = True
elif f == popen.stderr:
err += data
if data == '':
errDone = True
else: # POLLHUP or something unexpected
if f == popen.stdout:
outDone = True
elif f == popen.stderr:
errDone = True
poller.unregister(fd)
returncode = popen.wait()
debug(out, err, returncode)
return out, err, returncode
def do_px(self, line):
"""Execute a Python statement.
Node names may be used, e.g.: px print h1.cmd('ls')"""
try:
exec(line, globals(), self.getLocals())
except Exception, e:
output(str(e) + '\n')
def do_placement( self, _line ):
"Describe node placement"
mn = self.mn
nodes = mn.hosts + mn.switches + mn.controllers
for server in mn.servers:
names = [ n.name for n in nodes if hasattr( n, 'server' )
and n.server == server ]
output( '%s: %s\n' % ( server, ' '.join( names ) ) )
def do_dpctl(self, line):
"""Run dpctl (or ovs-ofctl) command on all switches.
Usage: dpctl command [arg1] [arg2] ..."""
args = line.split()
if len(args) < 1:
error('usage: dpctl command [arg1] [arg2] ...\n')
return
for sw in self.mn.switches:
output('*** ' + sw.name + ' ' + ('-' * 72) + '\n')
output(sw.dpctl(*args))
def do_status( self, _line ):
"Report on node shell status"
nodes = self.mn.hosts + self.mn.switches
for node in nodes:
node.shell.poll()
exited = [ node for node in nodes
if node.shell.returncode is not None ]
if exited:
for node in exited:
output( '%s has exited with code %d\n'
% ( node, node.shell.returncode ) )
else:
output( 'All nodes are still running.\n' )
def iperf(self, hosts=None, l4Type='TCP', udpBw='10M', fmt=None,
seconds=5, port=5001):
"""Run iperf between two hosts.
hosts: list of hosts; if None, uses first and last hosts
l4Type: string, one of [ TCP, UDP ]
udpBw: bandwidth target for UDP test
fmt: iperf format argument if any
seconds: iperf time to transmit
port: iperf port
returns: two-element array of [ server, client ] speeds
note: send() is buffered, so client rate can be much higher than
the actual transmission rate; on an unloaded system, server
rate should be much closer to the actual receive rate"""
hosts = hosts or [self.hosts[0], self.hosts[-1]]
assert len(hosts) == 2
client, server = hosts
output('*** Iperf: testing', l4Type, 'bandwidth between',
client, 'and', server, '\n')
server.cmd('killall -9 iperf')
iperfArgs = 'iperf -p %d ' % port
bwArgs = ''
if l4Type == 'UDP':
iperfArgs += '-u '
bwArgs = '-b ' + udpBw + ' '
elif l4Type != 'TCP':
raise Exception('Unexpected l4 type: %s' % l4Type)
if fmt:
iperfArgs += '-f %s ' % fmt
server.sendCmd(iperfArgs + '-s')
if l4Type == 'TCP':
if not waitListening(client, server.IP(), port):
raise Exception('Could not connect to iperf on port %d'
% port)
cliout = client.cmd(iperfArgs + '-t %d -c ' % seconds +
server.IP() + ' ' + bwArgs)
debug('Client output: %s\n' % cliout)
servout = ''
# We want the last *b/sec from the iperf server output
# for TCP, there are two fo them because of waitListening
count = 2 if l4Type == 'TCP' else 1
while len(re.findall('/sec', servout)) < count:
servout += server.monitor(timeoutms=5000)
server.sendInt()
servout += server.waitOutput()
debug('Server output: %s\n' % servout)
result = [self._parseIperf(servout), self._parseIperf(cliout)]
if l4Type == 'UDP':
result.insert(0, udpBw)
output('*** Results: %s\n' % result)
return result
def runCpuLimitTest(self, cpu, duration=5):
"""run CPU limit test with 'while true' processes.
cpu: desired CPU fraction of each host
duration: test duration in seconds (integer)
returns a single list of measured CPU fractions as floats.
"""
cores = int(quietRun('nproc'))
pct = cpu * 100
info('*** Testing CPU %.0f%% bandwidth limit\n' % pct)
hosts = self.hosts
cores = int(quietRun('nproc'))
# number of processes to run a while loop on per host
num_procs = int(ceil(cores * cpu))
pids = {}
for h in hosts:
pids[h] = []
for _core in range(num_procs):
h.cmd('while true; do a=1; done &')
pids[h].append(h.cmd('echo $!').strip())
outputs = {}
time = {}
# get the initial cpu time for each host
for host in hosts:
outputs[host] = []
with open('/sys/fs/cgroup/cpuacct/%s/cpuacct.usage' %
host, 'r') as f:
time[host] = float(f.read())
for _ in range(duration):
sleep(1)
for host in hosts:
with open('/sys/fs/cgroup/cpuacct/%s/cpuacct.usage' %
host, 'r') as f:
readTime = float(f.read())
outputs[host].append(((readTime - time[host])
/ 1000000000) / cores * 100)
time[host] = readTime
for h, pids in pids.items():
for pid in pids:
h.cmd('kill -9 %s' % pid)
cpu_fractions = []
for _host, outputs in outputs.items():
for pct in outputs:
cpu_fractions.append(pct)
output('*** Results: %s\n' % cpu_fractions)
return cpu_fractions
def dumpNodeConnections(nodes):
"Dump connections to/from nodes."
def dumpConnections(node):
"Helper function: dump connections to node"
for intf in node.intfList():
output(' %s:' % intf)
if intf.link:
intfs = [intf.link.intf1, intf.link.intf2]
intfs.remove(intf)
output(intfs[0])
else:
output(' ')
for node in nodes:
output(node.name)
dumpConnections(node)
output('\n')
def dumpPorts(switches):
"dump interface to openflow port mappings for each switch"
for switch in switches:
output('%s ' % switch.name)
for intf in switch.intfList():
port = switch.ports[intf]
output('%s:%d ' % (intf, port))
output('\n')
def waitForNode(self, node):
"Wait for a node to finish, and print its output."
# Pollers
nodePoller = poll()
nodePoller.register(node.stdout)
bothPoller = poll()
bothPoller.register(self.stdin, POLLIN)
bothPoller.register(node.stdout, POLLIN)
if self.isatty():
# Buffer by character, so that interactive
# commands sort of work
quietRun('stty -isig -icanon min 1')
while node.shell:
try:
bothPoller.poll()
# XXX BL: this doesn't quite do what we want.
if False and self.inputFile:
key = self.inputFile.read(1)
if key is not '':
node.write(key)
else:
self.inputFile = None
if isReadable(self.inPoller):
key = self.stdin.read(1)
node.write(key)
if isReadable(nodePoller):
data = node.monitor()
output(data)
if not node.waiting:
quietRun('stty isig')
break
except KeyboardInterrupt:
# There is an at least one race condition here, since
# it's possible to interrupt ourselves after we've
# read data but before it has been printed.
node.sendInt()
def dumpConnections(node):
"Helper function: dump connections to node"
for intf in node.intfList():
output(' %s:' % intf)
if intf.link:
intfs = [intf.link.intf1, intf.link.intf2]
intfs.remove(intf)
output(intfs[0])
else:
output(' ')
def printConnections(switches):
"Compactly print connected nodes to each switch"
for sw in switches:
output('%s: ' % sw)
for intf in sw.intfList():
link = intf.link
if link:
intf1, intf2 = link.intf1, link.intf2
remote = intf1 if intf1.node != sw else intf2
output('%s(%s) ' % (remote.node, sw.ports[intf]))
output('\n')
def do_py(self, line):
"""Evaluate a Python expression.
Node names may be used, e.g.: py h1.cmd('ls')"""
try:
result = eval(line, globals(), self.getLocals())
if not result:
return
elif isinstance(result, str):
output(result + '\n')
else:
output(repr(result) + '\n')
except Exception, e:
output(str(e) + '\n')
def do_EOF(self, line):
"Exit"
output('\n')
return self.do_exit(line)
def do_help(self, line):
"Describe available CLI commands."
Cmd.do_help(self, line)
if line is '':
output(self.helpStr)
def pingFull(self, hosts=None, timeout=None, manualdestip=None):
"""Ping between all specified hosts and return all data.
hosts: list of hosts
timeout: time to wait for a response, as string
returns: all ping data; see function body."""
# should we check if running?
# Each value is a tuple: (src, dsd, [all ping outputs])
all_outputs = []
if not hosts:
hosts = self.hosts
output('*** Ping: testing ping reachability\n')
for node in hosts:
output('%s -> ' % node.name)
if manualdestip is not None:
opts = ''
if timeout:
opts = '-W %s' % timeout
result = node.cmd('ping -c1 %s %s' % (opts, manualdestip))
outputs = self._parsePingFull(result)
sent, received, rttmin, rttavg, rttmax, rttdev = outputs
all_outputs.append((node, manualdestip, outputs))
output(('%s ' % manualdestip) if received else 'X ')
output('\n')
else:
for dest in hosts:
if node != dest:
opts = ''
if timeout:
opts = '-W %s' % timeout
result = node.cmd('ping -c1 %s %s' % (opts, dest.IP()))
outputs = self._parsePingFull(result)
sent, received, rttmin, rttavg, rttmax, rttdev = outputs
all_outputs.append((node, dest, outputs))
output(('%s ' % dest.name) if received else 'X ')
output('\n')
output("*** Results: \n")
for outputs in all_outputs:
src, dest, ping_outputs = outputs
sent, received, rttmin, rttavg, rttmax, rttdev = ping_outputs
output(" %s->%s: %s/%s, " % (src, dest, sent, received))
output("rtt min/avg/max/mdev %0.3f/%0.3f/%0.3f/%0.3f ms\n" %
(rttmin, rttavg, rttmax, rttdev))
return all_outputs
def ping(self, hosts=None, timeout=None, manualdestip=None):
"""Ping between all specified hosts.
hosts: list of hosts
timeout: time to wait for a response, as string
manualdestip: sends pings from each h in hosts to manualdestip
returns: ploss packet loss percentage"""
# should we check if running?
packets = 0
lost = 0
ploss = None
if not hosts:
hosts = self.hosts
output('*** Ping: testing ping reachability\n')
for node in hosts:
output('%s -> ' % node.name)
if manualdestip is not None:
opts = ''
if timeout:
opts = '-W %s' % timeout
result = node.cmd('ping -c1 %s %s' %
(opts, manualdestip))
sent, received = self._parsePing(result)
packets += sent
if received > sent:
error('*** Error: received too many packets')
error('%s' % result)
node.cmdPrint('route')
exit(1)
lost += sent - received
output(('%s ' % manualdestip) if received else 'X ')
else:
for dest in hosts:
if node != dest:
opts = ''
if timeout:
opts = '-W %s' % timeout
if dest.intfs:
result = node.cmd('ping -c1 %s %s' %
(opts, dest.IP()))
sent, received = self._parsePing(result)
else:
sent, received = 0, 0
packets += sent
if received > sent:
error('*** Error: received too many packets')
error('%s' % result)
node.cmdPrint('route')
exit(1)
lost += sent - received
output(('%s ' % dest.name) if received else 'X ')
output('\n')
if packets > 0:
ploss = 100.0 * lost / packets
received = packets - lost
output("*** Results: %i%% dropped (%d/%d received)\n" %
(ploss, received, packets))
else:
ploss = 0
output("*** Warning: No packets sent\n")
return ploss
def do_nodes(self, _line):
"List all nodes."
nodes = ' '.join(sorted(self.mn))
output('available nodes are: \n%s\n' % nodes)
def do_traffic(self, line):
switch_map = {}
intf_map = {}
for switch in self.mn.switches:
switch_map.update(switch.traffic_by_interface())
for link in self.mn.links:
intf_map[link.intf1.name] = link.intf2.name
for node in self.mn.hosts:
output(node.name + '\n')
for intf in node.nameToIntf.keys():
output(" " + intf + '\n')
output(" received:\n")
map_rx = switch_map[intf_map[intf]]['tx']
for attr in map_rx.keys():
output(" " + attr + ":" + str(map_rx[attr]) + '\n')
output(" transmitted:\n")
map_tx = switch_map[intf_map[intf]]['rx']
for attr in map_tx.keys():
output(" " + attr + ":" + str(map_tx[attr]) + '\n')
def do_print(self, line):
"print input line"
output('%s\n' % line)
def do_dump(self, _line):
"Dump node info."
for node in self.mn.values():
output('%s\n' % repr(node))
def do_intfs(self, _line):
"List interfaces."
for node in self.mn.values():
output('%s: %s\n' % (node.name, ','.join(node.intfNames())))
