def parse_cpus(self, lscpu_output):
"""
:param lscpu_output: string, output of `lscpu -p`
"""
if self.options.cpus: # no need to detect topology if user gave us cpu list
return
self.topology = Topology(lscpu_output=lscpu_output)
self.color = Color(self.topology, self.options.color)
if not any([self.options.socket is not None, self.options.cpus]):
self.socket_detect()
self.pci.devices = self.pci.node_dev_dict([self.options.dev], False)
def __init__(self):
BaseTop.__init__(self)
self.tops = {
'irqtop': IrqTop(),
'softnet_stat_top': SoftnetStatTop(),
'softirq_top': Softirqs(),
'link-rate': LinkRateTop(),
}
self.parse_options()
self.topology = Topology(fake=self.options.random,
lscpu_output=self.options.lscpu_output)
self.color = Color(self.topology)
for top in self.tops.values():
top.topology = self.topology
top.color = self.color
def post_optparse(self):
""" Asserting and applying parsing options """
self.options.devices = self.devices_list()
if not self.options.devices:
raise ValueError("No devices've been specified")
if self.options.rx_only:
self.stats = [
stat for stat in self.stats if stat.filename.startswith('rx')]
if self.options.simple_mode:
simple_stats = ('rx_packets', 'rx_bytes', 'rx_errors', 'tx_packets', 'tx_bytes', 'tx_errors')
self.stats = [
stat for stat in self.stats if stat.filename in simple_stats]
self.unit_change()
self.header = self.make_header()
self.align_map = ['l'] + ['r'] * (len(self.header) - 1)
if not self.pci:
self.pci = PCI()
self.pci.devices = self.pci.node_dev_dict(self.options.devices, self.options.random)
self.color = Color(topology=None, enabled=self.options.color)
def colorize_net_tx(net_tx):
""" :returns: highlighted by warning/error net_tx string """
return Color.colorize(net_tx, 20000, 30000)
def colorize_net_rx(net_rx):
""" :returns: highlighted by warning/error net_rx string """
return Color.colorize(net_rx, 40000, 80000)
class BaseTop(object):
""" Base class for all these top-like utils. """
current = None
previous = None
diff = None
header = Color.wrap('Press CTRL-C to exit...\n', Color.GREY)
options = None
file_arg = None
file_value = None
topology = None
color = None
@staticmethod
def make_base_parser(parser=None):
""" That should be explicitly called in __main__ part of any top-like utils """
if not parser:
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--interval', default=1, type=int,
help='Interval between screen renew in seconds.')
parser.add_argument('-n', '--iterations', dest='iterations', default=60, type=int,
help='Count of screen\'s renews, -1 - infinite loop.')
parser.add_argument('--no-delta-mode', action='store_false', dest='delta_mode',
default=True, help="Shows metrics' values instead of growth.")
parser.add_argument('--no-delta-small-hide', action='store_false',
dest='delta_small_hide', default=True,
help='Prevent lines with only small changes or without'
'changes at all from hiding.')
parser.add_argument('-l', '--delta-small-hide-limit', default=80, type=int,
help='Hides lines with only changes less than this limit')
parser.add_argument('--no-color', dest='color', default=True, action='store_false',
help="Don't highlight NUMA nodes or sockets")
parser.add_argument('--spaces', default=False, action='store_true',
help="Add spaces in numbers' representation, e.g. '1234567' "
"will be '1 234 567'")
parser.add_argument('--random', default=False, action='store_true',
help='Shows random diff data instead of real evaluation. '
'Helpful for testing on static files')
parser.add_argument('--no-clear', default=True, dest='clear', action='store_false',
help="Don't clear screen after each iteration. "
"May be useful in scripts/logging to file.")
parser.add_argument('--lscpu-output', help='Specify file with lscpu -p output')
return parser
def make_parser(self, parser=None):
if type(self) == BaseTop:
raise TypeError('make_parser should not be called directly by BaseTop')
if not parser:
parser = BaseTop.make_base_parser()
parser.add_argument(self.file_arg, default=self.file_value, help='Option for testing on MacOS purpose.')
return parser
def tick(self):
""" Gathers new data + evaluate diff between current & previous data """
self.previous = self.current
self.current = self.parse()
if all((self.previous, self.current)):
self.eval()
def list_diff(self, current, previous):
""" It's strange that there is no [3,4,3] - [1,2,1] -> [2,2,2] in standard library """
if self.options.random:
return [randint(0, 10000) for _ in current]
return [data - previous[n] for n, data in enumerate(current)]
def run(self):
""" Default main()-like function for specific top-like utils except meta-utils. """
infinite = -1
if self.options.iterations != infinite:
self.options.iterations += 1
try:
while self.options.iterations > 0 or self.options.iterations == infinite:
if self.options.iterations != infinite:
self.options.iterations -= 1
sleep(self.options.interval)
self.tick()
if self.options.clear:
system('clear')
if self.diff:
print_(self)
except KeyboardInterrupt:
print_()
exit(0)
def repr_source(self):
return self.diff if self.options.delta_mode else self.current
@staticmethod
def int(item):
return int(item) if item.isdigit() else item
def spaces(self, number, sep=' '):
""" 1234567 -> 1 234 567 """
if not self.options.spaces:
return number
output = str()
while number / 1000 > 0:
output = str(number % 1000).zfill(3) + sep + output
number /= 1000
return (str(number % 1000) + sep + output).strip()
def __repr_table__(self, table):
if self.options.clear:
return BaseTop.header + str(table)
return str(table)
def default_init(self, topology=None):
BaseTop.__init__(self)
self.topology = topology
self.color = Color(self.topology)
def default_post_optparse(self):
if not self.topology:
self.topology = Topology(fake=self.options.random)
self.color = Color(self.topology, self.options.color)
@abstractmethod
def parse(self):
""" Should read some file(s) into python structure (dict/list) """
@abstractmethod
def eval(self):
""" Should evaluate self.diff using self.previous / self.current """
@abstractmethod
def __repr__(self):
""" Should return string, representing self.diff """
def main(self):
""" Default entry point for most of top-like utils """
self.options = self.make_parser().parse_args()
if hasattr(self, 'post_optparse'):
self.post_optparse()
self.run()
def default_post_optparse(self):
if not self.topology:
self.topology = Topology(fake=self.options.random)
self.color = Color(self.topology, self.options.color)
def default_init(self, topology=None):
BaseTop.__init__(self)
self.topology = topology
self.color = Color(self.topology)
def colorize_stat(stat, value):
return Color.colorize(value, 1, 1) if 'errors' in stat.filename or 'dropped' in stat.filename else value
class RSSLadder(CPUBasedTune):
""" Distributor of queues' interrupts by multiple CPUs """
topology = None
color = None
interrupts_file = None
def __init__(self, argv=None):
if argv:
sys.argv = [sys.argv[0]] + argv
CPUBasedTune.__init__(self)
self.interrupts_file, lscpu_output = self.parse()
if not exists(self.interrupts_file): # unit-tests
return
self.parse_cpus(lscpu_output)
self.eval()
def parse(self):
""" Detects sources of system data """
interrupts_file = '/proc/interrupts'
lscpu_output = None
if self.options.test_dir:
interrupts_file = join(self.options.test_dir, 'interrupts')
lscpu_output_filename = join(self.options.test_dir, 'lscpu_output')
lscpu_output = open(lscpu_output_filename).read()
# Popen.stdout: python 2.7 returns <str>, 3.6 returns <bytes>
# read() in both cases return <str>
if isinstance(lscpu_output, bytes):
lscpu_output = str(lscpu_output)
return interrupts_file, lscpu_output
def eval(self):
""" Top of all the logic, decide what to do and then apply new settings """
interrupts = open(self.interrupts_file).readlines()
for postfix in sorted(self.queue_postfixes_detect(interrupts)):
self.apply(self.__eval(postfix, interrupts))
def apply(self, decision):
"""
'* 4' is in case of NIC has more queues than socket has CPUs
:param decision: list of tuples(irq, queue_name, socket)
"""
affinity = list(decision)
cpus = [socket_cpu for irq, queue, socket_cpu in affinity]
if len(set(cpus)) != len(cpus):
warning = 'WARNING: some CPUs process multiple queues, consider reduce queue count for this network device'
if self.options.color:
print_(self.color.wrap(warning, Color.YELLOW))
else:
print_(warning)
for irq, queue_name, socket_cpu in affinity:
print_(" - {0}: queue {1} (irq {2}) bound to CPU{3}".format(
self.dev_colorize(), queue_name, irq,
self.cpu_colorize(socket_cpu)))
if self.options.dry_run:
continue
filename = '/proc/irq/{0}/smp_affinity_list'.format(irq)
with open(filename, 'w') as irq_file:
irq_file.write(str(socket_cpu))
def __eval(self, postfix, interrupts):
"""
:param postfix: '-TxRx-'
:return: list of tuples(irq, queue_name, socket)
"""
print_('- distribute interrupts of {0} ({1}) on socket {2}'.format(
self.options.dev, postfix, self.options.socket))
queue_regex = r'{0}{1}[^ \n]+'.format(self.options.dev, postfix)
rss_cpus = self.rss_cpus_detect()
for _ in xrange(self.options.offset):
rss_cpus.pop()
for line in interrupts:
queue_name = re.findall(queue_regex, line)
if queue_name:
yield any2int(line.split()[0]), queue_name[0], rss_cpus.pop()
def parse_cpus(self, lscpu_output):
"""
:param lscpu_output: string, output of `lscpu -p`
"""
if self.options.cpus: # no need to detect topology if user gave us cpu list
return
self.topology = Topology(lscpu_output=lscpu_output)
self.color = Color(self.topology, self.options.color)
if not any([self.options.socket is not None, self.options.cpus]):
self.socket_detect()
self.pci.devices = self.pci.node_dev_dict([self.options.dev], False)
def queue_postfix_extract(self, line):
"""
:param line: '31312 0 0 0 blabla eth0-TxRx-0'
:return: '-TxRx-'
"""
queue_regex = r'{0}[^ \n]+'.format(self.options.dev)
queue_name = re.findall(queue_regex, line)
if queue_name:
return re.sub(r'({0}|[0-9])'.format(self.options.dev), '',
queue_name[0])
def queue_postfixes_detect(self, interrupts):
"""
self.dev: eth0
:return: '-TxRx-'
"""
return set([
line for line in
[self.queue_postfix_extract(line) for line in interrupts] if line
])
def rss_cpus_detect(self):
"""
:return: list of cpu ids to use with current queues distribution
"""
rss_cpus = self.options.cpus if self.options.cpus else self.cpus_detect_real(
)
rss_cpus.reverse()
return rss_cpus * MAX_QUEUE_PER_DEVICE
def dev_colorize(self):
"""
:return: highlighted by NUMA-node name of the device
"""
if not self.pci or not self.options.color or not self.pci.devices:
return self.options.dev
dev_color = self.pci.devices.get(self.options.dev)
color = self.color.COLORS_NODE.get(dev_color)
return self.color.wrap(self.options.dev, color)
def cpu_colorize(self, cpu):
"""
:param cpu: cpu number (0)
:return: highlighted by NUMA-node cpu number.
"""
if not self.topology:
return cpu
return self.color.wrap(cpu, self.color.colorize_cpu(cpu))
def parse_options(self):
"""
:return: parsed arguments
"""
parser = CPUBasedTune.make_parser()
parser.add_argument('--no-color',
help='Disable all highlights',
dest='color',
action='store_false',
default=True)
parser.add_argument(
'-o',
'--offset',
type=int,
default=0,
help=
'If you have 2 NICs with 4 queues and 1 socket with 8 cpus, you may be want '
'distribution like this: eth0: [0, 1, 2, 3]; eth1: [4, 5, 6, 7]; '
'so run: rss-ladder-test eth0; rss-ladder-test --offset=4 eth1')
return parser.parse_args()
def colorize_irq_per_cpu(irq_per_cpu):
""" :returns: highlighted by warning/error irq string """
return Color.colorize(irq_per_cpu, 40000, 80000)
class NetworkTop(BaseTop):
""" Global top-like util that combine information from 4 other tops """
def __init__(self):
BaseTop.__init__(self)
self.tops = {
'irqtop': IrqTop(),
'softnet_stat_top': SoftnetStatTop(),
'softirq_top': Softirqs(),
'link-rate': LinkRateTop(),
}
self.parse_options()
self.topology = Topology(fake=self.options.random,
lscpu_output=self.options.lscpu_output)
self.color = Color(self.topology)
for top in self.tops.values():
top.topology = self.topology
top.color = self.color
def parse(self):
"""
:return: dict with parsed results for each top-like object.
"""
return dict((top_name, _top.parse())
for top_name, _top in iteritems(self.tops))
def eval(self):
"""
:return: evaluates diff for each top-like object.
"""
if all((self.current, self.previous)):
self.diff = dict((top_name, _top.diff)
for top_name, _top in iteritems(self.tops))
def tick(self):
self.previous = self.current
self.current = self.parse()
for _top in itervalues(self.tops):
_top.tick()
self.eval()
def __repr__(self):
output = [
BaseTop.header,
self.__repr_irq(),
self.__repr_cpu(),
self.__repr_dev(),
]
if not self.options.clear:
del output[0]
return '\n'.join(output)
def parse_options(self):
""" Tricky way to gather all options in one util without conflicts, parse them and do some logic after parse """
parser = BaseTop.make_base_parser()
for top in itervalues(self.tops):
parser = top.make_parser(parser)
self.options = parser.parse_args()
if self.options.lscpu_output:
self.options.lscpu_output = open(self.options.lscpu_output).read()
for top in itervalues(self.tops):
top.options = self.options
if hasattr(top, 'post_optparse'):
top.post_optparse()
def __repr_dev(self):
top = self.tops.get('link-rate')
table = make_table(top.make_header(), top.align_map, top.make_rows())
return self.color.wrap_header('Network devices') + str(table)
def __repr_irq(self):
top = self.tops.get('irqtop')
output_lines, cpu_count = top.make_rows()
align_map = top.make_align_map(cpu_count)
table = make_table(output_lines[0], align_map, output_lines[1:])
return self.color.wrap_header('/proc/interrupts') + str(table)
def __repr_cpu(self):
irqtop = self.tops.get('irqtop')
softirq_top = self.tops.get('softirq_top')
softnet_stat_top = self.tops.get('softnet_stat_top')
# all these evaluations are better to put in softirqs.parse()
active_cpu = softirq_top.__active_cpu_count__(softirq_top.current)
softnet_stat_top_output = softnet_stat_top.repr_source()
network_output = zip(irqtop.diff_total,
softirq_top.repr_source()['NET_RX'][:active_cpu],
softirq_top.repr_source()['NET_TX'][:active_cpu],
softnet_stat_top_output)
fields = [
'CPU',
'Interrupts',
'NET RX',
'NET TX',
'total',
'dropped',
'time_squeeze',
'cpu_collision',
'received_rps',
]
fields = [self.color.bright(word) for word in fields]
rows = self.__repr_cpu_make_rows(irqtop, network_output, softirq_top,
softnet_stat_top)
table = make_table(fields, ['l'] + ['r'] * (len(fields) - 1), rows)
return self.color.wrap_header('Load per cpu:') + str(table)
def __repr_cpu_make_rows(self, irqtop, network_output, softirq_top,
softnet_stat_top):
return [[
self.color.wrap('CPU{0}'.format(stat.cpu),
self.color.colorize_cpu(stat.cpu)),
irqtop.colorize_irq_per_cpu(irq),
softirq_top.colorize_net_rx(net_rx),
softirq_top.colorize_net_tx(net_tx),
softnet_stat_top.colorize_total(stat.total),
softnet_stat_top.colorize_dropped(stat.dropped),
softnet_stat_top.colorize_time_squeeze(stat.time_squeeze),
softnet_stat_top.colorize_cpu_collision(stat.cpu_collision),
stat.received_rps
] for irq, net_rx, net_tx, stat in network_output]
def colorize_cpu_collision(cpu_collision):
""" :returns: highlighted by warning/error cpu_collision string """
return Color.colorize(cpu_collision, 1, 1000)
def colorize_time_squeeze(time_squeeze):
""" :returns: highlighted by warning/error time_squeeze string """
return Color.colorize(time_squeeze, 1, 300)
def colorize_dropped(dropped):
""" :returns: highlighted by warning/error dropped string """
return Color.colorize(dropped, 1, 1)
def colorize_total(total):
""" :returns: highlighted by warning/error total string """
return Color.colorize(total, 300000, 900000)
