def parse_cpus(self, lscpu_output): """ :param lscpu_output: string, output of `lscpu -p` """ if self.options.cpus: # no need to detect topology if user gave us cpu list return self.topology = Topology(lscpu_output=lscpu_output) self.color = Color(self.topology, self.options.color) if not any([self.options.socket is not None, self.options.cpus]): self.socket_detect() self.pci.devices = self.pci.node_dev_dict([self.options.dev], False)
def __init__(self): BaseTop.__init__(self) self.tops = { 'irqtop': IrqTop(), 'softnet_stat_top': SoftnetStatTop(), 'softirq_top': Softirqs(), 'link-rate': LinkRateTop(), } self.parse_options() self.topology = Topology(fake=self.options.random, lscpu_output=self.options.lscpu_output) self.color = Color(self.topology) for top in self.tops.values(): top.topology = self.topology top.color = self.color
def post_optparse(self): """ Asserting and applying parsing options """ self.options.devices = self.devices_list() if not self.options.devices: raise ValueError("No devices've been specified") if self.options.rx_only: self.stats = [ stat for stat in self.stats if stat.filename.startswith('rx')] if self.options.simple_mode: simple_stats = ('rx_packets', 'rx_bytes', 'rx_errors', 'tx_packets', 'tx_bytes', 'tx_errors') self.stats = [ stat for stat in self.stats if stat.filename in simple_stats] self.unit_change() self.header = self.make_header() self.align_map = ['l'] + ['r'] * (len(self.header) - 1) if not self.pci: self.pci = PCI() self.pci.devices = self.pci.node_dev_dict(self.options.devices, self.options.random) self.color = Color(topology=None, enabled=self.options.color)
def colorize_net_tx(net_tx): """ :returns: highlighted by warning/error net_tx string """ return Color.colorize(net_tx, 20000, 30000)
def colorize_net_rx(net_rx): """ :returns: highlighted by warning/error net_rx string """ return Color.colorize(net_rx, 40000, 80000)
class BaseTop(object): """ Base class for all these top-like utils. """ current = None previous = None diff = None header = Color.wrap('Press CTRL-C to exit...\n', Color.GREY) options = None file_arg = None file_value = None topology = None color = None @staticmethod def make_base_parser(parser=None): """ That should be explicitly called in __main__ part of any top-like utils """ if not parser: parser = argparse.ArgumentParser() parser.add_argument('-i', '--interval', default=1, type=int, help='Interval between screen renew in seconds.') parser.add_argument('-n', '--iterations', dest='iterations', default=60, type=int, help='Count of screen\'s renews, -1 - infinite loop.') parser.add_argument('--no-delta-mode', action='store_false', dest='delta_mode', default=True, help="Shows metrics' values instead of growth.") parser.add_argument('--no-delta-small-hide', action='store_false', dest='delta_small_hide', default=True, help='Prevent lines with only small changes or without' 'changes at all from hiding.') parser.add_argument('-l', '--delta-small-hide-limit', default=80, type=int, help='Hides lines with only changes less than this limit') parser.add_argument('--no-color', dest='color', default=True, action='store_false', help="Don't highlight NUMA nodes or sockets") parser.add_argument('--spaces', default=False, action='store_true', help="Add spaces in numbers' representation, e.g. '1234567' " "will be '1 234 567'") parser.add_argument('--random', default=False, action='store_true', help='Shows random diff data instead of real evaluation. ' 'Helpful for testing on static files') parser.add_argument('--no-clear', default=True, dest='clear', action='store_false', help="Don't clear screen after each iteration. " "May be useful in scripts/logging to file.") parser.add_argument('--lscpu-output', help='Specify file with lscpu -p output') return parser def make_parser(self, parser=None): if type(self) == BaseTop: raise TypeError('make_parser should not be called directly by BaseTop') if not parser: parser = BaseTop.make_base_parser() parser.add_argument(self.file_arg, default=self.file_value, help='Option for testing on MacOS purpose.') return parser def tick(self): """ Gathers new data + evaluate diff between current & previous data """ self.previous = self.current self.current = self.parse() if all((self.previous, self.current)): self.eval() def list_diff(self, current, previous): """ It's strange that there is no [3,4,3] - [1,2,1] -> [2,2,2] in standard library """ if self.options.random: return [randint(0, 10000) for _ in current] return [data - previous[n] for n, data in enumerate(current)] def run(self): """ Default main()-like function for specific top-like utils except meta-utils. """ infinite = -1 if self.options.iterations != infinite: self.options.iterations += 1 try: while self.options.iterations > 0 or self.options.iterations == infinite: if self.options.iterations != infinite: self.options.iterations -= 1 sleep(self.options.interval) self.tick() if self.options.clear: system('clear') if self.diff: print_(self) except KeyboardInterrupt: print_() exit(0) def repr_source(self): return self.diff if self.options.delta_mode else self.current @staticmethod def int(item): return int(item) if item.isdigit() else item def spaces(self, number, sep=' '): """ 1234567 -> 1 234 567 """ if not self.options.spaces: return number output = str() while number / 1000 > 0: output = str(number % 1000).zfill(3) + sep + output number /= 1000 return (str(number % 1000) + sep + output).strip() def __repr_table__(self, table): if self.options.clear: return BaseTop.header + str(table) return str(table) def default_init(self, topology=None): BaseTop.__init__(self) self.topology = topology self.color = Color(self.topology) def default_post_optparse(self): if not self.topology: self.topology = Topology(fake=self.options.random) self.color = Color(self.topology, self.options.color) @abstractmethod def parse(self): """ Should read some file(s) into python structure (dict/list) """ @abstractmethod def eval(self): """ Should evaluate self.diff using self.previous / self.current """ @abstractmethod def __repr__(self): """ Should return string, representing self.diff """ def main(self): """ Default entry point for most of top-like utils """ self.options = self.make_parser().parse_args() if hasattr(self, 'post_optparse'): self.post_optparse() self.run()
def default_post_optparse(self): if not self.topology: self.topology = Topology(fake=self.options.random) self.color = Color(self.topology, self.options.color)
def default_init(self, topology=None): BaseTop.__init__(self) self.topology = topology self.color = Color(self.topology)
def colorize_stat(stat, value): return Color.colorize(value, 1, 1) if 'errors' in stat.filename or 'dropped' in stat.filename else value
class RSSLadder(CPUBasedTune): """ Distributor of queues' interrupts by multiple CPUs """ topology = None color = None interrupts_file = None def __init__(self, argv=None): if argv: sys.argv = [sys.argv[0]] + argv CPUBasedTune.__init__(self) self.interrupts_file, lscpu_output = self.parse() if not exists(self.interrupts_file): # unit-tests return self.parse_cpus(lscpu_output) self.eval() def parse(self): """ Detects sources of system data """ interrupts_file = '/proc/interrupts' lscpu_output = None if self.options.test_dir: interrupts_file = join(self.options.test_dir, 'interrupts') lscpu_output_filename = join(self.options.test_dir, 'lscpu_output') lscpu_output = open(lscpu_output_filename).read() # Popen.stdout: python 2.7 returns <str>, 3.6 returns <bytes> # read() in both cases return <str> if isinstance(lscpu_output, bytes): lscpu_output = str(lscpu_output) return interrupts_file, lscpu_output def eval(self): """ Top of all the logic, decide what to do and then apply new settings """ interrupts = open(self.interrupts_file).readlines() for postfix in sorted(self.queue_postfixes_detect(interrupts)): self.apply(self.__eval(postfix, interrupts)) def apply(self, decision): """ '* 4' is in case of NIC has more queues than socket has CPUs :param decision: list of tuples(irq, queue_name, socket) """ affinity = list(decision) cpus = [socket_cpu for irq, queue, socket_cpu in affinity] if len(set(cpus)) != len(cpus): warning = 'WARNING: some CPUs process multiple queues, consider reduce queue count for this network device' if self.options.color: print_(self.color.wrap(warning, Color.YELLOW)) else: print_(warning) for irq, queue_name, socket_cpu in affinity: print_(" - {0}: queue {1} (irq {2}) bound to CPU{3}".format( self.dev_colorize(), queue_name, irq, self.cpu_colorize(socket_cpu))) if self.options.dry_run: continue filename = '/proc/irq/{0}/smp_affinity_list'.format(irq) with open(filename, 'w') as irq_file: irq_file.write(str(socket_cpu)) def __eval(self, postfix, interrupts): """ :param postfix: '-TxRx-' :return: list of tuples(irq, queue_name, socket) """ print_('- distribute interrupts of {0} ({1}) on socket {2}'.format( self.options.dev, postfix, self.options.socket)) queue_regex = r'{0}{1}[^ \n]+'.format(self.options.dev, postfix) rss_cpus = self.rss_cpus_detect() for _ in xrange(self.options.offset): rss_cpus.pop() for line in interrupts: queue_name = re.findall(queue_regex, line) if queue_name: yield any2int(line.split()[0]), queue_name[0], rss_cpus.pop() def parse_cpus(self, lscpu_output): """ :param lscpu_output: string, output of `lscpu -p` """ if self.options.cpus: # no need to detect topology if user gave us cpu list return self.topology = Topology(lscpu_output=lscpu_output) self.color = Color(self.topology, self.options.color) if not any([self.options.socket is not None, self.options.cpus]): self.socket_detect() self.pci.devices = self.pci.node_dev_dict([self.options.dev], False) def queue_postfix_extract(self, line): """ :param line: '31312 0 0 0 blabla eth0-TxRx-0' :return: '-TxRx-' """ queue_regex = r'{0}[^ \n]+'.format(self.options.dev) queue_name = re.findall(queue_regex, line) if queue_name: return re.sub(r'({0}|[0-9])'.format(self.options.dev), '', queue_name[0]) def queue_postfixes_detect(self, interrupts): """ self.dev: eth0 :return: '-TxRx-' """ return set([ line for line in [self.queue_postfix_extract(line) for line in interrupts] if line ]) def rss_cpus_detect(self): """ :return: list of cpu ids to use with current queues distribution """ rss_cpus = self.options.cpus if self.options.cpus else self.cpus_detect_real( ) rss_cpus.reverse() return rss_cpus * MAX_QUEUE_PER_DEVICE def dev_colorize(self): """ :return: highlighted by NUMA-node name of the device """ if not self.pci or not self.options.color or not self.pci.devices: return self.options.dev dev_color = self.pci.devices.get(self.options.dev) color = self.color.COLORS_NODE.get(dev_color) return self.color.wrap(self.options.dev, color) def cpu_colorize(self, cpu): """ :param cpu: cpu number (0) :return: highlighted by NUMA-node cpu number. """ if not self.topology: return cpu return self.color.wrap(cpu, self.color.colorize_cpu(cpu)) def parse_options(self): """ :return: parsed arguments """ parser = CPUBasedTune.make_parser() parser.add_argument('--no-color', help='Disable all highlights', dest='color', action='store_false', default=True) parser.add_argument( '-o', '--offset', type=int, default=0, help= 'If you have 2 NICs with 4 queues and 1 socket with 8 cpus, you may be want ' 'distribution like this: eth0: [0, 1, 2, 3]; eth1: [4, 5, 6, 7]; ' 'so run: rss-ladder-test eth0; rss-ladder-test --offset=4 eth1') return parser.parse_args()
def colorize_irq_per_cpu(irq_per_cpu): """ :returns: highlighted by warning/error irq string """ return Color.colorize(irq_per_cpu, 40000, 80000)
class NetworkTop(BaseTop): """ Global top-like util that combine information from 4 other tops """ def __init__(self): BaseTop.__init__(self) self.tops = { 'irqtop': IrqTop(), 'softnet_stat_top': SoftnetStatTop(), 'softirq_top': Softirqs(), 'link-rate': LinkRateTop(), } self.parse_options() self.topology = Topology(fake=self.options.random, lscpu_output=self.options.lscpu_output) self.color = Color(self.topology) for top in self.tops.values(): top.topology = self.topology top.color = self.color def parse(self): """ :return: dict with parsed results for each top-like object. """ return dict((top_name, _top.parse()) for top_name, _top in iteritems(self.tops)) def eval(self): """ :return: evaluates diff for each top-like object. """ if all((self.current, self.previous)): self.diff = dict((top_name, _top.diff) for top_name, _top in iteritems(self.tops)) def tick(self): self.previous = self.current self.current = self.parse() for _top in itervalues(self.tops): _top.tick() self.eval() def __repr__(self): output = [ BaseTop.header, self.__repr_irq(), self.__repr_cpu(), self.__repr_dev(), ] if not self.options.clear: del output[0] return '\n'.join(output) def parse_options(self): """ Tricky way to gather all options in one util without conflicts, parse them and do some logic after parse """ parser = BaseTop.make_base_parser() for top in itervalues(self.tops): parser = top.make_parser(parser) self.options = parser.parse_args() if self.options.lscpu_output: self.options.lscpu_output = open(self.options.lscpu_output).read() for top in itervalues(self.tops): top.options = self.options if hasattr(top, 'post_optparse'): top.post_optparse() def __repr_dev(self): top = self.tops.get('link-rate') table = make_table(top.make_header(), top.align_map, top.make_rows()) return self.color.wrap_header('Network devices') + str(table) def __repr_irq(self): top = self.tops.get('irqtop') output_lines, cpu_count = top.make_rows() align_map = top.make_align_map(cpu_count) table = make_table(output_lines[0], align_map, output_lines[1:]) return self.color.wrap_header('/proc/interrupts') + str(table) def __repr_cpu(self): irqtop = self.tops.get('irqtop') softirq_top = self.tops.get('softirq_top') softnet_stat_top = self.tops.get('softnet_stat_top') # all these evaluations are better to put in softirqs.parse() active_cpu = softirq_top.__active_cpu_count__(softirq_top.current) softnet_stat_top_output = softnet_stat_top.repr_source() network_output = zip(irqtop.diff_total, softirq_top.repr_source()['NET_RX'][:active_cpu], softirq_top.repr_source()['NET_TX'][:active_cpu], softnet_stat_top_output) fields = [ 'CPU', 'Interrupts', 'NET RX', 'NET TX', 'total', 'dropped', 'time_squeeze', 'cpu_collision', 'received_rps', ] fields = [self.color.bright(word) for word in fields] rows = self.__repr_cpu_make_rows(irqtop, network_output, softirq_top, softnet_stat_top) table = make_table(fields, ['l'] + ['r'] * (len(fields) - 1), rows) return self.color.wrap_header('Load per cpu:') + str(table) def __repr_cpu_make_rows(self, irqtop, network_output, softirq_top, softnet_stat_top): return [[ self.color.wrap('CPU{0}'.format(stat.cpu), self.color.colorize_cpu(stat.cpu)), irqtop.colorize_irq_per_cpu(irq), softirq_top.colorize_net_rx(net_rx), softirq_top.colorize_net_tx(net_tx), softnet_stat_top.colorize_total(stat.total), softnet_stat_top.colorize_dropped(stat.dropped), softnet_stat_top.colorize_time_squeeze(stat.time_squeeze), softnet_stat_top.colorize_cpu_collision(stat.cpu_collision), stat.received_rps ] for irq, net_rx, net_tx, stat in network_output]
def colorize_cpu_collision(cpu_collision): """ :returns: highlighted by warning/error cpu_collision string """ return Color.colorize(cpu_collision, 1, 1000)
def colorize_time_squeeze(time_squeeze): """ :returns: highlighted by warning/error time_squeeze string """ return Color.colorize(time_squeeze, 1, 300)
def colorize_dropped(dropped): """ :returns: highlighted by warning/error dropped string """ return Color.colorize(dropped, 1, 1)
def colorize_total(total): """ :returns: highlighted by warning/error total string """ return Color.colorize(total, 300000, 900000)