def infoService(stats, builds, visibilities, cluster, **ignore): prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "Service Information" column_names = ('node' , 'build' , 'cluster_size' , 'cluster_visibility' , '_cluster_integrity' , ('free-pct-disk', 'Free Disk%') , ('free-pct-memory', 'Free Mem%') , '_migrates' , ('_paxos_principal', 'Principal') , '_objects' , '_uptime') t = Table(title, column_names) t.addDataSource('_migrates' ,lambda data: "(%s,%s)"%(row['migrate_progress_send'] ,row['migrate_progress_recv'])) t.addDataSource('_objects' ,Extractors.sifExtractor('objects')) t.addDataSource('_cluster_integrity' , lambda data: True if row['cluster_integrity'] == 'true' else False) t.addDataSource('_uptime', Extractors.timeExtractor('uptime')) t.addCellAlert('cluster_visibility' , lambda data: data['cluster_visibility'] is not True) t.addCellAlert('_cluster_integrity' ,lambda data: data['cluster_integrity'] != 'true') t.addCellAlert('free-pct-disk' ,lambda data: int(data['free-pct-disk']) < 40) t.addCellAlert('free-pct-memory' ,lambda data: int(data['free-pct-memory']) < 40) t.addCellAlert('node' ,lambda data: data['real_node_id'] == principal , color=terminal.fg_green) for node_key, n_stats in stats.iteritems(): if isinstance(n_stats, Exception): n_stats = {} node = cluster.getNode(node_key)[0] row = n_stats row['real_node_id'] = node.node_id row['node'] = prefixes[node_key] try: paxos_node = cluster.getNode(row['paxos_principal'])[0] row['_paxos_principal'] = prefixes[paxos_node.key] except KeyError: # The principal is a node we currently do not know about # So return the principal ID try: row['_paxos_principal'] = row['paxos_principal'] except KeyError: pass build = builds[node_key] if not isinstance(build, Exception): row['build'] = build if node_key in visibilities: row['cluster_visibility'] = visibilities[node_key] t.insertRow(row) print t
def infoXDR(stats, builds, xdr_enable, cluster, **ignore): if not max(xdr_enable.itervalues()): return prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "XDR Information" column_names = ('node' ,'build' ,('_bytes-shipped', 'Data Shipped') ,'_free-dlog-pct' ,('_lag-secs', 'Lag (sec)') ,'_req-outstanding' ,'_req-relog' ,'_req-shipped' ,'cur_throughput' ,('latency_avg_ship', 'Avg Latency (ms)') ,'_xdr-uptime') t = Table(title, column_names) t.addDataSource('_xdr-uptime', Extractors.timeExtractor('xdr-uptime')) t.addDataSource('_bytes-shipped', Extractors.byteExtractor('esmt-bytes-shipped')) t.addDataSource('_lag-secs', Extractors.timeExtractor('timediff_lastship_cur_secs')) t.addDataSource('_req-outstanding', Extractors.sifExtractor('stat_recs_outstanding')) t.addDataSource('_req-relog', Extractors.sifExtractor('stat_recs_relogged')) t.addDataSource('_req-shipped', Extractors.sifExtractor('stat_recs_shipped')) # Highligh red if lag is more than 30 seconds t.addCellAlert('_lag-secs' , lambda data: int(data['timediff_lastship_cur_secs']) >= 300) t.addCellAlert('node' ,lambda data: data['real_node_id'] == principal , color=terminal.fg_green) row = None for node_key, row in stats.iteritems(): if isinstance(row, Exception): row = {} node = cluster.getNode(node_key)[0] if xdr_enable[node_key]: if row: row['build'] = builds[node_key] row['_free-dlog-pct'] = row['free-dlog-pct'][:-1] else: row = {} row['node-id'] = node.node_id row['real_node_id'] = node.node_id else: continue row['node'] = prefixes[node_key] t.insertRow(row) print t
def infoNamespace(stats, cluster, **ignore): prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "Namespace Information" column_names = ('node' ,'namespace' ,('available_pct', 'Avail%') ,('evicted-objects', 'Evictions') ,'_objects' ,'repl-factor' ,'stop-writes' ,('_used-bytes-disk', 'Disk Used') ,('_used-disk-pct', 'Disk Used%') ,('high-water-disk-pct', 'HWM Disk%') ,('_used-bytes-memory', 'Mem Used') ,('_used-mem-pct', 'Mem Used%') ,('high-water-memory-pct', 'HWM Mem%') ,('stop-writes-pct', 'Stop Writes%')) t = Table(title, column_names, group_by=1) t.addDataSource('_used-bytes-disk' ,Extractors.byteExtractor('used-bytes-disk')) t.addDataSource('_used-bytes-memory' ,Extractors.byteExtractor( 'used-bytes-memory')) t.addDataSource('_objects' ,Extractors.sifExtractor('objects')) t.addDataSource('_used-disk-pct' , lambda data: 100 - int(data['free-pct-disk'])) t.addDataSource('_used-mem-pct' , lambda data: 100 - int(data['free-pct-memory'])) t.addCellAlert('available_pct' , lambda data: int(data['available_pct']) <= 10) t.addCellAlert('stop-writes' , lambda data: data['stop-writes'] != 'false') t.addCellAlert('_used-disk-pct' , lambda data: int(data['_used-disk-pct']) >= int(data['high-water-disk-pct'])) t.addCellAlert('_used-mem-pct' , lambda data: (100 - int(data['free-pct-memory'])) >= int(data['high-water-memory-pct'])) t.addCellAlert('_used-disk-pct' , lambda data: (100 - int(data['free-pct-disk'])) >= int(data['high-water-disk-pct'])) t.addCellAlert('node' ,lambda data: data['real_node_id'] == principal , color=terminal.fg_green) for node_key, n_stats in stats.iteritems(): node = cluster.getNode(node_key)[0] if isinstance(n_stats, Exception): t.insertRow({'real_node_id':node.node_id , 'node':prefixes[node_key]}) continue for ns, ns_stats in n_stats.iteritems(): if isinstance(ns_stats, Exception): row = {} else: row = ns_stats row['namespace'] = ns row['real_node_id'] = node.node_id row['node'] = prefixes[node_key] t.insertRow(row) print t
def infoService(stats, builds, visibilities, cluster, **ignore): prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "Service Information" column_names = ('node', 'build', 'cluster_size', 'cluster_visibility', '_cluster_integrity', ('free-pct-disk', 'Free Disk%'), ('free-pct-memory', 'Free Mem%'), ('_migrates', 'Migrates (tx,rx,q)'), ('_paxos_principal', 'Principal'), '_objects', '_uptime') t = Table(title, column_names) t.addDataSource( '_migrates', lambda data: "(%s,%s,%s)" % (row.get('migrate_tx_partitions_remaining', False) or row[ 'migrate_progress_send'], row.get('migrate_rx_partitions_remaining', False) or row[ 'migrate_progress_recv'], int(row.get('migrate_progress_send', 0)) + int( row.get('migrate_progress_recv', 0)))) t.addDataSource('_objects', Extractors.sifExtractor('objects')) t.addDataSource( '_cluster_integrity', lambda data: True if row['cluster_integrity'] == 'true' else False) t.addDataSource('_uptime', Extractors.timeExtractor('uptime')) t.addCellAlert('cluster_visibility', lambda data: data['cluster_visibility'] is not True) t.addCellAlert('_cluster_integrity', lambda data: data['cluster_integrity'] != 'true') t.addCellAlert('free-pct-disk', lambda data: int(data['free-pct-disk']) < 40) t.addCellAlert('free-pct-memory', lambda data: int(data['free-pct-memory']) < 40) t.addCellAlert('node', lambda data: data['real_node_id'] == principal, color=terminal.fg_green) for node_key, n_stats in stats.iteritems(): if isinstance(n_stats, Exception): n_stats = {} node = cluster.getNode(node_key)[0] row = n_stats row['real_node_id'] = node.node_id row['node'] = prefixes[node_key] try: paxos_node = cluster.getNode(row['paxos_principal'])[0] row['_paxos_principal'] = prefixes[paxos_node.key] except KeyError: # The principal is a node we currently do not know about # So return the principal ID try: row['_paxos_principal'] = row['paxos_principal'] except KeyError: pass build = builds[node_key] if not isinstance(build, Exception): row['build'] = build if node_key in visibilities: row['cluster_visibility'] = visibilities[node_key] t.insertRow(row) print t
def infoXDR(stats, builds, xdr_enable, cluster, **ignore): if not max(xdr_enable.itervalues()): return prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "XDR Information" column_names = ('node', 'build', ('_bytes-shipped', 'Data Shipped'), '_free-dlog-pct', ('_lag-secs', 'Lag (sec)'), '_req-outstanding', '_req-relog', '_req-shipped', 'cur_throughput', ('latency_avg_ship', 'Avg Latency (ms)'), '_xdr-uptime') t = Table(title, column_names) t.addDataSource('_xdr-uptime', Extractors.timeExtractor(('xdr-uptime', 'xdr_uptime'))) t.addDataSource( '_bytes-shipped', Extractors.byteExtractor( ('esmt-bytes-shipped', 'esmt_bytes_shipped'))) t.addDataSource('_lag-secs', Extractors.timeExtractor('timediff_lastship_cur_secs')) t.addDataSource('_req-outstanding', Extractors.sifExtractor('stat_recs_outstanding')) t.addDataSource('_req-relog', Extractors.sifExtractor('stat_recs_relogged')) t.addDataSource('_req-shipped', Extractors.sifExtractor('stat_recs_shipped')) # Highligh red if lag is more than 30 seconds t.addCellAlert( '_lag-secs', lambda data: int(data['timediff_lastship_cur_secs']) >= 300) t.addCellAlert('node', lambda data: data['real_node_id'] == principal, color=terminal.fg_green) row = None for node_key, row in stats.iteritems(): if isinstance(row, Exception): row = {} node = cluster.getNode(node_key)[0] if xdr_enable[node_key]: if row: row['build'] = builds[node_key] if 'free_dlog_pct' in row: row['_free-dlog-pct'] = row['free_dlog_pct'][:-1] else: row['_free-dlog-pct'] = row['free-dlog-pct'][:-1] else: row = {} row['node-id'] = node.node_id row['real_node_id'] = node.node_id else: continue row['node'] = prefixes[node_key] t.insertRow(row) print t
def infoNamespace(stats, cluster, **ignore): prefixes = cluster.getPrefixes() principal = cluster.getExpectedPrincipal() title = "Namespace Information" column_names = ('node', 'namespace', ('available_pct', 'Avail%'), ('evicted-objects', 'Evictions'), ('_master-objects', 'Master Objects'), ('_prole-objects', 'Replica Objects'), 'repl-factor', 'stop-writes', ('_used-bytes-disk', 'Disk Used'), ('_used-disk-pct', 'Disk Used%'), ('high-water-disk-pct', 'HWM Disk%'), ('_used-bytes-memory', 'Mem Used'), ('_used-mem-pct', 'Mem Used%'), ('high-water-memory-pct', 'HWM Mem%'), ('stop-writes-pct', 'Stop Writes%')) t = Table(title, column_names, group_by=1) t.addDataSource('_used-bytes-disk', Extractors.byteExtractor('used-bytes-disk')) t.addDataSource('_used-bytes-memory', Extractors.byteExtractor('used-bytes-memory')) t.addDataSource('_master-objects', Extractors.sifExtractor('master-objects')) t.addDataSource('_prole-objects', Extractors.sifExtractor('prole-objects')) t.addDataSource('_used-disk-pct', lambda data: 100 - int(data['free-pct-disk'])) t.addDataSource('_used-mem-pct', lambda data: 100 - int(data['free-pct-memory'])) t.addCellAlert('available_pct', lambda data: int(data['available_pct']) <= 10) t.addCellAlert('stop-writes', lambda data: data['stop-writes'] != 'false') t.addCellAlert( '_used-disk-pct', lambda data: int(data['_used-disk-pct']) >= int( data['high-water-disk-pct'])) t.addCellAlert( '_used-mem-pct', lambda data: (100 - int(data['free-pct-memory'])) >= int(data['high-water-memory-pct'])) t.addCellAlert( '_used-disk-pct', lambda data: (100 - int(data['free-pct-disk'])) >= int(data['high-water-disk-pct'])) t.addCellAlert('node', lambda data: data['real_node_id'] == principal, color=terminal.fg_green) for node_key, n_stats in stats.iteritems(): node = cluster.getNode(node_key)[0] if isinstance(n_stats, Exception): t.insertRow({ 'real_node_id': node.node_id, 'node': prefixes[node_key] }) continue for ns, ns_stats in n_stats.iteritems(): if isinstance(ns_stats, Exception): row = {} else: row = ns_stats row['namespace'] = ns row['real_node_id'] = node.node_id row['node'] = prefixes[node_key] t.insertRow(row) print t