def print_report(self,
top=None,
alloc=False,
collate=True,
start=None,
end=None):
""" analyze and report on ttop files """
parser = TTopParser(start=start, end=end)
print("ttop version %s" % VERSION)
print()
for file in self.files:
log = open(file, 'r')
if env.DEBUG:
print("parsing", file)
for total, threads in parser.parse(log):
if alloc:
print('{0:<40} {1:<10} {2:<10} {3:<10}'.format(
total['date'].strftime("%Y-%m-%d %H:%M:%S"), 'Threads',
'Alloc/s',
"Total: " + format_bytes(total['heap_rate'])))
else:
print('{0:<40} {1:<10} {2:<10} {3:<10}'.format(
total['date'].strftime("%Y-%m-%d %H:%M:%S"), 'Threads',
'CPU%', "Total: " + str(total['app_cpu']) + '%'))
print('=' * 80)
combined = threads
if collate:
combined = self.collate_threads(threads)
ordered = []
if alloc:
ordered = sorted(combined.items(),
key=lambda k: k[1]['heap_rate'],
reverse=True)
else:
ordered = sorted(combined.items(),
key=lambda k: k[1]['total_cpu'],
reverse=True)
if top:
ordered = ordered[0:top]
for name, value in ordered:
count = 1
if collate:
count = int(value['thread_count'])
if alloc:
print('{0:<40} {1:<10} {2:<10} {3:<10}'.format(
name, count, format_bytes(value['heap_rate']),
textbar(total['heap_rate'], value['heap_rate'])))
else:
print('{0:<40} {1:<10} {2:<10} {3:<10}'.format(
name, count, value['total_cpu'],
textbar(total['app_cpu'], value['total_cpu'])))
print()
def read_jvm_based_parameters(jvm_args):
"""finds the max heap, system ram and gc type from the jvm args.
the last system ram argument is used as this tends to be duplicate frequently"""
gc_type = None
ram_in_mb = None
heap_size = None
for args in jvm_args:
if args.startswith('-Ddse.system_memory_in_mb'):
value = jvm_args[args]
#note it's, very very common to have multiple of these in the configuration
# we are only interested in the last one
# diffs on jvm args will reveal configuration differences between nodes
if value:
ram_in_mb = humanize.format_bytes(int(value[-1]) * 1024**2, 0)
if args == '-XX:+UseG1GC':
if gc_type:
#should never see this
gc_type = "%s/G1Gc" % gc_type
else:
gc_type = "G1GC"
elif args == "-XX:+UseConcMarkSweepGC":
if gc_type:
#should never see this
gc_type = "CMS/%s" % gc_type
else:
gc_type = "CMS"
elif args.startswith('-Xmx'):
heap_size = "".join(args[4:])
return (heap_size, gc_type, ram_in_mb)
def _format_read_ahead(drive_map):
if not drive_map:
return NA
result = []
for ra_bytes in drive_map:
result.append(humanize.format_bytes(ra_bytes))
return ", ".join(result)
def format_partition_bytes(stat, index):
"""writes out the bytes of the stat at index specified will return null if not present"""
if not stat or not index or len(stat) < index + 1:
return NA
val = stat[index]
if not val:
return NA
return humanize.format_bytes(val)
def format_largest_table(config):
"""outputs the biggest table in a pleasing format"""
largest_table = config.get("largest_table")
if not largest_table or len(largest_table) < 3:
return NA
node = largest_table[0]
table = largest_table[1]
max_table = largest_table[2]
if table and max_table:
max_table_fmt = humanize.format_bytes(max_table)
return "%s (%s %s)" % (table, max_table_fmt, node)
return NA
def generate_report(parsed):
"""generates a report from the result of parsing a
tarball or series of files"""
calculated = calculate_report(parsed)
report = []
report.append("")
report.append("NOTE: as of version 0.3.13 all evictions with duration of 0 ms " + \
"are deducted from the item eviction count and are not part of the " + \
"eviction freq or duration calculations")
report.append("")
if not calculated.get('start_log'):
report.append("start log: 'None'")
else:
report.append("start log: '%s'" % \
calculated.get('start_log').strftime(dates.CASSANDRA_LOG_FORMAT))
if not calculated.get('last_log'):
report.append("end log: 'None'")
else:
report.append("end log: '%s'" % \
calculated.get('last_log').strftime(dates.CASSANDRA_LOG_FORMAT))
report.append("")
node_info = calculated.get('node_info', [])
generate_recommendations(report, node_info)
if not node_info:
return "\nNo Logs Found For Processing\n"
table = []
table.append(["node", "Avg time between evictions", "Avg eviction duration", \
"Times (byte/item) limit reached", "Most recent limit (byte/item)", "Log duration"])
table.append(["----", "--------------------------", "---------------------", \
"-------------------------------", "-----------------------------", "------------"])
for node in node_info:
table.append([node.name, humanize.format_millis(node.avg_evict_freq), \
humanize.format_millis(node.avg_evict_duration), \
"%i/%i" % \
(node.byte_limit, node.item_limit), \
"%s/%i" % \
(humanize.format_bytes(node.last_byte_limit), node.last_item_limit), \
humanize.format_millis(node.log_duration)])
humanize.pad_table(table)
for row in table:
report.append(" ".join(row))
report.append("") #provides empty line after last line
return "\n".join(report)
def test_format_bytes_mb():
"""convert to mb"""
assert humanize.format_bytes(1024**2) == "1.00 mb"
assert humanize.format_bytes(1024.4**2) == "1.00 mb"
def test_format_bytes_none():
"""undefined input"""
assert humanize.format_bytes(None) == "Not available"
def print_report(self,
top=None,
alloc=False,
collate=True,
start=None,
end=None):
""" analyze and report on ttop files """
parser = TTopParser(start=start, end=end)
print("ttop version %s" % VERSION)
print()
table = []
for file in self.files:
with diag.FileWithProgress(file) as log:
table.append([])
if env.DEBUG:
print("parsing", file)
for total, threads in parser.parse(log):
if alloc:
table.append([
total["date"].strftime("%Y-%m-%d %H:%M:%S"),
"Threads",
"Alloc/s",
"Total: " + format_bytes(total["heap_rate"]),
])
else:
table.append([
total["date"].strftime("%Y-%m-%d %H:%M:%S"),
"Threads",
"CPU%",
"Total: " + str(total["app_cpu"]) + "%",
])
header = "=" * 80
table.append([header])
combined = threads
if collate:
combined = self.collate_threads(threads)
ordered = []
if alloc:
ordered = sorted(
combined.items(),
key=lambda k: k[1]["heap_rate"],
reverse=True,
)
else:
ordered = sorted(
combined.items(),
key=lambda k: k[1]["total_cpu"],
reverse=True,
)
if top:
ordered = ordered[0:top]
for name, value in ordered:
count = 1
if collate:
count = int(value["thread_count"])
if alloc:
table.append([
name,
str(count),
format_bytes(value["heap_rate"]),
textbar(total["heap_rate"],
value["heap_rate"]),
])
else:
table.append([
name,
str(count),
"{:.2f}".format(value["total_cpu"]),
textbar(total["app_cpu"], value["total_cpu"]),
])
table.append([])
pad_table(table, extra_pad=1)
for row in table:
print("".join(row))
print()
def test_format_bytes_1000_gb(self):
"""convert to gb and add separtor"""
self.assertEqual(humanize.format_bytes((1024**3) * 1000),
"1,000.00 gb")
self.assertEqual(humanize.format_bytes((1024**3) * 1000.4),
"1,000.40 gb")
def test_format_bytes_1000_pb(self):
"""convert to tb and add separator"""
self.assertEqual(humanize.format_bytes((1024**5) * 1000),
"1,000.00 pb")
self.assertEqual(humanize.format_bytes((1024**5) * 1000.04),
"1,000.04 pb")
def test_format_bytes_1024(self):
"""convert to k"""
self.assertEqual(humanize.format_bytes(1024), "1.00 kb")
self.assertEqual(humanize.format_bytes(1024.4), "1.00 kb")
def test_format_bytes_1000_mb(self):
"""convert to mb and add separator"""
self.assertEqual(humanize.format_bytes((1024**2) * 1000),
"1,000.00 mb")
self.assertEqual(humanize.format_bytes((1024**2) * 1000.4),
"1,000.40 mb")
def test_format_bytes_999point5(self):
"""round"""
self.assertEqual(humanize.format_bytes(999.4), "999 bytes")
self.assertEqual(humanize.format_bytes(999.5), "1,000 bytes")
def __repr__(self):
return "%s ops / %s data" % (format_num(
self.ops), format_bytes(self.data))
def test_format_bytes_1024():
"""convert to k"""
assert humanize.format_bytes(1024) == "1.00 kb"
assert humanize.format_bytes(1024.4) == "1.00 kb"
def test_format_bytes_1023():
"""add separator"""
assert humanize.format_bytes(1023) == "1,023 bytes"
assert humanize.format_bytes(1023.4) == "1,023 bytes"
def test_format_bytes_999point5():
"""round"""
assert humanize.format_bytes(999.4) == "999 bytes"
assert humanize.format_bytes(999.5) == "1,000 bytes"
def test_format_bytes_999(self):
"""leave alone"""
self.assertEqual(humanize.format_bytes(999), "999 bytes")
def test_format_bytes_1000_mb():
"""convert to mb and add separator"""
assert humanize.format_bytes((1024**2) * 1000) == "1,000.00 mb"
assert humanize.format_bytes((1024**2) * 1000.4) == "1,000.40 mb"
def test_format_bytes_1023(self):
"""add separator"""
self.assertEqual(humanize.format_bytes(1023), "1,023 bytes")
self.assertEqual(humanize.format_bytes(1023.4), "1,023 bytes")
def test_format_bytes_gb():
"""convert to gb"""
assert humanize.format_bytes(1024**3) == "1.00 gb"
assert humanize.format_bytes((1024**3) * 1.04) == "1.04 gb"
def test_format_bytes_mb(self):
"""convert to mb"""
self.assertEqual(humanize.format_bytes(1024**2), "1.00 mb")
self.assertEqual(humanize.format_bytes(1024.4**2), "1.00 mb")
def test_format_bytes_1000_gb():
"""convert to gb and add separtor"""
assert humanize.format_bytes((1024**3) * 1000) == "1,000.00 gb"
assert humanize.format_bytes((1024**3) * 1000.4) == "1,000.40 gb"
def test_format_bytes_gb(self):
"""convert to gb"""
self.assertEqual(humanize.format_bytes(1024**3), "1.00 gb")
self.assertEqual(humanize.format_bytes((1024**3) * 1.04), "1.04 gb")
def test_format_bytes_tb():
"""convert to tb"""
assert humanize.format_bytes(1024**4) == "1.00 tb"
assert humanize.format_bytes((1024**4) * 1.04) == "1.04 tb"
def test_format_bytes_tb(self):
"""convert to tb"""
self.assertEqual(humanize.format_bytes(1024**4), "1.00 tb")
self.assertEqual(humanize.format_bytes((1024**4) * 1.04), "1.04 tb")
def test_format_bytes_1000_pb():
"""convert to tb and add separator"""
assert humanize.format_bytes((1024**5) * 1000) == "1,000.00 pb"
assert humanize.format_bytes((1024**5) * 1000.04) == "1,000.04 pb"
def test_format_bytes_none(self):
"""undefined input"""
self.assertEqual(humanize.format_bytes(None), "Not available")
def test_format_bytes_999():
"""leave alone"""
assert humanize.format_bytes(999) == "999 bytes"