def print_module(module,
options,
gap_info_str,
start_addr,
end_addr,
unloaded=False):
global tainted_count
tainted = not is_our_module(module)
if options.shows_tainted and not tainted:
return
if tainted:
tainted_count = tainted_count + 1
crashcolor.set_color(crashcolor.LIGHTRED)
if unloaded:
crashcolor.set_color(crashcolor.MAGENTA | crashcolor.BOLD)
if member_offset("struct module", "core_layout") > -1:
core_size = module.core_layout.size
else:
core_size = module.core_size
print("0x%x %-25s %10d %s" %
(long(module), module.name, core_size, gap_info_str))
if options.shows_addr:
print(" " * 3, end="")
crashcolor.set_color(crashcolor.UNDERLINE)
print("addr range : 0x%x - 0x%x" % (start_addr, end_addr))
crashcolor.set_color(crashcolor.RESET)
def show_cgroup_tree_from_cgroup_roots(cgroup_roots, options):
global empty_count
global cgroup_count
empty_count = 0
cgroup_count = 0
crashcolor.set_color(crashcolor.BLUE)
print("** cgroup tree **")
crashcolor.set_color(crashcolor.RESET)
for cgroup_root in readSUListFromHead(cgroup_roots,
'root_list',
'struct cgroup_root',
maxel=1000000):
top_cgroup = cgroup_root.cgrp
curlimit = sys.getrecursionlimit()
sys.setrecursionlimit(1000)
print_cgroup_entry(top_cgroup, top_cgroup, 0, options)
sys.setrecursionlimit(curlimit)
print("-" * 70)
crashcolor.set_color(crashcolor.BLUE)
print("Total number of cgroup(s) = %d, %d had 0 count" %
(cgroup_count, empty_count))
crashcolor.set_color(crashcolor.RESET)
def print_handler(tab_str, handler_type, handler_addr, kp):
if handler_addr != 0:
handler_name = addr2sym(handler_addr)
mod_name = get_module_name(handler_name)
print("\t%s%s_handler = 0x%x (%s)%s" %
(tab_str, handler_type, handler_addr, handler_name, mod_name))
if handler_name is not None and handler_name.endswith("_kretprobe"):
kretprobe = readSU("struct kretprobe", kp)
ret_handler_name = addr2sym(kretprobe.handler)
mod_name = get_module_name(ret_handler_name)
if mod_name != "":
crashcolor.set_color(crashcolor.LIGHTRED)
print("\t\t%skretprobe.handler = 0x%x (%s)%s" %
(tab_str, kretprobe.handler, ret_handler_name, mod_name))
crashcolor.set_color(crashcolor.RESET)
else:
jprobe = readSU("struct jprobe", kp)
if jprobe.entry != 0:
entry_handler_name = addr2sym(jprobe.entry)
mod_name = get_module_name(entry_handler_name)
if mod_name != "":
crashcolor.set_color(crashcolor.LIGHTRED)
if entry_handler_name != None:
print(
"\t\t%sjprobe.entry = 0x%x (%s)%s" %
(tab_str, jprobe.entry, entry_handler_name, mod_name))
crashcolor.set_color(crashcolor.RESET)
def show_mem_cgroup(mem_cgroup_addr, depth, idx):
space_str = "\t" * depth
if mem_cgroup_addr == 0:
crashcolor.set_color(crashcolor.BLUE)
print("%s%d: mem_cgroup 0x0" % (space_str, idx))
crashcolor.set_color(crashcolor.RESET)
return
try:
mem_cgroup = readSU("struct mem_cgroup", mem_cgroup_addr)
except:
return
print("%s%d: mem_cgroup 0x%x : id = %d, refcnt = %d, dead_count = %d" %
(space_str, idx, mem_cgroup, mem_cgroup.id,
mem_cgroup.refcnt.counter, mem_cgroup.dead_count.counter))
def show_task_group():
global empty_count
count = 0
empty_count = 0
for task_group in readSUListFromHead(sym2addr('task_groups'), 'list',
'struct task_group'):
css = readSU('struct cgroup_subsys_state', task_group.css)
if (css == 0):
continue
if (css.cgroup == 0):
continue
cgroup = readSU('struct cgroup', css.cgroup)
if (cgroup == 0):
continue
count = count + 1
cgroup_name = ""
cgroup_counter = 0
if member_offset("struct cgroup", "dentry") > -1:
if (cgroup.dentry != 0):
cgroup_name = dentry_to_filename(cgroup.dentry)
cgroup_counter = cgroup.count.counter
elif member_offset("struct cgroup", "kn") > -1:
cgroup_name = cgroup.kn.name
cgroup_counter = cgroup.kn.count
if cgroup_name == "":
cgroup_name = "<default>"
if cgroup_counter == 0:
crashcolor.set_color(crashcolor.RED)
empty_count = empty_count + 1
else:
crashcolor.set_color(crashcolor.RESET)
print("task_group = 0x%x, cgroup = 0x%x, counter=%d\n\t(%s)" %
(task_group, cgroup, cgroup_counter, cgroup_name))
crashcolor.set_color(crashcolor.RESET)
print("-" * 70)
crashcolor.set_color(crashcolor.BLUE)
print("Total number of task_group(s) = %d, %d had 0 count" %
(count, empty_count))
crashcolor.set_color(crashcolor.RESET)
def show_callgraph_func(func_name, depth, first, options):
global branch_locations
print_branch(depth, first)
print_str = ("{%s} " % (func_name))
start_char = func_name[0]
if start_char == '*' or start_char == '0':
crashcolor.set_color(crashcolor.BLUE)
print("%s" % (print_str), end='')
if start_char == '*' or start_char == '0':
crashcolor.set_color(crashcolor.RESET)
if (len(branch_locations) <= depth):
branch_locations.append(len(print_str))
else:
branch_locations[depth] = len(print_str)
return 1
def show_tasks_memusage(options):
mem_usage_dict = {}
if (options.nogroup):
crash_command = "ps"
else:
crash_command = "ps -G"
result = exec_crash_command(crash_command)
result_lines = result.splitlines(True)
total_rss = 0
for i in range(1, len(result_lines) - 1):
if (result_lines[i].find('>') == 0):
result_lines[i] = result_lines[i].replace('>', ' ', 1)
result_line = result_lines[i].split()
if (len(result_line) < 9):
continue
pname = result_line[8]
rss = result_line[7]
total_rss = total_rss + int(rss)
if (pname in mem_usage_dict):
rss = mem_usage_dict[pname] + int(rss)
mem_usage_dict[pname] = int(rss)
sorted_usage = sorted(mem_usage_dict.items(),
key=operator.itemgetter(1), reverse=True)
print("=" * 70)
print("%-14s %-s" % (" [ RSS usage ]", "[ Process name ]"))
print("=" * 70)
min_number = 10
if (options.all):
min_number = len(sorted_usage) - 1
for i in range(0, min(len(sorted_usage) - 1, min_number)):
print("%10s KiB %-s" %
(sorted_usage[i][1], sorted_usage[i][0]))
print("=" * 70)
crashcolor.set_color(crashcolor.BLUE)
print("Total memory usage from user-space = %.2f GiB" %
(total_rss/1048576))
crashcolor.set_color(crashcolor.RESET)
def print_task(task, depth, first, options):
global pid_cnt
global branch_locations
if (task == None):
return
pid_cnt = pid_cnt + 1
thread_str = ""
if (options.print_thread):
thread_count = get_thread_count(task)
if (thread_count > 1):
if (task.tgid == task.pid):
thread_str = "---%d*[{%s}]" % (thread_count, task.comm)
else:
return 0
print_branch(depth, first)
comm_str = ""
if (task.comm != 0):
comm_str = task.comm
task_color = task_state_color(task.state)
if task_color != crashcolor.RESET:
crashcolor.set_color(task_color)
print_str = ("%s%s%s%s " %
(comm_str,
"(" + str(task.pid) + ")"
if options.print_pid else "",
"[" + task_state_str(task.state) +"]"
if options.print_state else "",
thread_str))
print ("%s" % (print_str), end='')
if task_color != crashcolor.RESET:
crashcolor.set_color(crashcolor.RESET)
if (len(branch_locations) <= depth):
branch_locations.append(len(print_str))
else:
branch_locations[depth] = len(print_str)
return 1
def show_rules_list():
global modules
count = len(modules)
if count == 0:
print("No rules available for this vmcore")
return
print("-" * 75)
for module in modules:
crashcolor.set_color(crashcolor.BLUE)
print("[%s]" % (module.__name__), end='')
crashcolor.set_color(crashcolor.RESET)
try:
print(": %s" % (module.description()))
except:
print(": No description available")
print("-" * 75)
print("There are %d rules available for this vmcore" % (count))
print("=" * 75)
def show_network_interfaces(options):
for dev in netdevice.dev_base_list():
dev_addr = "Not available"
if (dev.dev_addr != 0):
dev_addr = ("%02x:%02x:%02x:%02x:%02x:%02x" %
(dev.dev_addr[0], dev.dev_addr[1],
dev.dev_addr[2], dev.dev_addr[3],
dev.dev_addr[4], dev.dev_addr[5]))
netdev_ops = dev.netdev_ops
netdev_ops_str = addr2sym(netdev_ops)
driver_module = ""
crashout = exec_crash_command("sym %s" %
(netdev_ops_str))
driver_module = crashout.split()[-1]
if (driver_module[0] != "["):
driver_module = "<built-in>"
print ("0x%016x : %s (%s) managed by " %
(dev, dev.name, dev_addr), end='')
crashcolor.set_color(crashcolor.BLUE)
print ("%s" % driver_module)
crashcolor.set_color(crashcolor.RESET)
if (options.network_details):
print ("\tnetdev_ops = %s (0x%x)" %
(netdev_ops_str, netdev_ops))
master_name = ""
if (member_offset("struct net_device", "master") >= 0):
if (dev.master != 0):
master_name = dev.master.name
else:
master_name = "Not available"
print("\tMTU: %d, Master = <%s>" %
(dev.mtu, master_name))
print("\tIRQ: %d, num_tx_queues = %d, real_num_tx_queues = %d" %
(dev.irq, dev.num_tx_queues, dev.real_num_tx_queues))
def hangcheck_display(options, args):
get_task_list(options, args)
hung_task_timeout_usecs = readSymbol(
"sysctl_hung_task_timeout_secs") * 1000000
task_list_sorted = sorted(un_task_list, key=getKey, reverse=False)
hung_task_count = 0
for taskObj in task_list_sorted:
runtime = get_useconds(taskObj["runtime"])
if runtime >= hung_task_timeout_usecs * 2:
crashcolor.set_color(crashcolor.LIGHTRED)
hung_task_count = hung_task_count + 1
elif runtime >= hung_task_timeout_usecs:
crashcolor.set_color(crashcolor.BLUE)
hung_task_count = hung_task_count + 1
print(taskObj["raw"])
if options.detail == True:
show_task_details(taskObj)
if options.files == True:
show_task_files(taskObj)
crashcolor.set_color(crashcolor.RESET)
task_count = len(un_task_list)
if task_count > 0:
print("=" * 60)
task_s = "s" if task_count > 1 else ""
task_hung_s = "s" if hung_task_count > 1 else ""
print(
"Total %d task%s were in D state. %d task%s were in D state longer than %d seconds"
% (task_count, task_s, hung_task_count, task_hung_s,
hung_task_timeout_usecs / 1000000))
def check_syscall_table(options):
set_invalid_start_list()
sys_call_table, max_syscalls = sys_call_table_info()
hook_call_no = 0
trap_call_no = 0
idx = 0
while True:
if max_syscalls > 0:
call_addr = sys_call_table[idx]
else:
call_addr = readULong(sys_call_table + idx * 8)
# sys_call_table boundary has the value 'warn_bad_vsyscall'
check_call_addr = call_addr
if call_addr != 0 and syscall_double_read == True:
check_call_addr = readULong(call_addr)
if check_call_addr == syscall_end_mark:
break
if call_addr == 0:
continue
result = exec_crash_command("sym 0x%x" % call_addr)
if result.startswith("sym"):
break
words = result.split()
filepath = get_file_path(words)
if len(words) == 4 and not words[3].startswith("/"):
hook_call_no = hook_call_no + 1
crashcolor.set_color(crashcolor.LIGHTRED)
print("%3d %s" % (idx, result), end='')
else:
dis_result = exec_crash_command("dis 0x%x 1" % call_addr)
dis_words = dis_result.split()
if len(dis_words) >= 4 and dis_words[2].strip(
) in invalid_start_list:
trap_call_no = trap_call_no + 1
crashcolor.set_color(crashcolor.BLUE)
print("%3d %s %s %-25s %s" %
(idx, words[0], words[1], words[2], filepath))
print("\t%s" % (dis_result[dis_result.find(dis_words[2]):]),
end='')
crashcolor.set_color(crashcolor.RESET)
idx = idx + 1
if max_syscalls > 0 and idx >= max_syscalls:
break
if hook_call_no > 0 or trap_call_no > 0:
print("=" * 75)
if hook_call_no > 0:
print("%d system calls were replaced" % hook_call_no)
if trap_call_no > 0:
print("%d system calls were modified" % trap_call_no)
else:
print("No issues detected")
def all_filesystem_info(options):
fs_state_list = {}
super_blocks = sym2addr("super_blocks")
for sb in readSUListFromHead(super_blocks, "s_list", "struct super_block"):
frozen = -1
if (member_offset('struct super_block', 's_writers') >= 0):
frozen = sb.s_writers.frozen
elif (member_offset('struct super_block', 's_frozen') >= 0):
frozen = sb.s_frozen
frozen_str = get_frozen_str(frozen)
fs_state_list[frozen_str] = fs_state_list[
frozen_str] + 1 if frozen_str in fs_state_list else 1
vfsmnt_addr = get_vfsmount_from_sb(sb)
mnt_flags = 0
if (vfsmnt_addr != -1):
vfsmnt = readSU("struct vfsmount", vfsmnt_addr)
mnt_flags = vfsmnt.mnt_flags
if frozen_str == "SB_FREEZE_COMPLETE":
crashcolor.set_color(crashcolor.LIGHTRED)
elif frozen_str == "SB_FREEZE_WRITE":
crashcolor.set_color(crashcolor.LIGHTCYAN)
print("SB: 0x%14x, frozen=%s, %s (%s) [%s], (%s)" %
(sb, frozen_str, dentry_to_filename(sb.s_root), sb.s_id,
sb.s_type.name, get_mount_options(mnt_flags)))
crashcolor.set_color(crashcolor.RESET)
print("")
for frozen_str in fs_state_list:
print("%25s = %d" % (frozen_str, fs_state_list[frozen_str]))
def hvballoon_mem(options):
try:
pa = readSymbol('dm_device');
if (pa == 0):
return
except:
print ("Hyper-V balloon symbol does not exist")
return
if options.show_details:
baddr = sym2addr('dm_device')
balloon_result = exec_crash_command('struct hv_dynmem_device.state,num_pages_ballooned 0x%x' % (baddr))
print ('%s' % (balloon_result))
class hv_dm_state(enum.Enum):
DM_INITIALIZING = 0
DM_INITIALIZED = 1
DM_BALLOON_UP = 2
DM_BALLOON_DOWN = 3
DM_HOT_ADD = 4
DM_INIT_ERRO = 5
crashcolor.set_color(crashcolor.LIGHTRED)
num_pages_ballooned = pa.num_pages_ballooned
state = hv_dm_state(pa.state)
print ("== Hyper-V Ballooning Info ==")
print ("driver address = %s" % (pa))
print ("state = %s" % (state.name))
print ("allocated size (pages) = %d" % num_pages_ballooned)
print ("allocated size (bytes) = %d, (%.2fGB)" %
(num_pages_ballooned * crash.PAGESIZE,
((num_pages_ballooned * crash.PAGESIZE)/1024/1024/1024)))
num_pages = pa.balloon_wrk.num_pages
print ("required target (pages) = %d" % num_pages)
print ("required target (bytes) = %d, (%.2fGB)" %
(num_pages * crash.PAGESIZE,
((num_pages * crash.PAGESIZE)/1024/1024/1024)))
crashcolor.set_color(crashcolor.RESET)
print ("\n")
def show_vm(options):
private_mem_pages = 0
shared_mem_pages = 0
result_str = exec_crash_command("vm")
result_lines = result_str.splitlines()
total_lines = len(result_lines)
if total_lines < 4: # For kernel tasks
print(result_str)
return
for i in range(0, 3):
print(result_lines[i])
print("%10s %s" % ("", result_lines[3]))
for i in range(4, total_lines):
words = result_lines[i].split()
size = int(words[2], 16) - int(words[1], 16)
size_str = get_size_str(size, True)
print("%10s %s" % (size_str, result_lines[i]), end="")
if options.details:
pages_list = show_vm_details(options, words)
private_mem_pages = private_mem_pages + pages_list[0]
shared_mem_pages = shared_mem_pages + pages_list[1]
print(", P: %d, S: %d" % (pages_list[0], pages_list[1]), end="")
vm_ops = readSU("struct vm_area_struct", int(words[0], 16)).vm_ops
try:
vm_ops_name = " (" + addr2sym(vm_ops) + ")"
except:
vm_ops_name = ""
print(", %x%s" % (vm_ops, vm_ops_name), end="")
print("")
crashcolor.set_color(crashcolor.RESET)
if options.details:
print("\n\tPrivate memory pages = %d" % private_mem_pages)
print("\n\tShared memory pages = %d" % shared_mem_pages)
def all_filesystem_info(options):
super_blocks = sym2addr("super_blocks")
for sb in readSUListFromHead(super_blocks, "s_list", "struct super_block"):
frozen = -1
if (member_offset('struct super_block', 's_writers') >= 0):
frozen = sb.s_writers.frozen
elif (member_offset('struct super_block', 's_frozen') >= 0):
frozen = sb.s_frozen
frozen_str = get_frozen_str(frozen)
vfsmnt_addr = get_vfsmount_from_sb(sb)
mnt_flags = 0
if (vfsmnt_addr != -1):
vfsmnt = readSU("struct vfsmount", vfsmnt_addr)
mnt_flags = vfsmnt.mnt_flags
if frozen_str != "SB_UNFROZEN":
crashcolor.set_color(crashcolor.LIGHTRED)
print("SB: 0x%14x, frozen=%s, %s (%s) [%s], (%s)" %
(sb, frozen_str, dentry_to_filename(sb.s_root), sb.s_id,
sb.s_type.name, get_mount_options(mnt_flags)))
crashcolor.set_color(crashcolor.RESET)
def run_rules():
global modules
global sysinfo
issue_count = 0
for module in modules:
try:
result_list = module.run_rule(sysinfo)
if result_list != None:
issue_count = issue_count + len(result_list)
print_result(result_list)
except:
print("Error running rule %s" % (module))
if issue_count > 0:
print("*" * 75)
crashcolor.set_color(crashcolor.RED | crashcolor.BLINK)
print("\tWARNING: %d issue%s detected" %
(issue_count, "s" if issue_count > 1 else ""))
crashcolor.set_color(crashcolor.RESET)
print("*" * 75)
else:
print("No issues detected")
def vmw_mem():
try:
pa = readSymbol('balloon')
if (pa == 0):
return
except:
print("VMware balloon symbol does not exist")
return
baddr = sym2addr('balloon')
balloon_result = exec_crash_command(
'struct vmballoon.size,target,stats 0x%x' % (baddr))
print('%s' % (balloon_result))
crashcolor.set_color(crashcolor.LIGHTRED)
print("allocated size (pages) = %d" % pa.size)
print("allocated size (bytes) = %d, (%.2fGB)" %
(pa.size * crash.PAGESIZE,
((pa.size * crash.PAGESIZE) / 1024 / 1024 / 1024)))
print("required target (pages) = %d" % pa.target)
print("required target (bytes) = %d, (%.2fGB)" %
(pa.target * crash.PAGESIZE,
((pa.target * crash.PAGESIZE) / 1024 / 1024 / 1024)))
crashcolor.set_color(crashcolor.RESET)
print("")
if (member_offset(pa, "n_refused_pages") > -1):
print("refuesed pages = %d" % pa.n_refused_pages)
print("rate_alloc = %d" % pa.rate_alloc)
if (member_offset(pa, "rate_free") > -1):
print("rate_free = %d" % pa.rate_free)
print("\n")
"""
def try_get_module_struct(options):
module, loaded_module = find_module_struct(options.module_addr)
if module is not None:
print("Found the below module")
print("\tstruct module 0x%x" % module)
print("\tname : %s" % module.name)
print("\tstatus : %s" % loaded_module)
if options.show_symtab:
show_manual_module_detail(options, module)
else:
crashcolor.set_color(crashcolor.RED)
print("\nCannot find module structure for %s" % options.module_addr)
crashcolor.set_color(crashcolor.RESET)
module_addr_min = 0
module_addr_max = 0
module_addr = int(options.module_addr, 16)
try:
module_addr_min = readSymbol("module_addr_min")
module_addr_max = readSymbol("module_addr_max")
except:
pass
if module_addr_max == 0:
try:
mod_tree = readSymbol("mod_tree")
module_addr_min = mod_tree.addr_min
module_addr_max = mod_tree.addr_max
except:
pass
if (module_addr_min < module_addr) and (module_addr < module_addr_max):
print("\nThis address belongs to module allocation memory range")
print("\tmodule address min: 0x%x" % (module_addr_min))
print("\tmodule address max: 0x%x" % (module_addr_max))
print_last_unloaded_module()
def check_syscall_table(options):
sys_call_table, max_syscalls = sys_call_table_info()
invalid_start_list = ["jmp", "callq"]
arch = sys_info.machine
if arch.startswith("arm") or arch.startswith("ppc"):
#invalid_start_list = [ "bl" ]
pass # As the normal call also has 'bl' in some calls, not use it yet
hook_call_no = 0
trap_call_no = 0
idx = 0
for sys_call in sys_call_table:
if sys_call == None:
break
call_addr = sym2addr(sys_call)
result = exec_crash_command("sym 0x%x" % call_addr)
if result != None and result.startswith("sym"):
break
words = result.split()
filepath = get_file_path(words)
if len(words) == 4 and not words[3].startswith("/"):
hook_call_no = hook_call_no + 1
crashcolor.set_color(crashcolor.LIGHTRED)
print("%3d %s" % (idx, result), end='')
else:
dis_result = exec_crash_command("dis 0x%x 1" % call_addr)
dis_words = dis_result.split()
if len(dis_words) >= 4 and dis_words[2].strip(
) in invalid_start_list:
trap_call_no = trap_call_no + 1
crashcolor.set_color(crashcolor.BLUE)
print("%3d %s %s %-25s %s" %
(idx, words[0], words[1], words[2], filepath))
print("\t%s" % (dis_result[dis_result.find(dis_words[2]):]),
end='')
elif options.syscall_check == False:
print("%3d %s %s %-25s %s" %
(idx, words[0], words[1], words[2], filepath))
crashcolor.set_color(crashcolor.RESET)
idx = idx + 1
if hook_call_no > 0 or trap_call_no > 0:
print("=" * 75)
if hook_call_no > 0:
print("%d system calls were replaced" % hook_call_no)
if trap_call_no > 0:
print("%d system calls were modified" % trap_call_no)
elif options.syscall_check:
print("No issues detected")
def get_size_str(size, coloring=False):
size_str = ""
if size > (1024 * 1024 * 1024): # GiB
size_str = "%d GiB" % (size / (1024 * 1024 * 1024))
if coloring == True:
crashcolor.set_color(crashcolor.RED)
elif size > (1024 * 1024): # MiB
size_str = "%d MiB" % (size / (1024 * 1024))
if coloring == True:
crashcolor.set_color(crashcolor.MAGENTA)
elif size > (1024): # KiB
size_str = "%d KiB" % (size / (1024))
if coloring == True:
crashcolor.set_color(crashcolor.GREEN)
else:
size_str = "%d B" % (size)
return size_str
def show_tlb(options):
cpuinfo_addrs = percpu.get_cpu_var("cpu_tlbstate")
for cpu, addr in enumerate(cpuinfo_addrs):
tlb_state = readSU("struct tlb_state", addr)
task = 0
if tlb_state.active_mm > 0:
active_mm = readSU("struct mm_struct", tlb_state.active_mm)
task = active_mm.owner
if tlb_state.state == TLBSTATE_LAZY:
crashcolor.set_color(crashcolor.BLUE)
elif tlb_state.state == TLBSTATE_OK:
crashcolor.set_color(crashcolor.LIGHTGREEN)
if task != 0:
task_name = task.comm
else:
task_name = ""
print("CPU %3d : state = %d [%-13s], active_mm = 0x%x (%s)" %
(cpu, tlb_state.state, tlb_str(
tlb_state.state), tlb_state.active_mm, task_name))
crashcolor.set_color(crashcolor.RESET)
def lockup_display(reverse_sort, show_tasks, options):
rqlist = Tasks.getRunQueues()
rqsorted = sorted(rqlist, key=getKey, reverse=reverse_sort)
if (reverse_sort):
now = rqsorted[0].Timestamp
else:
now = rqsorted[-1].Timestamp
try:
watchdog_thresh = readSymbol("watchdog_thresh")
softlockup_thresh = watchdog_thresh * 2
except:
try:
softlockup_thresh = readSymbol("softlockup_thresh")
watchdog_thresh = 10
except:
watchdog_thresh = -1
for rq in rqsorted:
prio = rq.curr.prio
if (rq.curr.policy != 0):
prio = rq.curr.rt_priority
delayed_time = (now - rq.Timestamp) / 1000000000
if watchdog_thresh > 0:
if delayed_time >= softlockup_thresh:
crashcolor.set_color(crashcolor.RED)
elif delayed_time >= watchdog_thresh:
crashcolor.set_color(crashcolor.BLUE)
print("CPU %3d: %10.2f sec behind by "
"0x%x, %s [%s:%3d] (%d in queue)" %
(rq.cpu, delayed_time, rq.curr, rq.curr.comm,
get_task_policy_str(rq.curr.policy), prio, rq.nr_running))
if (show_tasks):
show_task_list(rq, reverse_sort, options)
crashcolor.set_color(crashcolor.RESET)
def print_last_unloaded_module():
last_unloaded_module = readSymbol("last_unloaded_module")
if len(last_unloaded_module) > 0:
crashcolor.set_color(crashcolor.BLUE)
print("\n\tLast unloaded module : %s" % (last_unloaded_module))
crashcolor.set_color(crashcolor.RESET)
def print_cgroup_entry(top_cgroup, cur_cgroup, idx, options):
global empty_count
global cgroup_count
if (idx > 0 and top_cgroup == cur_cgroup):
return
if (cur_cgroup == 0):
return
e_count = 0
for css_addr in cur_cgroup.subsys:
if (css_addr == 0):
continue
subsys = readSU("struct cgroup_subsys_state", css_addr)
if (subsys == 0):
continue
if (subsys.cgroup == 0):
continue
cgroup = subsys.cgroup
cgroup_name = ""
cgroup_counter = 0
if member_offset("struct cgroup", "dentry") > -1:
if (cgroup.dentry != 0):
cgroup_name = dentry_to_filename(cgroup.dentry)
elif member_offset("struct cgroup", "kn") > -1:
cgroup_name = cgroup.kn.name
cgroup_counter = cgroup_task_count(cgroup)
if cgroup_name == "":
cgroup_name = "<default>"
if cgroup_counter == 0:
crashcolor.set_color(crashcolor.RED)
e_count = 1
else:
crashcolor.set_color(crashcolor.RESET)
print ("%s%s%s at 0x%x (%d)" %
(" " * idx, "+--" if idx > 0 else "",
cgroup_name, cgroup, cgroup_counter))
if options.show_detail:
print_cgroup_details(idx, cgroup)
if options.task_list:
cgroup_task_list(cgroup, idx)
# if (cgroup.parent == 0):
# top_cgroup = cgroup
head_of_list = None
if member_offset("struct cgroup", "children") > -1:
head_of_list = cgroup.children
struct_name = "struct cgroup"
elif member_offset("struct cgroup_subsys_state", "children") > -1:
head_of_list = subsys.children
struct_name = "struct cgroup_subsys_state"
for childaddr in readSUListFromHead(head_of_list,
'sibling', struct_name,
maxel=1000000):
if struct_name == "struct cgroup":
cgroup = readSU(struct_name, childaddr)
else:
subsys_state = readSU(struct_name, childaddr)
cgroup = subsys_state.cgroup
print_cgroup_entry(top_cgroup, cgroup, idx + 1, options)
# if (idx == 0):
# print ("")
if (cgroup == top_cgroup):
continue
cgroup_count = cgroup_count + 1
empty_count = empty_count + e_count
crashcolor.set_color(crashcolor.RESET)
return
def show_audit_watch_details(audit_entry):
crashcolor.set_color(crashcolor.GREEN)
print("\twatch path: %s" % (audit_entry.rule.watch.path))
crashcolor.set_color(crashcolor.RESET)
def print_sym_list_section(title, sym_list, show_contents, show_strings,
min_line_length):
crashcolor.set_color(crashcolor.BLUE)
print(title)
crashcolor.set_color(crashcolor.RESET)
start_addr = end_addr = ""
for sym_entry in sym_list:
if start_addr == "":
start_addr = sym_entry[0]
end_addr = sym_entry[0]
print("0x%s %s %s" % (sym_entry[0], sym_entry[1], sym_entry[2]))
if show_contents == False:
continue
data = "0x%016x" % readULong(int(sym_entry[0], 16))
first_byte = str(chr(readU8(int(sym_entry[0], 16))))
try:
if data.startswith("0xf"):
data_str = exec_crash_command("sym %s" % (data))
if data_str.startswith("sym:"):
data_str = check_slab(data, "<no symbol>")
else:
data_str = "sym: %s" % data_str.split()[2]
elif first_byte.isprintable():
try:
data_str = exec_crash_command("rd -a 0x%s" %
(sym_entry[0]))
data_str = "\"%s\"" % data_str.split()[1]
except:
data_str = "= %d" % int(data, 16)
pass
else:
data_str = "= %d" % int(data, 16)
except:
data_str = "= %d" % int(data, 16)
pass
crashcolor.set_color(crashcolor.YELLOW)
print("\t( %s %s )" % (data, data_str))
crashcolor.set_color(crashcolor.RESET)
if show_strings == True and start_addr != "" and start_addr != end_addr:
try:
result = exec_crash_command("rd 0x%s -e 0x%s -a" %\
(start_addr, end_addr))
except:
result = ""
lines = result.splitlines()
crashcolor.set_color(crashcolor.GREEN)
for line in lines:
words = line.split()
if len(words) > 1:
dataindex = line.index(words[0]) + len(words[0])
content_len = len(line[dataindex:].strip())
if content_len >= min_line_length:
print(line)
crashcolor.set_color(crashcolor.RESET)
def show_cgroup_tree_from_rootnode(rootnode, options):
global empty_count
global cgroup_count
crashcolor.set_color(crashcolor.BLUE)
print ("** cgroup subsystems **")
crashcolor.set_color(crashcolor.RESET)
for cgroup_subsys in readSUListFromHead(rootnode.subsys_list,
'sibling',
'struct cgroup_subsys',
maxel=1000000):
print ("%s (0x%x)" % (cgroup_subsys.name, cgroup_subsys))
print ("")
crashcolor.set_color(crashcolor.BLUE)
print ("** cgroup tree **")
crashcolor.set_color(crashcolor.RESET)
top_cgroup = rootnode.top_cgroup
curlimit = sys.getrecursionlimit()
sys.setrecursionlimit(1000)
print_cgroup_entry(top_cgroup, top_cgroup, 0, options)
sys.setrecursionlimit(curlimit)
print ("-" * 70)
crashcolor.set_color(crashcolor.BLUE)
print ("Total number of cgroup(s) = %d, %d had 0 count" %
(cgroup_count, empty_count))
crashcolor.set_color(crashcolor.RESET)
def disasm(ins_addr, o, args, cmd_path_list):
global asm_color_dict
global funcname
global stackaddr_list
global stack_op_dict
path_list = cmd_path_list.split(':')
disasm_path = ""
for path in path_list:
if os.path.exists(path + "/disasm.py"):
disasm_path = path + "/disasm.py"
break
if disasm_path == "":
print("Can't find disasm.py in path")
return
options = ""
if (o.noaction == False):
options = "-l"
if (o.reverse):
options = options + " -r"
cmd_options = ""
if (o.sourceonly):
cmd_options = cmd_options + " -s"
if (o.fullsource):
cmd_options = cmd_options + " -f"
if (not o.reverse):
options = options + " -r"
if ins_addr.startswith("/"):
ins_addr = ins_addr[1:] # get rid of the first slash
if ":" in ins_addr or \
(not ins_addr.startswith(".") and "." in ins_addr): # It's for ppc
if ":" not in ins_addr: # Let's make fake line number
ins_addr = ins_addr + ": 0"
else:
words = ins_addr.split(":")
ins_addr = ""
for column in words:
if ins_addr == "":
ins_addr = column + ":"
else:
ins_addr = ins_addr + " " + column
for line_number in args[1:]:
ins_addr = ins_addr + " " + line_number
kernel_ver, release_ver = get_kernel_version()
disasm_str = "/usr/src/debug/kernel-%s/linux-%s/%s" % \
(kernel_ver, release_ver, ins_addr)
else:
command_str = "dis %s %s" % (options, ins_addr)
disasm_str = exec_crash_command(command_str)
if (disasm_str.startswith("symbol not found")):
print(disasm_str)
return
result_str = ""
if (o.noaction or not disasm_str.startswith("/")):
result_str = disasm_str
else:
python_list = {"python", "python3", "python2"}
for python_cmd in python_list:
if (is_command_exist(python_cmd)):
kerver, relver = get_kernel_version()
ver_line = ""
if kerver.find(".rt") >= 0: # rt kernel
relver = relver[:relver.find("-")]
ver_line = "/usr/src/debug/kernel-%s/linux-%s/" % \
(relver, kerver)
else:
ver_line = disasm_str.splitlines()[0]
disasm_str = ver_line + "\n" + disasm_str
result_str = crashhelper.run_gdb_command("!echo '%s' | %s %s %s" % \
(disasm_str, python_cmd, \
disasm_path, cmd_options))
break
if (o.graph):
result_str = draw_branches(result_str, o.jump_op_list)
set_stack_data(disasm_str, ins_addr) # To retreive stack data
if o.stackaddr != "":
stackaddr_list = [int(o.stackaddr, 16)]
set_asm_colors()
crashcolor.set_color(crashcolor.RESET)
for one_line in result_str.splitlines():
idx = one_line.find("0x")
if idx >= 0:
line = one_line[idx:]
graph = one_line[:idx]
is_disasm_line = True
else: # source line
prog = re.compile(r"(?P<line_number>[0-9]+)")
m = prog.search(one_line)
line = None
if m is not None:
line = m.group("line_number")
if line == None:
prog = re.compile(r"/[a-zA-Z]")
line = prog.match(one_line)
if line == None:
line = one_line
idx = one_line.find(line)
line = one_line[idx:]
graph = one_line[:idx]
is_disasm_line = False
idx = 2
default_color = crashcolor.RESET
for char in graph:
color = default_color
idx = idx + 1
if char == '+':
default_color = idx
color = default_color
elif char == '|':
color = idx
elif char == '-' or char == '=':
color = default_color
elif char == '>' or char == '*':
color = crashcolor.RED
else:
color = crashcolor.RESET
crashcolor.set_color(color)
print(char, end='')
if idx == crashcolor.MAX_COLOR:
idx = 2
if idx == 7: # For black background situation
idx = idx + 1
if is_disasm_line == True:
line = interpret_one_line(line) # Retreive stack data if possible
words = line.split()
if len(words) > 2:
if (o.symbol and is_address(words[-1])
== True): # Translate address into symbol
line = ("%s%s" % (words[-1], find_symbol(words[-1]))).join(
line.rsplit(words[-1], 1))
color_str = get_colored_asm(words[2].strip())
idx = line.find(words[2], len(words[0]) + len(words[1]) + 1)
print(line[:idx], end='')
if color_str != None:
crashcolor.set_color(color_str)
print(line[idx:idx + len(words[2])], end='')
if color_str != None:
crashcolor.set_color(operand_color)
if len(words) >= 4: # Not handling callq or jmp.
line = line[idx:]
print("%s" % line[len(words[2]):line.find(words[3])], end='')
idx = line.find(words[3])
op_list = words[3].split(",")
line = line[idx:]
for i in range(0, len(op_list)):
opval = op_list[i]
color_str = get_colored_arg(opval)
if color_str == None:
crashcolor.set_color(crashcolor.RESET)
else:
crashcolor.set_color(color_str)
if i < len(op_list) - 1:
next_idx = line.find(op_list[i + 1], len(opval))
else:
next_idx = len(opval)
print(line[:next_idx], end='')
if color_str == None:
crashcolor.set_color(crashcolor.RESET)
else:
crashcolor.set_color(operand_color)
line = line[next_idx:]
crashcolor.set_color(crashcolor.RESET)
if (is_disasm_line):
comment_idx = line.find(";")
if comment_idx > -1:
print(line[:comment_idx], end='')
crashcolor.set_color(crashcolor.LIGHTYELLOW)
print(line[comment_idx:])
else:
comment_idx = line.find("<")
if comment_idx > -1:
print(line[:comment_idx], end='')
crashcolor.set_color(crashcolor.LIGHTMAGENTA)
print(line[comment_idx:])
else:
print("%s" % line)
else:
print(line)
crashcolor.set_color(crashcolor.RESET)
else:
print(line[idx + len(words[2]):])
crashcolor.set_color(crashcolor.RESET)
else:
print(line)
crashcolor.set_color(crashcolor.RESET)
def set_color_for_particular_types(rule_type):
if rule_type in colored_type_list:
crashcolor.set_color(crashcolor.RED)
