def parse_arenas():
global jeheap
jeheap.arenas[:] = []
for i in range(0, jeheap.narenas):
current_arena = jemalloc.arena(0, i, [])
try:
current_arena.addr = \
dbg.to_int(dbg.eval_expr(dbg.arena_expr % (i)))
except:
print('[shadow] error: cannot evaluate arenas[%d]') % (i)
sys.exit()
for j in range(0, jeheap.nbins):
nrg = 0
run_sz = 0
reg_size = 0
reg_offset = 0
end_addr = 0
try:
expr = dbg.arena_reg_size_expr % (i, j)
reg_size = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_reg0_offset_expr % (i, j)
reg_offset = dbg.to_int(dbg.eval_expr(expr))
except:
# XXX: for now assume it's a standalone variant; we
# need to do some error checking here too.
jeheap.STANDALONE = true
expr = dbg.arena_bin_info_reg_size_expr % (j)
reg_size = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_bin_info_nregs_expr % (j)
nrg = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_bin_info_run_size_expr % (j)
run_sz = dbg.to_int(dbg.eval_expr(expr))
try:
expr = dbg.arena_runcur_expr % (i, j)
runcur_addr = runcur = dbg.to_int(dbg.eval_expr(expr))
end_addr = runcur_addr + run_sz
if runcur != 0:
current_run = \
jemalloc.arena_run(runcur, end_addr, run_sz, 0, \
int(reg_size), reg_offset, nrg, 0, [])
current_bin = jemalloc.arena_bin(0, j, current_run)
current_bin.addr = \
dbg.to_int(dbg.eval_expr(dbg.arena_bin_addr_expr % (i, j)))
current_arena.bins.append(current_bin)
else:
# no regions for this size class yet, therefore no runcur
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
except:
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
continue
# add arena to the list of arenas
jeheap.arenas.append(current_arena)
def parse_run(run_addr, proc=none):
'''Given a run's address return a jemalloc.arena_run object'''
global jeheap
new_run = jemalloc.arena_run()
new_run.start = run_addr
try:
new_run.bin_addr = dbg.read_memory(new_run.start, jeheap.DWORD_SIZE,
proc)
if jeheap.STANDALONE == false:
new_run.size = dbg.read_memory(new_run.bin_addr + \
(6 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
new_run.end = new_run.start + new_run.size
new_run.region_size = dbg.read_memory(new_run.bin_addr + \
(5 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
new_run.total_regions = dbg.read_memory(new_run.bin_addr + \
(7 * jeheap.DWORD_SIZE), jemalloc.INT_SIZE, proc)
if new_run.total_regions > 10000 or new_run.total_regions <= 0:
return none
except:
# print('[shadow] error parsing the metadata of run 0x%08x' % (run_addr))
return none
# XXX: this isn't correct on jemalloc standalone *debug* variant
try:
new_run.free_regions = dbg.read_memory(new_run.start + \
jeheap.DWORD_SIZE + jemalloc.INT_SIZE, jemalloc.INT_SIZE, proc)
except:
# print('[shadow] error parsing the free regions of run 0x%08x' % (run_addr))
new_run.free_regions = 0
if new_run.free_regions < 0:
new_run.free_regions = 0
# delete the run's regions
new_run.regions[:] = []
# parse the run's regions
new_run.reg0_offset = dbg.read_memory(new_run.bin_addr + \
(9 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
if new_run.reg0_offset > 10000 or new_run.reg0_offset <= 0:
return none
first_region_addr = reg0_addr = run_addr + new_run.reg0_offset
regs_mask_bits = (new_run.total_regions / 8) + 1
regs_mask_addr = 0
regs_mask_str = ''
if dbg_engine == 'gdb':
regs_mask_addr = dbg.to_int(dbg.execute(dbg.regs_mask_addr_expr % \
(run_addr)))
regs_mask_str = dbg.execute(dbg.regs_mask_addr_bits_expr % \
(regs_mask_bits, regs_mask_addr))
elif dbg_engine == 'pykd':
regs_mask_addr = dbg.to_int(dbg.eval_expr(dbg.regs_mask_addr_expr % \
(run_addr)))
regs_mask_str = dbg.execute(dbg.regs_mask_addr_bits_expr % \
(regs_mask_addr, regs_mask_bits))
else: # lldb
regs_mask_str = ''
regs_mask = ''
if dbg_engine == 'gdb':
for line in regs_mask_str.splitlines():
line = line[line.find(dbg.address_separator) + \
len(dbg.address_separator) : line.find('\n')]
line = line.replace('\n', '')
line = line.replace('\t', '')
line = line.replace(' ', '')
regs_mask += line
elif dbg_engine == 'pykd':
lines = regs_mask_str.splitlines()
lines = lines[2:]
for line in lines:
line = line[line.find(dbg.address_separator) + \
len(dbg.address_separator) : \
line.rfind(dbg.address_separator)]
line = line.replace('\n', '')
line = line.replace('\t', '')
line = line.replace(' ', '')
regs_mask += line
else: # lldb
regs_mask = ''
new_run.regs_mask = regs_mask
first_region = jemalloc.region(0, first_region_addr, \
int(new_run.regs_mask[0]))
try:
first_region.content_preview = hex(dbg.read_memory(first_region.addr, \
jemalloc.INT_SIZE, proc)).rstrip('L')
except:
print('[shadow] error reading the first dword of region 0x%08x' \
% (first_region.addr))
first_region.content_preview = ''
new_run.regions.append(first_region)
for i in range(1, new_run.total_regions):
try:
current_region = jemalloc.region(i, 0, int(new_run.regs_mask[i]))
except:
current_region = jemalloc.region(i, 0, 0)
current_region.addr = reg0_addr + (i * new_run.region_size)
try:
current_region.content_preview = \
hex(dbg.read_memory(current_region.addr, jemalloc.INT_SIZE, proc)).rstrip('L')
except:
current_region.content_preview = ''
new_run.regions.append(current_region)
return new_run
def jeparse_all_runs(proc):
global jeheap
# number of pages a chunk occupies
chunk_npages = jeheap.chunk_size >> 12
# offset of bits in arena_chunk_map_t in double words
bitmap_offset = \
gdbutil.offsetof('arena_chunk_map_t', 'bits') / jeheap.DWORD_SIZE
# number of double words occupied by an arena_chunk_map_t
chunk_map_dwords = \
(bitmap_offset / jeheap.DWORD_SIZE) + 1
# prefix to use in gdb's examine command
if jeheap.DWORD_SIZE == 8:
dword_fmt = 'g'
else:
dword_fmt = 'w'
# the 12 least significant bits of each bitmap entry hold
# various flags for the corresponding run
flags_mask = (1 << 12) - 1
# delete the heap's runs' array
jeheap.runs[:] = []
for chunk in jeheap.chunks:
if not chunk.arena:
continue
try:
# parse the whole map at once to avoid gdb delays
expr = 'x/%d%sx ((arena_chunk_t *)%#x)->map' % \
(chunk_npages * chunk_map_dwords, dword_fmt, chunk.addr)
except:
print '[unmask_jemalloc] error: cannot read bitmap from chunk %#x' (chunk.addr)
sys.exit()
lines = (gdb.execute(expr, to_string = true)).split('\n')
dwords = []
for line in lines:
dwords += [int(dw, 16) for dw in line[line.find(':') + 1:].split()]
bitmap = [dwords[i] for i in range(bitmap_offset, len(dwords), bitmap_offset + 1)]
# traverse the bitmap
for mapelm in bitmap:
flags = mapelm & flags_mask
# flags == 1 means the chunk is small and the rest of the bits
# hold the actual run address
if flags == 1:
addr = mapelm & ~flags_mask
size = gdbutil.get_page_size()
# flags = 3 indicates a large chunk; calculate the run's address
# directly from the map element index and extract the run's size
elif flags == 3:
addr = chunk.addr + i * gdbutil.get_page_size()
size = mapelm & ~flags_mask
# run is not allocated? skip it
else:
continue
if addr not in [a for (a, s) in runs]:
# XXX: we need to parse run headers here with a
# dedicated function
new_run = jemalloc.arena_run(addr, 0, size, 0, 0, 0, 0, 0, [])
jeheap.runs.append(new_run)
def jeparse_arenas():
global jeheap
jeheap.arenas[:] = []
for i in range(0, jeheap.narenas):
current_arena = jemalloc.arena(0, i, [])
try:
current_arena.addr = \
gdbutil.to_int(gdb.parse_and_eval('arenas[%d]' % (i)))
except:
print '[unmask_jemalloc] error: cannot evaluate arenas[%d]' % (i)
sys.exit()
for j in range(0, jeheap.nbins):
nrg = 0
run_sz = 0
reg_size = 0
reg_offset = 0
end_addr = 0
try:
expr = 'arenas[%d].bins[%d].reg_size' % (i, j)
reg_size = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arenas[%d].bins[%d].reg0_offset' % (i, j)
reg_offset = \
gdbutil.to_int(gdb.parse_and_eval(expr))
except RuntimeError:
# XXX: for now assume it's a standalone variant; we
# need to do some error checking here too.
jeheap.STANDALONE = true
expr = 'arena_bin_info[%d].reg_size' % (j)
reg_size = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arena_bin_info[%d].nregs' % (j)
nrg = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arena_bin_info[%d].run_size' % (j)
run_sz = \
gdbutil.to_int(gdb.parse_and_eval(expr))
try:
expr = 'arenas[%d].bins[%d].runcur' % (i, j)
runcur_addr = runcur = \
gdbutil.to_int(gdb.parse_and_eval(expr))
end_addr = runcur_addr + run_sz
if runcur != 0:
current_run = \
jemalloc.arena_run(runcur, end_addr, run_sz, 0, \
int(reg_size), reg_offset, nrg, 0, [])
current_bin = jemalloc.arena_bin(0, j, current_run)
current_bin.addr = \
gdbutil.to_int(gdb.parse_and_eval('&arenas[%d].bins[%d]' % (i, j)))
current_arena.bins.append(current_bin)
else:
# no regions for this size class yet, therefore no runcur
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
except RuntimeError:
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
continue
# add arena to the list of arenas
jeheap.arenas.append(current_arena)
def jeparse_arenas():
global jeheap
jeheap.arenas[:] = []
for i in range(0, jeheap.narenas):
current_arena = jemalloc.arena(0, i, [])
try:
current_arena.addr = \
gdbutil.to_int(gdb.parse_and_eval('arenas[%d]' % (i)))
except:
print('[unmask_jemalloc] error: cannot evaluate arenas[%d]') % (i)
sys.exit()
for j in range(0, jeheap.nbins):
nrg = 0
run_sz = 0
reg_size = 0
reg_offset = 0
end_addr = 0
try:
expr = 'arenas[%d].bins[%d].reg_size' % (i, j)
reg_size = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arenas[%d].bins[%d].reg0_offset' % (i, j)
reg_offset = \
gdbutil.to_int(gdb.parse_and_eval(expr))
except RuntimeError:
# XXX: for now assume it's a standalone variant; we
# need to do some error checking here too.
jeheap.STANDALONE = true
expr = 'arena_bin_info[%d].reg_size' % (j)
reg_size = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arena_bin_info[%d].nregs' % (j)
nrg = \
gdbutil.to_int(gdb.parse_and_eval(expr))
expr = 'arena_bin_info[%d].run_size' % (j)
run_sz = \
gdbutil.to_int(gdb.parse_and_eval(expr))
try:
expr = 'arenas[%d].bins[%d].runcur' % (i, j)
runcur_addr = runcur = \
gdbutil.to_int(gdb.parse_and_eval(expr))
end_addr = runcur_addr + run_sz
if runcur != 0:
current_run = \
jemalloc.arena_run(runcur, end_addr, run_sz, 0, \
int(reg_size), reg_offset, nrg, 0, [])
current_bin = jemalloc.arena_bin(0, j, current_run)
current_bin.addr = \
gdbutil.to_int(gdb.parse_and_eval('&arenas[%d].bins[%d]' % (i, j)))
current_arena.bins.append(current_bin)
else:
# no regions for this size class yet, therefore no runcur
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
except RuntimeError:
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
continue
# add arena to the list of arenas
jeheap.arenas.append(current_arena)
def jeparse_all_runs(proc):
global jeheap
# number of pages a chunk occupies
chunk_npages = jeheap.chunk_size >> 12
# offset of bits in arena_chunk_map_t in double words
bitmap_offset = \
gdbutil.offsetof('arena_chunk_map_t', 'bits') / jeheap.DWORD_SIZE
# number of double words occupied by an arena_chunk_map_t
chunk_map_dwords = \
(bitmap_offset / jeheap.DWORD_SIZE) + 1
# prefix to use in gdb's examine command
if jeheap.DWORD_SIZE == 8:
dword_fmt = 'g'
else:
dword_fmt = 'w'
# the 12 least significant bits of each bitmap entry hold
# various flags for the corresponding run
flags_mask = (1 << 12) - 1
# delete the heap's runs' array
jeheap.runs[:] = []
for chunk in jeheap.chunks:
if not chunk.arena:
continue
try:
# parse the whole map at once to avoid gdb delays
expr = 'x/%d%sx ((arena_chunk_t *)%#x)->map' % \
(chunk_npages * chunk_map_dwords, dword_fmt, chunk.addr)
except:
print(
'[unmask_jemalloc] error: cannot read bitmap from chunk %#x' %
(chunk.addr))
sys.exit()
lines = (gdb.execute(expr, to_string=true)).split('\n')
dwords = []
i = 0
for line in lines:
dwords += [int(dw, 16) for dw in line[line.find(':') + 1:].split()]
bitmap = [dwords[i] for i in range(int(bitmap_offset), \
int(len(dwords)), int(bitmap_offset + 1))]
# traverse the bitmap
for mapelm in bitmap:
flags = mapelm & flags_mask
# flags == 1 means the chunk is small and the rest of the bits
# hold the actual run address
if flags == 1:
addr = mapelm & ~flags_mask
size = gdbutil.get_page_size()
# flags = 3 indicates a large chunk; calculate the run's address
# directly from the map element index and extract the run's size
elif flags == 3:
addr = chunk.addr + i * gdbutil.get_page_size()
size = mapelm & ~flags_mask
# run is not allocated? skip it
else:
continue
if addr not in [r.start for r in jeheap.runs]:
# XXX: we need to parse run headers here with a
# dedicated function
new_run = jemalloc.arena_run(addr, 0, size, 0, 0, 0, 0, 0, [])
jeheap.runs.append(new_run)
def parse_run(run_addr, proc=none):
"""Given a run's address return a jemalloc.arena_run object"""
global jeheap
new_run = jemalloc.arena_run()
new_run.start = run_addr
try:
new_run.bin_addr = dbg.read_memory(new_run.start, jeheap.DWORD_SIZE, proc)
if jeheap.STANDALONE == false:
new_run.size = dbg.read_memory(new_run.bin_addr + (6 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
new_run.end = new_run.start + new_run.size
new_run.region_size = dbg.read_memory(new_run.bin_addr + (5 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
new_run.total_regions = dbg.read_memory(new_run.bin_addr + (7 * jeheap.DWORD_SIZE), jemalloc.INT_SIZE, proc)
if new_run.total_regions > 10000 or new_run.total_regions <= 0:
return none
except:
# print('[shadow] error parsing the metadata of run 0x%08x' % (run_addr))
return none
# XXX: this isn't correct on jemalloc standalone *debug* variant
try:
new_run.free_regions = dbg.read_memory(
new_run.start + jeheap.DWORD_SIZE + jemalloc.INT_SIZE, jemalloc.INT_SIZE, proc
)
except:
# print('[shadow] error parsing the free regions of run 0x%08x' % (run_addr))
new_run.free_regions = 0
if new_run.free_regions < 0:
new_run.free_regions = 0
# delete the run's regions
new_run.regions[:] = []
# parse the run's regions
new_run.reg0_offset = dbg.read_memory(new_run.bin_addr + (9 * jeheap.DWORD_SIZE), jeheap.DWORD_SIZE, proc)
if new_run.reg0_offset > 10000 or new_run.reg0_offset <= 0:
return none
first_region_addr = reg0_addr = run_addr + new_run.reg0_offset
regs_mask_bits = (new_run.total_regions / 8) + 1
regs_mask_addr = 0
regs_mask_str = ""
if dbg_engine == "gdb":
regs_mask_addr = dbg.to_int(dbg.execute(dbg.regs_mask_addr_expr % (run_addr)))
regs_mask_str = dbg.execute(dbg.regs_mask_addr_bits_expr % (regs_mask_bits, regs_mask_addr))
elif dbg_engine == "pykd":
regs_mask_addr = dbg.to_int(dbg.eval_expr(dbg.regs_mask_addr_expr % (run_addr)))
regs_mask_str = dbg.execute(dbg.regs_mask_addr_bits_expr % (regs_mask_addr, regs_mask_bits))
else: # lldb
regs_mask_str = ""
regs_mask = ""
if dbg_engine == "gdb":
for line in regs_mask_str.splitlines():
line = line[line.find(dbg.address_separator) + len(dbg.address_separator) : line.find("\n")]
line = line.replace("\n", "")
line = line.replace("\t", "")
line = line.replace(" ", "")
regs_mask += line
elif dbg_engine == "pykd":
lines = regs_mask_str.splitlines()
lines = lines[2:]
for line in lines:
line = line[
line.find(dbg.address_separator) + len(dbg.address_separator) : line.rfind(dbg.address_separator)
]
line = line.replace("\n", "")
line = line.replace("\t", "")
line = line.replace(" ", "")
regs_mask += line
else: # lldb
regs_mask = ""
new_run.regs_mask = regs_mask
first_region = jemalloc.region(0, first_region_addr, int(new_run.regs_mask[0]))
try:
first_region.content_preview = hex(dbg.read_memory(first_region.addr, jemalloc.INT_SIZE, proc)).rstrip("L")
except:
print("[shadow] error reading the first dword of region 0x%08x" % (first_region.addr))
first_region.content_preview = ""
new_run.regions.append(first_region)
for i in range(1, new_run.total_regions):
try:
current_region = jemalloc.region(i, 0, int(new_run.regs_mask[i]))
except:
current_region = jemalloc.region(i, 0, 0)
current_region.addr = reg0_addr + (i * new_run.region_size)
try:
current_region.content_preview = hex(dbg.read_memory(current_region.addr, jemalloc.INT_SIZE, proc)).rstrip(
"L"
)
except:
current_region.content_preview = ""
new_run.regions.append(current_region)
return new_run
def parse_arenas():
global jeheap
jeheap.arenas[:] = []
for i in range(0, jeheap.narenas):
current_arena = jemalloc.arena(0, i, [])
try:
current_arena.addr = dbg.to_int(dbg.eval_expr(dbg.arena_expr % (i)))
except:
print("[shadow] error: cannot evaluate arenas[%d]") % (i)
sys.exit()
for j in range(0, jeheap.nbins):
nrg = 0
run_sz = 0
reg_size = 0
reg_offset = 0
end_addr = 0
try:
expr = dbg.arena_reg_size_expr % (i, j)
reg_size = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_reg0_offset_expr % (i, j)
reg_offset = dbg.to_int(dbg.eval_expr(expr))
except:
# XXX: for now assume it's a standalone variant; we
# need to do some error checking here too.
jeheap.STANDALONE = true
expr = dbg.arena_bin_info_reg_size_expr % (j)
reg_size = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_bin_info_nregs_expr % (j)
nrg = dbg.to_int(dbg.eval_expr(expr))
expr = dbg.arena_bin_info_run_size_expr % (j)
run_sz = dbg.to_int(dbg.eval_expr(expr))
try:
expr = dbg.arena_runcur_expr % (i, j)
runcur_addr = runcur = dbg.to_int(dbg.eval_expr(expr))
end_addr = runcur_addr + run_sz
if runcur != 0:
current_run = jemalloc.arena_run(runcur, end_addr, run_sz, 0, int(reg_size), reg_offset, nrg, 0, [])
current_bin = jemalloc.arena_bin(0, j, current_run)
current_bin.addr = dbg.to_int(dbg.eval_expr(dbg.arena_bin_addr_expr % (i, j)))
current_arena.bins.append(current_bin)
else:
# no regions for this size class yet, therefore no runcur
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
except:
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
continue
# add arena to the list of arenas
jeheap.arenas.append(current_arena)
def jeparse_arenas():
global jeheap
jeheap.arenas[:] = []
for i in range(0, jeheap.narenas):
current_arena = jemalloc.arena(0, i, [])
try:
current_arena.addr = \
gdbutil.to_int(gdb.parse_and_eval('je_arenas[%d]' % (i)))
except:
print('[unmask_jemalloc] error: cannot evaluate je_arenas[%d]') % (i)
sys.exit()
for j in range(0, jeheap.nbins):
nrg = 0
run_sz = 0
reg_size = 0
reg_offset = 0
end_addr = 0
jeheap.STANDALONE = true
reg_size = gdbutil.to_int(gdb.parse_and_eval('je_arena_bin_info[%d].reg_size' % (j)))
nrg = gdbutil.to_int(gdb.parse_and_eval('je_arena_bin_info[%d].nregs' % (j)))
run_sz = gdbutil.to_int(gdb.parse_and_eval('je_arena_bin_info[%d].run_size' % (j)))
reg_offset = gdbutil.to_int(gdb.parse_and_eval('je_arena_bin_info[%d].reg0_offset' % (j)))
try:
expr = 'je_arenas[%d].bins[%d].runcur' % (i, j)
runcur_addr = runcur = gdbutil.to_int(gdb.parse_and_eval(expr))
end_addr = runcur_addr + run_sz
if runcur != 0:
expr = 'je_arenas[%d].bins[%d].runcur.nfree' % (i, j)
nfree = gdbutil.to_int(gdb.parse_and_eval(expr))
current_run = \
jemalloc.arena_run(runcur, end_addr, run_sz, 0, \
int(reg_size), reg_offset, nrg, nfree, [])
current_bin = jemalloc.arena_bin(0, j, current_run)
current_bin.addr = \
gdbutil.to_int(gdb.parse_and_eval('&je_arenas[%d].bins[%d]' % (i, j)))
current_arena.bins.append(current_bin)
else:
# no regions for this size class yet, therefore no runcur
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
except RuntimeError:
current_run = jemalloc.arena_run()
current_bin = jemalloc.arena_bin(0, j, current_run)
current_arena.bins.append(current_bin)
continue
# add arena to the list of arenas
jeheap.arenas.append(current_arena)