def find_struct_in(self,
memoryMap,
structType,
hintOffset=0,
maxNum=10,
maxDepth=99):
'''
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with instance(structType)().isValid()
and confirming with instance(structType)().loadMembers()
returns POINTERS to structType instances.
'''
# update process mappings
log.debug("scanning 0x%lx --> 0x%lx %s" %
(memoryMap.start, memoryMap.end, memoryMap.pathname))
# where do we look
start = memoryMap.start
end = memoryMap.end
plen = ctypes.sizeof(ctypes.c_void_p) # use aligned words only
structlen = ctypes.sizeof(structType)
#ret vals
outputs = []
# alignement
if hintOffset in memoryMap: # absolute offset
align = hintOffset % plen
start = hintOffset - align
elif hintOffset != 0 and hintOffset < end - start: # relative offset
align = hintOffset % plen
start = start + (hintOffset - align)
# parse for structType on each aligned word
log.debug("checking 0x%lx-0x%lx by increment of %d" %
(start, (end - structlen), plen))
instance = None
import time
t0 = time.time()
p = 0
# xrange sucks. long int not ok
for offset in xrange(start, end - structlen, plen):
if offset % (1024 << 6) == 0:
p2 = offset - start
log.debug('processed %d bytes - %02.02f test/sec' %
(p2, (p2 - p) / (plen * (time.time() - t0))))
t0 = time.time()
p = p2
instance, validated = self.loadAt(memoryMap, offset, structType,
maxDepth)
if validated:
log.debug("found instance @ 0x%lx" % (offset))
# do stuff with it.
outputs.append((instance, offset))
if len(outputs) >= maxNum:
log.debug(
'Found enough instance. returning results. find_struct_in')
break
return outputs
def __iter__(self):
"""
Iterate over the allocated chunk of this heap mapping to find all valid matches
"""
log.debug(
'iterate allocated chunks in %s heap mapping for matching values',
self.get_search_mapping())
mapping = self.get_search_mapping()
i = 0
for vaddr, size in self._walker.get_user_allocations():
self._check_steps(i)
# check head of chunk
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr)
if b:
yield (vaddr, val)
if size < 2 * self._word_size:
continue
# check each offset in that allocated chunk
for vaddr_2 in xrange(vaddr + size, vaddr + size - self._word_size,
self._word_size):
i += 1
self._check_steps(i)
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr_2)
if b:
yield (vaddr_2, val)
return
def mergeDump(dumpFile):
log.info("Loading the mappings in the memory dump file.")
mappings = _openDumpfile(dumpFile)
if mappings is None:
return
heap, stack, mappings = mappings
# log.info('Make the signature.')
# sigMaker = SignatureMaker(mapping)
# sig = sigMaker.search()
# get pointers in stack
stackSearcher = PointerSearcher(stack)
stackSearcher.setTargetMapping(heap)
heapSearcher = PointerSearcher(heap)
pointersFromHeap = heapSearcher.search()
pointersFromStack = stackSearcher.search()
pointersFromHeap = sorted(pointersFromHeap)
pointersFromStack = sorted(pointersFromStack)
log.info("%d heap pointers in stack" % (len(pointersFromStack)))
log.info("%d heap pointers in heap" % (len(pointersFromHeap)))
# common ones
intersex = set(pointersFromHeap) & set(pointersFromStack)
log.info("%d heap pointers in both" % (len(intersex)))
# all
allpointers = []
# allpointers.extend(pointersFromHeap)
allpointers.extend(pointersFromStack)
allpointers = sorted(set(allpointers))
# give intervals between pointers
intervals = []
for p in xrange(1, len(allpointers) - 1):
val = allpointers[p] - allpointers[p - 1]
intervals.append(val)
return
def __iter__(self):
"""
Iterate over the allocated chunk of this heap mapping to find all valid matches
"""
log.debug('iterate allocated chunks in %s heap mapping for matching values', self.get_search_mapping())
mapping = self.get_search_mapping()
i = 0
for vaddr, size in self._walker.get_user_allocations():
self._check_steps(i)
# check head of chunk
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr)
if b:
yield (vaddr, val)
if size < 2*self._word_size:
continue
# check each offset in that allocated chunk
for vaddr_2 in xrange(vaddr+size, vaddr+size-self._word_size, self._word_size):
i+=1
self._check_steps(i)
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr_2)
if b:
yield (vaddr_2, val)
return
def mergeDump(dumpFile):
log.info('Loading the mappings in the memory dump file.')
mappings = _openDumpfile(dumpFile)
if mappings is None:
return
heap,stack,mappings = mappings
#log.info('Make the signature.')
#sigMaker = SignatureMaker(mapping)
#sig = sigMaker.search()
# get pointers in stack
stackSearcher = PointerSearcher(stack)
stackSearcher.setTargetMapping(heap)
heapSearcher = PointerSearcher(heap)
pointersFromHeap = heapSearcher.search()
pointersFromStack = stackSearcher.search()
pointersFromHeap = sorted(pointersFromHeap)
pointersFromStack = sorted(pointersFromStack)
log.info('%d heap pointers in stack'%( len(pointersFromStack) ))
log.info('%d heap pointers in heap'%( len(pointersFromHeap) ))
# common ones
intersex = set(pointersFromHeap) & set(pointersFromStack)
log.info('%d heap pointers in both'%( len(intersex) ))
# all
allpointers = []
#allpointers.extend(pointersFromHeap)
allpointers.extend(pointersFromStack)
allpointers = sorted(set(allpointers))
# give intervals between pointers
intervals=[]
for p in xrange(1,len(allpointers)-1):
val = allpointers[p] - allpointers[p-1]
intervals.append(val)
return
def __iter__(self):
''' Iterate over the mapping to return the signature of that memspace '''
log.debug('iterate %s mapping for matching values'%(self.getSearchMapping()))
for vaddr in xrange(self.getSearchMapping().start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
yield struct.pack('B',self.testMatch(vaddr))
return
def _search_in(self, mem_map, struct_type, nb=10, depth=99):
"""
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with Validator.isValid(instance)
and confirming with a Validator.load_members(instance)
we only look for user memory allocation chunks matching the
size of the structure.
returns POINTERS to structType instances.
"""
log.debug('Looking at %s (%x bytes)', mem_map, len(mem_map))
log.debug('look for %s', str(struct_type))
# prepare return values
outputs = []
# check the word size to use aligned words only
plen = self._memory_handler.get_target_platform().get_word_size()
my_ctypes = self._memory_handler.get_target_platform(
).get_target_ctypes()
# where do we look for that structure
finder = self._memory_handler.get_heap_finder()
walker = finder.get_heap_walker(mem_map)
struct_size = my_ctypes.sizeof(struct_type)
# get all allocated chunks
for addr, size in walker.get_user_allocations():
# FIXME, heap walker should give a hint
# minimum chunk size varies...
# if size != struct_size:
if size < struct_size:
log.debug("size %d < struct_size %d", size, struct_size)
continue
log.debug("testing 0x%lx", addr)
# try every aligned offset from there to the end of chunk
start = addr
end = start + size - struct_size + 1
# check if there is room (if size < struct_size)
if end < start:
log.debug('end < start')
continue
log.debug('xrange(%d, %d, %d) ', start, end, plen)
for offset in utils.xrange(start, end, plen):
# a - load and validate the record
# DEBUG
# offset = addr
log.debug('load_at(%d) ', offset)
instance, validated = self._load_at(mem_map, offset,
struct_type, depth)
if validated:
log.debug("found instance @ 0x%lx", offset)
# do stuff with it.
if self._update_cb is not None:
self._update_cb(instance, offset)
outputs.append((instance, offset))
# stop when time to stop
if len(outputs) >= nb:
log.debug('_search_in: Found enough instance.')
break
return outputs
def search(self):
''' returns the memspace signature. Dont forget to del that object, it's big. '''
self.values = b''
log.debug('search %s mapping for matching values'%(self.getSearchMapping()))
for vaddr in xrange(self.getSearchMapping().start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
self.values += struct.pack('B', self.testMatch(vaddr))
return self.values
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
log.debug("iterate %s mapping for matching values" % (self.getSearchMapping()))
for vaddr in xrange(self.getSearchMapping().start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
if self.testMatch(vaddr):
yield vaddr
return
def __iter__(self):
''' Iterate over the mapping to find all valid matches '''
log.debug('iterate %s mapping for matching values'%(self.getSearchMapping()))
for vaddr in xrange(self.getSearchMapping().start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
if self.testMatch(vaddr):
yield vaddr
return
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
start = self.getSearchMapping().start
for vaddr in xrange(start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
b, val = self.testMatch(vaddr) # expect a boolean, value tuple from testMatch
if b:
yield (vaddr, val)
return
def __iter__(self):
''' Iterate over the mapping to find all valid matches '''
start = self.getSearchMapping().start
for vaddr in xrange(start, self.getSearchMapping().end, self.WORDSIZE):
self._checkSteps(vaddr) # be verbose
b,val = self.testMatch(vaddr) # expect a boolean, value tuple from testMatch
if b:
yield (vaddr, val )
return
def _search_in(self, mem_map, struct_type, nb=10, depth=99, align=None):
"""
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with Validator.isValid(instance)
and confirming with a Validator.load_members(instance)
returns POINTERS to structType instances.
"""
log.debug('Looking at %s (%x bytes)', mem_map, len(mem_map))
log.debug('look for %s', str(struct_type))
# where do we look
start = mem_map.start
end = mem_map.end
# pointer len for alignment
plen = self._memory_handler.get_target_platform().get_word_size()
# # check the word size to use aligned words only
if align is None:
align = plen
else:
align = align - align % plen
# the struct cannot fit after that point.
my_ctypes = self._memory_handler.get_target_platform(
).get_target_ctypes()
end = end - my_ctypes.sizeof(struct_type) + 1
if end <= start:
raise ValueError("The record is too big for this memory mapping")
log.debug("scanning 0x%lx --> 0x%lx %s every %d bytes", start, end,
mem_map.pathname, plen)
# prepare return values
outputs = []
# parse for structType on each aligned word
t0 = time.time()
p = 0
# python 2.7 xrange doesn't handle long int. replace with ours.
for offset in utils.xrange(start, end, align):
# print a debug message every now and then
if offset % (1024 << 6) == 0:
p2 = offset - start
log.debug('processed %d bytes - %02.02f test/sec', p2,
(p2 - p) / (plen * (time.time() - t0)))
t0 = time.time()
p = p2
# a - load and validate the record
instance, validated = self._load_at(mem_map, offset, struct_type,
depth)
if validated:
log.debug("found instance @ 0x%lx", offset)
# do stuff with it.
if self._update_cb is not None:
self._update_cb(instance, offset)
outputs.append((instance, offset))
# stop when time to stop
if len(outputs) >= nb:
log.debug('_search_in: Found enough instance.')
break
return outputs
def find_struct_in(
self, memoryMap, structType, hintOffset=0, maxNum=10, maxDepth=99):
"""
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with instance(structType)().isValid()
and confirming with instance(structType)().loadMembers()
returns POINTERS to structType instances.
"""
import ctypes
# update process mappings
log.debug(
"scanning 0x%lx --> 0x%lx %s" %
(memoryMap.start, memoryMap.end, memoryMap.pathname))
# where do we look
start = memoryMap.start
end = memoryMap.end
plen = ctypes.sizeof(ctypes.c_void_p) # use aligned words only
structlen = ctypes.sizeof(structType)
# ret vals
outputs = []
# alignement
if hintOffset in memoryMap: # absolute offset
align = hintOffset % plen
start = hintOffset - align
elif hintOffset != 0 and hintOffset < end - start: # relative offset
align = hintOffset % plen
start = start + (hintOffset - align)
# parse for structType on each aligned word
log.debug(
"checking 0x%lx-0x%lx by increment of %d" %
(start, (end - structlen), plen))
instance = None
t0 = time.time()
p = 0
# xrange sucks. long int not ok
for offset in xrange(start, end - structlen, plen):
if offset % (1024 << 6) == 0:
p2 = offset - start
log.debug('processed %d bytes - %02.02f test/sec' %
(p2, (p2 - p) / (plen * (time.time() - t0))))
t0 = time.time()
p = p2
instance, validated = self.loadAt(
memoryMap, offset, structType, maxDepth)
if validated:
log.debug("found instance @ 0x%lx" % (offset))
# do stuff with it.
outputs.append((instance, offset))
if len(outputs) >= maxNum:
log.debug(
'Found enough instance. returning results. find_struct_in')
break
return outputs
def _search_in(self, mem_map, struct_type, nb=10, depth=99):
"""
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with Validator.isValid(instance)
and confirming with a Validator.load_members(instance)
we only look for user memory allocation chunks matching the
size of the structure.
returns POINTERS to structType instances.
"""
log.debug('Looking at %s (%x bytes)', mem_map, len(mem_map))
log.debug('look for %s', str(struct_type))
# prepare return values
outputs = []
# where do we look for that structure
finder = self._memory_handler.get_heap_finder()
walker = finder.get_heap_walker(mem_map)
# check the word size to use aligned words only
target = walker.get_target_platform()
plen = target.get_word_size()
my_ctypes = target.get_target_ctypes()
struct_size = my_ctypes.sizeof(struct_type)
# get all allocated chunks
for addr, size in walker.get_user_allocations():
# FIXME, heap walker should give a hint
# minimum chunk size varies...
if size < struct_size:
log.debug("size %d < struct_size %d", size, struct_size)
continue
log.debug("testing 0x%lx", addr)
# could change
mem_map = self._memory_handler.get_mapping_for_address(addr)
# try every aligned offset from there to the end of chunk
start = addr
end = start + size - struct_size + 1
# check if there is room (if size < struct_size)
if end < start:
log.debug('end < start')
continue
log.debug('xrange(%d, %d, %d) ', start, end, plen)
for offset in utils.xrange(start, end, plen):
# a - load and validate the record
log.debug('load_at(%d) ', offset)
instance, validated = self._load_at(mem_map, offset, struct_type, depth)
if validated:
log.debug("found instance @ 0x%lx", offset)
# do stuff with it.
if self._update_cb is not None:
self._update_cb(instance, offset)
outputs.append((instance, offset))
# stop when time to stop
if len(outputs) >= nb:
log.debug('_search_in: Found enough instance.')
break
return outputs
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
log.debug('iterate %s mapping for matching values', self.get_search_mapping())
mapping = self.get_search_mapping()
for i, vaddr in enumerate(xrange(mapping.start, mapping.end, self._word_size)):
self._check_steps(i) # be verbose
if self._matcher.test_match(mapping, vaddr):
yield vaddr
return
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
mapping = self.get_search_mapping()
for i, vaddr in enumerate(xrange(mapping.start, mapping.end, self._word_size)):
self._check_steps(i) # be verbose
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr)
if b:
yield (vaddr, val)
return
def test_xrange(self):
"""tests home made xrange that handles big ints.
Not an issue in Py 3"""
a = 11111111111111111111111111111111111111111111111111111111111111111111111111111111111111
b = a + 10
r = [x for x in utils.xrange(a, b)]
r2 = []
while a < b:
r2.append(a)
a += 1
self.assertEquals(r, r2)
def test_xrange(self):
"""tests home made xrange that handles big ints.
Not an issue in Py 3"""
a = 11111111111111111111111111111111111111111111111111111111111111111111111111111111111111
b = a + 10
r = [x for x in utils.xrange(a, b)]
r2 = []
while a < b:
r2.append(a)
a += 1
self.assertEquals(r, r2)
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
log.debug('iterate %s mapping for matching values',
self.get_search_mapping())
mapping = self.get_search_mapping()
for i, vaddr in enumerate(
xrange(mapping.start, mapping.end, self._word_size)):
self._check_steps(i) # be verbose
if self._matcher.test_match(mapping, vaddr):
yield vaddr
return
def __iter__(self):
""" Iterate over the mapping to find all valid matches """
mapping = self.get_search_mapping()
for i, vaddr in enumerate(
xrange(mapping.start, mapping.end, self._word_size)):
self._check_steps(i) # be verbose
# expect a boolean, value tuple from testMatch
b, val = self._matcher.test_match(mapping, vaddr)
if b:
yield (vaddr, val)
return
def _search_in(self, mem_map, struct_type, nb=10, depth=99, align=None):
"""
Looks for structType instances in memory, using :
hints from structType (default values, and such)
guessing validation with Validator.isValid(instance)
and confirming with a Validator.load_members(instance)
returns POINTERS to structType instances.
"""
log.debug('Looking at %s (%x bytes)', mem_map, len(mem_map))
log.debug('look for %s', str(struct_type))
# where do we look
start = mem_map.start
end = mem_map.end
# pointer len for alignment
plen = mem_map.get_target_platform().get_word_size()
# # check the word size to use aligned words only
if align is None:
align = plen
else:
align = align - align % plen
# the struct cannot fit after that point.
my_ctypes = mem_map.get_target_platform().get_target_ctypes()
end = end - my_ctypes.sizeof(struct_type) + 1
if end <= start:
raise ValueError("The record is too big for this memory mapping")
log.debug("scanning 0x%lx --> 0x%lx %s every %d bytes", start, end, mem_map.pathname, plen)
# prepare return values
outputs = []
# parse for structType on each aligned word
t0 = time.time()
p = 0
# python 2.7 xrange doesn't handle long int. replace with ours.
for offset in utils.xrange(start, end, align):
# print a debug message every now and then
if offset % (1024 << 6) == 0:
p2 = offset - start
log.debug('processed %d bytes - %02.02f test/sec', p2, (p2 - p) / (plen * (time.time() - t0)))
t0 = time.time()
p = p2
# a - load and validate the record
instance, validated = self._load_at(mem_map, offset, struct_type, depth)
if validated:
log.debug("found instance @ 0x%lx", offset)
# do stuff with it.
if self._update_cb is not None:
self._update_cb(instance, offset)
outputs.append((instance, offset))
# stop when time to stop
if len(outputs) >= nb:
log.debug('_search_in: Found enough instance.')
break
return outputs
def merge_dump(dumpname):
log.info('Loading the _memory_handler in the memory dump file.')
memory_handler = dump_loader.load(dumpname)
if memory_handler is None:
return
# FIXME - get stack on windows.
heap, stack, mappings = memory_handler
#log.info('Make the signature.')
#sigMaker = SignatureMaker(mapping)
#sig = sigMaker.search()
word_size = memory_handler.get_target_platform().get_word_size()
feedback = NoFeedback()
# get pointers in stack
stack_searcher = WordAlignedSearcher(
stack, matchers.PointerSearcher(memory_handler), feedback, word_size)
heap_searcher = WordAlignedSearcher(
heap, matchers.PointerSearcher(memory_handler), feedback, word_size)
pointersFromHeap = heap_searcher.search()
pointersFromStack = stack_searcher.search()
pointersFromHeap = sorted(pointersFromHeap)
pointersFromStack = sorted(pointersFromStack)
log.info('%d heap pointers in stack' % (len(pointersFromStack)))
log.info('%d heap pointers in heap' % (len(pointersFromHeap)))
# common ones
intersex = set(pointersFromHeap) & set(pointersFromStack)
log.info('%d heap pointers in both' % (len(intersex)))
# all
allpointers = []
# allpointers.extend(pointersFromHeap)
allpointers.extend(pointersFromStack)
allpointers = sorted(set(allpointers))
# give intervals between pointers
intervals = []
for p in xrange(1, len(allpointers) - 1):
val = allpointers[p] - allpointers[p - 1]
intervals.append(val)
return
def merge_dump(dumpname):
log.info('Loading the _memory_handler in the memory dump file.')
memory_handler = dump_loader.load(dumpname)
if memory_handler is None:
return
# FIXME - get stack on windows.
heap, stack, mappings = memory_handler
#log.info('Make the signature.')
#sigMaker = SignatureMaker(mapping)
#sig = sigMaker.search()
word_size = memory_handler.get_target_platform().get_word_size()
feedback = NoFeedback()
# get pointers in stack
stack_searcher = WordAlignedSearcher(stack, matchers.PointerSearcher(memory_handler), feedback, word_size)
heap_searcher = WordAlignedSearcher(heap, matchers.PointerSearcher(memory_handler), feedback, word_size)
pointersFromHeap = heap_searcher.search()
pointersFromStack = stack_searcher.search()
pointersFromHeap = sorted(pointersFromHeap)
pointersFromStack = sorted(pointersFromStack)
log.info('%d heap pointers in stack' % (len(pointersFromStack)))
log.info('%d heap pointers in heap' % (len(pointersFromHeap)))
# common ones
intersex = set(pointersFromHeap) & set(pointersFromStack)
log.info('%d heap pointers in both' % (len(intersex)))
# all
allpointers = []
# allpointers.extend(pointersFromHeap)
allpointers.extend(pointersFromStack)
allpointers = sorted(set(allpointers))
# give intervals between pointers
intervals = []
for p in xrange(1, len(allpointers) - 1):
val = allpointers[p] - allpointers[p - 1]
intervals.append(val)
return
