def test_offsetof(self):
"""returns the offset of a member fields in a record"""
ctypes = types.reload_ctypes(4, 4, 8)
class Y(ctypes.Structure):
_pack_ = True
_fields_ = [('a', ctypes.c_long),
('p', ctypes.POINTER(ctypes.c_int)),
('b', ctypes.c_ubyte)]
o = utils.offsetof(Y, 'b')
self.assertEquals(o, 8)
ctypes = types.reload_ctypes(8, 8, 16)
class X(ctypes.Structure):
_pack_ = True
_fields_ = [('a', ctypes.c_long),
('p', ctypes.POINTER(ctypes.c_int)),
('b', ctypes.c_ubyte)]
o = utils.offsetof(X, 'b')
self.assertEquals(o, 16)
class X2(ctypes.Union):
_pack_ = True
_fields_ = [('a', ctypes.c_long),
('p', ctypes.POINTER(ctypes.c_int)),
('b', ctypes.c_ubyte)]
o = utils.offsetof(X2, 'b')
self.assertEquals(o, 0)
pass
def iterateListField(self, mappings, fieldname, sentinels=[]):
'''
start from the field and iterate a list.
does not return self.'''
structType, offset = self._getListFieldInfo(fieldname)
# @ of the field
headAddr = self._orig_address_ + utils.offsetof( type(self), fieldname)
#log.info('Ignore headAddress self.%s at 0x%0.8x'%(fieldname, headAddr))
head = getattr(self, fieldname)
if not hasattr(head, '_iterateList'):
raise ValueError('Not an iterable field. Probably not declared as a list.')
from haystack import model # need my cache
done = [s for s in sentinels]+[headAddr]
for entry in head._iterateList(mappings):
# DO NOT think HEAD is a valid entry - FIXME
if entry in done:
continue
# save it
done.append(entry)
# @ of the struct, entry is not null, head._iterateList garantizes it.
link = entry + offset
#log.info('Read %s at 0x%0.8x instead of 0x%0.8x'%(fieldname, link, entry))
# use cache if possible, avoid loops.
st = model.getRef( structType, link)
#st._orig_addr_ = link
if st:
yield st
else:
raise ValueError('the structure has not been loaded, please use loadMembers.')
raise StopIteration
def test_ctypes_sizes(self):
''' road to faking POINTER :
get_subtype(attrtype) # checks for attrtype._type_
getaddress(attr) # check for address of attr.contents being a ctypes.xx.from_address(ptr_value)
'''
from haystack.reverse.win32 import win7heap
self.assertEquals(ctypes.sizeof(win7heap._HEAP_SEGMENT), 64)
self.assertEquals(ctypes.sizeof(win7heap._HEAP_ENTRY), 8)
self.assertEquals(ctypes.sizeof(ctypes.POINTER(None)), 4)
self.assertEquals(
ctypes.sizeof(ctypes.POINTER(win7heap._HEAP_TAG_ENTRY)), 4)
self.assertEquals(ctypes.sizeof(win7heap._LIST_ENTRY), 8)
self.assertEquals(
ctypes.sizeof(ctypes.POINTER(win7heap._HEAP_LIST_LOOKUP)), 4)
self.assertEquals(
ctypes.sizeof(ctypes.POINTER(win7heap._HEAP_PSEUDO_TAG_ENTRY)), 4)
self.assertEquals(ctypes.sizeof(ctypes.POINTER(win7heap._HEAP_LOCK)),
4)
self.assertEquals(ctypes.sizeof(ctypes.c_ubyte), 1)
self.assertEquals(ctypes.sizeof((ctypes.c_ubyte * 1)), 1)
self.assertEquals(ctypes.sizeof(win7heap._HEAP_COUNTERS), 84)
self.assertEquals(ctypes.sizeof(win7heap._HEAP_TUNING_PARAMETERS), 8)
self.assertEquals(ctypes.sizeof(win7heap.HEAP), 312)
self.assertEquals(utils.offsetof(win7heap.HEAP, 'Signature'), 100)
def _loadListEntries(self, fieldname, mappings, maxDepth):
'''
we need to load the pointed entry as a valid struct at the right offset,
and parse it.
When does it stop following FLink/BLink ?
sentinel is headAddr only
'''
structType, offset = self._getListFieldInfo(fieldname)
# DO NOT think HEAD is a valid entry.
# if its a ListEntries, self has already been loaded anyway.
headAddr = self._orig_address_ + utils.offsetof(type(self), fieldname)
head = getattr(self, fieldname)
for entry in head._iterateList(mappings):
# DO NOT think HEAD is a valid entry
if entry == headAddr:
continue
link = entry + offset
log.debug('got a element of list at %s 0x%x/0x%x offset:%d' %
(fieldname, entry, link, offset))
# use cache if possible, avoid loops.
#XXX
from haystack import model
ref = model.getRef(structType, link)
if ref: # struct has already been loaded, bail out
log.debug("%s loading from references cache %s/0x%lx" %
(fieldname, structType, link))
continue # do not reload
else:
# OFFSET read, specific to a LIST ENTRY model
memoryMap = utils.is_valid_address_value(
link, mappings, structType)
if memoryMap is False:
log.error(
'error while validating address 0x%x type:%s @end:0x%x'
% (link, structType.__name__,
link + ctypes.sizeof(structType)))
log.error('self : %s , fieldname : %s' %
(self.__class__.__name__, fieldname))
raise ValueError(
'error while validating address 0x%x type:%s @end:0x%x'
% (link, structType.__name__,
link + ctypes.sizeof(structType)))
st = memoryMap.readStruct(
link, structType) # point at the right offset
st._orig_addr_ = link
model.keepRef(st, structType, link)
log.debug("keepRef %s.%s @%x" % (structType, fieldname, link))
# load the list entry structure members
if not st.loadMembers(mappings, maxDepth - 1):
log.error('Error while loading members on %s' %
(self.__class__.__name__))
print st
raise ValueError('error while loading members')
return True
def _getListFieldInfo(self, fieldname):
'''
if fieldname is in listmember, return offset of fieldname.
if fieldname is in listhead, return offset of target field.
'''
if fieldname in self._listMember_:
return type(self), utils.offsetof( type(self), fieldname)
for fname,typ,typFieldname,offset in self._listHead_:
if fieldname == fname:
return typ, offset # FIXME: offset is also == utils.offsetof( typ, typFieldname)
raise TypeError('This field %s is not a list.'%(fieldname))
def _getListFieldInfo(self, fieldname):
'''
if fieldname is in listmember, return offset of fieldname.
if fieldname is in listhead, return offset of target field.
'''
if fieldname in self._listMember_:
return type(self), utils.offsetof(type(self), fieldname)
for fname, typ, typFieldname, offset in self._listHead_:
if fieldname == fname:
return typ, offset # FIXME: offset is also == utils.offsetof( typ, typFieldname)
raise TypeError('This field %s is not a list.' % (fieldname))
def _loadListEntries(self, fieldname, mappings, maxDepth):
"""
we need to load the pointed entry as a valid struct at the right offset,
and parse it.
When does it stop following FLink/BLink ?
sentinel is headAddr only
"""
import ctypes
structType, offset = self._getListFieldInfo(fieldname)
# FIXME offset == utils.offsetof(type(self), fieldname)
# DO NOT think HEAD is a valid entry.
# if its a ListEntries, self has already been loaded anyway.
headAddr = self._orig_address_ + utils.offsetof(type(self), fieldname)
head = getattr(self, fieldname)
for entry in head._iterateList(mappings):
# DO NOT think HEAD is a valid entry
if entry == headAddr:
continue
link = entry + offset
log.debug("got a element of list at %s 0x%x/0x%x offset:%d" % (fieldname, entry, link, offset))
# use cache if possible, avoid loops.
ref = mappings.getRef(structType, link)
if ref: # struct has already been loaded, bail out
log.debug("%s loading from references cache %s/0x%lx" % (fieldname, structType, link))
continue # do not reload
else:
# OFFSET read, specific to a LIST ENTRY model
memoryMap = mappings.is_valid_address_value(link, structType)
if memoryMap is False:
log.error(
"error while validating address 0x%x type:%s @end:0x%x"
% (link, structType.__name__, link + ctypes.sizeof(structType))
)
log.error("self : %s , fieldname : %s" % (self.__class__.__name__, fieldname))
raise ValueError(
"error while validating address 0x%x type:%s @end:0x%x"
% (link, structType.__name__, link + ctypes.sizeof(structType))
)
st = memoryMap.readStruct(link, structType) # point at the right offset
st._orig_address_ = link
mappings.keepRef(st, structType, link)
log.debug("keepRef %s.%s @%x" % (structType, fieldname, link))
# load the list entry structure members
if not st.loadMembers(mappings, maxDepth - 1):
log.error("Error while loading members on %s" % (self.__class__.__name__))
# print st
raise ValueError("error while loading members")
return True
def test_ctypes_sizes(self):
''' road to faking POINTER :
get_subtype(attrtype) # checks for attrtype._type_
getaddress(attr) # check for address of attr.contents being a ctypes.xx.from_address(ptr_value)
'''
from haystack.reverse.win32 import win7heap
self.assertEquals( ctypes.sizeof( win7heap._HEAP_SEGMENT), 64 )
self.assertEquals( ctypes.sizeof( win7heap._HEAP_ENTRY), 8 )
self.assertEquals( ctypes.sizeof( ctypes.POINTER(None)), 4 )
self.assertEquals( ctypes.sizeof( ctypes.POINTER(win7heap._HEAP_TAG_ENTRY)), 4 )
self.assertEquals( ctypes.sizeof( win7heap._LIST_ENTRY), 8 )
self.assertEquals( ctypes.sizeof( ctypes.POINTER(win7heap._HEAP_LIST_LOOKUP)), 4 )
self.assertEquals( ctypes.sizeof( ctypes.POINTER(win7heap._HEAP_PSEUDO_TAG_ENTRY)), 4 )
self.assertEquals( ctypes.sizeof( ctypes.POINTER(win7heap._HEAP_LOCK)), 4 )
self.assertEquals( ctypes.sizeof( ctypes.c_ubyte), 1 )
self.assertEquals( ctypes.sizeof( (ctypes.c_ubyte*1)), 1 )
self.assertEquals( ctypes.sizeof( win7heap._HEAP_COUNTERS), 84 )
self.assertEquals( ctypes.sizeof( win7heap._HEAP_TUNING_PARAMETERS), 8 )
self.assertEquals( ctypes.sizeof( win7heap.HEAP ) , 312 )
self.assertEquals( utils.offsetof( win7heap.HEAP , 'Signature') , 100 )
def iterateListField(self, mappings, fieldname, sentinels=[]):
'''
start from the field and iterate a list.
does not return self.'''
structType, offset = self._getListFieldInfo(fieldname)
# @ of the field
headAddr = self._orig_address_ + utils.offsetof(type(self), fieldname)
#log.info('Ignore headAddress self.%s at 0x%0.8x'%(fieldname, headAddr))
head = getattr(self, fieldname)
if not hasattr(head, '_iterateList'):
raise ValueError(
'Not an iterable field. Probably not declared as a list.')
from haystack import model # need my cache
done = [s for s in sentinels] + [headAddr]
for entry in head._iterateList(mappings):
# DO NOT think HEAD is a valid entry - FIXME
if entry in done:
continue
# save it
done.append(entry)
# @ of the struct, entry is not null, head._iterateList garantizes it.
link = entry + offset
#log.info('Read %s at 0x%0.8x instead of 0x%0.8x'%(fieldname, link, entry))
# use cache if possible, avoid loops.
st = model.getRef(structType, link)
#st._orig_addr_ = link
if st:
yield st
else:
raise ValueError(
'the structure has not been loaded, please use loadMembers.'
)
raise StopIteration
def test_ctypes_sizes(self):
# You have to import after ctypes has been tuned ( mapping loader )
from haystack.structures.win32 import win7heapwalker, win7heap
ctypes = self._mappings.config.ctypes
self.assertEquals(ctypes.sizeof(win7heap.HEAP_SEGMENT), 64)
self.assertEquals(ctypes.sizeof(win7heap.HEAP_ENTRY), 8)
self.assertEquals(ctypes.sizeof(ctypes.POINTER(None)), 4)
self.assertEquals(ctypes.sizeof(
ctypes.POINTER(win7heap.HEAP_TAG_ENTRY)), 4)
self.assertEquals(ctypes.sizeof(win7heap.LIST_ENTRY), 8)
self.assertEquals(ctypes.sizeof(
ctypes.POINTER(win7heap.HEAP_LIST_LOOKUP)), 4)
self.assertEquals(ctypes.sizeof(
ctypes.POINTER(win7heap.HEAP_PSEUDO_TAG_ENTRY)), 4)
self.assertEquals(ctypes.sizeof(ctypes.POINTER(win7heap.HEAP_LOCK)), 4)
self.assertEquals(ctypes.sizeof(ctypes.c_ubyte), 1)
self.assertEquals(ctypes.sizeof((ctypes.c_ubyte * 1)), 1)
self.assertEquals(ctypes.sizeof(win7heap.HEAP_COUNTERS), 84)
self.assertEquals(ctypes.sizeof(win7heap.HEAP_TUNING_PARAMETERS), 8)
self.assertEquals(ctypes.sizeof(win7heap.HEAP), 312)
self.assertEquals(utils.offsetof(win7heap.HEAP, 'Signature'), 100)
def _loadMember(self, attr, attrname, attrtype, mappings, maxDepth):
ctypes = self._mappings_.config.ctypes
# skip static void_p data members
if not self._isLoadableMember(attr, attrname, attrtype):
log.debug("%s %s not loadable bool(attr) = %s" % (attrname, attrtype,
bool(attr)))
return True
# load it, fields are valid
elif ctypes.is_struct_type(attrtype) or ctypes.is_union_type(attrtype):
# its an embedded record. Bytes are already loaded.
offset = utils.offsetof(type(self), attrname)
log.debug('st: %s %s is STRUCT at @%x' % (attrname, attrtype,
self._orig_address_ + offset))
# TODO pydoc for impl.
attr._orig_address_ = self._orig_address_ + offset
attr._mappings = mappings
if not attr.loadMembers(mappings, maxDepth - 1):
log.debug(
"st: %s %s not valid, error while loading inner struct" %
(attrname, attrtype))
return False
log.debug("st: %s %s inner struct LOADED " % (attrname, attrtype))
return True
elif ctypes.is_array_of_basic_type(attrtype):
return True
elif ctypes.is_array_type(attrtype):
log.debug('a: %s is arraytype %s recurse load' % (attrname,
repr(attr)))
attrLen = len(attr)
if attrLen == 0:
return True
elType = type(attr[0])
for i in range(0, attrLen):
# FIXME BUG DOES NOT WORK
# offsetof("%s[%d]") is called, and %s exists, not %s[%d]
# if not self._loadMember(attr[i], "%s[%d]"%(attrname,i),
# elType, mappings, maxDepth):
if not self._loadMember(
attr[i], attrname, elType, mappings, maxDepth):
return False
return True
# we have PointerType here . Basic or complex
# exception cases
elif ctypes.is_function_type(attrtype):
pass
# FIXME
elif ctypes.is_cstring_type(attrtype):
# can't use basic c_char_p because we can't load in foreign memory
# FIXME, you need to keep a ref to this ctring if
# your want _mappings_ to exists
# or just mandate mappings in toString
attr_obj_address = utils.get_pointee_address(attr.ptr)
if not bool(attr_obj_address):
log.debug('%s %s is a CString, the pointer is null (validation '
'must have occurred earlier)' % (attrname, attr))
return True
memoryMap = mappings.is_valid_address_value(attr_obj_address)
if not memoryMap:
log.warning('Error on addr while fetching a CString.'
'should not happen')
return False
ref = mappings.getRef(ctypes.CString, attr_obj_address)
if ref:
log.debug("%s %s loading from references cache %s/0x%lx" % (attrname,
attr, ctypes.CString, attr_obj_address))
return True
max_size = min(
self.MAX_CSTRING_SIZE,
memoryMap.end -
attr_obj_address)
log.debug('%s %s is defined as a CString, loading %d bytes from 0x%lx '
'is_valid_address %s' % (attrname, attr, max_size, attr_obj_address,
mappings.is_valid_address_value(attr_obj_address)))
txt, truncated = memoryMap.readCString(attr_obj_address, max_size)
if truncated:
log.warning(
'buffer size was too small for this CString: %d' %
(max_size))
# that will SEGFAULT attr.string = txt - instead keepRef to String
mappings.keepRef(txt, ctypes.CString, attr_obj_address)
log.debug(
'kept CString ref for "%s" at @%x' %
(txt, attr_obj_address))
return True
# not functionType, it's not loadable
elif ctypes.is_pointer_type(attrtype):
_attrType = get_subtype(attrtype)
attr_obj_address = utils.get_pointee_address(attr)
####
# memcpy and save objet ref + pointer in attr
# we know the field is considered valid, so if it's not in
# memory_space, we can ignore it
memoryMap = mappings.is_valid_address(attr, _attrType)
if(not memoryMap):
# big BUG Badaboum, why did pointer changed validity/value ?
log.warning(
"%s %s not loadable 0x%lx but VALID " %
(attrname, attr, attr_obj_address))
return True
ref = mappings.getRef(_attrType, attr_obj_address)
if ref:
log.debug(
"%s %s loading from references cache %s/0x%lx" %
(attrname, attr, _attrType, attr_obj_address))
# DO NOT CHANGE STUFF SOUPID attr.contents = ref. attr.contents
# will SEGFAULT
return True
log.debug(
"%s %s loading from 0x%lx (is_valid_address: %s)" %
(attrname, attr, attr_obj_address, memoryMap))
# Read the struct in memory and make a copy to play with.
# DO NOT COPY THE STRUCT, we have a working readStruct for that...
# ERRROR
# attr.contents=_attrType.from_buffer_copy(memoryMap.readStruct(attr_obj_address,
# _attrType ))
contents = memoryMap.readStruct(attr_obj_address, _attrType)
# save that validated and loaded ref and original addr so we dont
# need to recopy it later
mappings.keepRef(contents, _attrType, attr_obj_address)
log.debug(
"keepRef %s.%s @%x" %
(_attrType, attrname, attr_obj_address))
log.debug(
"%s %s loaded memcopy from 0x%lx to 0x%lx" %
(attrname,
attr,
attr_obj_address,
(utils.get_pointee_address(attr))))
# recursive validation checks on new struct
if not bool(attr):
log.warning(
'Member %s is null after copy: %s' %
(attrname, attr))
return True
# go and load the pointed struct members recursively
subtype = utils.get_subtype(attrtype)
if (ctypes.is_basic_type(subtype) or
ctypes.is_array_of_basic_type(subtype)):
# do nothing
return True
elif (ctypes.is_array_type(subtype) or
ctypes.is_pointer_type(subtype)):
return self._loadMember(
contents, 'pointee', subtype, mappings, maxDepth - 1)
if not contents.loadMembers(mappings, maxDepth - 1):
log.debug('member %s was not loaded' % (attrname))
# invalidate the cache ref.
mappings.delRef(_attrType, attr_obj_address)
return False
return True
# TATAFN
return True