def analyze_function(self, address):
chunks = YaToolIDATools.get_function_chunks(address)
for (ch_start, ch_end) in chunks:
if idc.AnalyzeArea(ch_start, ch_end) != 1:
logger.error(
"[0x%08X] idc.AnalyzeArea failed [0x%08X->0x%08X" %
(address, ch_start, ch_end))
def walk_datas():
for seg_start in idautils.Segments():
seg_end = idc.SegEnd(seg_start)
for ea in YaToolIDATools.address_range_get_items(seg_start, seg_end):
flags = idc.GetFlags(ea)
func = idaapi.get_func(ea)
if idaapi.isFunc(flags) or (func and idc.isCode(flags)):
# function
continue
if not func and idc.isCode(flags):
# code
continue
yield ea
def make_struc_member(self,
object_version,
address,
member_type=ya.OBJECT_TYPE_STRUCT_MEMBER):
struc_object_id = object_version.get_parent_object_id()
struc_id = 0
try:
struc_id = self.struc_ids[struc_object_id]
except:
return
is_union = struc_id in self.union_ids
offset = address
if is_union:
last_offset = idc.GetLastMember(struc_id)
if last_offset == idc.BADADDR:
last_offset = -1
if last_offset < offset:
for i in xrange(last_offset + 1, offset + 1):
idc.AddStrucMember(struc_id, "yaco_filler_%d" % i, 0,
idc.FF_BYTE | idc.FF_DATA, -1, 1)
# ensure that 'offset' fields are present
member_size = object_version.get_size()
member_name = object_version.get_name()
flags = object_version.get_object_flags()
if idc.isStruct(flags):
# if the sub field is a struct, it must have a single Xref field with the struct object id
try:
sub_struc_object_id = object_version.getXRefIdsAt(0, 0)[0]
sub_struc_id = self.struc_ids[sub_struc_object_id]
# logger.debug("%20s: adding sub member at offset 0x%08X,
# size=0x%08X (sub=0x%.016X, size=0x%08X) with name %s" %
# (
# idc.GetStrucName(struc_id), offset, member_size, sub_struc_id,
# idc.GetStrucSize(sub_struc_id), object_version.get_name()
# ))
sub_struc_size = idc.GetStrucSize(sub_struc_id)
if sub_struc_size == 0:
logger.error(
"%20s: adding sub member at offset 0x%08X, size=0x%08X "
"(sub=0x%.016X, size=0x%08X) with name %s : sub struc size is ZERO"
% (idc.GetStrucName(struc_id), offset, member_size,
sub_struc_id, idc.GetStrucSize(sub_struc_id),
object_version.get_name()))
else:
nitems = member_size / sub_struc_size
YaToolIDATools.SetStrucmember(struc_id, member_name,
offset, flags, sub_struc_id,
nitems)
except KeyError:
logger.error(
"Error while looking for sub struc in struc %s, offset 0x%08X (field name='%s')"
% (self.hash_provider.hash_to_string(struc_object_id),
offset, object_version.get_name()))
traceback.print_exc()
elif idc.isEnum0(flags):
# an enum is applied here
try:
sub_enum_object_id = object_version.getXRefIdsAt(0, 0)[0]
sub_enum_id = self.enum_ids[sub_enum_object_id]
name_ok = idc.SetMemberName(struc_id, offset, member_name)
if name_ok is not True:
logger.debug(
"Error while setting member name (enum) : "
"(struc=%s, member=%s, offset=0x%08X, mflags=%d, msize=%d, tid=0x%016X"
% (name_ok, idc.GetStrucName(struc_id), member_name,
offset, flags, member_size, sub_struc_id))
else:
sub_enum_size = idc.GetEnumWidth(sub_enum_id)
if sub_enum_size == 0:
sub_enum_size = member_size
nitems = member_size / sub_enum_size
ret = idc.SetMemberType(struc_id, offset, flags,
sub_enum_id, nitems)
if ret == 0:
logger.debug(
"Error while setting member type (enum) : "
"(struc=%s, member=%s, offset=0x%08X, mflags=%d, msize=%d, tid=0x%016X"
% (ret, idc.GetStrucName(struc_id), member_name,
offset, flags, member_size, sub_struc_id))
except KeyError:
logger.error(
"Error while looking for sub enum in struc %s, offset 0x%08X (field name='%s')"
% (struc_object_id, offset, member_name))
traceback.print_exc()
else:
# logger.debug("%20s: adding member at offset 0x%08X, size=0x%08X with name %s" %
# (
# idc.GetStrucName(struc_id), offset, member_size, object_version.get_name()
# ))
tid = -1
if idc.isASCII(flags):
logger.debug(
"object: %s : %s" % (self.hash_provider.hash_to_string(
object_version.get_id()), object_version.get_name()))
try:
tid = object_version.get_string_type()
except KeyError:
tid = idc.ASCSTR_C
name_ok = idc.SetMemberName(struc_id, offset, member_name)
if name_ok is not True:
logger.debug(
"Error while setting member name :" +
" (struc_id=0x%08X, struc=%s, member=%s, offset=0x%08X, mflags=%d, msize=%d)"
% (struc_id, idc.GetStrucName(struc_id), member_name,
offset, flags, member_size))
else:
item_size = YaToolIDATools.get_field_size(flags, tid)
nitems = member_size / item_size
# IDA BUG : 4-byte chars are stored as 2 double words, thus me must
# multiply nitem by 2!
ret = idc.SetMemberType(struc_id, offset, flags, tid, nitems)
if ret == 0:
logger.debug(
"Error while setting member type :" +
" (struc=%s, member=%s, offset=0x%08X, mflags=%d, msize=%d)"
% (idc.GetStrucName(struc_id), member_name, offset,
flags, member_size))
try:
repeatable_headercomment = self.sanitize_comment_to_ascii(
object_version.get_header_comment(True))
idc.SetMemberComment(struc_id, offset, repeatable_headercomment, 1)
except KeyError:
pass
try:
nonrepeatable_headercomment = self.sanitize_comment_to_ascii(
object_version.get_header_comment(False))
idc.SetMemberComment(struc_id, offset, nonrepeatable_headercomment,
0)
except KeyError:
pass
member_id = idc.GetMemberId(struc_id, offset)
self.set_struct_member_type(object_version, member_id)
if object_version.get_type() == ya.OBJECT_TYPE_STRUCT_MEMBER:
self.strucmember_ids[object_version.get_id()] = member_id
def clear_struc_fields(self,
struc_id,
struc_size,
xref_keys,
is_union=False,
member_type=ya.OBJECT_TYPE_STRUCT_MEMBER,
name_offset=0):
idc.BeginTypeUpdating(idc.UTP_STRUCT)
last_offset = idc.GetLastMember(struc_id)
# get existing member offsets
field_offsets = set()
for (xref_offset, xref_operand) in xref_keys:
field_offsets.add(xref_offset)
new_offsets = set()
struc = idaapi.get_struc(struc_id)
# create missing members first (prevent from deleting all members)
for offset in field_offsets:
member = idaapi.get_member(struc, offset)
if member is not None and member.soff < offset:
# we have a member above this member that is too big and contain this member
# clear it!
if DEBUG_EXPORTER:
logger.debug(
"reduce field : set_member_type(0x%08X, 0x%08X), overlapping 0x%08X",
struc_id, member.soff, offset)
idaapi.set_member_type(struc, member.soff, idc.FF_BYTE, None,
1)
member = idaapi.get_member(struc, offset)
if member is None or idaapi.get_member_name(member.id) is None:
new_offsets.add(offset)
member_name = YaToolIDATools.get_default_struc_member_name(
member_type, offset, name_offset)
if offset == last_offset and offset == struc_size:
field_size = 0
else:
field_size = 1
if DEBUG_EXPORTER:
logger.debug(
"AddStrucMember(0x%08X, '%s', 0x%08X, idc.FF_BYTE, -1, 0x%08X), name_offset=%d",
struc_id, member_name, offset, field_size, name_offset)
retval = idc.AddStrucMember(struc_id, member_name, offset,
idc.FF_BYTE, -1, field_size)
if retval != 0:
logger.error(
"Error %d with idc.AddStrucMember(0x%08X, '%s', 0x%08X,"
"idc.FF_BYTE, -1, 0x%08X), name_offset=%d", retval,
struc_id, member_name, offset, field_size, name_offset)
elif DEBUG_EXPORTER:
logger.debug("Member exists : (0x%08X, '%s', 0x%08X, 0x%08X)",
struc_id, idc.GetMemberName(struc_id,
offset), offset,
idc.GetMemberSize(struc_id, offset))
kept_offsets = field_offsets - new_offsets
# clear kept members
# split the loop since we will modify the structure while iterating
offsets = set()
for (offset, member_name) in YaToolIDATools.struc_member_list(
struc_id, is_union):
offsets.add(offset)
for offset in offsets:
if offset in kept_offsets:
# This member already existed and is kept
if offset == last_offset and offset == struc_size:
# this is the last field, and it is a variable sized structure
field_size = 0
else:
field_size = 1
if member_type == ya.OBJECT_TYPE_STRUCT_MEMBER:
strucmember_id = self.hash_provider.get_struc_member_id(
struc_id, offset)
elif member_type == ya.OBJECT_TYPE_STACKFRAME_MEMBER:
strucmember_id = self.hash_provider.get_stackframe_member_object_id(
struc_id, offset)
else:
logger.error("Bad member_type : %d" % member_type)
if strucmember_id not in self.strucmember_ids:
# It is not necessary to clear the member if it is presnet in the resolved_objects
if DEBUG_EXPORTER:
logger.debug(
"SetStrucmember(0x%08X, None, 0x%08X, idc.FF_BYTE, -1, 0x%08X, name_offset=%s)",
struc_id, offset, field_size, name_offset)
YaToolIDATools.SetStrucmember(struc_id,
None,
offset,
idc.FF_BYTE,
-1,
field_size,
member_type=member_type,
name_offset=name_offset)
idc.SetMemberComment(struc_id, offset, "", 0)
idc.SetMemberComment(struc_id, offset, "", 1)
elif offset not in new_offsets:
if (member_type != ya.OBJECT_TYPE_STACKFRAME_MEMBER
or not idaapi.is_special_member(
idc.GetMemberId(struc_id, offset))):
if DEBUG_EXPORTER:
logger.debug(
"DelStrucMember(0x%08X, 0x%08X) (=%s:%s)",
struc_id, offset, idc.GetStrucName(struc_id),
idc.GetMemberName(struc_id, offset))
idc.DelStrucMember(struc_id, offset)
else:
# in new_offsets : just created
pass
idc.EndTypeUpdating(idc.UTP_STRUCT)
def make_enum(self, object_version, address):
enum_name = object_version.get_name()
object_id = object_version.get_id()
# build flags
bitfield = False
flags = object_version.get_object_flags()
if flags & 0x1 == 0x1:
bitfield = True
flags = flags & ~0x1
try:
enum_width = object_version.get_size()
except KeyError:
enum_width = None
# check if enum already exists
enum_id = idc.GetEnum(enum_name)
enum_xrefs_ids = object_version.get_xrefed_ids()
logger.debug(
"%s:%d : Check here that enum_xrefs_ids is a set(YaToolObjectId) and correctly used"
% (__file__, inspect.currentframe().f_lineno))
if enum_id != idc.BADADDR:
# enum already exists, deleting all members
for (const_id, const_value,
bmask) in YaToolIDATools.enum_member_iterate_all(enum_id):
if bmask == idc.BADADDR:
bmask = -1
const_name = idc.GetConstName(const_id)
if (enum_name, const_name,
const_value) not in self.enum_member_ids:
idc.DelConstEx(enum_id, const_value, 0, bmask)
elif self.hash_provider.get_enum_member_id(
enum_id, enum_name, const_id, const_name,
const_value) not in enum_xrefs_ids:
logger.debug("deleting not found constant : %s/%s" %
(enum_name, const_name))
idc.DelConstEx(enum_id, const_value, 0, bmask)
"""
if the enum "bitfield" state has changed : change it!
We can't access the functions
"""
if idc.IsBitfield(enum_id) != bitfield:
idaapi.set_enum_bf(enum_id, bitfield)
else:
# create enum
# SetEnumFlag return "'module' object has no attribute 'set_enum_flag'".
enum_id = idc.AddEnum(address, enum_name, flags)
if enum_width is not None:
idc.SetEnumWidth(enum_id, enum_width)
if bitfield:
idc.SetEnumBf(enum_id, 1)
# process comment
try:
repeatable_headercomment = self.sanitize_comment_to_ascii(
object_version.get_header_comment(True))
idc.SetEnumCmt(enum_id, repeatable_headercomment, 1)
except KeyError:
pass
try:
nonrepeatable_headercomment = self.sanitize_comment_to_ascii(
object_version.get_header_comment(False))
idc.SetEnumCmt(enum_id, nonrepeatable_headercomment, 0)
except KeyError:
pass
self.enum_ids[object_id] = enum_id
self.hash_provider.put_hash_struc_or_enum(enum_id, object_id)
logger.debug("adding enum id %s : '0x%.016X'" %
(self.hash_provider.hash_to_string(object_id), enum_id))
def make_data(self, object_version, address):
size = 0
try:
size = object_version.get_size()
except KeyError:
pass
flags = None
try:
flags = object_version.get_object_flags()
except KeyError:
pass
if size == 0:
idc.MakeUnkn(address, idc.DOUNK_EXPAND)
else:
if flags is not None:
if idc.isASCII(flags):
try:
str_type = object_version.get_string_type()
YaToolIDATools.check_and_set_str_type(str_type)
except KeyError:
pass
idc.MakeStr(address, address + size)
idc.SetFlags(address, flags)
if idc.isStruct(flags):
found = False
for xref_offset_operand, xref_id_attr in object_version.get_xrefed_id_map(
).iteritems():
(xref_offset, xref_operand) = xref_offset_operand
for xref_hash, xref_attrs in xref_id_attr:
if xref_hash in self.struc_ids:
struc_id = self.struc_ids[xref_hash]
if DEBUG_EXPORTER:
logger.debug(
"making unknown from 0x%08X to 0x%08X"
% (address, address + size))
idaapi.do_unknown_range(
address, size, idc.DOUNK_DELNAMES)
# idc.MakeUnkn(address, DOUNK_SIMPLE | idc.DOUNK_DELNAMES)
# for i in xrange(1, size):
# MakeName(address + i, "")
# idc.MakeUnkn(address + i, DOUNK_SIMPLE | idc.DOUNK_DELNAMES)
# idc.MakeStructEx uses idaapi.doStruct (but asks for name while
# we already have the struc id)
if DEBUG_EXPORTER:
logger.debug(
"Making struc at %s : %s (sizeof(%s)=0x%08X, size=0x%08X, flags=0x%08X"
% (self.yatools.address_to_hex_string(
address),
self.yatools.address_to_hex_string(
struc_id),
idaapi.get_struc_name(struc_id),
idc.GetStrucSize(struc_id), size,
flags))
idc.SetCharPrm(idc.INF_AUTO, True)
idc.Wait()
if idaapi.doStruct(address, size,
struc_id) == 0:
if DEBUG_EXPORTER:
logger.warning("Making struc failed")
idc.SetCharPrm(idc.INF_AUTO, False)
# idc.SetFlags(address, flags)
found = True
else:
logger.error(
"bad struc flags : idc.isStruct is true but no xref available for object %s"
% self.hash_provider.hash_to_string(
object_version.get_id()))
if not found:
logger.error(
"bad struc flags : idc.isStruct is true "
"but no struc available for object %s (%s)" %
(self.hash_provider.hash_to_string(
object_version.get_id()),
object_version.get_name()))
else:
idc.MakeData(address, flags & (idc.DT_TYPE | idc.MS_0TYPE),
size, 0)
else:
idc.MakeData(address, idc.FF_BYTE, size, 0)
self.make_name(object_version, address, False)
self.set_type(object_version, address)
def save(self):
start_time = time.time()
YaToolIDATools.update_bookmarks()
"""
TODO : improve cache re-generation
pb : we should not regenerate the whole cache everytime
pb : when we load strucmembers (from the cache) and they are
later deleted, we get stalled XML files (they are not referenced
in the parent struc/stackframe, which is good, but they still
exist)
*do not store objects here : store them in the memory exporter
*make 3 pass :
-delete deleted objects
-create updated objects
-create new objects
"""
logger.debug("YaToolIDAHooks.save()")
db = ya.MakeStdModel()
memory_exporter = db.visitor
if VALIDATE_EXPORTED_XML:
memory_exporter = ya.MakeMultiplexerDebugger(db.visitor)
memory_exporter.visit_start()
"""
First, find modified informations : marked positions, comments, ...
"""
# some marked comment may have been deleted
self.new_marked_pos = set()
for i in xrange(1, 1024):
if idc.GetMarkedPos(i) == idc.BADADDR:
break
else:
self.new_marked_pos.add(idc.GetMarkedPos(i))
# get remove marked comments
for removed_marked_pos in (self.marked_pos - self.new_marked_pos):
self.addresses_to_process.add(removed_marked_pos)
self.repo_manager.add_auto_comment(removed_marked_pos, "Removed marked comment")
# process comments
for (ea, value) in self.comments_to_process.iteritems():
# do not save comments coming from function prototype
# if not idaapi.is_tilcmt(ea):
self.addresses_to_process.add(ea)
self.repo_manager.add_auto_comment(ea, value)
"""
Next, export strucs and enums
This will also delete unneeded files
"""
self.save_strucs(memory_exporter)
self.save_enums(memory_exporter)
for ea in self.addresses_to_process:
self.ida_model.clear_exported_ea(ea)
segs = set()
for ea in self.segment_address_to_process:
segs.add(idc.SegStart(ea))
for ea in segs:
self.ida_model.clear_segment_item_cache(ea)
self.ida_model.clear_exported_segment_id(ea)
"""
explore IDA yacoHooks for logged ea
"""
for ea in self.addresses_to_process:
self.ida_model.accept_ea(memory_exporter, 0, ea)
for seg_ea_start, seg_ea_end in self.updated_segments:
self.ida_model.accept_segment(memory_exporter, 0, seg_ea_start, seg_ea_end, export_chunks=True)
memory_exporter.visit_end()
"""
#before saving, we remove all cache (some files may have been deleted)
order = ("struc", "strucmember", "enum", "enum_member", "segment", "function",
"stackframe", "stackframe_member", "basic_block", "data", "code")
for obj_type in order:
current_dir = os.path.join(self.idb_directory, "cache", obj_type)
if not os.path.isdir(current_dir):
continue
for f in os.listdir(current_dir):
os.remove(os.path.join(current_dir, f))
"""
logger.debug("Exporting from memory to XML")
# now export to XML
xml_exporter = ya.MakeXmlExporter(os.path.join(self.repo_manager.idb_directory, "cache"))
if VALIDATE_EXPORTED_XML_2:
db.model.accept(ya.MakePathDebuggerVisitor("SaveXMLValidator", ya.MakeExporterValidatorVisitor(), False))
db.model.accept(xml_exporter)
end_time = time.time()
logger.debug("YaCo saved in %d seconds." % (end_time - start_time))