def open_coverage_xref(self, address, dctx=None):
"""
Open the 'Coverage Xref' dialog for a given address.
"""
lctx = self.get_context(dctx)
# show the coverage xref dialog
dialog = CoverageXref(lctx.director, address)
if not dialog.exec_():
return
# activate the user selected xref (if one was double clicked)
if dialog.selected_coverage:
lctx.director.select_coverage(dialog.selected_coverage)
return
# load a coverage file from disk
disassembler.show_wait_box("Loading coverage from disk...")
created_coverage, errors = lctx.director.load_coverage_files(
[dialog.selected_filepath],
disassembler.replace_wait_box
)
if not created_coverage:
lmsg("No coverage files could be loaded...")
disassembler.hide_wait_box()
warn_errors(errors)
return
disassembler.replace_wait_box("Selecting coverage...")
lctx.director.select_coverage(created_coverage[0].name)
disassembler.hide_wait_box()
def aggregate_drcov_batch(self, drcov_list):
"""
Aggregate a given list of DrcovData into a single coverage mapping.
See create_coverage_from_drcov_list(...) for more verbose comments.
"""
errors = []
# create a new coverage set to manually aggregate data into
coverage = DatabaseCoverage(self._palette)
for i, drcov_data in enumerate(drcov_list, 1):
# keep the user informed about our progress while aggregating
disassembler.replace_wait_box(
"Aggregating batch data %u/%u" % (i, len(drcov_list))
)
# normalize coverage data to the open database
try:
addresses = self._normalize_drcov_data(drcov_data)
except Exception as e:
errors.append((self.ERROR_COVERAGE_ABSENT, drcov_data.filepath))
lmsg("Failed to normalize coverage %s" % drcov_data.filepath)
lmsg("- %s" % e)
continue
# aggregate the addresses into the output coverage mapping
coverage.add_addresses(addresses, False)
# return the created coverage name
return (coverage, errors)
def metadata_progress(completed, total):
"""
Handler for metadata collection callback, updates progress dialog.
"""
disassembler.replace_wait_box(
"Collected metadata for %u/%u Functions" % (completed, total)
)
def _parse_module_symbol(self, f, reg, all_lines):
disassembler.replace_wait_box("Building module_symbol metadata...")
len_idx = 1
while 1:
try:
module_line = f.readline().strip()
if ("Opened" in module_line
or "start" in module_line) and "!" not in module_line:
break
if module_line == None or module_line == "":
break
else:
len_idx = len_idx + 1
disassembler.replace_wait_box(
"Building module symbol metadata %u/%u" %
(len_idx, all_lines))
regMatch = reg.match(module_line)
linebits = regMatch.groupdict()
dic = {}
for k, v in linebits.iteritems():
if k == "addr":
dic["addr"] = v
if k == "symbol":
module = v[0:v.find("!")]
symbol = v[v.find("!") + 1:]
dic["module"] = module
dic["symbol"] = symbol
self.all_symbol.append(dic)
except Exception as e:
continue
def check_root_trace(self,all_Excute_Function_list):
all_len = len(all_Excute_Function_list)
len_idx = 0
for cur_cov in all_Excute_Function_list:
len_idx = len_idx + 1
self.check_sub_trace(cur_cov, cur_cov.sub_func_coverage )
if len_idx % 50 == 0 or len_idx == all_len:
disassembler.replace_wait_box("Optimizing execute trace metadata %u/%u" % (len_idx, all_len))
def refresh_metadata(self):
"""
Hard refresh of the director and table metadata layers.
"""
disassembler.replace_wait_box("Building execute trace metadata...")
self._model._director.refresh_execute_trace(self._model)
# ensure the table's model gets refreshed
disassembler.replace_wait_box("Refreshing execute trace Overview...")
def _fix_symbol(self, f):
len_idx = 1
all_lines = len(self.all_symbol)
for dic in self.all_symbol:
len_idx = len_idx + 1
disassembler.replace_wait_box(
"Fixing module symbol metadata %u/%u" % (len_idx, all_lines))
for md in self.modules:
if md.name == dic["module"]:
offset = int(dic["addr"], 16) - int(md.base, 16)
md.SymbolDic[offset] = dic["symbol"].split(
' ')[0] if "" in dic["symbol"] else dic["symbol"]
def refresh_metadata(self):
"""
Hard refresh of the director and table metadata layers.
"""
disassembler.show_wait_box("Building database metadata...")
self._model._director.refresh()
# ensure the table's model gets refreshed
disassembler.replace_wait_box("Refreshing Coverage Overview...")
self._model.refresh()
# all done
disassembler.hide_wait_box()
def _parse_symbol_file(self, filepath):
with open(filepath, "rb") as f:
all_lines = len(f.read().splitlines())
f.seek(0)
rexp = "(?P<addr>[0-9a-fA-F]+)(\s+)(?P<symbol>.+)"
reg = re.compile(rexp)
self._parse_module_symbol(f, reg, all_lines)
self._parse_symbol_header(f)
self._parse_module_table(f)
self._fix_symbol(f)
disassembler.replace_wait_box(
"Building module_symbol metadata Successfully")
def _refresh_database_coverage(self):
"""
Refresh all the database coverage mappings managed by the director.
"""
logger.debug("Refreshing database coverage mappings")
for i, name in enumerate(self.all_names, 1):
logger.debug(" - %s" % name)
disassembler.replace_wait_box("Refreshing coverage mapping %u/%u" %
(i, len(self.all_names)))
coverage = self.get_coverage(name)
coverage.update_metadata(self.metadata)
coverage.refresh()
def interactive_load_file(self, dctx=None):
"""
Perform the user-interactive loading of individual coverage files.
"""
lctx = self.get_context(dctx)
#
# kick off an asynchronous metadata refresh. this will run in the
# background while the user is selecting which coverage files to load
#
future = lctx.metadata.refresh_async(progress_callback=metadata_progress)
#
# we will now prompt the user with an interactive file dialog so they
# can select the coverage files they would like to load from disk
#
filenames = lctx.select_coverage_files()
if not filenames:
lctx.metadata.abort_refresh()
return
#
# to begin mapping the loaded coverage data, we require that the
# asynchronous database metadata refresh has completed. if it is
# not done yet, we will block here until it completes.
#
# a progress dialog depicts the work remaining in the refresh
#
disassembler.show_wait_box("Building database metadata...")
lctx.metadata.go_synchronous()
await_future(future)
#
# now that the database metadata is available, we can use the director
# to load and normalize the selected coverage files
#
disassembler.replace_wait_box("Loading coverage from disk...")
created_coverage, errors = lctx.director.load_coverage_files(filenames, disassembler.replace_wait_box)
#
# if the director failed to map any coverage, the user probably
# provided bad files. emit any warnings and bail...
#
if not created_coverage:
lmsg("No coverage files could be loaded...")
disassembler.hide_wait_box()
warn_errors(errors)
return
#
# activate the first of the newly loaded coverage file(s). this is the
# one that will be visible in the coverage overview once opened
#
disassembler.replace_wait_box("Selecting coverage...")
lctx.director.select_coverage(created_coverage[0].name)
# all done! pop the coverage overview to show the user their results
disassembler.hide_wait_box()
lmsg("Successfully loaded %u coverage file(s)..." % len(created_coverage))
self.open_coverage_overview(lctx.dctx)
# finally, emit any notable issues that occurred during load
warn_errors(errors, lctx.director.suppressed_errors)
def interactive_load_batch(self, dctx=None):
"""
Perform the user-interactive loading of a coverage batch.
"""
lctx = self.get_context(dctx)
#
# kick off an asynchronous metadata refresh. this will run in the
# background while the user is selecting which coverage files to load
#
future = lctx.metadata.refresh_async(progress_callback=metadata_progress)
#
# we will now prompt the user with an interactive file dialog so they
# can select the coverage files they would like to load from disk
#
filepaths = lctx.select_coverage_files()
if not filepaths:
lctx.director.metadata.abort_refresh()
return
# prompt the user to name the new coverage aggregate
default_name = "BATCH_%s" % lctx.director.peek_shorthand()
ok, batch_name = prompt_string(
"Batch Name:",
"Please enter a name for this coverage",
default_name
)
#
# if user didn't enter a name for the batch (or hit cancel) we should
# abort the loading process...
#
if not (ok and batch_name):
lmsg("User failed to enter a name for the batch coverage...")
lctx.director.metadata.abort_refresh()
return
#
# to begin mapping the loaded coverage data, we require that the
# asynchronous database metadata refresh has completed. if it is
# not done yet, we will block here until it completes.
#
# a progress dialog depicts the work remaining in the refresh
#
disassembler.show_wait_box("Building database metadata...")
lctx.metadata.go_synchronous()
await_future(future)
#
# now that the database metadata is available, we can use the director
# to normalize and condense (aggregate) all the coverage data
#
disassembler.replace_wait_box("Loading coverage from disk...")
batch_coverage, errors = lctx.director.load_coverage_batch(
filepaths,
batch_name,
disassembler.replace_wait_box
)
# if batch creation fails...
if not batch_coverage:
lmsg("Creation of batch '%s' failed..." % batch_name)
disassembler.hide_wait_box()
warn_errors(errors)
return
# select the newly created batch coverage
disassembler.replace_wait_box("Selecting coverage...")
lctx.director.select_coverage(batch_name)
# all done! pop the coverage overview to show the user their results
disassembler.hide_wait_box()
lmsg("Successfully loaded batch %s..." % batch_name)
self.open_coverage_overview(lctx.dctx)
# finally, emit any notable issues that occurred during load
warn_errors(errors, lctx.director.suppressed_errors)
def interactive_load_file(self):
"""
Perform the user-interactive loading of individual coverage files.
"""
self.palette.refresh_colors()
#
# kick off an asynchronous metadata refresh. this will run in the
# background while the user is selecting which coverage files to load
#
future = self.director.refresh_metadata(
progress_callback=metadata_progress)
#
# we will now prompt the user with an interactive file dialog so they
# can select the coverage files they would like to load from disk
#
filenames = self._select_coverage_files()
#
# load the selected coverage files from disk (if any), returning a list
# of loaded DrcovData objects (which contain coverage data)
#
disassembler.show_wait_box("Loading coverage from disk...")
drcov_list = load_coverage_files(filenames)
if not drcov_list:
disassembler.hide_wait_box()
self.director.metadata.abort_refresh()
return
#
# to begin mapping the loaded coverage data, we require that the
# asynchronous database metadata refresh has completed. if it is
# not done yet, we will block here until it completes.
#
# a progress dialog depicts the work remaining in the refresh
#
disassembler.replace_wait_box("Building database metadata...")
await_future(future)
# insert the loaded drcov data objects into the director
created_coverage, errors = self.director.create_coverage_from_drcov_list(
drcov_list)
#
# if the director failed to map any coverage, the user probably
# provided bad files. emit any warnings and bail...
#
if not created_coverage:
lmsg("No coverage files could be loaded...")
disassembler.hide_wait_box()
warn_errors(errors)
return
#
# activate the first of the newly loaded coverage file(s). this is the
# one that will be visible in the coverage overview once opened
#
disassembler.replace_wait_box("Selecting coverage...")
self.director.select_coverage(created_coverage[0])
# all done! pop the coverage overview to show the user their results
disassembler.hide_wait_box()
lmsg("Successfully loaded %u coverage file(s)..." %
len(created_coverage))
self.open_coverage_overview()
# finally, emit any notable issues that occurred during load
warn_errors(errors)
def interactive_load_batch(self):
"""
Perform the user-interactive loading of a coverage batch.
"""
self.palette.refresh_colors()
#
# kick off an asynchronous metadata refresh. this will run in the
# background while the user is selecting which coverage files to load
#
future = self.director.refresh_metadata(
progress_callback=metadata_progress)
#
# we will now prompt the user with an interactive file dialog so they
# can select the coverage files they would like to load from disk
#
filenames = self._select_coverage_files()
#
# load the selected coverage files from disk (if any), returning a list
# of loaded DrcovData objects (which contain coverage data)
#
drcov_list = load_coverage_files(filenames)
if not drcov_list:
self.director.metadata.abort_refresh()
return
# prompt the user to name the new coverage aggregate
default_name = "BATCH_%s" % self.director.peek_shorthand()
ok, coverage_name = prompt_string(
"Batch Name:", "Please enter a name for this coverage",
default_name)
#
# if user didn't enter a name for the batch (or hit cancel) we should
# abort the loading process...
#
if not (ok and coverage_name):
lmsg("User failed to enter a name for the loaded batch...")
self.director.metadata.abort_refresh()
return
#
# to begin mapping the loaded coverage data, we require that the
# asynchronous database metadata refresh has completed. if it is
# not done yet, we will block here until it completes.
#
# a progress dialog depicts the work remaining in the refresh
#
disassembler.show_wait_box("Building database metadata...")
await_future(future)
#
# now that the database metadata is available, we can use the director
# to normalize and condense (aggregate) all the coverage data
#
new_coverage, errors = self.director.aggregate_drcov_batch(drcov_list)
#
# finally, we can inject the aggregated coverage data into the
# director under the user specified batch name
#
disassembler.replace_wait_box("Mapping coverage...")
self.director.create_coverage(coverage_name, new_coverage.data)
# select the newly created batch coverage
disassembler.replace_wait_box("Selecting coverage...")
self.director.select_coverage(coverage_name)
# all done! pop the coverage overview to show the user their results
disassembler.hide_wait_box()
lmsg("Successfully loaded batch %s..." % coverage_name)
self.open_coverage_overview()
# finally, emit any notable issues that occurred during load
warn_errors(errors)
def create_coverage_from_drcov_list(self, drcov_list):
"""
Create a number of database coverage mappings from a list of DrcovData.
Returns a tuple of (created_coverage, errors)
"""
created_coverage = []
errors = []
#
# stop the director's aggregate from updating. this will prevent the
# aggregate from recomputing after each individual mapping is created.
# instead, we will wait till *all* have been created, computing the
# new aggregate at the very end. this is far more performant.
#
self.suspend_aggregation()
#
# loop through the coverage data we been given (drcov_list), and begin
# the normalization process to translate / filter / flatten its blocks
# into a generic format the director can consume (a list of addresses)
#
for i, drcov_data in enumerate(drcov_list, 1):
# keep the user informed about our progress while loading coverage
disassembler.replace_wait_box(
"Normalizing and mapping coverage %u/%u" % (i, len(drcov_list))
)
#
# translate the coverage data's basic block addresses to the
# imagebase of the open database, and flatten the blocks to a
# list of instruction addresses
#
try:
coverage_data = self._normalize_drcov_data(drcov_data)
except ValueError as e:
errors.append((self.ERROR_COVERAGE_ABSENT, drcov_data.filepath))
lmsg("Failed to normalize coverage %s" % drcov_data.filepath)
lmsg("- %s" % e)
continue
#
# before injecting the new coverage data (now a list of instruction
# addresses), we check to see if there is an existing coverage
# object under the same name.
#
# if there is an existing coverage mapping, odds are that the user
# is probably re-loading the same coverage file in which case we
# simply overwrite the old DatabaseCoverage object.
#
# but we have to be careful for the case where the user loads a
# coverage file from a different directory, but under the same name
#
# e.g:
# - C:\coverage\foo.log
# - C:\coverage\testing\foo.log
#
# in these cases, we will append a suffix to the new coverage file
#
coverage_name = os.path.basename(drcov_data.filepath)
coverage = self.get_coverage(coverage_name)
# assign a suffix to the coverage name in the event of a collision
if coverage and coverage.filepath != drcov_data.filepath:
for i in xrange(2,0x100000):
new_name = "%s_%u" % (coverage_name, i)
if not self.get_coverage(new_name):
break
coverage_name = new_name
#
# finally, we can ask the director to create a coverage mapping
# from the data we have pre-processed for it
#
coverage = self.create_coverage(
coverage_name,
coverage_data,
drcov_data.filepath
)
created_coverage.append(coverage_name)
# warn when loaded coverage appears to be poorly mapped (suspicious)
if coverage.suspicious:
errors.append((self.ERROR_COVERAGE_SUSPICIOUS, drcov_data.filepath))
lmsg("Badly mapped coverage %s" % drcov_data.filepath)
# warn when loaded coverage (for this module) appears to be empty
if not len(coverage.nodes):
errors.append((self.ERROR_COVERAGE_ABSENT, drcov_data.filepath))
lmsg("No relevant coverage data in %s" % drcov_data.filepath)
#
# resume the director's aggregation service, triggering an update to
# recompute the aggregate with the newly loaded coverage
#
disassembler.replace_wait_box("Recomputing coverage aggregate...")
self.resume_aggregation()
# done
return (created_coverage, errors)
def buidExecuteTrace(self):
dic_bb = {}
all_len=len(self.orgdata.basic_blocks)
for bb in self.orgdata.basic_blocks:
if not dic_bb.has_key(bb.thread_id):
dic_bb[bb.thread_id]=[]
dic_bb[bb.thread_id].append(bb)
disassembler.replace_wait_box("Building execute trace metadata...")
len_idx = 0
for ke,ve in dic_bb.iteritems():
per_thread_Excute_Function_list=[]
per_thread_Excute_Function_list_cache=[]
bb_start = 0
bb_end = 0
cur_func = None
check_bb = False
if len(ve)>3000000:
for bb_idx, bb in enumerate(ve):
if bb_idx % 200000 ==0:
per_thread_Excute_Function_list_cache=self.shrink_Excute_Function_list_cache(per_thread_Excute_Function_list_cache)
len_idx = len_idx + 1
if len_idx % 30000 == 0:
disassembler.replace_wait_box("Building execute trace metadata %u/%u" % (len_idx, all_len))
if bb.call_type == lighthouse.core.BASIC_BLOCK:
bb_start = bb.start
bb_end = bb.start+bb.size
check_bb = True
if bb.call_type == lighthouse.core.FUNC_DIRECT_CALL or bb.call_type == lighthouse.core.FUNC_INDIRECT_CALL:
if cur_func is not None and ((check_bb is True and bb.start <= bb_end and bb.start >= bb_start) or (
check_bb is False and bb.start >= bb_end)):
sub_cov = ExecuteFunctionCoverage(cur_func, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = sub_cov
elif cur_func is None:
root_cov = ExecuteFunctionCoverage(None, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = root_cov
else:
root_cov = ExecuteFunctionCoverage(None, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = root_cov
if cur_func is not None and cur_func.trace_depth < 100:
per_thread_Excute_Function_list.append(cur_func)
per_thread_Excute_Function_list_cache.append(cur_func)
bb_start = bb.start
bb_end =bb.to
check_bb=False
if bb.call_type == lighthouse.core.FUNC_RETURN:
ret_cov = self.getReturnCaller(per_thread_Excute_Function_list_cache,bb.to, bb_idx,bb.thread_id,bb.mod_id_to)
if ret_cov is not None:
cur_func =ret_cov
else:
cur_func=None
bb_start = bb.start
bb_end = bb.to
check_bb = False
else:
for bb_idx, bb in enumerate(ve):
len_idx = len_idx + 1
if len_idx % 50 == 0 or len_idx == all_len:
disassembler.replace_wait_box("Building execute trace metadata %u/%u" % (len_idx, all_len))
if bb.call_type == lighthouse.core.BASIC_BLOCK:
bb_start = bb.start
bb_end = bb.start + bb.size
check_bb = True
if bb.call_type == lighthouse.core.FUNC_DIRECT_CALL or bb.call_type == lighthouse.core.FUNC_INDIRECT_CALL:
if cur_func is not None:
sub_cov = ExecuteFunctionCoverage(cur_func, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = sub_cov
elif cur_func is None:
root_cov = ExecuteFunctionCoverage(None, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = root_cov
else:
tmp_func = self.getFuncCaller(per_thread_Excute_Function_list, bb.start, bb_idx,
bb.thread_id, bb.mod_id)
if tmp_func is not None:
sub_cov = ExecuteFunctionCoverage(None, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = sub_cov
else:
root_cov = ExecuteFunctionCoverage(None, bb_idx, bb.thread_id, bb.mod_id, bb.start,
bb.mod_id_to, bb.to, bb.ret)
cur_func = root_cov
if cur_func is not None:
per_thread_Excute_Function_list.append(cur_func)
bb_start = bb.start
bb_end = bb.to
check_bb = False
if bb.call_type == lighthouse.core.FUNC_RETURN:
ret_cov = self.getReturnCaller(per_thread_Excute_Function_list, bb.to, bb_idx, bb.thread_id,
bb.mod_id_to)
if ret_cov is not None:
cur_func = ret_cov
else:
cur_func = None
bb_start = bb.start
bb_end = bb.to
check_bb = False
self.all_Excute_Function_list.extend(per_thread_Excute_Function_list)
self.check_root_trace(self.all_Excute_Function_list)
