def extract_all_user_names(filename=None):
"""
Get all user-named labels inside IDA. Also prints into output window.
:return: dictionary of all user named labels: label_name -> ea
"""
results = {}
output = ''
for ea, name in idautils.Names():
if ida_kernwin.user_cancelled():
return results
if '_' in name:
if name.split('_')[0] in ('def', 'sub', 'loc', 'jpt', 'j',
'nullsub'):
continue
flags = ida_bytes.get_full_flags(ea)
if ida_bytes.has_user_name(flags):
results[name] = ea
output += '{} = 0x{:08x};\n'.format(name, ea)
if filename is not None:
with open(filename, 'w') as f:
f.write(output)
return results
def update(self, text):
"""emit progress update
check if user cancelled action, raise exception for parent function to catch
"""
if ida_kernwin.user_cancelled():
raise UserCancelledError("user cancelled")
self.progress.emit("extracting features from %s" % text)
def _process(self, i):
n = self._add_node(i)
self.n_processed += 1
if n < 0:
return n
if len(self.parents) > 1:
lp = self.parents.back().obj_id
for k_obj_id, v in self.reverse.items():
if k_obj_id == lp:
p = v
break
self.cg.add_edge(p, n)
if self.n_items:
if self.n_processed >= self.n_items:
ida_kernwin.hide_wait_box()
if ida_kernwin.user_cancelled():
return 1
return 0
def load_capa_function_results(self):
""" """
if not self.rules_cache or not self.ruleset_cache:
# only reload rules if caches are empty
if not self.load_capa_rules():
return False
else:
logger.info(
'Using cached ruleset, click "Reset" to reload rules from disk.'
)
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("loading IDA extractor")
try:
# must use extractor to get function, as capa analysis requires casted object
extractor = CapaExplorerFeatureExtractor()
except Exception as e:
logger.error("Failed to load IDA feature extractor (error: %s)" %
e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("extracting function features")
try:
f = idaapi.get_func(idaapi.get_screen_ea())
if f:
f = extractor.get_function(f.start_ea)
self.rulegen_current_function = f
func_features, bb_features = find_func_features(f, extractor)
self.rulegen_func_features_cache = collections.defaultdict(
set, copy.copy(func_features))
self.rulegen_bb_features_cache = collections.defaultdict(
dict, copy.copy(bb_features))
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("matching function/basic block rule scope")
try:
# add function and bb rule matches to function features, for display purposes
func_matches, bb_matches = find_func_matches(
f, self.ruleset_cache, func_features, bb_features)
for (name, res) in itertools.chain(func_matches.items(),
bb_matches.items()):
rule = self.ruleset_cache[name]
if rule.meta.get("capa/subscope-rule"):
continue
for (ea, _) in res:
func_features[capa.features.common.MatchedRule(
name)].add(ea)
except Exception as e:
logger.error(
"Failed to match function/basic block rule scope (error: %s)"
% e)
return False
else:
func_features = {}
except UserCancelledError:
logger.info("User cancelled analysis.")
return False
except Exception as e:
logger.error("Failed to extract function features (error: %s)" % e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("extracting file features")
try:
file_features = find_file_features(extractor)
self.rulegen_file_features_cache = collections.defaultdict(
dict, copy.copy(file_features))
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("matching file rule scope")
try:
# add file matches to file features, for display purposes
for (name, res) in find_file_matches(self.ruleset_cache,
file_features).items():
rule = self.ruleset_cache[name]
if rule.meta.get("capa/subscope-rule"):
continue
for (ea, _) in res:
file_features[capa.features.common.MatchedRule(
name)].add(ea)
except Exception as e:
logger.error("Failed to match file scope rules (error: %s)" %
e)
return False
except Exception as e:
logger.error("Failed to extract file features (error: %s)" % e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("rendering views")
try:
# load preview and feature tree
self.view_rulegen_preview.load_preview_meta(
f.start_ea if f else None,
settings.user.get(CAPA_SETTINGS_RULEGEN_AUTHOR,
"<insert_author>"),
settings.user.get(CAPA_SETTINGS_RULEGEN_SCOPE, "function"),
)
self.view_rulegen_features.load_features(file_features,
func_features)
# self.view_rulegen_header_label.setText("Function Features (%s)" % trim_function_name(f))
self.set_view_status_label("capa rules directory: %s (%d rules)" %
(settings.user[CAPA_SETTINGS_RULE_PATH],
len(self.rules_cache)))
except Exception as e:
logger.error("Failed to render views (error: %s)" % e)
return False
return True
def load_capa_results(self, use_cache=False):
"""run capa analysis and render results in UI
note: this function must always return, exception or not, in order for plugin to safely close the IDA
wait box
"""
if not use_cache:
# new analysis, new doc
self.doc = None
self.process_total = 0
self.process_count = 1
def slot_progress_feature_extraction(text):
"""slot function to handle feature extraction progress updates"""
update_wait_box("%s (%d of %d)" %
(text, self.process_count, self.process_total))
self.process_count += 1
extractor = CapaExplorerFeatureExtractor()
extractor.indicator.progress.connect(
slot_progress_feature_extraction)
update_wait_box("calculating analysis")
try:
self.process_total += len(tuple(extractor.get_functions()))
except Exception as e:
logger.error("Failed to calculate analysis (error: %s).", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("loading rules")
if not self.load_capa_rules():
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("extracting features")
try:
meta = capa.ida.helpers.collect_metadata()
capabilities, counts = capa.main.find_capabilities(
self.ruleset_cache, extractor, disable_progress=True)
meta["analysis"].update(counts)
except UserCancelledError:
logger.info("User cancelled analysis.")
return False
except Exception as e:
logger.error(
"Failed to extract capabilities from database (error: %s)",
e)
return False
update_wait_box("checking for file limitations")
try:
# support binary files specifically for x86/AMD64 shellcode
# warn user binary file is loaded but still allow capa to process it
# TODO: check specific architecture of binary files based on how user configured IDA processors
if idaapi.get_file_type_name() == "Binary file":
logger.warning("-" * 80)
logger.warning(" Input file appears to be a binary file.")
logger.warning(" ")
logger.warning(
" capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA."
)
logger.warning(
" This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64."
)
logger.warning(
" If you don't know the input file type, you can try using the `file` utility to guess it."
)
logger.warning("-" * 80)
capa.ida.helpers.inform_user_ida_ui(
"capa encountered file type warnings during analysis")
if capa.main.has_file_limitation(self.ruleset_cache,
capabilities,
is_standalone=False):
capa.ida.helpers.inform_user_ida_ui(
"capa encountered file limitation warnings during analysis"
)
except Exception as e:
logger.error(
"Failed to check for file limitations (error: %s)", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("rendering results")
try:
self.doc = capa.render.result_document.convert_capabilities_to_result_document(
meta, self.ruleset_cache, capabilities)
except Exception as e:
logger.error("Failed to render results (error: %s)", e)
return False
try:
self.model_data.render_capa_doc(
self.doc, self.view_show_results_by_function.isChecked())
self.set_view_status_label("capa rules directory: %s (%d rules)" %
(settings.user[CAPA_SETTINGS_RULE_PATH],
len(self.rules_cache)))
except Exception as e:
logger.error("Failed to render results (error: %s)", e)
return False
return True
def load_capa_rules(self):
""" """
self.ruleset_cache = None
self.rules_cache = None
try:
# resolve rules directory - check self and settings first, then ask user
if not os.path.exists(
settings.user.get(CAPA_SETTINGS_RULE_PATH, "")):
idaapi.info(
"Please select a file directory containing capa rules.")
path = self.ask_user_directory()
if not path:
logger.warning(
"You must select a file directory containing capa rules before analysis can be run. The standard collection of capa rules can be downloaded from https://github.com/fireeye/capa-rules."
)
return False
settings.user[CAPA_SETTINGS_RULE_PATH] = path
except Exception as e:
logger.error("Failed to load capa rules (error: %s).", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
rule_path = settings.user[CAPA_SETTINGS_RULE_PATH]
try:
# TODO refactor: this first part is identical to capa.main.get_rules
if not os.path.exists(rule_path):
raise IOError(
"rule path %s does not exist or cannot be accessed" %
rule_path)
rule_paths = []
if os.path.isfile(rule_path):
rule_paths.append(rule_path)
elif os.path.isdir(rule_path):
for root, dirs, files in os.walk(rule_path):
if ".github" in root:
# the .github directory contains CI config in capa-rules
# this includes some .yml files
# these are not rules
continue
for file in files:
if not file.endswith(".yml"):
if not (file.startswith(".git") or file.endswith(
(".git", ".md", ".txt"))):
# expect to see .git* files, readme.md, format.md, and maybe a .git directory
# other things maybe are rules, but are mis-named.
logger.warning("skipping non-.yml file: %s",
file)
continue
rule_path = os.path.join(root, file)
rule_paths.append(rule_path)
rules = []
total_paths = len(rule_paths)
for (i, rule_path) in enumerate(rule_paths):
update_wait_box("loading capa rules from %s (%d of %d)" %
(settings.user[CAPA_SETTINGS_RULE_PATH], i + 1,
total_paths))
if ida_kernwin.user_cancelled():
raise UserCancelledError("user cancelled")
try:
rule = capa.rules.Rule.from_yaml_file(rule_path)
except capa.rules.InvalidRule:
raise
else:
rule.meta["capa/path"] = rule_path
if capa.main.is_nursery_rule_path(rule_path):
rule.meta["capa/nursery"] = True
rules.append(rule)
_rules = copy.copy(rules)
ruleset = capa.rules.RuleSet(_rules)
except UserCancelledError:
logger.info("User cancelled analysis.")
return False
except Exception as e:
capa.ida.helpers.inform_user_ida_ui(
"Failed to load capa rules from %s" %
settings.user[CAPA_SETTINGS_RULE_PATH])
logger.error("Failed to load rules from %s (error: %s).",
settings.user[CAPA_SETTINGS_RULE_PATH], e)
logger.error(
"Make sure your file directory contains properly formatted capa rules. You can download the standard collection of capa rules from https://github.com/fireeye/capa-rules."
)
settings.user[CAPA_SETTINGS_RULE_PATH] = ""
return False
self.ruleset_cache = ruleset
self.rules_cache = rules
return True
# Note: this try/except block below is just there to
# let us (at Hex-Rays) test this script in various
# situations.
try:
perform_decompilation = under_test__perform_decompilation
except:
pass
step_sleep = 0.5
ida_kernwin.show_wait_box("Processing")
try:
all_eas = list(idautils.Functions())
neas = len(all_eas)
for i, ea in enumerate(all_eas):
if ida_kernwin.user_cancelled():
break
ida_kernwin.replace_wait_box("Processing; step %d/%d" % (i+1, neas))
if perform_decompilation:
try:
ida_hexrays.decompile(ida_funcs.get_func(ea))
except ida_hexrays.DecompilationFailure as df:
print("Decompilation failure: %s" % df)
time.sleep(step_sleep * random.random())
finally:
ida_kernwin.hide_wait_box()
def load_capa_results(self):
"""run capa analysis and render results in UI
note: this function must always return, exception or not, in order for plugin to safely close the IDA
wait box
"""
# new analysis, new doc
self.doc = None
self.process_total = 0
self.process_count = 1
def update_wait_box(text):
"""update the IDA wait box"""
ida_kernwin.replace_wait_box("capa explorer...%s" % text)
def slot_progress_feature_extraction(text):
"""slot function to handle feature extraction progress updates"""
update_wait_box("%s (%d of %d)" %
(text, self.process_count, self.process_total))
self.process_count += 1
extractor = CapaExplorerFeatureExtractor()
extractor.indicator.progress.connect(slot_progress_feature_extraction)
update_wait_box("calculating analysis")
try:
self.process_total += len(tuple(extractor.get_functions()))
except Exception as e:
logger.error("Failed to calculate analysis (error: %s).", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("loading rules")
try:
# resolve rules directory - check self and settings first, then ask user
if not self.rule_path:
if "rule_path" in settings and os.path.exists(
settings["rule_path"]):
self.rule_path = settings["rule_path"]
else:
idaapi.info(
"Please select a file directory containing capa rules."
)
rule_path = self.ask_user_directory()
if not rule_path:
logger.warning(
"You must select a file directory containing capa rules before analysis can be run. The standard collection of capa rules can be downloaded from https://github.com/fireeye/capa-rules."
)
return False
self.rule_path = rule_path
settings.user["rule_path"] = rule_path
except Exception as e:
logger.error("Failed to load capa rules (error: %s).", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
rule_path = self.rule_path
try:
if not os.path.exists(rule_path):
raise IOError(
"rule path %s does not exist or cannot be accessed" %
rule_path)
rule_paths = []
if os.path.isfile(rule_path):
rule_paths.append(rule_path)
elif os.path.isdir(rule_path):
for root, dirs, files in os.walk(rule_path):
if ".github" in root:
# the .github directory contains CI config in capa-rules
# this includes some .yml files
# these are not rules
continue
for file in files:
if not file.endswith(".yml"):
if not (file.endswith(".md")
or file.endswith(".git")
or file.endswith(".txt")):
# expect to see readme.md, format.md, and maybe a .git directory
# other things maybe are rules, but are mis-named.
logger.warning("skipping non-.yml file: %s",
file)
continue
rule_path = os.path.join(root, file)
rule_paths.append(rule_path)
rules = []
total_paths = len(rule_paths)
for (i, rule_path) in enumerate(rule_paths):
update_wait_box("loading capa rules from %s (%d of %d)" %
(self.rule_path, i + 1, total_paths))
if ida_kernwin.user_cancelled():
raise UserCancelledError("user cancelled")
try:
rule = capa.rules.Rule.from_yaml_file(rule_path)
except capa.rules.InvalidRule:
raise
else:
rule.meta["capa/path"] = rule_path
if capa.main.is_nursery_rule_path(rule_path):
rule.meta["capa/nursery"] = True
rules.append(rule)
rule_count = len(rules)
rules = capa.rules.RuleSet(rules)
except UserCancelledError:
logger.info("User cancelled analysis.")
return False
except Exception as e:
capa.ida.helpers.inform_user_ida_ui(
"Failed to load capa rules from %s" % self.rule_path)
logger.error("Failed to load rules from %s (error: %s).",
self.rule_path, e)
logger.error(
"Make sure your file directory contains properly formatted capa rules. You can download the standard collection of capa rules from https://github.com/fireeye/capa-rules."
)
self.rule_path = ""
settings.user.del_value("rule_path")
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("extracting features")
try:
meta = capa.ida.helpers.collect_metadata()
capabilities, counts = capa.main.find_capabilities(
rules, extractor, disable_progress=True)
meta["analysis"].update(counts)
except UserCancelledError:
logger.info("User cancelled analysis.")
return False
except Exception as e:
logger.error(
"Failed to extract capabilities from database (error: %s)", e)
return False
update_wait_box("checking for file limitations")
try:
# support binary files specifically for x86/AMD64 shellcode
# warn user binary file is loaded but still allow capa to process it
# TODO: check specific architecture of binary files based on how user configured IDA processors
if idaapi.get_file_type_name() == "Binary file":
logger.warning("-" * 80)
logger.warning(" Input file appears to be a binary file.")
logger.warning(" ")
logger.warning(
" capa currently only supports analyzing binary files containing x86/AMD64 shellcode with IDA."
)
logger.warning(
" This means the results may be misleading or incomplete if the binary file loaded in IDA is not x86/AMD64."
)
logger.warning(
" If you don't know the input file type, you can try using the `file` utility to guess it."
)
logger.warning("-" * 80)
capa.ida.helpers.inform_user_ida_ui(
"capa encountered file type warnings during analysis")
if capa.main.has_file_limitation(rules,
capabilities,
is_standalone=False):
capa.ida.helpers.inform_user_ida_ui(
"capa encountered file limitation warnings during analysis"
)
except Exception as e:
logger.error("Failed to check for file limitations (error: %s)", e)
return False
if ida_kernwin.user_cancelled():
logger.info("User cancelled analysis.")
return False
update_wait_box("rendering results")
try:
self.doc = capa.render.convert_capabilities_to_result_document(
meta, rules, capabilities)
self.model_data.render_capa_doc(self.doc)
self.render_capa_doc_mitre_summary()
self.enable_controls()
self.set_view_status_label("capa rules directory: %s (%d rules)" %
(self.rule_path, rule_count))
except Exception as e:
logger.error("Failed to render results (error: %s)", e)
return False
return True
