def run_kordesii_decoder(self, decoder_name):
"""
Run the specified kordesii decoder against the file data. The reporter object is returned
and can be accessed as necessary to obtain output files, etc.
:param decoder_name: name of the decoder to run
:return: Instance of the kordesii_reporter.
:raises RuntimeError: If kordesii is not installed.
"""
if not kordesiireporter:
raise RuntimeError('Please install kordesii to use this function.')
logger.info('Running {} kordesii decoder on file {}.'.format(decoder_name, self.file_name))
kordesii_reporter = kordesiireporter(base64outputfiles=True, enableidalog=True)
kordesii_reporter.run_decoder(decoder_name, data=self.file_data)
for message in kordesii_reporter.get_debug():
logger.info('[kordesii_debug] {}'.format(message))
for message in kordesii_reporter.get_errors():
logger.error('[kordesii_error] {}'.format(message))
decrypted_strings = kordesii_reporter.get_strings()
if not decrypted_strings:
logger.warn(
'No decrypted strings were returned by the decoder for file {}.'.format(self.file_name))
return kordesii_reporter
def descriptions():
"""
List descriptions of decoder modules
"""
try:
response.content_type = "application/json"
reporter = kordesiireporter(decoderdir=DECODER_DIR, base64outputfiles=True)
output = {"decoders": reporter.list_decoders()}
return reporter.pprint(output)
except Exception as e:
output = {'error': traceback.format_exc()}
logger.error("descriptions %s" % (traceback.format_exc()))
return output
def __run_decoder(name, data, filename, append_output_text=True):
output = {}
logger.info("__run_decoder %s %s %s" %
(name, filename, hashlib.md5(data).hexdigest()))
try:
reporter = kordesiireporter(decoderdir=DECODER_DIR,
base64outputfiles=True)
# Since we want the marked up IDB returned using the original filename, we
# want to pass in a file to the reporter instead of data.
tempdir = tempfile.mkdtemp(prefix="kordesii-server_tempdir-")
file_path = os.path.join(tempdir, filename)
with open(file_path, "wb") as f:
f.write(data)
# Run decoder
reporter.run_decoder(name, filename=file_path)
# Since we used our own temp directory to pass in a file, we have to
# clean it up manually.
try:
shutil.rmtree(tempdir, ignore_errors=True)
except Exception as e:
reporter.debug("Failed to purge server temp dir: %s, %s" %
(tempdir, str(e)))
# Format and return results
output = reporter.metadata
if reporter.errors:
output["error"] = str(reporter.errors)
for error in reporter.errors:
logger.error("__run_decoder %s %s %s" %
(name, filename, error))
if append_output_text:
output["output_text"] = reporter.get_output_text()
return output
except Exception as e:
output = {}
output['error'] = traceback.format_exc()
logger.error("__run_decoder %s %s %s" %
(name, filename, traceback.format_exc()))
return output
def make_opt_parser():
'''
create a option parser to handle command line inputs
'''
usage_str = 'usage: %s [options] FILE' % (os.path.basename(sys.argv[0]))
description = "DC3-Kordesii Framework: utility for executing string decoder modules"
opt_parser = optparse.OptionParser(usage_str, description=description)
default_decoderdir = ''
#create reporter to get default paths, ignore if this fails
try:
default_reporter = kordesiireporter()
default_decoderdir = default_reporter.decoderdir
except Exception as e:
traceback.print_exc()
opt_parser.add_option(
'-a',
'--autonomous',
action='store_false',
default=True,
dest='autonomous',
help='Launch IDA without autonomous mode so IDA GUI appears.')
opt_parser.add_option('-c',
'--hidedebug',
action="store_true",
default=False,
dest='hidedebug',
help='Hide debug messages in output')
opt_parser.add_option('-p',
'--decoder',
action='store',
type='string',
default='',
dest='decoder',
help='string decoder to call')
opt_parser.add_option(
'-e',
'--enableidalog',
action="store_true",
default=False,
dest='enableidalog',
help='include the log contents produced by IDA in the results')
opt_parser.add_option(
'-f',
'--includefileinfo',
action='store_true',
default=False,
dest='includefilename',
help=
'include input file information such as filename, hashes, and compile time in parser output'
)
opt_parser.add_option(
'-g',
'--disabletempcleanup',
action='store_true',
default=False,
dest='disabletempcleanup',
help=
'Disable cleanup of framework created temp files including managed tempdir'
)
opt_parser.add_option(
'-j',
'--jsonoutput',
action='store_true',
default=False,
dest='jsonoutput',
help='Enable json output for parser reports (instead of formatted text)'
)
opt_parser.add_option('-l',
'--list',
action="store_true",
default=False,
dest='list',
help='list all kordesii string decoders')
opt_parser.add_option('-m',
'--tempdir',
action='store',
type='string',
metavar='DIR',
default=tempfile.gettempdir(),
dest='tempdir',
help='temp directory' + ' [default: %default]')
opt_parser.add_option('-d',
'--decoderdir',
action='store',
type='string',
metavar='DIR',
default=default_decoderdir,
dest='decoderdir',
help='decoders directory' + ' [default: %default]')
opt_parser.add_option(
'-t',
'--timeout',
action='store',
type=int,
default=10,
dest='timeout',
help=
'Timeout for running IDA (default = %default). A timout of 0 disables '
+ 'the timeout.')
opt_parser.add_option(
'-x',
'--disabletxtcleanup',
action='store_true',
default=False,
dest='disabletxtcleanup',
help=
'Disable cleanup of txt files generated by IDA (ida_strings.txt, ida_log.txt, ida_debug.txt)'
)
opt_parser.add_option(
'-y',
'--enableidbcleanup',
action='store_true',
default=False,
dest='enableidbcleanup',
help='Enable cleanup of IDB files and any IDA component files')
opt_parser.add_option(
'-z',
'--enableoutputfilecleanup',
action='store_true',
default=False,
dest='enableoutputfilecleanup',
help='Enable any cleanup of unique output files generated by a decoder'
)
return opt_parser
def main():
'''
Takes args from the command line, runs IDA, and returns with IDA's returncode on success or a message
on failure.
'''
opt_parse = make_opt_parser()
options, args = opt_parse.parse_args()
# If we can not create reporter object there is very little we can do. Just die immediately.
try:
reporter = kordesiireporter(
decoderdir=options.decoderdir,
tempdir=options.tempdir,
disabletempcleanup=options.disabletempcleanup,
disabledebug=options.hidedebug,
enableidalog=options.enableidalog)
except Exception as e:
error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % (
traceback.format_exc())
if options.jsonoutput:
print('{"errors": ["%s"]}' % (error_message))
else:
print(error_message)
sys.exit(1)
# List out decoder names and exit
if options.list:
decoders = reporter.list_decoders()
if options.jsonoutput:
if reporter.errors:
decoders.append({"errors": reporter.errors})
print reporter.pprint(decoders)
else:
for name in decoders:
print(name)
if reporter.errors:
print("")
print("Errors:")
for error in reporter.errors:
print(" %s" % (error))
return
# Currently only allow one file to be passed in
if not args or len(args) != 1:
opt_parse.print_help()
return
# Run decoder
if options.decoder:
# Grab file from arguments
input_file = os.path.abspath(args[0])
# Verify provided decoder name is valid
decoder = options.decoder
try:
decoder_path = reporter.get_decoder_path(decoder)
except ValueError as e:
print "Error: {}".format(e)
return
# Ensure the decoder script will close IDA properly
if "idc.Exit(" not in open(decoder_path, 'r').read():
print "Error: The decoder will leave IDA open indefinitely. Ensure it has a call to 'idc.Exit'."
# IDA doesn't like backslashes in it's argv.
input_file = input_file.replace('\\', '/').strip()
decoder_path = decoder_path.replace('\\', '/').strip()
# Run the decoder
reporter.run_decoder(
decoder,
input_file,
timeout=options.timeout,
autonomous=options.autonomous,
cleanup_txt_files=not options.disabletxtcleanup,
cleanup_output_files=options.enableoutputfilecleanup,
cleanup_idb_files=options.enableidbcleanup)
# Output results
if options.jsonoutput:
output = reporter.metadata
if reporter.errors:
output["errors"] = reporter.errors
if reporter.ida_log:
output["ida_log"] = reporter.ida_log
print reporter.pprint(output)
else:
reporter.output_text()
def main():
''' Run tool. '''
print ''
# Get command line arguments
argparser = get_arg_parser()
args = argparser.parse_args()
# Configure reporter based on args
reporter = kordesiireporter(enableidalog=True)
# Configure test object
tester = kordesiitester(reporter=reporter, results_dir=args.test_case_dir)
valid_decoder_names = reporter.list_decoders()
decoders = []
if args.decoder_name:
if args.decoder_name in valid_decoder_names:
decoders = [args.decoder_name]
else:
print "Error: Invalid decoder name(s) specified. decoder names are case sensitive."
exit(1)
if args.all_tests:
decoders = valid_decoder_names
if not decoders:
print "You must specify the decoder to test (or test all decoders)"
exit(2)
if args.decoder_name:
results_file_path = tester.get_results_filepath(args.decoder_name)
# Gather all our input files
input_files = []
if args.input_file:
input_files = read_input_list(args.input_file)
# Default is to run test cases
if args.run_tests:
print "Running test cases. May take a while..."
all_passed, test_results = tester.run_tests(
decoders,
filter(None, args.field_names.split(",")),
ignore_field_names=filter(None,
args.exclude_field_names.split(",")))
print "All Passed = {0}\n".format(all_passed)
if not args.silent:
if args.only_failed_tests:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=False,
json_format=args.json)
else:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=True,
json_format=args.json)
if all_passed:
exit(0)
else:
exit(1)
#add files to test cases
elif args.delete:
removed_files = tester.remove_test_results(args.decoder_name,
input_files)
for filename in removed_files:
print("Removing results for %s in %s" %
(filename, results_file_path))
elif args.decoder_name and (args.update or
(not args.delete and input_files)):
if args.update:
input_files.extend(tester.list_test_files(args.decoder_name))
for input_file in input_files:
tester.gen_results(decoder_name=args.decoder_name,
input_file_path=input_file)
if len(reporter.metadata) > 1 and len(reporter.errors) == 0:
print("Updating results for %s in %s" %
(input_file, results_file_path))
tester.update_test_results(results_file_path=results_file_path,
results_data=reporter.metadata,
replace=True)
elif len(reporter.metadata) > 1 and len(reporter.errors) > 0:
print("Error occurred for %s in %s, not updating" %
(input_file, results_file_path))
else:
print("Empty results for %s in %s, not updating" %
(input_file, results_file_path))
else:
argparser.print_help()
def main():
""" Run tool. """
print('')
# Get command line arguments
argparser = get_arg_parser()
args = argparser.parse_args()
# Configure reporter based on args
reporter = kordesiireporter(enableidalog=True)
# Configure test object
tester = kordesiitester(reporter=reporter, results_dir=args.test_case_dir)
valid_decoder_names = reporter.list_decoders()
decoders = []
if args.decoder_name:
if args.decoder_name in valid_decoder_names:
decoders = [args.decoder_name]
else:
print(
"Error: Invalid decoder name(s) specified. decoder names are case sensitive."
)
exit(1)
if args.all_tests:
decoders = valid_decoder_names
if not decoders:
print("You must specify the decoder to test (or test all decoders)")
exit(2)
if args.decoder_name:
results_file_path = tester.get_results_filepath(args.decoder_name)
# Gather all our input files
input_files = []
if args.input_file:
input_files = read_input_list(args.input_file)
# Default is to run test cases
if args.run_tests:
print("Running test cases. May take a while...")
start_time = timeit.default_timer()
test_infos = []
json_list = []
all_passed = True
for passed, test_result, test_info in tester.run_tests(
decoders,
filter(None, args.field_names.split(",")),
ignore_field_names=filter(None,
args.exclude_field_names.split(",")),
nprocs=args.nprocs):
test_infos.append(test_info)
if not passed:
all_passed = False
if not args.silent:
sys.stdout.write(
"{finished}/{total} - {decoder} {filename} {run_time:.4f}s\n"
.format(**test_info))
sys.stdout.flush()
if not passed or not args.only_failed_tests:
# TODO: Refactor support for json.
if args.json:
json_list.append(
tester.format_test_result(test_result,
json_format=True))
else:
display = tester.format_test_result(test_result)
print(
display.encode(locale.getpreferredencoding(),
'replace'))
if args.json:
print(json.dumps(json_list))
# Don't count calculating the stats and printing them as test running time
end_time = timeit.default_timer()
if not args.silent:
print('\nTest stats:')
print('\nTop 10 Slowest Test Cases:')
# Cases sorted slowest first
sorted_cases = sorted(test_infos,
key=lambda x: x['run_time'],
reverse=True)
for i, info in enumerate(sorted_cases[:10]):
print('{:2}. {} {} {:.4f}s'.format(i + 1, info['decoder'],
info['filename'],
info['run_time']))
print('\nTop 10 Fastest Test Cases:')
for i, info in enumerate(list(reversed(sorted_cases))[:10]):
print('{:2}. {} {} {:.4f}s'.format(i + 1, info['decoder'],
info['filename'],
info['run_time']))
run_times = [info['run_time'] for info in test_infos]
print('\nMean Running Time: {:.4}s'.format(
sum(run_times) / len(test_infos)))
print('Median Running Time: {:.4f}s'.format(_median(run_times)))
print()
print("Total Running Time: {}".format(
datetime.timedelta(seconds=end_time - start_time)))
print("All Passed = {0}\n".format(all_passed))
exit(0 if all_passed else 1)
# add files to test cases
elif args.decoder_name and args.delete:
removed_files = tester.remove_test_results(args.decoder_name,
input_files)
for filename in removed_files:
print("Removing results for %s in %s" %
(filename, results_file_path))
elif args.decoder_name and (args.update or
(not args.delete and input_files)):
if args.update:
input_files.extend(tester.list_test_files(args.decoder_name))
for input_file in input_files:
tester.gen_results(decoder_name=args.decoder_name,
input_file_path=input_file)
if len(reporter.metadata) > 1 and len(reporter.errors) == 0:
print("Updating results for %s in %s" %
(input_file, results_file_path))
tester.update_test_results(results_file_path=results_file_path,
results_data=reporter.metadata,
replace=True)
elif len(reporter.metadata) > 1 and len(reporter.errors) > 0:
print("Error occurred for %s in %s, not updating: " %
(input_file, results_file_path))
print("\n".join(reporter.get_debug()))
print("\n".join(reporter.errors))
else:
print("Empty results for %s in %s, not updating" %
(input_file, results_file_path))
else:
argparser.print_help()