def descriptions():
'''
List descriptions of parser modules
'''
try:
response.content_type = "application/json"
reporter = malwareconfigreporter(base64outputfiles = True, disableoutputfiles = True)
return reporter.pprint(reporter.get_parser_descriptions())
except Exception as e:
output = {}
output['errors'] = [traceback.format_exc()]
logger.error("descriptions %s" % (traceback.format_exc()))
return output
def __run_parser(name, data = '', modargs = ''):
output = {}
logger.info("__run_parser %s %s" % (name, hashlib.md5(data).hexdigest()))
try:
reporter = malwareconfigreporter(base64outputfiles = True)
kwargs = {}
if modargs:
kwargs = dict(json.loads(modargs))
reporter.run_parser(name, data = data, **kwargs)
output = reporter.metadata
if reporter.errors:
output["errors"] = reporter.errors
for error in reporter.errors:
logger.error("__run_parser %s %s" % (name, error))
return output
except Exception as e:
output = {}
output['errors'] = [traceback.format_exc()]
logger.error("__run_parser %s %s" % (name, traceback.format_exc()))
return output
def process_file(self, file_path, CAPE_files, append_file):
"""Process file.
@return: nothing
"""
strings = []
buf = self.options.get("buffer", BUFSIZE)
if file_path.endswith("_info.txt"):
return
texttypes = [
"ASCII",
"Windows Registry text",
"XML document text",
"Unicode text",
]
if os.path.exists(file_path + "_info.txt"):
with open(file_path + "_info.txt", 'r') as f:
metastring = f.readline()
else:
metastring = ""
file_info = File(file_path, metastring).get_all()
# Get the file data
with open(file_info["path"], "r") as drop_open:
filedata = drop_open.read(buf + 1)
if len(filedata) > buf:
file_info["data"] = binascii.b2a_hex(filedata[:buf] +
" <truncated>")
else:
file_info["data"] = binascii.b2a_hex(filedata)
metastrings = metastring.split(",")
if len(metastrings) > 1:
file_info["pid"] = metastrings[1]
if len(metastrings) > 2:
file_info["process_path"] = metastrings[2]
file_info["process_name"] = metastrings[2].split("\\")[-1]
if len(metastrings) > 3:
file_info["module_path"] = metastrings[3]
file_info["cape_type_code"] = 0
file_info["cape_type"] = ""
if metastrings != "":
try:
file_info["cape_type_code"] = int(metastrings[0])
except Exception as e:
pass
if file_info["cape_type_code"] == COMPRESSION:
file_info["cape_type"] = "Decompressed PE Image"
if file_info["cape_type_code"] == INJECTION_PE:
file_info["cape_type"] = "Injected PE Image"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split(
"\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == INJECTION_SHELLCODE:
file_info["cape_type"] = "Injected Shellcode/Data"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split(
"\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == EXTRACTION_PE:
file_info["cape_type"] = "Extracted PE Image"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
if file_info["cape_type_code"] == EXTRACTION_SHELLCODE:
file_info["cape_type"] = "Extracted Shellcode"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# PlugX
if file_info["cape_type_code"] == PLUGX_CONFIG:
file_info["cape_type"] = "PlugX Config"
plugx_parser = plugx.PlugXConfig()
config_output = plugx_parser.parse_config(
filedata, len(filedata))
if config_output:
file_info["plugx_config"] = config_output
append_file = True
if file_info["cape_type_code"] == PLUGX_PAYLOAD:
file_info["cape_type"] = "PlugX Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# EvilGrab
if file_info["cape_type_code"] == EVILGRAB_PAYLOAD:
file_info["cape_type"] = "EvilGrab Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if file_info["cape_type_code"] == EVILGRAB_DATA:
file_info["cape_type"] = "EvilGrab Data"
append_file = True
# UPX
if file_info["cape_type_code"] == UPX:
file_info["cape_type"] = "Unpacked PE Image"
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# Process CAPE Yara hits
for hit in file_info["cape_yara"]:
cape_name = hit["name"]
try:
file_info["cape_type"] = hit["meta"]["cape_type"]
except:
#log.error("CAPE Yara signature has no CAPE type metadata: %s", cape_name)
file_info["cape_type"] = "CAPE Detection: <Type missing>"
# UPX Check and unpack
if cape_name == 'UPX':
log.info(
"CAPE: Found UPX Packed sample - attempting to unpack")
unpacked_file = upx_unpack(filedata)
if unpacked_file and os.path.exists(unpacked_file):
unpacked_yara = File(unpacked_file).get_yara(
CAPE_YARA_RULEPATH)
for unpacked_hit in unpacked_yara:
unpacked_name = unpacked_hit["name"]
if unpacked_name == 'UPX':
# Failed to unpack
log.info("CAPE: Failed to unpack UPX")
os.unlink(unpacked_file)
break
if not os.path.exists(self.CAPE_path):
os.makedirs(self.CAPE_path)
newname = os.path.join(self.CAPE_path,
os.path.basename(unpacked_file))
os.rename(unpacked_file, newname)
infofd = open(newname + "_info.txt", "a")
infofd.write(os.path.basename(unpacked_file) + "\n")
infofd.close()
# Recursive process of unpacked file
self.process_file(newname, CAPE_files, True)
# Java Dropper Check
#if cape_name == 'JavaDropper':
# log.info("CAPE: Found Java Dropped, attemping to unpack")
# unpacked_file = JavaDropper.run(unpacked_file)
# cape_name = yara_scan(unpacked_file)
#
# if cape_name == 'JavaDropper':
# log.info("CAPE: Failed to unpack JavaDropper")
# #return
# Attempt to import a parser for the yara hit
# DC3-MWCP
try:
mwcp = malwareconfigreporter.malwareconfigreporter()
kwargs = {}
mwcp.run_parser(cape_name, data=filedata, **kwargs)
if mwcp.errors == []:
log.info("CAPE: Imported DC3-MWCP parser %s", cape_name)
mwcp_loaded = True
else:
for error in mwcp.errors:
#log.info("CAPE: DC3-MWCP parser error: %s", error.readline())
log.info("CAPE: DC3-MWCP parser error: %s", error)
mwcp_loaded = False
except ImportError:
mwcp_loaded = False
# malwareconfig
try:
malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules",
"processing", "parsers",
"malwareconfig")
file, pathname, description = imp.find_module(
cape_name, [malwareconfig_parsers])
module = imp.load_module(cape_name, file, pathname,
description)
malwareconfig_loaded = True
log.info("CAPE: Imported malwareconfig.com parser %s",
cape_name)
except ImportError:
#log.error("CAPE: Unable to import malwareconfig.com parser %s", cape_name)
malwareconfig_loaded = False
# Get config data
if mwcp_loaded:
try:
file_info["cape_config"] = convert(mwcp.metadata)
file_info["cape_name"] = format(cape_name)
append_file = True
except Exception as e:
log.error(
"CAPE: DC3-MWCP config parsing error with %s: %s",
cape_name, e)
elif malwareconfig_loaded:
try:
file_info["cape_config"] = {}
for (key, value) in module.config(filedata).iteritems():
file_info["cape_config"].update({key: [value]})
file_info["cape_name"] = format(cape_name)
append_file = True
except Exception as e:
log.error("CAPE: malwareconfig parsing error with %s: %s",
cape_name, e)
if append_file == True:
CAPE_files.append(file_info)
def process_file(self, file_path, CAPE_output, append_file):
"""Process file.
@return: file_info
"""
global cape_config
cape_name = ""
strings = []
buf = self.options.get("buffer", BUFSIZE)
if file_path.endswith("_info.txt"):
return
texttypes = [
"ASCII",
"Windows Registry text",
"XML document text",
"Unicode text",
]
if os.path.exists(file_path + "_info.txt"):
with open(file_path + "_info.txt", 'r') as f:
metastring = f.readline()
else:
metastring=""
file_info = File(file_path, metastring).get_all()
# Get the file data
with open(file_info["path"], "r") as file_open:
file_data = file_open.read(buf + 1)
if len(file_data) > buf:
file_info["data"] = binascii.b2a_hex(file_data[:buf] + " <truncated>")
else:
file_info["data"] = binascii.b2a_hex(file_data)
metastrings = metastring.split(",")
if len(metastrings) > 1:
file_info["pid"] = metastrings[1]
if len(metastrings) > 2:
file_info["process_path"] = metastrings[2]
file_info["process_name"] = metastrings[2].split("\\")[-1]
if len(metastrings) > 3:
file_info["module_path"] = metastrings[3]
file_info["cape_type_code"] = 0
file_info["cape_type"] = ""
if metastrings != "":
try:
file_info["cape_type_code"] = int(metastrings[0])
except Exception as e:
pass
if file_info["cape_type_code"] == COMPRESSION:
file_info["cape_type"] = "Decompressed PE Image"
if file_info["cape_type_code"] == INJECTION_PE:
file_info["cape_type"] = "Injected PE Image"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split("\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == INJECTION_SHELLCODE:
file_info["cape_type"] = "Injected Shellcode/Data"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split("\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == INJECTION_SECTION:
file_info["cape_type"] = "Injected Section"
if len(metastrings) > 4:
file_info["section_handle"] = metastrings[4]
if file_info["cape_type_code"] == EXTRACTION_PE:
file_info["cape_type"] = "Extracted PE Image"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
if file_info["cape_type_code"] == EXTRACTION_SHELLCODE:
file_info["cape_type"] = "Extracted Shellcode"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# PlugX
if file_info["cape_type_code"] == PLUGX_CONFIG:
file_info["cape_type"] = "PlugX Config"
plugx_parser = plugx.PlugXConfig()
plugx_config = plugx_parser.parse_config(file_data, len(file_data))
if not "cape_config" in cape_config and plugx_config:
cape_config["cape_config"] = {}
for key, value in plugx_config.items():
cape_config["cape_config"].update({key: [value]})
cape_name = "PlugX"
else:
log.error("CAPE: PlugX config parsing failure - size many not be handled.")
append_file = False
if file_info["cape_type_code"] == PLUGX_PAYLOAD:
file_info["cape_type"] = "PlugX Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# EvilGrab
if file_info["cape_type_code"] == EVILGRAB_PAYLOAD:
file_info["cape_type"] = "EvilGrab Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if file_info["cape_type_code"] == EVILGRAB_DATA:
cape_name = "EvilGrab"
file_info["cape_type"] = "EvilGrab Data"
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
if file_info["size"] == 256 or file_info["size"] == 260:
ConfigItem = "filepath"
ConfigData = format(file_data)
cape_config["cape_config"].update({ConfigItem: [ConfigData]})
if file_info["size"] > 0x1000:
append_file = True
else:
append_file = False
# Sedreco
if file_info["cape_type_code"] == SEDRECO_DATA:
cape_name = "Sedreco"
cape_config["cape_type"] = "Sedreco Config"
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
if len(metastrings) > 4:
SedrecoConfigIndex = metastrings[4]
if SedrecoConfigIndex == '0x0':
ConfigItem = "Timer1"
elif SedrecoConfigIndex == '0x1':
ConfigItem = "Timer2"
elif SedrecoConfigIndex == '0x2':
ConfigItem = "Computer Name"
elif SedrecoConfigIndex == '0x3':
ConfigItem = "C&C1"
elif SedrecoConfigIndex == '0x4':
ConfigItem = "C&C2"
elif SedrecoConfigIndex == '0x5':
ConfigItem = "Operation Name"
elif SedrecoConfigIndex == '0x6':
ConfigItem = "Keylogger MaxBuffer"
elif SedrecoConfigIndex == '0x7':
ConfigItem = "Keylogger MaxTimeout"
elif SedrecoConfigIndex == '0x8':
ConfigItem = "Keylogger Flag"
elif SedrecoConfigIndex == '0x9':
ConfigItem = "C&C3"
else:
ConfigItem = "Unknown"
ConfigData = format(file_data)
if ConfigData:
cape_config["cape_config"].update({ConfigItem: [ConfigData]})
append_file = False
# Cerber
if file_info["cape_type_code"] == CERBER_CONFIG:
file_info["cape_type"] = "Cerber Config"
cape_config["cape_type"] = "Cerber Config"
cape_name = "Cerber"
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
ConfigItem = "JSON Data"
parsed = json.loads(file_data.rstrip(b'\0'))
ConfigData = json.dumps(parsed, indent=4, sort_keys=True)
cape_config["cape_config"].update({ConfigItem: [ConfigData]})
append_file = True
if file_info["cape_type_code"] == CERBER_PAYLOAD:
file_info["cape_type"] = "Cerber Payload"
cape_config["cape_type"] = "Cerber Payload"
cape_name = "Cerber"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
append_file = True
# UPX package output
if file_info["cape_type_code"] == UPX:
file_info["cape_type"] = "Unpacked PE Image"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# Process CAPE Yara hits
for hit in file_info["cape_yara"]:
# Check to see if file is packed with UPX
if hit["name"] == "UPX":
log.info("CAPE: Found UPX Packed sample - attempting to unpack")
self.upx_unpack(file_data, CAPE_output)
# Check for a payload or config hit
try:
if "payload" in hit["meta"]["cape_type"].lower() or "config" in hit["meta"]["cape_type"].lower():
file_info["cape_type"] = hit["meta"]["cape_type"]
cape_name = hit["name"]
except:
pass
type_strings = file_info["type"].split()
if "-bit" not in file_info["cape_type"]:
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# Attempt to import a parser for the hit
# DC3-MWCP
mwcp_loaded = False
if cape_name:
try:
mwcp = malwareconfigreporter.malwareconfigreporter(analysis_path=self.analysis_path)
kwargs = {}
mwcp.run_parser(cape_name, data=file_data, **kwargs)
if mwcp.errors == []:
log.info("CAPE: Imported DC3-MWCP parser %s", cape_name)
mwcp_loaded = True
else:
error_lines = mwcp.errors[0].split("\n")
for line in error_lines:
if line.startswith('ImportError: '):
log.info("CAPE: DC3-MWCP parser: %s", line.split(': ')[1])
except ImportError:
pass
# malwareconfig
malwareconfig_loaded = False
if cape_name and mwcp_loaded == False:
try:
malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules", "processing", "parsers", "malwareconfig")
file, pathname, description = imp.find_module(cape_name,[malwareconfig_parsers])
module = imp.load_module(cape_name, file, pathname, description)
malwareconfig_loaded = True
log.info("CAPE: Imported malwareconfig.com parser %s", cape_name)
except ImportError:
log.info("CAPE: malwareconfig.com parser: No module named %s", cape_name)
# Get config data
if mwcp_loaded:
try:
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
cape_config["cape_config"] = convert(mwcp.metadata)
else:
cape_config["cape_config"].update(convert(mwcp.metadata))
except Exception as e:
log.error("CAPE: DC3-MWCP config parsing error with %s: %s", cape_name, e)
elif malwareconfig_loaded:
try:
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
malwareconfig_config = module.config(file_data)
if isinstance(malwareconfig_config, list):
for (key, value) in module.config(file_data)[0].iteritems():
cape_config["cape_config"].update({key: [value]})
elif isinstance(malwareconfig_config, dict):
for (key, value) in module.config(file_data).iteritems():
cape_config["cape_config"].update({key: [value]})
except Exception as e:
log.error("CAPE: malwareconfig parsing error with %s: %s", cape_name, e)
if "cape_config" in cape_config:
if cape_config["cape_config"] == {}:
del cape_config["cape_config"]
if cape_name:
if "cape_config" in cape_config:
cape_config["cape_name"] = format(cape_name)
if not "cape" in self.results:
if cape_name != "UPX":
self.results["cape"] = cape_name
if append_file == True:
CAPE_output.append(file_info)
return file_info
def main():
''' Run tool. '''
print ''
# Setup
mwcproot = ""
if os.path.dirname(sys.argv[0]):
mwcproot = os.path.dirname(sys.argv[0])
# Get command line arguments
argparser = get_arg_parser(mwcproot)
args, input_files = argparser.parse_known_args()
# Configure reporter based on args
if args.resource_dir:
reporter = malwareconfigreporter(resourcedir=args.resource_dir,
disableoutputfiles=True)
else:
reporter = malwareconfigreporter(disableoutputfiles=True)
# Configure test object
tester = malwareconfigtester(reporter=reporter,
results_dir=args.test_case_dir)
parser_descriptions = reporter.get_parser_descriptions()
valid_parser_names = [x[0] for x in parser_descriptions]
parsers = []
if args.parser_name:
if args.parser_name in valid_parser_names:
parsers = [args.parser_name]
else:
print "Error: Invalid parser name(s) specified. Parser names are case sensitive."
exit(1)
if args.all_tests:
parsers = valid_parser_names
if not parsers:
print "You must specify the parser to run (or run all parsers)"
exit(2)
if args.parser_name:
results_file_path = tester.get_results_filepath(args.parser_name)
#gather all our input files
if args.input_file:
input_files = read_input_list(input_files[0])
# Default is to run test cases
if args.run_tests:
print "Running test cases. May take a while..."
all_passed, test_results = tester.run_tests(
parsers,
filter(None, args.field_names.split(",")),
ignore_field_names=filter(None,
args.exclude_field_names.split(",")))
print "All Passed = {0}\n".format(all_passed)
if not args.silent:
if args.only_failed_tests:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=False,
verbose=args.verbose,
json_format=args.json)
else:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=True,
verbose=args.verbose,
json_format=args.json)
if all_passed:
exit(0)
else:
exit(1)
#add files to test cases
elif args.delete:
removed_files = tester.remove_test_results(args.parser_name,
input_files)
for filename in removed_files:
print("Removing results for %s in %s" %
(filename, results_file_path))
elif args.parser_name and (args.update or
(not args.delete and input_files)):
if args.update:
input_files.extend(tester.list_test_files(args.parser_name))
for input_file in input_files:
metadata = tester.gen_results(parser_name=args.parser_name,
input_file_path=input_file)
if len(metadata) > 1 and len(reporter.errors) == 0:
print("Updating results for %s in %s" %
(input_file, results_file_path))
tester.update_test_results(results_file_path=results_file_path,
results_data=metadata,
replace=True)
elif len(metadata) > 1 and len(reporter.errors) > 0:
print("Error occurred for %s in %s, not updating" %
(input_file, results_file_path))
else:
print("Empty results for %s in %s, not updating" %
(input_file, results_file_path))
else:
argparser.print_help()
def process_file(self, file_path, CAPE_output, append_file):
"""Process file.
@return: file_info
"""
global cape_config
cape_name = ""
strings = []
buf = self.options.get("buffer", BUFSIZE)
if file_path.endswith("_info.txt"):
return
texttypes = [
"ASCII",
"Windows Registry text",
"XML document text",
"Unicode text",
]
if os.path.exists(file_path + "_info.txt"):
with open(file_path + "_info.txt", 'r') as f:
metastring = f.readline()
else:
metastring = ""
file_info = File(file_path, metastring).get_all()
# Get the file data
with open(file_info["path"], "r") as file_open:
filedata = file_open.read(buf + 1)
if len(filedata) > buf:
file_info["data"] = binascii.b2a_hex(filedata[:buf] +
" <truncated>")
else:
file_info["data"] = binascii.b2a_hex(filedata)
metastrings = metastring.split(",")
if len(metastrings) > 1:
file_info["pid"] = metastrings[1]
if len(metastrings) > 2:
file_info["process_path"] = metastrings[2]
file_info["process_name"] = metastrings[2].split("\\")[-1]
if len(metastrings) > 3:
file_info["module_path"] = metastrings[3]
file_info["cape_type_code"] = 0
file_info["cape_type"] = ""
if metastrings != "":
try:
file_info["cape_type_code"] = int(metastrings[0])
except Exception as e:
pass
if file_info["cape_type_code"] == COMPRESSION:
file_info["cape_type"] = "Decompressed PE Image"
if file_info["cape_type_code"] == INJECTION_PE:
file_info["cape_type"] = "Injected PE Image"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split(
"\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == INJECTION_SHELLCODE:
file_info["cape_type"] = "Injected Shellcode/Data"
if len(metastrings) > 4:
file_info["target_path"] = metastrings[4]
file_info["target_process"] = metastrings[4].split(
"\\")[-1]
file_info["target_pid"] = metastrings[5]
if file_info["cape_type_code"] == EXTRACTION_PE:
file_info["cape_type"] = "Extracted PE Image"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
if file_info["cape_type_code"] == EXTRACTION_SHELLCODE:
file_info["cape_type"] = "Extracted Shellcode"
if len(metastrings) > 4:
file_info["virtual_address"] = metastrings[4]
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# PlugX
if file_info["cape_type_code"] == PLUGX_CONFIG:
file_info["cape_type"] = "PlugX Config"
plugx_parser = plugx.PlugXConfig()
plugx_config = plugx_parser.parse_config(
filedata, len(filedata))
if not "cape_config" in cape_config and plugx_config:
cape_config["cape_config"] = {}
for key, value in plugx_config.items():
cape_config["cape_config"].update({key: [value]})
cape_name = "PlugX"
append_file = False
if file_info["cape_type_code"] == PLUGX_PAYLOAD:
file_info["cape_type"] = "PlugX Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# EvilGrab
if file_info["cape_type_code"] == EVILGRAB_PAYLOAD:
file_info["cape_type"] = "EvilGrab Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if file_info["cape_type_code"] == EVILGRAB_DATA:
cape_name = "EvilGrab"
file_info["cape_type"] = "EvilGrab Data"
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
if file_info["size"] == 256 or file_info["size"] == 260:
ConfigItem = "filepath"
ConfigData = format(filedata)
cape_config["cape_config"].update(
{ConfigItem: [ConfigData]})
if file_info["size"] > 0x1000:
append_file = True
else:
append_file = False
# Sedreco
if file_info["cape_type_code"] == SEDRECO_DATA:
cape_name = "Sedreco"
cape_config["cape_type"] = "Sedreco Config"
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
if len(metastrings) > 4:
SedrecoConfigIndex = metastrings[4]
if SedrecoConfigIndex == '0x0':
ConfigItem = "Timer1"
elif SedrecoConfigIndex == '0x1':
ConfigItem = "Timer2"
elif SedrecoConfigIndex == '0x2':
ConfigItem = "Computer Name"
elif SedrecoConfigIndex == '0x3':
ConfigItem = "C&C1"
elif SedrecoConfigIndex == '0x4':
ConfigItem = "C&C2"
elif SedrecoConfigIndex == '0x5':
ConfigItem = "Operation Name"
elif SedrecoConfigIndex == '0x6':
ConfigItem = "Keylogger MaxBuffer"
elif SedrecoConfigIndex == '0x7':
ConfigItem = "Keylogger MaxTimeout"
elif SedrecoConfigIndex == '0x8':
ConfigItem = "Keylogger Flag"
elif SedrecoConfigIndex == '0x9':
ConfigItem = "C&C3"
else:
ConfigItem = "Unknown"
ConfigData = format(filedata)
if ConfigData:
cape_config["cape_config"].update(
{ConfigItem: [ConfigData]})
append_file = False
# UPX
if file_info["cape_type_code"] == UPX:
file_info["cape_type"] = "Unpacked PE Image"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# Process CAPE Yara hits
for hit in file_info["cape_yara"]:
cape_name = hit["name"]
try:
file_info["cape_type"] = hit["meta"]["cape_type"]
except:
file_info["cape_type"] = cape_name + " Payload"
type_strings = file_info["type"].split()
if type_strings[0] == ("PE32+"):
file_info["cape_type"] += ": 64-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
file_info["cape_type"] += ": 32-bit "
if type_strings[2] == ("(DLL)"):
file_info["cape_type"] += "DLL"
else:
file_info["cape_type"] += "executable"
# UPX Check and unpack
if cape_name == 'UPX':
log.info(
"CAPE: Found UPX Packed sample - attempting to unpack")
unpacked_file = upx_unpack(filedata)
if unpacked_file and os.path.exists(unpacked_file):
unpacked_yara = File(unpacked_file).get_yara(
CAPE_YARA_RULEPATH)
for unpacked_hit in unpacked_yara:
unpacked_name = unpacked_hit["name"]
if unpacked_name == 'UPX':
# Failed to unpack
log.info("CAPE: Failed to unpack UPX")
os.unlink(unpacked_file)
break
if not os.path.exists(self.CAPE_path):
os.makedirs(self.CAPE_path)
newname = os.path.join(self.CAPE_path,
os.path.basename(unpacked_file))
os.rename(unpacked_file, newname)
infofd = open(newname + "_info.txt", "a")
infofd.write(os.path.basename(unpacked_file) + "\n")
infofd.close()
# Recursive process of unpacked file
upx_extract = self.process_file(newname, CAPE_output, True)
if upx_extract["type"]:
upx_extract["cape_type"] = "UPX-extracted "
type_strings = upx_extract["type"].split()
if type_strings[0] == ("PE32+"):
upx_extract["cape_type"] += " 64-bit "
if type_strings[2] == ("(DLL)"):
upx_extract["cape_type"] += "DLL"
else:
upx_extract["cape_type"] += "executable"
if type_strings[0] == ("PE32"):
upx_extract["cape_type"] += " 32-bit "
if type_strings[2] == ("(DLL)"):
upx_extract["cape_type"] += "DLL"
else:
upx_extract["cape_type"] += "executable"
# Attempt to import a parser for the yara hit
# DC3-MWCP
try:
mwcp = malwareconfigreporter.malwareconfigreporter()
kwargs = {}
mwcp.run_parser(cape_name, data=filedata, **kwargs)
if mwcp.errors == []:
log.info("CAPE: Imported DC3-MWCP parser %s", cape_name)
mwcp_loaded = True
else:
error_lines = mwcp.errors[0].split("\n")
for line in error_lines:
if line.startswith('ImportError: '):
log.info("CAPE: DC3-MWCP parser: %s",
line.split(': ')[1])
mwcp_loaded = False
except ImportError:
mwcp_loaded = False
# malwareconfig
try:
malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules",
"processing", "parsers",
"malwareconfig")
file, pathname, description = imp.find_module(
cape_name, [malwareconfig_parsers])
module = imp.load_module(cape_name, file, pathname,
description)
malwareconfig_loaded = True
log.info("CAPE: Imported malwareconfig.com parser %s",
cape_name)
except ImportError:
#log.info("CAPE: No malwareconfig.com parser for %s", cape_name)
malwareconfig_loaded = False
# Get config data
if mwcp_loaded:
try:
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
cape_config["cape_config"] = convert(mwcp.metadata)
else:
cape_config["cape_config"].update(
convert(mwcp.metadata))
except Exception as e:
log.error(
"CAPE: DC3-MWCP config parsing error with %s: %s",
cape_name, e)
elif malwareconfig_loaded:
try:
if not "cape_config" in cape_config:
cape_config["cape_config"] = {}
malwareconfig_config = module.config(filedata)
if isinstance(malwareconfig_config, list):
for (key,
value) in module.config(filedata)[0].iteritems():
cape_config["cape_config"].update({key: [value]})
elif isinstance(malwareconfig_config, dict):
for (key,
value) in module.config(filedata).iteritems():
cape_config["cape_config"].update({key: [value]})
except Exception as e:
log.error("CAPE: malwareconfig parsing error with %s: %s",
cape_name, e)
if cape_name:
cape_config["cape_name"] = format(cape_name)
if not "cape" in self.results:
#self.results["cape"] = []
self.results["cape"] = cape_name
#if cape_name not in self.results["cape"]:
# self.results["cape"].append(cape_name)
if append_file == True:
CAPE_output.append(file_info)
return file_info
def main():
''' Run tool. '''
print ''
# Setup
mwcproot = ""
if os.path.dirname(sys.argv[0]):
mwcproot = os.path.dirname(sys.argv[0])
# Get command line arguments
argparser = get_arg_parser(mwcproot)
args, input_files = argparser.parse_known_args()
# Configure reporter based on args
if args.resource_dir:
reporter = malwareconfigreporter(resourcedir = args.resource_dir,
disableoutputfiles = True)
else:
reporter = malwareconfigreporter(disableoutputfiles = True)
# Configure test object
tester = malwareconfigtester(reporter = reporter, results_dir = args.test_case_dir)
parser_descriptions = reporter.get_parser_descriptions()
valid_parser_names = [x[0] for x in parser_descriptions]
parsers = []
if args.parser_name:
if args.parser_name in valid_parser_names:
parsers = [ args.parser_name ]
else:
print "Error: Invalid parser name(s) specified. Parser names are case sensitive."
exit(1)
if args.all_tests:
parsers = valid_parser_names
if not parsers:
print "You must specify the parser to run (or run all parsers)"
exit(2)
if args.parser_name:
results_file_path = tester.get_results_filepath(args.parser_name)
#gather all our input files
if args.input_file:
input_files = read_input_list(input_files[0])
# Default is to run test cases
if args.run_tests:
print "Running test cases. May take a while..."
all_passed, test_results = tester.run_tests(parsers, filter(None,args.field_names.split(",")),
ignore_field_names = filter(None, args.exclude_field_names.split(",")))
print "All Passed = {0}\n".format(all_passed)
if not args.silent:
if args.only_failed_tests:
tester.print_test_results(test_results,
failed_tests = True,
passed_tests = False,
verbose = args.verbose,
json_format = args.json)
else:
tester.print_test_results(test_results,
failed_tests = True,
passed_tests = True,
verbose = args.verbose,
json_format = args.json)
if all_passed:
exit(0)
else:
exit(1)
#add files to test cases
elif args.delete:
removed_files = tester.remove_test_results(args.parser_name, input_files)
for filename in removed_files:
print("Removing results for %s in %s" % (filename, results_file_path))
elif args.update or (not args.delete and input_files):
if args.update:
input_files.extend(tester.list_test_files(args.parser_name))
for input_file in input_files:
metadata = tester.gen_results(parser_name = args.parser_name, input_file_path = input_file)
if len(metadata) > 1 and len(reporter.errors) == 0:
print("Updating results for %s in %s" % (input_file, results_file_path))
tester.update_test_results(results_file_path = results_file_path,
results_data = metadata,
replace = True)
elif len(metadata) > 1 and len(reporter.errors) > 0:
print("Error occurred for %s in %s, not updating" % (input_file, results_file_path))
else:
print("Empty results for %s in %s, not updating" % (input_file, results_file_path))
else:
argparser.print_help()
def main():
''' Run tool. '''
print ''
# Setup
mwcproot = ""
if os.path.dirname(sys.argv[0]):
mwcproot = os.path.dirname(sys.argv[0])
# Get command line arguments
argparser = get_arg_parser(mwcproot)
args, input_files = argparser.parse_known_args()
# Configure reporter based on args
if args.resource_dir:
reporter = malwareconfigreporter(resourcedir=args.resource_dir,
disableoutputfiles=True)
else:
reporter = malwareconfigreporter(disableoutputfiles=True)
# Configure test object
tester = malwareconfigtester(reporter=reporter,
results_dir=args.test_case_dir)
parser_descriptions = reporter.get_parser_descriptions()
valid_parser_names = [x[0] for x in parser_descriptions]
parsers = []
if args.parser_name:
if args.parser_name in valid_parser_names:
parsers = [args.parser_name]
else:
print "Error: Invalid parser name(s) specified. Parser names are case sensitive."
sys.exit(1)
if args.all_tests:
parsers = valid_parser_names
if not parsers:
print "You must specify a single parser or all parsers to run or update."
sys.exit(2)
if args.parser_name:
results_file_path = tester.get_results_filepath(args.parser_name)
# Gather all our input files
if args.input_file:
input_files = read_input_list(input_files[0])
# Run test cases
if args.run_tests:
print "Running test cases. May take a while..."
# Run tests
test_results = tester.run_tests(
parsers,
filter(None, args.field_names.split(",")),
ignore_field_names=filter(None,
args.exclude_field_names.split(",")))
# Determine if any test cases failed
all_passed = True
if any([not test_result.passed for test_result in test_results]):
all_passed = False
print "All Passed = {0}\n".format(all_passed)
if not args.silent:
if args.only_failed_tests:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=False,
json_format=args.json)
else:
tester.print_test_results(test_results,
failed_tests=True,
passed_tests=True,
json_format=args.json)
if all_passed:
sys.exit(0)
else:
sys.exit(1)
# Delete files from test cases
elif args.delete:
removed_files = tester.remove_test_results(args.parser_name,
input_files)
for filename in removed_files:
print u"Removing results for {} in {}".format(
filename, results_file_path).encode(
encoding=sys.stdout.encoding
if sys.stdout.encoding else locale.getpreferredencoding(),
errors="replace")
# Update previously existing test cases
elif args.update:
print "Updating test cases. May take a while..."
for parser in parsers:
results_file_path = tester.get_results_filepath(parser)
if os.path.isfile(results_file_path):
input_files = tester.list_test_files(parser)
else:
print "No test case file found for parser '{}'. No update could be made.".format(
parser)
continue
for input_file in input_files:
metadata = tester.gen_results(parser_name=parser,
input_file_path=input_file)
if len(metadata) > 1 and len(reporter.errors) == 0:
print u"Updating results for {} in {}".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
tester.update_test_results(
results_file_path=results_file_path,
results_data=metadata,
replace=True)
elif len(metadata) > 1 and len(reporter.errors) > 0:
print u"Error occurred for {} in {}, not updating".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
else:
print u"Empty results for {} in {}, not updating".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
# Add/update test cases for specified input files and specified parser
elif args.parser_name and (not args.delete and input_files):
for input_file in input_files:
metadata = tester.gen_results(parser_name=args.parser_name,
input_file_path=input_file)
if len(metadata) > 1 and len(reporter.errors) == 0:
print u"Updating results for {} in {}".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
tester.update_test_results(results_file_path=results_file_path,
results_data=metadata,
replace=True)
elif len(metadata) > 1 and len(reporter.errors) > 0:
print u"Error occurred for {} in {}, not updating".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
else:
print u"Empty results for {} in {}, not updating".format(
input_file, results_file_path).encode(
encoding=sys.stdout.encoding if sys.stdout.encoding
else locale.getpreferredencoding(),
errors="replace")
else:
argparser.print_help()
#!/usr/bin/env python
'''
Simple example to demonstrate use of the API provided by DC3-MWCP framework.
'''
#first, import the malwareconfigreporter class
from mwcp.malwareconfigreporter import malwareconfigreporter
#create an instance of the malwareconfigreporter class
reporter = malwareconfigreporter()
'''
The malwareconfigreporter object is the primary DC3-MWCP framework object, containing most input and output data
and controlling execution of the parser modules.
The most common parameters to provide are parserdir and resourcedir, depending upon your installation.
'''
#view location of resource and parser directories
print reporter.resourcedir
print reporter.parserdir
#view available parsers
print reporter.get_parser_descriptions()
#run the dummy config parser, view the output
reporter.run_parser("foo", "README.md")
#alternate, run on provided buffer:
reporter.run_parser("foo", data="lorem ipsum")
print reporter.pprint(reporter.metadata)
def main():
optparser = make_opt_parser()
options, args = optparser.parse_args()
#if we can not create reporter object there is very little we can do. Just die immediately.
try:
reporter = malwareconfigreporter(
parserdir=options.parserdir,
resourcedir=options.resourcedir,
outputdir=options.outputdir,
outputfile_prefix=options.outputfile_prefix,
tempdir=options.tempdir,
disabledebug=options.hidedebug,
disableoutputfiles=options.disableoutputfiles,
disabletempcleanup=options.disabletempcleanup,
base64outputfiles=options.base64outputfiles)
except Exception as e:
error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % (
traceback.format_exc())
if options.jsonoutput:
print('{"errors": ["%s"]}' % (error_message))
else:
print(error_message)
sys.exit(1)
if options.list:
descriptions = reporter.get_parser_descriptions()
if options.jsonoutput:
if reporter.errors:
descriptions.append({"errors": reporter.errors})
print reporter.pprint(descriptions)
else:
for name, author, description in sorted(descriptions):
print('%-25s %-8s %s' % (name, author, description))
if reporter.errors:
print("")
print("Errors:")
for error in reporter.errors:
print(" %s" % (error))
return
if options.fields:
if options.jsonoutput:
print reporter.pprint(reporter.fields)
else:
for key in sorted(reporter.fields):
print('%-20s %s' % (key, reporter.fields[key]['description']))
for example in reporter.fields[key]['examples']:
print('%s %s' % (" " * 24, json.dumps(example)))
return
if not args:
optparser.print_help()
return
if options.parser:
if options.filelistindirection:
if args[0] == "-":
inputfilelist = [line.rstrip() for line in sys.stdin]
else:
with open(args[0], "rb") as f:
inputfilelist = [line.rstrip() for line in f]
else:
inputfilelist = args
kwargs = {}
if options.kwargs_raw:
kwargs = dict(json.loads(options.kwargs_raw))
for key, value in kwargs.iteritems():
if value and len(value) > len("b64file("):
if value[:len("b64file("
)] == "b64file(" and value[-1:] == ")":
tmp_filename = value[len("b64file("):-1]
with open(tmp_filename, "rb") as f:
kwargs[key] = base64.b64encode(f.read())
for inputfilename in inputfilelist:
if inputfilename == "-":
reporter.run_parser(options.parser,
data=sys.stdin.read(),
**kwargs)
else:
reporter.run_parser(options.parser, inputfilename, **kwargs)
if options.includefilename:
reporter.metadata['inputfilename'] = inputfilename
reporter.metadata['md5'] = hashlib.md5(
reporter.data).hexdigest()
reporter.metadata['sha1'] = hashlib.sha1(
reporter.data).hexdigest()
reporter.metadata['sha256'] = hashlib.sha256(
reporter.data).hexdigest()
reporter.metadata['parser'] = options.parser
if reporter.pe:
reporter.metadata[
'compiletime'] = datetime.datetime.fromtimestamp(
reporter.pe.FILE_HEADER.TimeDateStamp).isoformat()
if options.jsonoutput:
output = reporter.metadata
if reporter.errors:
output["errors"] = reporter.errors
print reporter.pprint(output)
else:
reporter.output_text()
#!/usr/bin/env python
'''
Simple example to demonstrate use of the API provided by DC3-MWCP framework.
'''
#first, import the malwareconfigreporter class
from mwcp.malwareconfigreporter import malwareconfigreporter
#create an instance of the malwareconfigreporter class
reporter = malwareconfigreporter()
'''
The malwareconfigreporter object is the primary DC3-MWCP framework object, containing most input and output data
and controlling execution of the parser modules.
The most common parameters to provide are parserdir and resourcedir, depending upon your installation.
'''
#view location of resource and parser directories
print reporter.resourcedir
print reporter.parserdir
#view available parsers
print reporter.get_parser_descriptions()
#run the dummy config parser, view the output
reporter.run_parser("foo", "README.md")
#alternate, run on provided buffer:
reporter.run_parser("foo", data = "lorem ipsum")
print reporter.pprint(reporter.metadata)
def make_opt_parser():
'''
create a option parser to handle command line inputs
'''
usage_str = 'usage: %s [options] FILES' % (os.path.basename(sys.argv[0]))
description = "DC3-MWCP Framework: utility for executing parser modules"
opt_parser = optparse.OptionParser(usage_str, description=description)
#available opts:
#--c-e-----------q-s--v-xyz
#A-Z
default_parserdir = ''
default_resourcedir = ''
#create reporter to get default paths, ignore if this fails
try:
default_reporter = malwareconfigreporter()
default_parserdir = default_reporter.parserdir
default_resourcedir = default_reporter.resourcedir
except Exception as e:
pass
opt_parser.add_option('-p',
'--parser',
action='store',
type='string',
default='',
dest='parser',
help='malware config parser to call')
opt_parser.add_option('-l',
'--list',
action="store_true",
default=False,
dest='list',
help='list all malware config parsers')
opt_parser.add_option(
'-k',
'--listfields',
action="store_true",
default=False,
dest='fields',
help=
'list all standardized fields and examples. See resources/fields.json')
opt_parser.add_option('-a',
'--parserdir',
action='store',
type='string',
metavar='DIR',
default=default_parserdir,
dest='parserdir',
help='parsers directory' + ' [default: %default]')
opt_parser.add_option('-r',
'--resourcedir',
action='store',
type='string',
metavar='DIR',
default=default_resourcedir,
dest='resourcedir',
help='resources directory' + ' [default: %default]')
opt_parser.add_option('-o',
'--outputdir',
action='store',
type='string',
metavar='DIR',
default='',
dest='outputdir',
help='output directory' + ' [default: %default]')
opt_parser.add_option('-t',
'--tempdir',
action='store',
type='string',
metavar='DIR',
default=tempfile.gettempdir(),
dest='tempdir',
help='temp directory' + ' [default: %default]')
opt_parser.add_option(
'-j',
'--jsonoutput',
action='store_true',
default=False,
dest='jsonoutput',
help='Enable json output for parser reports (instead of formatted text)'
)
opt_parser.add_option('-n',
'--disableoutputfiles',
action="store_true",
default=False,
dest='disableoutputfiles',
help='disable writing output files to filesystem')
opt_parser.add_option(
'-g',
'--disabletempcleanup',
action='store_true',
default=False,
dest='disabletempcleanup',
help=
'Disable cleanup of framework created temp files including managed tempdir'
)
opt_parser.add_option(
'-f',
'--includefileinfo',
action='store_true',
default=False,
dest='includefilename',
help=
'include input file information such as filename, hashes, and compile time in parser output'
)
opt_parser.add_option('-d',
'--hidedebug',
action="store_true",
default=False,
dest='hidedebug',
help='Hide debug messages in output')
opt_parser.add_option(
'-u',
'--outputfileprefix',
action='store',
type='string',
metavar='FILENAME',
default='',
dest='outputfile_prefix',
help=
'string prepended to output files written to filesystem. specifying "md5"\
will cause output files to be prefixed with the md5 of the input file'
+ ' [default: %default]')
opt_parser.add_option(
'-i',
'--filelistindirection',
action="store_true",
default=False,
dest='filelistindirection',
help='input file contains a list of filenames to process')
opt_parser.add_option(
'-b',
'--base64outputfiles',
action="store_true",
default=False,
dest='base64outputfiles',
help='base64 encode output files and include in metadata')
opt_parser.add_option(
'-w',
'--kwargs',
action='store',
type='string',
metavar='JSON',
default='',
dest='kwargs_raw',
help='module keyword arguments as json encoded dictionary\
if values in the dictionary use the special paradigm "b64file(filename)", then \
filename is read, base64 encoded, and used as the value'
)
return opt_parser
def make_opt_parser():
'''
create a option parser to handle command line inputs
'''
usage_str = 'usage: %s [options] FILES' % (os.path.basename(sys.argv[0]))
description = "DC3-MWCP Framework: utility for executing parser modules"
opt_parser = optparse.OptionParser(usage_str, description = description)
#available opts:
#--c-e-----------q-s--v-xyz
#A-Z
default_parserdir = ''
default_resourcedir = ''
#create reporter to get default paths, ignore if this fails
try:
default_reporter = malwareconfigreporter()
default_parserdir = default_reporter.parserdir
default_resourcedir = default_reporter.resourcedir
except Exception as e:
pass
opt_parser.add_option('-p',
'--parser',
action = 'store',
type = 'string',
default = '',
dest = 'parser',
help = 'malware config parser to call')
opt_parser.add_option('-l',
'--list',
action = "store_true",
default = False,
dest = 'list',
help = 'list all malware config parsers')
opt_parser.add_option('-k',
'--listfields',
action = "store_true",
default = False,
dest = 'fields',
help = 'list all standardized fields and examples. See resources/fields.json')
opt_parser.add_option('-a',
'--parserdir',
action = 'store',
type = 'string',
metavar = 'DIR',
default = default_parserdir,
dest = 'parserdir',
help = 'parsers directory' + ' [default: %default]')
opt_parser.add_option('-r',
'--resourcedir',
action = 'store',
type = 'string',
metavar = 'DIR',
default = default_resourcedir,
dest = 'resourcedir',
help = 'resources directory' + ' [default: %default]')
opt_parser.add_option('-o',
'--outputdir',
action = 'store',
type = 'string',
metavar = 'DIR',
default = '',
dest = 'outputdir',
help = 'output directory' + ' [default: %default]')
opt_parser.add_option('-t',
'--tempdir',
action = 'store',
type = 'string',
metavar = 'DIR',
default = tempfile.gettempdir(),
dest = 'tempdir',
help = 'temp directory' + ' [default: %default]')
opt_parser.add_option('-j',
'--jsonoutput',
action = 'store_true',
default = False,
dest = 'jsonoutput',
help = 'Enable json output for parser reports (instead of formatted text)')
opt_parser.add_option('-n',
'--disableoutputfiles',
action = "store_true",
default = False,
dest = 'disableoutputfiles',
help = 'disable writing output files to filesystem')
opt_parser.add_option('-g',
'--disabletempcleanup',
action = 'store_true',
default = False,
dest = 'disabletempcleanup',
help = 'Disable cleanup of framework created temp files including managed tempdir')
opt_parser.add_option('-f',
'--includefileinfo',
action = 'store_true',
default = False,
dest = 'includefilename',
help = 'include input file information such as filename, hashes, and compile time in parser output')
opt_parser.add_option('-d',
'--hidedebug',
action = "store_true",
default = False,
dest = 'hidedebug',
help = 'Hide debug messages in output')
opt_parser.add_option('-u',
'--outputfileprefix',
action = 'store',
type = 'string',
metavar = 'FILENAME',
default = '',
dest = 'outputfile_prefix',
help = 'string prepended to output files written to filesystem. specifying "md5"\
will cause output files to be prefixed with the md5 of the input file' + ' [default: %default]')
opt_parser.add_option('-i',
'--filelistindirection',
action = "store_true",
default = False,
dest = 'filelistindirection',
help = 'input file contains a list of filenames to process')
opt_parser.add_option('-b',
'--base64outputfiles',
action = "store_true",
default = False,
dest = 'base64outputfiles',
help = 'base64 encode output files and include in metadata')
opt_parser.add_option('-w',
'--kwargs',
action = 'store',
type = 'string',
metavar = 'JSON',
default = '',
dest = 'kwargs_raw',
help = 'module keyword arguments as json encoded dictionary\
if values in the dictionary use the special paradigm "b64file(filename)", then \
filename is read, base64 encoded, and used as the value')
return opt_parser
def main():
optparser = make_opt_parser()
options, args = optparser.parse_args()
#if we can not create reporter object there is very little we can do. Just die immediately.
try:
reporter = malwareconfigreporter(parserdir = options.parserdir,
resourcedir = options.resourcedir,
outputdir = options.outputdir,
outputfile_prefix = options.outputfile_prefix,
tempdir = options.tempdir,
disabledebug = options.hidedebug,
disableoutputfiles = options.disableoutputfiles,
disabletempcleanup = options.disabletempcleanup,
base64outputfiles = options.base64outputfiles)
except Exception as e:
error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % (traceback.format_exc())
if options.jsonoutput:
print('{"errors": ["%s"]}' % (error_message))
else:
print(error_message)
sys.exit(1)
if options.list:
descriptions = reporter.get_parser_descriptions()
if options.jsonoutput:
if reporter.errors:
descriptions.append({"errors": reporter.errors})
print reporter.pprint(descriptions)
else:
for name, author, description in sorted(descriptions):
print('%-25s %-8s %s' % (name, author, description) )
if reporter.errors:
print("")
print("Errors:")
for error in reporter.errors:
print(" %s" % (error))
return
if options.fields:
if options.jsonoutput:
print reporter.pprint(reporter.fields)
else:
for key in sorted(reporter.fields):
print('%-20s %s' % (key, reporter.fields[key]['description']))
for example in reporter.fields[key]['examples']:
print('%s %s' % (" " * 24, json.dumps(example)))
return
if not args:
optparser.print_help()
return
if options.parser:
if options.filelistindirection:
if args[0] == "-":
inputfilelist = [ line.rstrip() for line in sys.stdin ]
else:
with open(args[0],"rb") as f:
inputfilelist = [ line.rstrip() for line in f ]
else:
inputfilelist = args
kwargs = {}
if options.kwargs_raw:
kwargs = dict(json.loads(options.kwargs_raw))
for key, value in kwargs.iteritems():
if value and len(value) > len("b64file("):
if value[:len("b64file(")] == "b64file(" and value[-1:] == ")":
tmp_filename = value[len("b64file("):-1]
with open(tmp_filename, "rb") as f:
kwargs[key] = base64.b64encode(f.read())
for inputfilename in inputfilelist:
if inputfilename == "-":
reporter.run_parser(options.parser, data=sys.stdin.read(), **kwargs)
else:
reporter.run_parser(options.parser, inputfilename, **kwargs)
if options.includefilename:
reporter.metadata['inputfilename'] = inputfilename
reporter.metadata['md5'] = hashlib.md5(reporter.data).hexdigest()
reporter.metadata['sha1'] = hashlib.sha1(reporter.data).hexdigest()
reporter.metadata['sha256'] = hashlib.sha256(reporter.data).hexdigest()
reporter.metadata['parser'] = options.parser
if reporter.pe:
reporter.metadata['compiletime'] = datetime.datetime.fromtimestamp(reporter.pe.FILE_HEADER.TimeDateStamp).isoformat()
if options.jsonoutput:
output = reporter.metadata
if reporter.errors:
output["errors"] = reporter.errors
print reporter.pprint(output)
else:
reporter.output_text()
def get_arg_parser():
'''
create a option parser to handle command line inputs
'''
usage_str = 'usage: %s [options] FILES_DIRS' % (os.path.basename(
sys.argv[0]))
description = "DC3-MWCP Framework: utility for executing parser modules"
parser = argparse.ArgumentParser(
description=description,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=usage_str)
default_parserdir = ''
default_resourcedir = ''
# Create reporter to get default paths, ignore if this fails
try:
default_reporter = malwareconfigreporter()
default_parserdir = default_reporter.parserdir
default_resourcedir = default_reporter.resourcedir
except Exception as e:
pass
parser.add_argument("-p",
default="",
type=str,
dest="parser",
help="Malware config parser to call.")
parser.add_argument("-l",
action="store_true",
default=False,
dest="list",
help="list all malware config parsers.")
parser.add_argument(
"-k",
action="store_true",
default=False,
dest="fields",
help=
"List all standardized fields and examples. See resources/fields.json")
parser.add_argument("-a",
metavar="DIR",
default=default_parserdir,
dest="parserdir",
help="Parsers directory" +
" [default: {}]".format(default_parserdir))
parser.add_argument("-r",
metavar="DIR",
default=default_resourcedir,
dest="resourcedir",
help="Resources directory" +
" [default: {}]".format(default_resourcedir))
parser.add_argument("-o",
metavar="DIR",
default="",
dest="outputdir",
help="Output directory.")
parser.add_argument("-c",
metavar="CSVWRITE",
default="",
dest="csvwrite",
help="Output CSV file.")
parser.add_argument("-t",
metavar="DIR",
default=tempfile.gettempdir(),
dest="tempdir",
help="Temp directory." +
" [default: {}]".format(tempfile.gettempdir()))
parser.add_argument(
"-j",
action="store_true",
default=False,
dest="jsonoutput",
help=
"Enable json output for parser reports (instead of formatted text).")
parser.add_argument("-n",
action="store_true",
default=False,
dest="disableoutputfiles",
help="Disable writing output files to filesystem.")
parser.add_argument(
"-g",
action="store_true",
default=False,
dest="disabletempcleanup",
help=
"Disable cleanup of framework created temp files including managed tempdir."
)
parser.add_argument(
"-f",
action="store_true",
default=False,
dest="includefilename",
help=
"Include file information such as filename, hashes, and compile time in parser output."
)
parser.add_argument("-d",
action="store_true",
default=False,
dest="hidedebug",
help="Hide debug messages in output.")
parser.add_argument(
"-u",
metavar="FILENAME",
default="",
dest="outputfile_prefix",
help=
"String prepended to output files written to filesystem. Specifying 'md5' will cause "
+
"output files to be prefixed with the md5 of the input file. When passing in multiple "
+
"files for analysis, the default will be 'md5'. Passing in a value with the -u option "
+
"or using the -U option can be used to override the 'md5' default for multiple files. "
+ "[default: (No prefix|md5)]")
parser.add_argument(
"-U",
action="store_true",
default=False,
dest="disableoutputfileprefix",
help=
"When in effect, parser output files will not have a filename prefix.")
parser.add_argument(
"-i",
action="store_true",
default=False,
dest="filelistindirection",
help="Input file contains a list of filenames to process.")
parser.add_argument(
"-b",
action="store_true",
default=False,
dest="base64outputfiles",
help="Base64 encode output files and include in metadata.")
parser.add_argument(
"-w",
metavar="JSON",
default="",
dest="kwargs_raw",
help="Module keyword arguments as json encoded dictionary " +
"if values in the dictionary use the special paradigm 'b64file(filename)', then "
+ "filename is read, base64 encoded, and used as the value)")
return parser
def main():
argparser = get_arg_parser()
args, input_files = argparser.parse_known_args()
# This is a preliminary check before creating the reporter to establish how output
# file prefixes should be set.
if args.disableoutputfileprefix:
args.outputfile_prefix = ''
elif args.filelistindirection or len(input_files) > 1 or any(
[os.path.isdir(x) for x in input_files]):
args.outputfile_prefix = 'md5'
# If we can not create reporter object there is very little we can do. Just die immediately.
try:
reporter = malwareconfigreporter(
parserdir=args.parserdir,
resourcedir=args.resourcedir,
outputdir=args.outputdir,
outputfile_prefix=args.outputfile_prefix,
tempdir=args.tempdir,
disabledebug=args.hidedebug,
disableoutputfiles=args.disableoutputfiles,
disabletempcleanup=args.disabletempcleanup,
base64outputfiles=args.base64outputfiles)
except Exception as e:
error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % (
traceback.format_exc())
if args.jsonoutput:
print('{"errors": ["{}"]}'.format(error_message))
else:
print(error_message)
sys.exit(1)
if args.list:
descriptions = reporter.get_parser_descriptions()
if args.jsonoutput:
if reporter.errors:
descriptions.append({"errors": reporter.errors})
print reporter.pprint(descriptions)
else:
for name, author, description in descriptions:
print('%-25s %-8s %s' % (name, author, description))
if reporter.errors:
print("")
print("Errors:")
for error in reporter.errors:
print(" {}".format(error))
return
if args.fields:
if args.jsonoutput:
print reporter.pprint(reporter.fields)
else:
for key in sorted(reporter.fields):
print('%-20s %s' % (key, reporter.fields[key]['description']))
for example in reporter.fields[key]['examples']:
print("{} {}".format(" " * 24, json.dumps(example)))
return
if not input_files:
argparser.print_help()
return
if args.parser:
if args.filelistindirection:
if input_files[0] == "-":
inputfilelist = [line.rstrip() for line in sys.stdin]
else:
with open(input_files[0], "rb") as f:
inputfilelist = [line.rstrip() for line in f]
else:
inputfilelist = []
for arg in input_files:
if os.path.isfile(arg):
inputfilelist.append(arg)
elif os.path.isdir(arg):
for root, dirs, files in os.walk(arg):
for file in files:
inputfilelist.append(os.path.join(root, file))
kwargs = {}
if args.kwargs_raw:
kwargs = dict(json.loads(args.kwargs_raw))
for key, value in kwargs.iteritems():
if value and len(value) > len("b64file("):
if value[:len("b64file("
)] == "b64file(" and value[-1:] == ")":
tmp_filename = value[len("b64file("):-1]
with open(tmp_filename, "rb") as f:
kwargs[key] = base64.b64encode(f.read())
json_accum = []
for inputfilename in inputfilelist:
if inputfilename == "-":
reporter.run_parser(args.parser,
data=sys.stdin.read(),
**kwargs)
else:
reporter.run_parser(args.parser, inputfilename, **kwargs)
if args.includefilename:
reporter.metadata['inputfilename'] = inputfilename
reporter.metadata['md5'] = hashlib.md5(
reporter.data).hexdigest()
reporter.metadata['sha1'] = hashlib.sha1(
reporter.data).hexdigest()
reporter.metadata['sha256'] = hashlib.sha256(
reporter.data).hexdigest()
reporter.metadata['parser'] = args.parser
if reporter.pe:
reporter.metadata[
'compiletime'] = datetime.datetime.fromtimestamp(
reporter.pe.FILE_HEADER.TimeDateStamp).isoformat()
output = reporter.metadata
if reporter.errors:
output["errors"] = reporter.errors
json_accum.append(output)
if not args.jsonoutput:
reporter.output_text()
if args.jsonoutput:
print reporter.pprint(
json_accum if len(json_accum) > 1 else json_accum[0])
if args.csvwrite:
csv_filename = args.csvwrite
if not csv_filename.endswith('.csv'):
csv_filename += '.csv'
if json_accum:
key_list = []
# Begin flushing out CSV column names. A column needs to exist for
# for each unique field in all the produced results.
# 1 key_list entry = 1 column name
for metadata in json_accum:
key_list.extend(metadata.keys())
additional_other_keys = []
# Flatten 'other' entries so nested values get their own columns,
# are more readable, and easier to individually analyze.
#
# Example:
# {'other': {"unique_entry": "value", "unique_key": "value2"}}
# Results in columns: other, other.unique_entry, other.unique_key
if 'other' in metadata:
other_dict = metadata['other']
additional_other_keys = [
'other.' + other_key
for other_key in other_dict.keys()
]
# Append the metadata to include these more isolated key value pairs
for i, value in enumerate(other_dict.values()):
metadata[additional_other_keys[i]] = value
key_list += additional_other_keys
# Make sure all column names are unique
key_list = list(set(key_list))
# Flatten 'outputfile' field into separate columns for easier viewing and analysis.
if 'outputfile' in key_list:
if args.base64outputfiles:
key_list = [
'outputfile.name', 'outputfile.description',
'outputfile.md5', 'outputfile.base64'
] + key_list
else:
key_list = [
'outputfile.name', 'outputfile.description',
'outputfile.md5'
] + key_list
# Add timestamp and input filename as first columns for readability. Remaining
# columns are sorted to sift through them easier.
scan_date = time.ctime()
if 'inputfilename' in key_list:
key_list.remove('inputfilename')
key_list = ['scan_date', 'inputfilename'] + sorted(key_list)
# Reformat result metadata to:
# 1. Populate the newly added fields to the keylist
# 2. Reformat values that are lists and dictionaries to be more readable
for i, metadata in enumerate(json_accum):
# Populate the newly created outputfile fields
if 'outputfile' in metadata.keys():
metadata['outputfile.name'] = [
outp[0] for outp in metadata['outputfile']
]
metadata['outputfile.description'] = [
outp[1] for outp in metadata['outputfile']
]
metadata['outputfile.md5'] = [
outp[2] for outp in metadata['outputfile']
]
if len(outp) > 3 and args.base64outputfiles is True:
metadata['outputfile.base64'] = [
outp[3] for outp in metadata['outputfile']
]
# Reformat lists and dictionaries as string values
for k, v in metadata.iteritems():
if isinstance(v, basestring):
metadata[k] = reporter.convert_to_unicode(v)
elif isinstance(v, list):
metadata[k] = u''
for j in v:
if not isinstance(j, basestring):
metadata[k] += u'{}\n'.format(u', '.join([
reporter.convert_to_unicode(item)
for item in j
]))
else:
metadata[k] += u'{}\n'.format(
reporter.convert_to_unicode(j))
metadata[k] = metadata[k].rstrip()
elif isinstance(v, dict):
metadata[k] = u''
for field, value in v.iteritems():
if isinstance(value, list):
value = u'[{}]'.format(u', '.join(value))
metadata[k] += u'{}: {}\n'.format(field, value)
metadata[k] = metadata[k].rstrip()
else:
metadata[k] = reporter.convert_to_unicode(v)
# Populate the newly added scan_date and inputfilename fields
metadata['scan_date'] = scan_date
if 'inputfilename' not in metadata.keys():
metadata['inputfilename'] = inputfilelist[i]
# Write reformatted metadata results to CSV
try:
with open(csv_filename, 'wb') as csvfile:
dw = csv.DictWriter(csvfile, fieldnames=key_list)
dw.writeheader()
dw.writerows(
[{k: v.encode('utf8')
for k, v in entry.items()}
for entry in json_accum])
except IOError as exc:
print('\nUnable to write %s (%s)' %
(csv_filename, exc.args[1]))
