def descriptions(): ''' List descriptions of parser modules ''' try: response.content_type = "application/json" reporter = malwareconfigreporter(base64outputfiles = True, disableoutputfiles = True) return reporter.pprint(reporter.get_parser_descriptions()) except Exception as e: output = {} output['errors'] = [traceback.format_exc()] logger.error("descriptions %s" % (traceback.format_exc())) return output
def __run_parser(name, data = '', modargs = ''): output = {} logger.info("__run_parser %s %s" % (name, hashlib.md5(data).hexdigest())) try: reporter = malwareconfigreporter(base64outputfiles = True) kwargs = {} if modargs: kwargs = dict(json.loads(modargs)) reporter.run_parser(name, data = data, **kwargs) output = reporter.metadata if reporter.errors: output["errors"] = reporter.errors for error in reporter.errors: logger.error("__run_parser %s %s" % (name, error)) return output except Exception as e: output = {} output['errors'] = [traceback.format_exc()] logger.error("__run_parser %s %s" % (name, traceback.format_exc())) return output
def process_file(self, file_path, CAPE_files, append_file): """Process file. @return: nothing """ strings = [] buf = self.options.get("buffer", BUFSIZE) if file_path.endswith("_info.txt"): return texttypes = [ "ASCII", "Windows Registry text", "XML document text", "Unicode text", ] if os.path.exists(file_path + "_info.txt"): with open(file_path + "_info.txt", 'r') as f: metastring = f.readline() else: metastring = "" file_info = File(file_path, metastring).get_all() # Get the file data with open(file_info["path"], "r") as drop_open: filedata = drop_open.read(buf + 1) if len(filedata) > buf: file_info["data"] = binascii.b2a_hex(filedata[:buf] + " <truncated>") else: file_info["data"] = binascii.b2a_hex(filedata) metastrings = metastring.split(",") if len(metastrings) > 1: file_info["pid"] = metastrings[1] if len(metastrings) > 2: file_info["process_path"] = metastrings[2] file_info["process_name"] = metastrings[2].split("\\")[-1] if len(metastrings) > 3: file_info["module_path"] = metastrings[3] file_info["cape_type_code"] = 0 file_info["cape_type"] = "" if metastrings != "": try: file_info["cape_type_code"] = int(metastrings[0]) except Exception as e: pass if file_info["cape_type_code"] == COMPRESSION: file_info["cape_type"] = "Decompressed PE Image" if file_info["cape_type_code"] == INJECTION_PE: file_info["cape_type"] = "Injected PE Image" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split( "\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == INJECTION_SHELLCODE: file_info["cape_type"] = "Injected Shellcode/Data" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split( "\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == EXTRACTION_PE: file_info["cape_type"] = "Extracted PE Image" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] if file_info["cape_type_code"] == EXTRACTION_SHELLCODE: file_info["cape_type"] = "Extracted Shellcode" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # PlugX if file_info["cape_type_code"] == PLUGX_CONFIG: file_info["cape_type"] = "PlugX Config" plugx_parser = plugx.PlugXConfig() config_output = plugx_parser.parse_config( filedata, len(filedata)) if config_output: file_info["plugx_config"] = config_output append_file = True if file_info["cape_type_code"] == PLUGX_PAYLOAD: file_info["cape_type"] = "PlugX Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # EvilGrab if file_info["cape_type_code"] == EVILGRAB_PAYLOAD: file_info["cape_type"] = "EvilGrab Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if file_info["cape_type_code"] == EVILGRAB_DATA: file_info["cape_type"] = "EvilGrab Data" append_file = True # UPX if file_info["cape_type_code"] == UPX: file_info["cape_type"] = "Unpacked PE Image" if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # Process CAPE Yara hits for hit in file_info["cape_yara"]: cape_name = hit["name"] try: file_info["cape_type"] = hit["meta"]["cape_type"] except: #log.error("CAPE Yara signature has no CAPE type metadata: %s", cape_name) file_info["cape_type"] = "CAPE Detection: <Type missing>" # UPX Check and unpack if cape_name == 'UPX': log.info( "CAPE: Found UPX Packed sample - attempting to unpack") unpacked_file = upx_unpack(filedata) if unpacked_file and os.path.exists(unpacked_file): unpacked_yara = File(unpacked_file).get_yara( CAPE_YARA_RULEPATH) for unpacked_hit in unpacked_yara: unpacked_name = unpacked_hit["name"] if unpacked_name == 'UPX': # Failed to unpack log.info("CAPE: Failed to unpack UPX") os.unlink(unpacked_file) break if not os.path.exists(self.CAPE_path): os.makedirs(self.CAPE_path) newname = os.path.join(self.CAPE_path, os.path.basename(unpacked_file)) os.rename(unpacked_file, newname) infofd = open(newname + "_info.txt", "a") infofd.write(os.path.basename(unpacked_file) + "\n") infofd.close() # Recursive process of unpacked file self.process_file(newname, CAPE_files, True) # Java Dropper Check #if cape_name == 'JavaDropper': # log.info("CAPE: Found Java Dropped, attemping to unpack") # unpacked_file = JavaDropper.run(unpacked_file) # cape_name = yara_scan(unpacked_file) # # if cape_name == 'JavaDropper': # log.info("CAPE: Failed to unpack JavaDropper") # #return # Attempt to import a parser for the yara hit # DC3-MWCP try: mwcp = malwareconfigreporter.malwareconfigreporter() kwargs = {} mwcp.run_parser(cape_name, data=filedata, **kwargs) if mwcp.errors == []: log.info("CAPE: Imported DC3-MWCP parser %s", cape_name) mwcp_loaded = True else: for error in mwcp.errors: #log.info("CAPE: DC3-MWCP parser error: %s", error.readline()) log.info("CAPE: DC3-MWCP parser error: %s", error) mwcp_loaded = False except ImportError: mwcp_loaded = False # malwareconfig try: malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules", "processing", "parsers", "malwareconfig") file, pathname, description = imp.find_module( cape_name, [malwareconfig_parsers]) module = imp.load_module(cape_name, file, pathname, description) malwareconfig_loaded = True log.info("CAPE: Imported malwareconfig.com parser %s", cape_name) except ImportError: #log.error("CAPE: Unable to import malwareconfig.com parser %s", cape_name) malwareconfig_loaded = False # Get config data if mwcp_loaded: try: file_info["cape_config"] = convert(mwcp.metadata) file_info["cape_name"] = format(cape_name) append_file = True except Exception as e: log.error( "CAPE: DC3-MWCP config parsing error with %s: %s", cape_name, e) elif malwareconfig_loaded: try: file_info["cape_config"] = {} for (key, value) in module.config(filedata).iteritems(): file_info["cape_config"].update({key: [value]}) file_info["cape_name"] = format(cape_name) append_file = True except Exception as e: log.error("CAPE: malwareconfig parsing error with %s: %s", cape_name, e) if append_file == True: CAPE_files.append(file_info)
def process_file(self, file_path, CAPE_output, append_file): """Process file. @return: file_info """ global cape_config cape_name = "" strings = [] buf = self.options.get("buffer", BUFSIZE) if file_path.endswith("_info.txt"): return texttypes = [ "ASCII", "Windows Registry text", "XML document text", "Unicode text", ] if os.path.exists(file_path + "_info.txt"): with open(file_path + "_info.txt", 'r') as f: metastring = f.readline() else: metastring="" file_info = File(file_path, metastring).get_all() # Get the file data with open(file_info["path"], "r") as file_open: file_data = file_open.read(buf + 1) if len(file_data) > buf: file_info["data"] = binascii.b2a_hex(file_data[:buf] + " <truncated>") else: file_info["data"] = binascii.b2a_hex(file_data) metastrings = metastring.split(",") if len(metastrings) > 1: file_info["pid"] = metastrings[1] if len(metastrings) > 2: file_info["process_path"] = metastrings[2] file_info["process_name"] = metastrings[2].split("\\")[-1] if len(metastrings) > 3: file_info["module_path"] = metastrings[3] file_info["cape_type_code"] = 0 file_info["cape_type"] = "" if metastrings != "": try: file_info["cape_type_code"] = int(metastrings[0]) except Exception as e: pass if file_info["cape_type_code"] == COMPRESSION: file_info["cape_type"] = "Decompressed PE Image" if file_info["cape_type_code"] == INJECTION_PE: file_info["cape_type"] = "Injected PE Image" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split("\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == INJECTION_SHELLCODE: file_info["cape_type"] = "Injected Shellcode/Data" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split("\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == INJECTION_SECTION: file_info["cape_type"] = "Injected Section" if len(metastrings) > 4: file_info["section_handle"] = metastrings[4] if file_info["cape_type_code"] == EXTRACTION_PE: file_info["cape_type"] = "Extracted PE Image" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] if file_info["cape_type_code"] == EXTRACTION_SHELLCODE: file_info["cape_type"] = "Extracted Shellcode" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # PlugX if file_info["cape_type_code"] == PLUGX_CONFIG: file_info["cape_type"] = "PlugX Config" plugx_parser = plugx.PlugXConfig() plugx_config = plugx_parser.parse_config(file_data, len(file_data)) if not "cape_config" in cape_config and plugx_config: cape_config["cape_config"] = {} for key, value in plugx_config.items(): cape_config["cape_config"].update({key: [value]}) cape_name = "PlugX" else: log.error("CAPE: PlugX config parsing failure - size many not be handled.") append_file = False if file_info["cape_type_code"] == PLUGX_PAYLOAD: file_info["cape_type"] = "PlugX Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # EvilGrab if file_info["cape_type_code"] == EVILGRAB_PAYLOAD: file_info["cape_type"] = "EvilGrab Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if file_info["cape_type_code"] == EVILGRAB_DATA: cape_name = "EvilGrab" file_info["cape_type"] = "EvilGrab Data" if not "cape_config" in cape_config: cape_config["cape_config"] = {} if file_info["size"] == 256 or file_info["size"] == 260: ConfigItem = "filepath" ConfigData = format(file_data) cape_config["cape_config"].update({ConfigItem: [ConfigData]}) if file_info["size"] > 0x1000: append_file = True else: append_file = False # Sedreco if file_info["cape_type_code"] == SEDRECO_DATA: cape_name = "Sedreco" cape_config["cape_type"] = "Sedreco Config" if not "cape_config" in cape_config: cape_config["cape_config"] = {} if len(metastrings) > 4: SedrecoConfigIndex = metastrings[4] if SedrecoConfigIndex == '0x0': ConfigItem = "Timer1" elif SedrecoConfigIndex == '0x1': ConfigItem = "Timer2" elif SedrecoConfigIndex == '0x2': ConfigItem = "Computer Name" elif SedrecoConfigIndex == '0x3': ConfigItem = "C&C1" elif SedrecoConfigIndex == '0x4': ConfigItem = "C&C2" elif SedrecoConfigIndex == '0x5': ConfigItem = "Operation Name" elif SedrecoConfigIndex == '0x6': ConfigItem = "Keylogger MaxBuffer" elif SedrecoConfigIndex == '0x7': ConfigItem = "Keylogger MaxTimeout" elif SedrecoConfigIndex == '0x8': ConfigItem = "Keylogger Flag" elif SedrecoConfigIndex == '0x9': ConfigItem = "C&C3" else: ConfigItem = "Unknown" ConfigData = format(file_data) if ConfigData: cape_config["cape_config"].update({ConfigItem: [ConfigData]}) append_file = False # Cerber if file_info["cape_type_code"] == CERBER_CONFIG: file_info["cape_type"] = "Cerber Config" cape_config["cape_type"] = "Cerber Config" cape_name = "Cerber" if not "cape_config" in cape_config: cape_config["cape_config"] = {} ConfigItem = "JSON Data" parsed = json.loads(file_data.rstrip(b'\0')) ConfigData = json.dumps(parsed, indent=4, sort_keys=True) cape_config["cape_config"].update({ConfigItem: [ConfigData]}) append_file = True if file_info["cape_type_code"] == CERBER_PAYLOAD: file_info["cape_type"] = "Cerber Payload" cape_config["cape_type"] = "Cerber Payload" cape_name = "Cerber" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" append_file = True # UPX package output if file_info["cape_type_code"] == UPX: file_info["cape_type"] = "Unpacked PE Image" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # Process CAPE Yara hits for hit in file_info["cape_yara"]: # Check to see if file is packed with UPX if hit["name"] == "UPX": log.info("CAPE: Found UPX Packed sample - attempting to unpack") self.upx_unpack(file_data, CAPE_output) # Check for a payload or config hit try: if "payload" in hit["meta"]["cape_type"].lower() or "config" in hit["meta"]["cape_type"].lower(): file_info["cape_type"] = hit["meta"]["cape_type"] cape_name = hit["name"] except: pass type_strings = file_info["type"].split() if "-bit" not in file_info["cape_type"]: if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # Attempt to import a parser for the hit # DC3-MWCP mwcp_loaded = False if cape_name: try: mwcp = malwareconfigreporter.malwareconfigreporter(analysis_path=self.analysis_path) kwargs = {} mwcp.run_parser(cape_name, data=file_data, **kwargs) if mwcp.errors == []: log.info("CAPE: Imported DC3-MWCP parser %s", cape_name) mwcp_loaded = True else: error_lines = mwcp.errors[0].split("\n") for line in error_lines: if line.startswith('ImportError: '): log.info("CAPE: DC3-MWCP parser: %s", line.split(': ')[1]) except ImportError: pass # malwareconfig malwareconfig_loaded = False if cape_name and mwcp_loaded == False: try: malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules", "processing", "parsers", "malwareconfig") file, pathname, description = imp.find_module(cape_name,[malwareconfig_parsers]) module = imp.load_module(cape_name, file, pathname, description) malwareconfig_loaded = True log.info("CAPE: Imported malwareconfig.com parser %s", cape_name) except ImportError: log.info("CAPE: malwareconfig.com parser: No module named %s", cape_name) # Get config data if mwcp_loaded: try: if not "cape_config" in cape_config: cape_config["cape_config"] = {} cape_config["cape_config"] = convert(mwcp.metadata) else: cape_config["cape_config"].update(convert(mwcp.metadata)) except Exception as e: log.error("CAPE: DC3-MWCP config parsing error with %s: %s", cape_name, e) elif malwareconfig_loaded: try: if not "cape_config" in cape_config: cape_config["cape_config"] = {} malwareconfig_config = module.config(file_data) if isinstance(malwareconfig_config, list): for (key, value) in module.config(file_data)[0].iteritems(): cape_config["cape_config"].update({key: [value]}) elif isinstance(malwareconfig_config, dict): for (key, value) in module.config(file_data).iteritems(): cape_config["cape_config"].update({key: [value]}) except Exception as e: log.error("CAPE: malwareconfig parsing error with %s: %s", cape_name, e) if "cape_config" in cape_config: if cape_config["cape_config"] == {}: del cape_config["cape_config"] if cape_name: if "cape_config" in cape_config: cape_config["cape_name"] = format(cape_name) if not "cape" in self.results: if cape_name != "UPX": self.results["cape"] = cape_name if append_file == True: CAPE_output.append(file_info) return file_info
def main(): ''' Run tool. ''' print '' # Setup mwcproot = "" if os.path.dirname(sys.argv[0]): mwcproot = os.path.dirname(sys.argv[0]) # Get command line arguments argparser = get_arg_parser(mwcproot) args, input_files = argparser.parse_known_args() # Configure reporter based on args if args.resource_dir: reporter = malwareconfigreporter(resourcedir=args.resource_dir, disableoutputfiles=True) else: reporter = malwareconfigreporter(disableoutputfiles=True) # Configure test object tester = malwareconfigtester(reporter=reporter, results_dir=args.test_case_dir) parser_descriptions = reporter.get_parser_descriptions() valid_parser_names = [x[0] for x in parser_descriptions] parsers = [] if args.parser_name: if args.parser_name in valid_parser_names: parsers = [args.parser_name] else: print "Error: Invalid parser name(s) specified. Parser names are case sensitive." exit(1) if args.all_tests: parsers = valid_parser_names if not parsers: print "You must specify the parser to run (or run all parsers)" exit(2) if args.parser_name: results_file_path = tester.get_results_filepath(args.parser_name) #gather all our input files if args.input_file: input_files = read_input_list(input_files[0]) # Default is to run test cases if args.run_tests: print "Running test cases. May take a while..." all_passed, test_results = tester.run_tests( parsers, filter(None, args.field_names.split(",")), ignore_field_names=filter(None, args.exclude_field_names.split(","))) print "All Passed = {0}\n".format(all_passed) if not args.silent: if args.only_failed_tests: tester.print_test_results(test_results, failed_tests=True, passed_tests=False, verbose=args.verbose, json_format=args.json) else: tester.print_test_results(test_results, failed_tests=True, passed_tests=True, verbose=args.verbose, json_format=args.json) if all_passed: exit(0) else: exit(1) #add files to test cases elif args.delete: removed_files = tester.remove_test_results(args.parser_name, input_files) for filename in removed_files: print("Removing results for %s in %s" % (filename, results_file_path)) elif args.parser_name and (args.update or (not args.delete and input_files)): if args.update: input_files.extend(tester.list_test_files(args.parser_name)) for input_file in input_files: metadata = tester.gen_results(parser_name=args.parser_name, input_file_path=input_file) if len(metadata) > 1 and len(reporter.errors) == 0: print("Updating results for %s in %s" % (input_file, results_file_path)) tester.update_test_results(results_file_path=results_file_path, results_data=metadata, replace=True) elif len(metadata) > 1 and len(reporter.errors) > 0: print("Error occurred for %s in %s, not updating" % (input_file, results_file_path)) else: print("Empty results for %s in %s, not updating" % (input_file, results_file_path)) else: argparser.print_help()
def process_file(self, file_path, CAPE_output, append_file): """Process file. @return: file_info """ global cape_config cape_name = "" strings = [] buf = self.options.get("buffer", BUFSIZE) if file_path.endswith("_info.txt"): return texttypes = [ "ASCII", "Windows Registry text", "XML document text", "Unicode text", ] if os.path.exists(file_path + "_info.txt"): with open(file_path + "_info.txt", 'r') as f: metastring = f.readline() else: metastring = "" file_info = File(file_path, metastring).get_all() # Get the file data with open(file_info["path"], "r") as file_open: filedata = file_open.read(buf + 1) if len(filedata) > buf: file_info["data"] = binascii.b2a_hex(filedata[:buf] + " <truncated>") else: file_info["data"] = binascii.b2a_hex(filedata) metastrings = metastring.split(",") if len(metastrings) > 1: file_info["pid"] = metastrings[1] if len(metastrings) > 2: file_info["process_path"] = metastrings[2] file_info["process_name"] = metastrings[2].split("\\")[-1] if len(metastrings) > 3: file_info["module_path"] = metastrings[3] file_info["cape_type_code"] = 0 file_info["cape_type"] = "" if metastrings != "": try: file_info["cape_type_code"] = int(metastrings[0]) except Exception as e: pass if file_info["cape_type_code"] == COMPRESSION: file_info["cape_type"] = "Decompressed PE Image" if file_info["cape_type_code"] == INJECTION_PE: file_info["cape_type"] = "Injected PE Image" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split( "\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == INJECTION_SHELLCODE: file_info["cape_type"] = "Injected Shellcode/Data" if len(metastrings) > 4: file_info["target_path"] = metastrings[4] file_info["target_process"] = metastrings[4].split( "\\")[-1] file_info["target_pid"] = metastrings[5] if file_info["cape_type_code"] == EXTRACTION_PE: file_info["cape_type"] = "Extracted PE Image" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] if file_info["cape_type_code"] == EXTRACTION_SHELLCODE: file_info["cape_type"] = "Extracted Shellcode" if len(metastrings) > 4: file_info["virtual_address"] = metastrings[4] type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # PlugX if file_info["cape_type_code"] == PLUGX_CONFIG: file_info["cape_type"] = "PlugX Config" plugx_parser = plugx.PlugXConfig() plugx_config = plugx_parser.parse_config( filedata, len(filedata)) if not "cape_config" in cape_config and plugx_config: cape_config["cape_config"] = {} for key, value in plugx_config.items(): cape_config["cape_config"].update({key: [value]}) cape_name = "PlugX" append_file = False if file_info["cape_type_code"] == PLUGX_PAYLOAD: file_info["cape_type"] = "PlugX Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # EvilGrab if file_info["cape_type_code"] == EVILGRAB_PAYLOAD: file_info["cape_type"] = "EvilGrab Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if file_info["cape_type_code"] == EVILGRAB_DATA: cape_name = "EvilGrab" file_info["cape_type"] = "EvilGrab Data" if not "cape_config" in cape_config: cape_config["cape_config"] = {} if file_info["size"] == 256 or file_info["size"] == 260: ConfigItem = "filepath" ConfigData = format(filedata) cape_config["cape_config"].update( {ConfigItem: [ConfigData]}) if file_info["size"] > 0x1000: append_file = True else: append_file = False # Sedreco if file_info["cape_type_code"] == SEDRECO_DATA: cape_name = "Sedreco" cape_config["cape_type"] = "Sedreco Config" if not "cape_config" in cape_config: cape_config["cape_config"] = {} if len(metastrings) > 4: SedrecoConfigIndex = metastrings[4] if SedrecoConfigIndex == '0x0': ConfigItem = "Timer1" elif SedrecoConfigIndex == '0x1': ConfigItem = "Timer2" elif SedrecoConfigIndex == '0x2': ConfigItem = "Computer Name" elif SedrecoConfigIndex == '0x3': ConfigItem = "C&C1" elif SedrecoConfigIndex == '0x4': ConfigItem = "C&C2" elif SedrecoConfigIndex == '0x5': ConfigItem = "Operation Name" elif SedrecoConfigIndex == '0x6': ConfigItem = "Keylogger MaxBuffer" elif SedrecoConfigIndex == '0x7': ConfigItem = "Keylogger MaxTimeout" elif SedrecoConfigIndex == '0x8': ConfigItem = "Keylogger Flag" elif SedrecoConfigIndex == '0x9': ConfigItem = "C&C3" else: ConfigItem = "Unknown" ConfigData = format(filedata) if ConfigData: cape_config["cape_config"].update( {ConfigItem: [ConfigData]}) append_file = False # UPX if file_info["cape_type_code"] == UPX: file_info["cape_type"] = "Unpacked PE Image" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # Process CAPE Yara hits for hit in file_info["cape_yara"]: cape_name = hit["name"] try: file_info["cape_type"] = hit["meta"]["cape_type"] except: file_info["cape_type"] = cape_name + " Payload" type_strings = file_info["type"].split() if type_strings[0] == ("PE32+"): file_info["cape_type"] += ": 64-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" if type_strings[0] == ("PE32"): file_info["cape_type"] += ": 32-bit " if type_strings[2] == ("(DLL)"): file_info["cape_type"] += "DLL" else: file_info["cape_type"] += "executable" # UPX Check and unpack if cape_name == 'UPX': log.info( "CAPE: Found UPX Packed sample - attempting to unpack") unpacked_file = upx_unpack(filedata) if unpacked_file and os.path.exists(unpacked_file): unpacked_yara = File(unpacked_file).get_yara( CAPE_YARA_RULEPATH) for unpacked_hit in unpacked_yara: unpacked_name = unpacked_hit["name"] if unpacked_name == 'UPX': # Failed to unpack log.info("CAPE: Failed to unpack UPX") os.unlink(unpacked_file) break if not os.path.exists(self.CAPE_path): os.makedirs(self.CAPE_path) newname = os.path.join(self.CAPE_path, os.path.basename(unpacked_file)) os.rename(unpacked_file, newname) infofd = open(newname + "_info.txt", "a") infofd.write(os.path.basename(unpacked_file) + "\n") infofd.close() # Recursive process of unpacked file upx_extract = self.process_file(newname, CAPE_output, True) if upx_extract["type"]: upx_extract["cape_type"] = "UPX-extracted " type_strings = upx_extract["type"].split() if type_strings[0] == ("PE32+"): upx_extract["cape_type"] += " 64-bit " if type_strings[2] == ("(DLL)"): upx_extract["cape_type"] += "DLL" else: upx_extract["cape_type"] += "executable" if type_strings[0] == ("PE32"): upx_extract["cape_type"] += " 32-bit " if type_strings[2] == ("(DLL)"): upx_extract["cape_type"] += "DLL" else: upx_extract["cape_type"] += "executable" # Attempt to import a parser for the yara hit # DC3-MWCP try: mwcp = malwareconfigreporter.malwareconfigreporter() kwargs = {} mwcp.run_parser(cape_name, data=filedata, **kwargs) if mwcp.errors == []: log.info("CAPE: Imported DC3-MWCP parser %s", cape_name) mwcp_loaded = True else: error_lines = mwcp.errors[0].split("\n") for line in error_lines: if line.startswith('ImportError: '): log.info("CAPE: DC3-MWCP parser: %s", line.split(': ')[1]) mwcp_loaded = False except ImportError: mwcp_loaded = False # malwareconfig try: malwareconfig_parsers = os.path.join(CUCKOO_ROOT, "modules", "processing", "parsers", "malwareconfig") file, pathname, description = imp.find_module( cape_name, [malwareconfig_parsers]) module = imp.load_module(cape_name, file, pathname, description) malwareconfig_loaded = True log.info("CAPE: Imported malwareconfig.com parser %s", cape_name) except ImportError: #log.info("CAPE: No malwareconfig.com parser for %s", cape_name) malwareconfig_loaded = False # Get config data if mwcp_loaded: try: if not "cape_config" in cape_config: cape_config["cape_config"] = {} cape_config["cape_config"] = convert(mwcp.metadata) else: cape_config["cape_config"].update( convert(mwcp.metadata)) except Exception as e: log.error( "CAPE: DC3-MWCP config parsing error with %s: %s", cape_name, e) elif malwareconfig_loaded: try: if not "cape_config" in cape_config: cape_config["cape_config"] = {} malwareconfig_config = module.config(filedata) if isinstance(malwareconfig_config, list): for (key, value) in module.config(filedata)[0].iteritems(): cape_config["cape_config"].update({key: [value]}) elif isinstance(malwareconfig_config, dict): for (key, value) in module.config(filedata).iteritems(): cape_config["cape_config"].update({key: [value]}) except Exception as e: log.error("CAPE: malwareconfig parsing error with %s: %s", cape_name, e) if cape_name: cape_config["cape_name"] = format(cape_name) if not "cape" in self.results: #self.results["cape"] = [] self.results["cape"] = cape_name #if cape_name not in self.results["cape"]: # self.results["cape"].append(cape_name) if append_file == True: CAPE_output.append(file_info) return file_info
def main(): ''' Run tool. ''' print '' # Setup mwcproot = "" if os.path.dirname(sys.argv[0]): mwcproot = os.path.dirname(sys.argv[0]) # Get command line arguments argparser = get_arg_parser(mwcproot) args, input_files = argparser.parse_known_args() # Configure reporter based on args if args.resource_dir: reporter = malwareconfigreporter(resourcedir = args.resource_dir, disableoutputfiles = True) else: reporter = malwareconfigreporter(disableoutputfiles = True) # Configure test object tester = malwareconfigtester(reporter = reporter, results_dir = args.test_case_dir) parser_descriptions = reporter.get_parser_descriptions() valid_parser_names = [x[0] for x in parser_descriptions] parsers = [] if args.parser_name: if args.parser_name in valid_parser_names: parsers = [ args.parser_name ] else: print "Error: Invalid parser name(s) specified. Parser names are case sensitive." exit(1) if args.all_tests: parsers = valid_parser_names if not parsers: print "You must specify the parser to run (or run all parsers)" exit(2) if args.parser_name: results_file_path = tester.get_results_filepath(args.parser_name) #gather all our input files if args.input_file: input_files = read_input_list(input_files[0]) # Default is to run test cases if args.run_tests: print "Running test cases. May take a while..." all_passed, test_results = tester.run_tests(parsers, filter(None,args.field_names.split(",")), ignore_field_names = filter(None, args.exclude_field_names.split(","))) print "All Passed = {0}\n".format(all_passed) if not args.silent: if args.only_failed_tests: tester.print_test_results(test_results, failed_tests = True, passed_tests = False, verbose = args.verbose, json_format = args.json) else: tester.print_test_results(test_results, failed_tests = True, passed_tests = True, verbose = args.verbose, json_format = args.json) if all_passed: exit(0) else: exit(1) #add files to test cases elif args.delete: removed_files = tester.remove_test_results(args.parser_name, input_files) for filename in removed_files: print("Removing results for %s in %s" % (filename, results_file_path)) elif args.update or (not args.delete and input_files): if args.update: input_files.extend(tester.list_test_files(args.parser_name)) for input_file in input_files: metadata = tester.gen_results(parser_name = args.parser_name, input_file_path = input_file) if len(metadata) > 1 and len(reporter.errors) == 0: print("Updating results for %s in %s" % (input_file, results_file_path)) tester.update_test_results(results_file_path = results_file_path, results_data = metadata, replace = True) elif len(metadata) > 1 and len(reporter.errors) > 0: print("Error occurred for %s in %s, not updating" % (input_file, results_file_path)) else: print("Empty results for %s in %s, not updating" % (input_file, results_file_path)) else: argparser.print_help()
def main(): ''' Run tool. ''' print '' # Setup mwcproot = "" if os.path.dirname(sys.argv[0]): mwcproot = os.path.dirname(sys.argv[0]) # Get command line arguments argparser = get_arg_parser(mwcproot) args, input_files = argparser.parse_known_args() # Configure reporter based on args if args.resource_dir: reporter = malwareconfigreporter(resourcedir=args.resource_dir, disableoutputfiles=True) else: reporter = malwareconfigreporter(disableoutputfiles=True) # Configure test object tester = malwareconfigtester(reporter=reporter, results_dir=args.test_case_dir) parser_descriptions = reporter.get_parser_descriptions() valid_parser_names = [x[0] for x in parser_descriptions] parsers = [] if args.parser_name: if args.parser_name in valid_parser_names: parsers = [args.parser_name] else: print "Error: Invalid parser name(s) specified. Parser names are case sensitive." sys.exit(1) if args.all_tests: parsers = valid_parser_names if not parsers: print "You must specify a single parser or all parsers to run or update." sys.exit(2) if args.parser_name: results_file_path = tester.get_results_filepath(args.parser_name) # Gather all our input files if args.input_file: input_files = read_input_list(input_files[0]) # Run test cases if args.run_tests: print "Running test cases. May take a while..." # Run tests test_results = tester.run_tests( parsers, filter(None, args.field_names.split(",")), ignore_field_names=filter(None, args.exclude_field_names.split(","))) # Determine if any test cases failed all_passed = True if any([not test_result.passed for test_result in test_results]): all_passed = False print "All Passed = {0}\n".format(all_passed) if not args.silent: if args.only_failed_tests: tester.print_test_results(test_results, failed_tests=True, passed_tests=False, json_format=args.json) else: tester.print_test_results(test_results, failed_tests=True, passed_tests=True, json_format=args.json) if all_passed: sys.exit(0) else: sys.exit(1) # Delete files from test cases elif args.delete: removed_files = tester.remove_test_results(args.parser_name, input_files) for filename in removed_files: print u"Removing results for {} in {}".format( filename, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") # Update previously existing test cases elif args.update: print "Updating test cases. May take a while..." for parser in parsers: results_file_path = tester.get_results_filepath(parser) if os.path.isfile(results_file_path): input_files = tester.list_test_files(parser) else: print "No test case file found for parser '{}'. No update could be made.".format( parser) continue for input_file in input_files: metadata = tester.gen_results(parser_name=parser, input_file_path=input_file) if len(metadata) > 1 and len(reporter.errors) == 0: print u"Updating results for {} in {}".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") tester.update_test_results( results_file_path=results_file_path, results_data=metadata, replace=True) elif len(metadata) > 1 and len(reporter.errors) > 0: print u"Error occurred for {} in {}, not updating".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") else: print u"Empty results for {} in {}, not updating".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") # Add/update test cases for specified input files and specified parser elif args.parser_name and (not args.delete and input_files): for input_file in input_files: metadata = tester.gen_results(parser_name=args.parser_name, input_file_path=input_file) if len(metadata) > 1 and len(reporter.errors) == 0: print u"Updating results for {} in {}".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") tester.update_test_results(results_file_path=results_file_path, results_data=metadata, replace=True) elif len(metadata) > 1 and len(reporter.errors) > 0: print u"Error occurred for {} in {}, not updating".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") else: print u"Empty results for {} in {}, not updating".format( input_file, results_file_path).encode( encoding=sys.stdout.encoding if sys.stdout.encoding else locale.getpreferredencoding(), errors="replace") else: argparser.print_help()
#!/usr/bin/env python ''' Simple example to demonstrate use of the API provided by DC3-MWCP framework. ''' #first, import the malwareconfigreporter class from mwcp.malwareconfigreporter import malwareconfigreporter #create an instance of the malwareconfigreporter class reporter = malwareconfigreporter() ''' The malwareconfigreporter object is the primary DC3-MWCP framework object, containing most input and output data and controlling execution of the parser modules. The most common parameters to provide are parserdir and resourcedir, depending upon your installation. ''' #view location of resource and parser directories print reporter.resourcedir print reporter.parserdir #view available parsers print reporter.get_parser_descriptions() #run the dummy config parser, view the output reporter.run_parser("foo", "README.md") #alternate, run on provided buffer: reporter.run_parser("foo", data="lorem ipsum") print reporter.pprint(reporter.metadata)
def main(): optparser = make_opt_parser() options, args = optparser.parse_args() #if we can not create reporter object there is very little we can do. Just die immediately. try: reporter = malwareconfigreporter( parserdir=options.parserdir, resourcedir=options.resourcedir, outputdir=options.outputdir, outputfile_prefix=options.outputfile_prefix, tempdir=options.tempdir, disabledebug=options.hidedebug, disableoutputfiles=options.disableoutputfiles, disabletempcleanup=options.disabletempcleanup, base64outputfiles=options.base64outputfiles) except Exception as e: error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % ( traceback.format_exc()) if options.jsonoutput: print('{"errors": ["%s"]}' % (error_message)) else: print(error_message) sys.exit(1) if options.list: descriptions = reporter.get_parser_descriptions() if options.jsonoutput: if reporter.errors: descriptions.append({"errors": reporter.errors}) print reporter.pprint(descriptions) else: for name, author, description in sorted(descriptions): print('%-25s %-8s %s' % (name, author, description)) if reporter.errors: print("") print("Errors:") for error in reporter.errors: print(" %s" % (error)) return if options.fields: if options.jsonoutput: print reporter.pprint(reporter.fields) else: for key in sorted(reporter.fields): print('%-20s %s' % (key, reporter.fields[key]['description'])) for example in reporter.fields[key]['examples']: print('%s %s' % (" " * 24, json.dumps(example))) return if not args: optparser.print_help() return if options.parser: if options.filelistindirection: if args[0] == "-": inputfilelist = [line.rstrip() for line in sys.stdin] else: with open(args[0], "rb") as f: inputfilelist = [line.rstrip() for line in f] else: inputfilelist = args kwargs = {} if options.kwargs_raw: kwargs = dict(json.loads(options.kwargs_raw)) for key, value in kwargs.iteritems(): if value and len(value) > len("b64file("): if value[:len("b64file(" )] == "b64file(" and value[-1:] == ")": tmp_filename = value[len("b64file("):-1] with open(tmp_filename, "rb") as f: kwargs[key] = base64.b64encode(f.read()) for inputfilename in inputfilelist: if inputfilename == "-": reporter.run_parser(options.parser, data=sys.stdin.read(), **kwargs) else: reporter.run_parser(options.parser, inputfilename, **kwargs) if options.includefilename: reporter.metadata['inputfilename'] = inputfilename reporter.metadata['md5'] = hashlib.md5( reporter.data).hexdigest() reporter.metadata['sha1'] = hashlib.sha1( reporter.data).hexdigest() reporter.metadata['sha256'] = hashlib.sha256( reporter.data).hexdigest() reporter.metadata['parser'] = options.parser if reporter.pe: reporter.metadata[ 'compiletime'] = datetime.datetime.fromtimestamp( reporter.pe.FILE_HEADER.TimeDateStamp).isoformat() if options.jsonoutput: output = reporter.metadata if reporter.errors: output["errors"] = reporter.errors print reporter.pprint(output) else: reporter.output_text()
#!/usr/bin/env python ''' Simple example to demonstrate use of the API provided by DC3-MWCP framework. ''' #first, import the malwareconfigreporter class from mwcp.malwareconfigreporter import malwareconfigreporter #create an instance of the malwareconfigreporter class reporter = malwareconfigreporter() ''' The malwareconfigreporter object is the primary DC3-MWCP framework object, containing most input and output data and controlling execution of the parser modules. The most common parameters to provide are parserdir and resourcedir, depending upon your installation. ''' #view location of resource and parser directories print reporter.resourcedir print reporter.parserdir #view available parsers print reporter.get_parser_descriptions() #run the dummy config parser, view the output reporter.run_parser("foo", "README.md") #alternate, run on provided buffer: reporter.run_parser("foo", data = "lorem ipsum") print reporter.pprint(reporter.metadata)
def make_opt_parser(): ''' create a option parser to handle command line inputs ''' usage_str = 'usage: %s [options] FILES' % (os.path.basename(sys.argv[0])) description = "DC3-MWCP Framework: utility for executing parser modules" opt_parser = optparse.OptionParser(usage_str, description=description) #available opts: #--c-e-----------q-s--v-xyz #A-Z default_parserdir = '' default_resourcedir = '' #create reporter to get default paths, ignore if this fails try: default_reporter = malwareconfigreporter() default_parserdir = default_reporter.parserdir default_resourcedir = default_reporter.resourcedir except Exception as e: pass opt_parser.add_option('-p', '--parser', action='store', type='string', default='', dest='parser', help='malware config parser to call') opt_parser.add_option('-l', '--list', action="store_true", default=False, dest='list', help='list all malware config parsers') opt_parser.add_option( '-k', '--listfields', action="store_true", default=False, dest='fields', help= 'list all standardized fields and examples. See resources/fields.json') opt_parser.add_option('-a', '--parserdir', action='store', type='string', metavar='DIR', default=default_parserdir, dest='parserdir', help='parsers directory' + ' [default: %default]') opt_parser.add_option('-r', '--resourcedir', action='store', type='string', metavar='DIR', default=default_resourcedir, dest='resourcedir', help='resources directory' + ' [default: %default]') opt_parser.add_option('-o', '--outputdir', action='store', type='string', metavar='DIR', default='', dest='outputdir', help='output directory' + ' [default: %default]') opt_parser.add_option('-t', '--tempdir', action='store', type='string', metavar='DIR', default=tempfile.gettempdir(), dest='tempdir', help='temp directory' + ' [default: %default]') opt_parser.add_option( '-j', '--jsonoutput', action='store_true', default=False, dest='jsonoutput', help='Enable json output for parser reports (instead of formatted text)' ) opt_parser.add_option('-n', '--disableoutputfiles', action="store_true", default=False, dest='disableoutputfiles', help='disable writing output files to filesystem') opt_parser.add_option( '-g', '--disabletempcleanup', action='store_true', default=False, dest='disabletempcleanup', help= 'Disable cleanup of framework created temp files including managed tempdir' ) opt_parser.add_option( '-f', '--includefileinfo', action='store_true', default=False, dest='includefilename', help= 'include input file information such as filename, hashes, and compile time in parser output' ) opt_parser.add_option('-d', '--hidedebug', action="store_true", default=False, dest='hidedebug', help='Hide debug messages in output') opt_parser.add_option( '-u', '--outputfileprefix', action='store', type='string', metavar='FILENAME', default='', dest='outputfile_prefix', help= 'string prepended to output files written to filesystem. specifying "md5"\ will cause output files to be prefixed with the md5 of the input file' + ' [default: %default]') opt_parser.add_option( '-i', '--filelistindirection', action="store_true", default=False, dest='filelistindirection', help='input file contains a list of filenames to process') opt_parser.add_option( '-b', '--base64outputfiles', action="store_true", default=False, dest='base64outputfiles', help='base64 encode output files and include in metadata') opt_parser.add_option( '-w', '--kwargs', action='store', type='string', metavar='JSON', default='', dest='kwargs_raw', help='module keyword arguments as json encoded dictionary\ if values in the dictionary use the special paradigm "b64file(filename)", then \ filename is read, base64 encoded, and used as the value' ) return opt_parser
def make_opt_parser(): ''' create a option parser to handle command line inputs ''' usage_str = 'usage: %s [options] FILES' % (os.path.basename(sys.argv[0])) description = "DC3-MWCP Framework: utility for executing parser modules" opt_parser = optparse.OptionParser(usage_str, description = description) #available opts: #--c-e-----------q-s--v-xyz #A-Z default_parserdir = '' default_resourcedir = '' #create reporter to get default paths, ignore if this fails try: default_reporter = malwareconfigreporter() default_parserdir = default_reporter.parserdir default_resourcedir = default_reporter.resourcedir except Exception as e: pass opt_parser.add_option('-p', '--parser', action = 'store', type = 'string', default = '', dest = 'parser', help = 'malware config parser to call') opt_parser.add_option('-l', '--list', action = "store_true", default = False, dest = 'list', help = 'list all malware config parsers') opt_parser.add_option('-k', '--listfields', action = "store_true", default = False, dest = 'fields', help = 'list all standardized fields and examples. See resources/fields.json') opt_parser.add_option('-a', '--parserdir', action = 'store', type = 'string', metavar = 'DIR', default = default_parserdir, dest = 'parserdir', help = 'parsers directory' + ' [default: %default]') opt_parser.add_option('-r', '--resourcedir', action = 'store', type = 'string', metavar = 'DIR', default = default_resourcedir, dest = 'resourcedir', help = 'resources directory' + ' [default: %default]') opt_parser.add_option('-o', '--outputdir', action = 'store', type = 'string', metavar = 'DIR', default = '', dest = 'outputdir', help = 'output directory' + ' [default: %default]') opt_parser.add_option('-t', '--tempdir', action = 'store', type = 'string', metavar = 'DIR', default = tempfile.gettempdir(), dest = 'tempdir', help = 'temp directory' + ' [default: %default]') opt_parser.add_option('-j', '--jsonoutput', action = 'store_true', default = False, dest = 'jsonoutput', help = 'Enable json output for parser reports (instead of formatted text)') opt_parser.add_option('-n', '--disableoutputfiles', action = "store_true", default = False, dest = 'disableoutputfiles', help = 'disable writing output files to filesystem') opt_parser.add_option('-g', '--disabletempcleanup', action = 'store_true', default = False, dest = 'disabletempcleanup', help = 'Disable cleanup of framework created temp files including managed tempdir') opt_parser.add_option('-f', '--includefileinfo', action = 'store_true', default = False, dest = 'includefilename', help = 'include input file information such as filename, hashes, and compile time in parser output') opt_parser.add_option('-d', '--hidedebug', action = "store_true", default = False, dest = 'hidedebug', help = 'Hide debug messages in output') opt_parser.add_option('-u', '--outputfileprefix', action = 'store', type = 'string', metavar = 'FILENAME', default = '', dest = 'outputfile_prefix', help = 'string prepended to output files written to filesystem. specifying "md5"\ will cause output files to be prefixed with the md5 of the input file' + ' [default: %default]') opt_parser.add_option('-i', '--filelistindirection', action = "store_true", default = False, dest = 'filelistindirection', help = 'input file contains a list of filenames to process') opt_parser.add_option('-b', '--base64outputfiles', action = "store_true", default = False, dest = 'base64outputfiles', help = 'base64 encode output files and include in metadata') opt_parser.add_option('-w', '--kwargs', action = 'store', type = 'string', metavar = 'JSON', default = '', dest = 'kwargs_raw', help = 'module keyword arguments as json encoded dictionary\ if values in the dictionary use the special paradigm "b64file(filename)", then \ filename is read, base64 encoded, and used as the value') return opt_parser
def main(): optparser = make_opt_parser() options, args = optparser.parse_args() #if we can not create reporter object there is very little we can do. Just die immediately. try: reporter = malwareconfigreporter(parserdir = options.parserdir, resourcedir = options.resourcedir, outputdir = options.outputdir, outputfile_prefix = options.outputfile_prefix, tempdir = options.tempdir, disabledebug = options.hidedebug, disableoutputfiles = options.disableoutputfiles, disabletempcleanup = options.disabletempcleanup, base64outputfiles = options.base64outputfiles) except Exception as e: error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % (traceback.format_exc()) if options.jsonoutput: print('{"errors": ["%s"]}' % (error_message)) else: print(error_message) sys.exit(1) if options.list: descriptions = reporter.get_parser_descriptions() if options.jsonoutput: if reporter.errors: descriptions.append({"errors": reporter.errors}) print reporter.pprint(descriptions) else: for name, author, description in sorted(descriptions): print('%-25s %-8s %s' % (name, author, description) ) if reporter.errors: print("") print("Errors:") for error in reporter.errors: print(" %s" % (error)) return if options.fields: if options.jsonoutput: print reporter.pprint(reporter.fields) else: for key in sorted(reporter.fields): print('%-20s %s' % (key, reporter.fields[key]['description'])) for example in reporter.fields[key]['examples']: print('%s %s' % (" " * 24, json.dumps(example))) return if not args: optparser.print_help() return if options.parser: if options.filelistindirection: if args[0] == "-": inputfilelist = [ line.rstrip() for line in sys.stdin ] else: with open(args[0],"rb") as f: inputfilelist = [ line.rstrip() for line in f ] else: inputfilelist = args kwargs = {} if options.kwargs_raw: kwargs = dict(json.loads(options.kwargs_raw)) for key, value in kwargs.iteritems(): if value and len(value) > len("b64file("): if value[:len("b64file(")] == "b64file(" and value[-1:] == ")": tmp_filename = value[len("b64file("):-1] with open(tmp_filename, "rb") as f: kwargs[key] = base64.b64encode(f.read()) for inputfilename in inputfilelist: if inputfilename == "-": reporter.run_parser(options.parser, data=sys.stdin.read(), **kwargs) else: reporter.run_parser(options.parser, inputfilename, **kwargs) if options.includefilename: reporter.metadata['inputfilename'] = inputfilename reporter.metadata['md5'] = hashlib.md5(reporter.data).hexdigest() reporter.metadata['sha1'] = hashlib.sha1(reporter.data).hexdigest() reporter.metadata['sha256'] = hashlib.sha256(reporter.data).hexdigest() reporter.metadata['parser'] = options.parser if reporter.pe: reporter.metadata['compiletime'] = datetime.datetime.fromtimestamp(reporter.pe.FILE_HEADER.TimeDateStamp).isoformat() if options.jsonoutput: output = reporter.metadata if reporter.errors: output["errors"] = reporter.errors print reporter.pprint(output) else: reporter.output_text()
def get_arg_parser(): ''' create a option parser to handle command line inputs ''' usage_str = 'usage: %s [options] FILES_DIRS' % (os.path.basename( sys.argv[0])) description = "DC3-MWCP Framework: utility for executing parser modules" parser = argparse.ArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter, usage=usage_str) default_parserdir = '' default_resourcedir = '' # Create reporter to get default paths, ignore if this fails try: default_reporter = malwareconfigreporter() default_parserdir = default_reporter.parserdir default_resourcedir = default_reporter.resourcedir except Exception as e: pass parser.add_argument("-p", default="", type=str, dest="parser", help="Malware config parser to call.") parser.add_argument("-l", action="store_true", default=False, dest="list", help="list all malware config parsers.") parser.add_argument( "-k", action="store_true", default=False, dest="fields", help= "List all standardized fields and examples. See resources/fields.json") parser.add_argument("-a", metavar="DIR", default=default_parserdir, dest="parserdir", help="Parsers directory" + " [default: {}]".format(default_parserdir)) parser.add_argument("-r", metavar="DIR", default=default_resourcedir, dest="resourcedir", help="Resources directory" + " [default: {}]".format(default_resourcedir)) parser.add_argument("-o", metavar="DIR", default="", dest="outputdir", help="Output directory.") parser.add_argument("-c", metavar="CSVWRITE", default="", dest="csvwrite", help="Output CSV file.") parser.add_argument("-t", metavar="DIR", default=tempfile.gettempdir(), dest="tempdir", help="Temp directory." + " [default: {}]".format(tempfile.gettempdir())) parser.add_argument( "-j", action="store_true", default=False, dest="jsonoutput", help= "Enable json output for parser reports (instead of formatted text).") parser.add_argument("-n", action="store_true", default=False, dest="disableoutputfiles", help="Disable writing output files to filesystem.") parser.add_argument( "-g", action="store_true", default=False, dest="disabletempcleanup", help= "Disable cleanup of framework created temp files including managed tempdir." ) parser.add_argument( "-f", action="store_true", default=False, dest="includefilename", help= "Include file information such as filename, hashes, and compile time in parser output." ) parser.add_argument("-d", action="store_true", default=False, dest="hidedebug", help="Hide debug messages in output.") parser.add_argument( "-u", metavar="FILENAME", default="", dest="outputfile_prefix", help= "String prepended to output files written to filesystem. Specifying 'md5' will cause " + "output files to be prefixed with the md5 of the input file. When passing in multiple " + "files for analysis, the default will be 'md5'. Passing in a value with the -u option " + "or using the -U option can be used to override the 'md5' default for multiple files. " + "[default: (No prefix|md5)]") parser.add_argument( "-U", action="store_true", default=False, dest="disableoutputfileprefix", help= "When in effect, parser output files will not have a filename prefix.") parser.add_argument( "-i", action="store_true", default=False, dest="filelistindirection", help="Input file contains a list of filenames to process.") parser.add_argument( "-b", action="store_true", default=False, dest="base64outputfiles", help="Base64 encode output files and include in metadata.") parser.add_argument( "-w", metavar="JSON", default="", dest="kwargs_raw", help="Module keyword arguments as json encoded dictionary " + "if values in the dictionary use the special paradigm 'b64file(filename)', then " + "filename is read, base64 encoded, and used as the value)") return parser
def main(): argparser = get_arg_parser() args, input_files = argparser.parse_known_args() # This is a preliminary check before creating the reporter to establish how output # file prefixes should be set. if args.disableoutputfileprefix: args.outputfile_prefix = '' elif args.filelistindirection or len(input_files) > 1 or any( [os.path.isdir(x) for x in input_files]): args.outputfile_prefix = 'md5' # If we can not create reporter object there is very little we can do. Just die immediately. try: reporter = malwareconfigreporter( parserdir=args.parserdir, resourcedir=args.resourcedir, outputdir=args.outputdir, outputfile_prefix=args.outputfile_prefix, tempdir=args.tempdir, disabledebug=args.hidedebug, disableoutputfiles=args.disableoutputfiles, disabletempcleanup=args.disabletempcleanup, base64outputfiles=args.base64outputfiles) except Exception as e: error_message = "Error loading DC3-MWCP reporter object, please check installation: %s" % ( traceback.format_exc()) if args.jsonoutput: print('{"errors": ["{}"]}'.format(error_message)) else: print(error_message) sys.exit(1) if args.list: descriptions = reporter.get_parser_descriptions() if args.jsonoutput: if reporter.errors: descriptions.append({"errors": reporter.errors}) print reporter.pprint(descriptions) else: for name, author, description in descriptions: print('%-25s %-8s %s' % (name, author, description)) if reporter.errors: print("") print("Errors:") for error in reporter.errors: print(" {}".format(error)) return if args.fields: if args.jsonoutput: print reporter.pprint(reporter.fields) else: for key in sorted(reporter.fields): print('%-20s %s' % (key, reporter.fields[key]['description'])) for example in reporter.fields[key]['examples']: print("{} {}".format(" " * 24, json.dumps(example))) return if not input_files: argparser.print_help() return if args.parser: if args.filelistindirection: if input_files[0] == "-": inputfilelist = [line.rstrip() for line in sys.stdin] else: with open(input_files[0], "rb") as f: inputfilelist = [line.rstrip() for line in f] else: inputfilelist = [] for arg in input_files: if os.path.isfile(arg): inputfilelist.append(arg) elif os.path.isdir(arg): for root, dirs, files in os.walk(arg): for file in files: inputfilelist.append(os.path.join(root, file)) kwargs = {} if args.kwargs_raw: kwargs = dict(json.loads(args.kwargs_raw)) for key, value in kwargs.iteritems(): if value and len(value) > len("b64file("): if value[:len("b64file(" )] == "b64file(" and value[-1:] == ")": tmp_filename = value[len("b64file("):-1] with open(tmp_filename, "rb") as f: kwargs[key] = base64.b64encode(f.read()) json_accum = [] for inputfilename in inputfilelist: if inputfilename == "-": reporter.run_parser(args.parser, data=sys.stdin.read(), **kwargs) else: reporter.run_parser(args.parser, inputfilename, **kwargs) if args.includefilename: reporter.metadata['inputfilename'] = inputfilename reporter.metadata['md5'] = hashlib.md5( reporter.data).hexdigest() reporter.metadata['sha1'] = hashlib.sha1( reporter.data).hexdigest() reporter.metadata['sha256'] = hashlib.sha256( reporter.data).hexdigest() reporter.metadata['parser'] = args.parser if reporter.pe: reporter.metadata[ 'compiletime'] = datetime.datetime.fromtimestamp( reporter.pe.FILE_HEADER.TimeDateStamp).isoformat() output = reporter.metadata if reporter.errors: output["errors"] = reporter.errors json_accum.append(output) if not args.jsonoutput: reporter.output_text() if args.jsonoutput: print reporter.pprint( json_accum if len(json_accum) > 1 else json_accum[0]) if args.csvwrite: csv_filename = args.csvwrite if not csv_filename.endswith('.csv'): csv_filename += '.csv' if json_accum: key_list = [] # Begin flushing out CSV column names. A column needs to exist for # for each unique field in all the produced results. # 1 key_list entry = 1 column name for metadata in json_accum: key_list.extend(metadata.keys()) additional_other_keys = [] # Flatten 'other' entries so nested values get their own columns, # are more readable, and easier to individually analyze. # # Example: # {'other': {"unique_entry": "value", "unique_key": "value2"}} # Results in columns: other, other.unique_entry, other.unique_key if 'other' in metadata: other_dict = metadata['other'] additional_other_keys = [ 'other.' + other_key for other_key in other_dict.keys() ] # Append the metadata to include these more isolated key value pairs for i, value in enumerate(other_dict.values()): metadata[additional_other_keys[i]] = value key_list += additional_other_keys # Make sure all column names are unique key_list = list(set(key_list)) # Flatten 'outputfile' field into separate columns for easier viewing and analysis. if 'outputfile' in key_list: if args.base64outputfiles: key_list = [ 'outputfile.name', 'outputfile.description', 'outputfile.md5', 'outputfile.base64' ] + key_list else: key_list = [ 'outputfile.name', 'outputfile.description', 'outputfile.md5' ] + key_list # Add timestamp and input filename as first columns for readability. Remaining # columns are sorted to sift through them easier. scan_date = time.ctime() if 'inputfilename' in key_list: key_list.remove('inputfilename') key_list = ['scan_date', 'inputfilename'] + sorted(key_list) # Reformat result metadata to: # 1. Populate the newly added fields to the keylist # 2. Reformat values that are lists and dictionaries to be more readable for i, metadata in enumerate(json_accum): # Populate the newly created outputfile fields if 'outputfile' in metadata.keys(): metadata['outputfile.name'] = [ outp[0] for outp in metadata['outputfile'] ] metadata['outputfile.description'] = [ outp[1] for outp in metadata['outputfile'] ] metadata['outputfile.md5'] = [ outp[2] for outp in metadata['outputfile'] ] if len(outp) > 3 and args.base64outputfiles is True: metadata['outputfile.base64'] = [ outp[3] for outp in metadata['outputfile'] ] # Reformat lists and dictionaries as string values for k, v in metadata.iteritems(): if isinstance(v, basestring): metadata[k] = reporter.convert_to_unicode(v) elif isinstance(v, list): metadata[k] = u'' for j in v: if not isinstance(j, basestring): metadata[k] += u'{}\n'.format(u', '.join([ reporter.convert_to_unicode(item) for item in j ])) else: metadata[k] += u'{}\n'.format( reporter.convert_to_unicode(j)) metadata[k] = metadata[k].rstrip() elif isinstance(v, dict): metadata[k] = u'' for field, value in v.iteritems(): if isinstance(value, list): value = u'[{}]'.format(u', '.join(value)) metadata[k] += u'{}: {}\n'.format(field, value) metadata[k] = metadata[k].rstrip() else: metadata[k] = reporter.convert_to_unicode(v) # Populate the newly added scan_date and inputfilename fields metadata['scan_date'] = scan_date if 'inputfilename' not in metadata.keys(): metadata['inputfilename'] = inputfilelist[i] # Write reformatted metadata results to CSV try: with open(csv_filename, 'wb') as csvfile: dw = csv.DictWriter(csvfile, fieldnames=key_list) dw.writeheader() dw.writerows( [{k: v.encode('utf8') for k, v in entry.items()} for entry in json_accum]) except IOError as exc: print('\nUnable to write %s (%s)' % (csv_filename, exc.args[1]))