def parse_reports(resultlist, groups=[], ugly=True, includeMetadata=False, python=False): """Turn report dictionaries into json output. Returns a string. resultlist - A list of the scan return values groups - A list of modules types that will be grouped together for the report ugly - If True the return json will not be formatted includeMetadata - If True module metadata will be included in the report python - If true a python dictionary is returned instead of a json string """ files = {} metadatas = {} for item in resultlist: if item is not None: (result, metadata) = item else: continue for (fname, hit) in result: if fname not in files: files[fname] = {} # Group together module results if configured if metadata['Type'] in groups: if not files[fname].get(metadata['Type'], False): files[fname][metadata['Type']] = {} files[fname][metadata['Type']][metadata['Name']] = hit # Else put it in the root of the file else: files[fname][metadata['Name']] = hit # This is to prevent some modules from showing in metadata reports. if includeMetadata: if metadata['Name'] not in metadatas and metadata.get( "Include", True): metadatas[metadata['Name']] = metadata if includeMetadata: finaldata = {"Files": files, "Metadata": metadatas} else: finaldata = files if python: return finaldata finaldata = convert_encoding(finaldata) if not ugly: return json.dumps(finaldata, sort_keys=True, indent=3, ensure_ascii=False) else: return json.dumps(finaldata, sort_keys=True, separators=(',', ':'), ensure_ascii=False)
def scan(filelist, conf=DEFAULTCONF): results = [] libmagicresults, libmagicmeta = REQUIRES[0] for fname, libmagicresult in libmagicresults: if fname not in filelist: print("DEBUG: File not in filelist") if not libmagicresult.startswith('PE32'): continue result = {} pe = pefile.PE(fname) result['pehash'] = _get_pehash(pe) check, sha = _get_rich_header(pe) if check: result['rich_header_checksum'] = check if sha: result['rich_header_sha256'] = sha if callable(getattr(pe, 'get_imphash', None)): try: result['import_hash'] = pe.get_imphash() except: pass if hasattr(pe, 'DIRECTORY_ENTRY_RESOURCE'): result['resource_data'] = _dump_resource_data("ROOT", pe.DIRECTORY_ENTRY_RESOURCE, pe, False) result['sections'] = _get_sections(pe) if hasattr(pe, 'DIRECTORY_ENTRY_IMPORT'): result['imports'] = _get_imports(pe) else: result['imports'] = None if hasattr(pe, 'DIRECTORY_ENTRY_EXPORT'): result['exports'] = _get_exports(pe) else: result['exports'] = None result['pe_timestamp'] = _get_timestamp(pe) if hasattr(pe, 'DIRECTORY_ENTRY_DEBUG'): result['debug_info'] = _get_debug_info(pe) if hasattr(pe, 'VS_VERSIONINFO'): result['version_info'] = _get_version_info(pe) if hasattr(pe, 'DIRECTORY_ENTRY_TLS'): ret = _get_tls_info(pe) if ret: result['tls_callback_info'] = ret result = convert_encoding(result) results.append((fname, result)) metadata = {} metadata["Name"] = NAME metadata["Type"] = TYPE metadata["Version"] = pefile.__version__ metadata["Include"] = False return (results, metadata)
def parse_reports(resultlist, groups=[], ugly=True, includeMetadata=False, python=False): """Turn report dictionaries into json output. Returns a string. resultlist - A list of the scan return values groups - A list of modules types that will be grouped together for the report ugly - If True the return json will not be formatted includeMetadata - If True module metadata will be included in the report python - If true a python dictionary is returned instead of a json string """ files = {} metadatas = {} for item in resultlist: if item is not None: (result, metadata) = item else: continue for (fname, hit) in result: if fname not in files: files[fname] = {} # Group together module results if configured if metadata['Type'] in groups: if not files[fname].get(metadata['Type'], False): files[fname][metadata['Type']] = {} files[fname][metadata['Type']][metadata['Name']] = hit # Else put it in the root of the file else: files[fname][metadata['Name']] = hit # This is to prevent some modules from showing in metadata reports. if includeMetadata: if metadata['Name'] not in metadatas and metadata.get("Include", True): metadatas[metadata['Name']] = metadata if includeMetadata: finaldata = {"Files": files, "Metadata": metadatas} else: finaldata = files if python: return finaldata finaldata = convert_encoding(finaldata) if not ugly: return json.dumps(finaldata, sort_keys=True, indent=3, ensure_ascii=False) else: return json.dumps(finaldata, sort_keys=True, separators=(',', ':'), ensure_ascii=False)
def _main(): # Force all prints to go to stderr stdout = sys.stdout sys.stdout = sys.stderr # Import dependencies only needed by _main() import zipfile # Get args args = _parse_args() # Set verbose if args.verbose: global VERBOSE VERBOSE = args.verbose # Checks if user is trying to initialize if args.Files == ['init'] and not os.path.isfile('init'): _init(args) if not os.path.isfile(args.config): config_init(args.config) # Make sure report is not a dir if args.json: if os.path.isdir(args.json): print('ERROR:', args.json, 'is a directory, a file is expected') return False # Parse the file list parsedlist = parseFileList(args.Files, recursive=args.recursive) # Unzip zip files if asked to if args.extractzips: for fname in parsedlist: if zipfile.is_zipfile(fname): unzip_dir = os.path.join('_tmp', os.path.basename(fname)) z = zipfile.ZipFile(fname) if PY3: args.password = bytes(args.password, 'utf-8') try: z.extractall(path=unzip_dir, pwd=args.password) for uzfile in z.namelist(): parsedlist.append(os.path.join(unzip_dir, uzfile)) except RuntimeError as e: print("ERROR: Failed to extract ", fname, ' - ', e, sep='') parsedlist.remove(fname) # Resume from report if args.resume: i = len(parsedlist) try: reportfile = codecs.open(args.json, 'r', 'utf-8') except Exception as e: print("ERROR: Could not open report file") exit(1) for line in reportfile: line = json.loads(line) for fname in line: if fname in parsedlist: parsedlist.remove(fname) reportfile.close() i = i - len(parsedlist) if VERBOSE: print("Skipping", i, "files which are in the report already") # Do multiple runs if there are too many files filelists = [] if len(parsedlist) > args.numberper: while len(parsedlist) > args.numberper: filelists.append(parsedlist[:args.numberper]) parsedlist = parsedlist[args.numberper:] if parsedlist: filelists.append(parsedlist) for filelist in filelists: # Record start time for metadata starttime = str(datetime.datetime.now()) # Run the multiscan results = multiscan(filelist, recursive=None, configfile=args.config) # We need to read in the config for the parseReports call Config = configparser.SafeConfigParser() Config.optionxform = str Config.read(args.config) config = _get_main_config(Config) # Make sure we have a group-types if "group-types" not in config: config["group-types"] = [] elif not config["group-types"]: config["group-types"] = [] # Add in script metadata endtime = str(datetime.datetime.now()) # For windows compatibility try: username = os.getlogin() except: username = os.getenv('USERNAME') results.append(([], {"Name": "MultiScanner", "Start Time": starttime, "End Time": endtime, # "Command Line":list2cmdline(sys.argv), "Run by": username })) if args.show or not stdout.isatty(): # TODO: Make this output something readable # Parse Results report = parse_reports(results, groups=config["group-types"], ugly=args.ugly, includeMetadata=args.metadata) # Print report try: print(convert_encoding(report, encoding='ascii', errors='replace'), file=stdout) stdout.flush() except Exception as e: print('ERROR: Can\'t print report -', e) report = parse_reports(results, groups=config["group-types"], includeMetadata=args.metadata, python=True) update_conf = None if args.json: update_conf = {'File': {'path': args.json}} if args.json.endswith('.gz') or args.json.endswith('.gzip'): update_conf['File']['gzip'] = True if 'storage-config' not in config: config["storage-config"] = None storage_handle = storage.StorageHandler(configfile=config["storage-config"], config=update_conf) storage_handle.store(report) storage_handle.close() # Cleanup zip extracted files if args.extractzips: shutil.rmtree('_tmp')
def _main(): # Force all prints to go to stderr stdout = sys.stdout sys.stdout = sys.stderr # Import dependencies only needed by _main() import zipfile # Get args args = _parse_args() # Set verbose if args.verbose: global VERBOSE VERBOSE = args.verbose # Checks if user is trying to initialize if str(args.Files) == "['init']" and not os.path.isfile('init'): _init(args) if not os.path.isfile(args.config): config_init(args.config) # Make sure report is not a dir if args.json: if os.path.isdir(args.json): print('ERROR:', args.json, 'is a directory, a file is expected') return False # Parse the file list parsedlist = parseFileList(args.Files, recursive=args.recursive) # Unzip zip files if asked to if args.extractzips: for fname in parsedlist: if zipfile.is_zipfile(fname): unzip_dir = os.path.join('_tmp', os.path.basename(fname)) z = zipfile.ZipFile(fname) if PY3: args.password = bytes(args.password, 'utf-8') try: z.extractall(path=unzip_dir, pwd=args.password) for uzfile in z.namelist(): parsedlist.append(os.path.join(unzip_dir, uzfile)) except RuntimeError as e: print("ERROR: Failed to extract ", fname, ' - ', e, sep='') parsedlist.remove(fname) # Resume from report if args.resume: i = len(parsedlist) try: reportfile = codecs.open(args.json, 'r', 'utf-8') except Exception as e: print("ERROR: Could not open report file") exit(1) for line in reportfile: line = json.loads(line) for fname in line: if fname in parsedlist: parsedlist.remove(fname) reportfile.close() i = i - len(parsedlist) if VERBOSE: print("Skipping", i, "files which are in the report already") # Do multiple runs if there are too many files filelists = [] if len(parsedlist) > args.numberper: while len(parsedlist) > args.numberper: filelists.append(parsedlist[:args.numberper]) parsedlist = parsedlist[args.numberper:] if parsedlist: filelists.append(parsedlist) for filelist in filelists: # Record start time for metadata starttime = str(datetime.datetime.now()) # Run the multiscan results = multiscan(filelist, recursive=None, configfile=args.config) # We need to read in the config for the parseReports call Config = configparser.SafeConfigParser() Config.optionxform = str Config.read(args.config) config = _get_main_config(Config) # Make sure we have a group-types if "group-types" not in config: config["group-types"] = [] elif not config["group-types"]: config["group-types"] = [] # Add in script metadata endtime = str(datetime.datetime.now()) # For windows compatibility try: username = os.getlogin() except: username = os.getenv('USERNAME') results.append(( [], { "Name": "MultiScanner", "Start Time": starttime, "End Time": endtime, # "Command Line":list2cmdline(sys.argv), "Run by": username })) if args.show or not stdout.isatty(): # TODO: Make this output something readable # Parse Results report = parse_reports(results, groups=config["group-types"], ugly=args.ugly, includeMetadata=args.metadata) # Print report try: print(convert_encoding(report, encoding='ascii', errors='replace'), file=stdout) stdout.flush() except Exception as e: print('ERROR: Can\'t print report -', e) report = parse_reports(results, groups=config["group-types"], includeMetadata=args.metadata, python=True) update_conf = None if args.json: update_conf = {'File': {'path': args.json}} if args.json.endswith('.gz') or args.json.endswith('.gzip'): update_conf['File']['gzip'] = True if 'storage-config' not in config: config["storage-config"] = None storage_handle = storage.StorageHandler( configfile=config["storage-config"], config=update_conf) storage_handle.store(report) storage_handle.close() # Cleanup zip extracted files if args.extractzips: shutil.rmtree('_tmp')