def diff_stats(sum_stats, old_stats): if len(sum_stats) != len(old_stats): print_warn("Stats corrupted for '" + clr.GRA + "%s" % sum_stats["afl_banner"] + clr.RST + "'!") return None diff_stat = { "fuzzers": len(sum_stats), "fuzzer_pid": 0, "execs_done": 0, "execs_per_sec": 0, "paths_total": 0, "paths_favored": 0, "pending_favs": 0, "pending_total": 0, "unique_crashes": 0, "unique_hangs": 0, "afl_banner": 0, "host": socket.gethostname()[:10], } for k in sum_stats.keys(): if k not in ["afl_banner", "host"]: diff_stat[k] = sum_stats[k] - old_stats[k] else: diff_stat[k] = sum_stats[k] return diff_stat
def diff_stats(sum_stats, old_stats): if len(sum_stats) != len(old_stats): print_warn("Stats corrupted for '" + clr.GRA + "%s" % sum_stats['afl_banner'] + clr.RST + "'!") return None diff_stat = { 'fuzzers': len(sum_stats), 'fuzzer_pid': 0, 'execs_done': 0, 'execs_per_sec': 0, 'paths_total': 0, 'paths_favored': 0, 'pending_favs': 0, 'pending_total': 0, 'unique_crashes': 0, 'unique_hangs': 0, 'afl_banner': 0, 'host': socket.gethostname()[:10] } for k in sum_stats.keys(): if k not in ['afl_banner', 'host']: diff_stat[k] = sum_stats[k] - old_stats[k] else: diff_stat[k] = sum_stats[k] return diff_stat
def parse_stat_file(stat_file): try: f = open(stat_file, "r") lines = f.readlines() f.close() stats = { 'fuzzer_pid': None, 'execs_done': None, 'execs_per_sec': None, 'paths_total': None, 'paths_favored': None, 'pending_favs': None, 'pending_total': None, 'unique_crashes': None, 'unique_hangs': None, 'afl_banner': None } for l in lines: for k in stats.keys(): if k != "fuzzer_pid": if k in l: stats[k] = l[16:].strip(" \r\n") else: if k in l: stats[k] = fuzzer_alive(int(l[16:].strip(" \r\n"))) return stats except FileNotFoundError as e: print_warn("Stat file " + clr.GRA + "%s" % e.filename + clr.RST + " not found!") return None
def invoke_dryrun(input_files, crash_dir, timeout_dir, target_cmd, timeout=60, num_threads=1): invalid_samples, timeout_samples = afl_vcrash.verify_samples(num_threads, input_files, target_cmd, timeout_secs=timeout) invalid_sample_set = set(invalid_samples+timeout_samples) input_sample_set = set(input_files) crashes_set = input_sample_set - invalid_sample_set crashes = list(crashes_set) if len(crashes) > 0: if not os.path.exists(crash_dir): os.makedirs(crash_dir, exist_ok=True) for c in crashes: shutil.move(c, os.path.join(crash_dir, os.path.basename(c))) print_warn("Moved %d crash samples from the corpus to %s." % (len(crashes), crash_dir)) if len(timeout_samples) > 0: if not os.path.exists(timeout_dir): os.makedirs(timeout_dir, exist_ok=True) for t in timeout_samples: shutil.move(t, os.path.join(timeout_dir, os.path.basename(t))) print_warn("Moved %d timeouts from the corpus to %s." % (len(timeout_samples), timeout_dir)) return
def load_stats(fuzzer_dir, summary=True): fuzzer_dir = os.path.abspath(os.path.expanduser(fuzzer_dir)) if not os.path.isdir(fuzzer_dir): print_warn("Invalid fuzzing directory specified: " + clr.GRA + "%s" % fuzzer_dir + clr.RST) return None fuzzer_stats = [] if os.path.isfile(os.path.join(fuzzer_dir, "fuzzer_stats")): # single afl-fuzz job stats = parse_stat_file(os.path.join(fuzzer_dir, "fuzzer_stats"), summary) if stats: fuzzer_stats.append(stats) else: fuzzer_inst = [] for fdir in os.listdir(fuzzer_dir): if os.path.isdir(os.path.join(fuzzer_dir, fdir)): fuzzer_inst.append(os.path.join(fuzzer_dir, fdir, "fuzzer_stats")) for stat_file in fuzzer_inst: stats = parse_stat_file(stat_file, summary) if stats: fuzzer_stats.append(stats) return fuzzer_stats
def invoke_dryrun(input_files, crash_dir, timeout_dir, target_cmd, num_threads=1): invalid_samples, timeout_samples = afl_vcrash.verify_samples(num_threads, input_files, target_cmd) invalid_sample_set = set(invalid_samples+timeout_samples) input_sample_set = set(input_files) crashes_set = input_sample_set - invalid_sample_set crashes = list(crashes_set) if len(crashes) > 0: if not os.path.exists(crash_dir): os.makedirs(crash_dir, exist_ok=True) for c in crashes: shutil.move(c, os.path.join(crash_dir, os.path.basename(c))) print_warn("Moved %d crash samples from the corpus to %s." % (len(crashes), crash_dir)) if len(timeout_samples) > 0: if not os.path.exists(timeout_dir): os.makedirs(timeout_dir, exist_ok=True) for t in timeout_samples: shutil.move(t, os.path.join(timeout_dir, os.path.basename(t))) print_warn("Moved %d timeouts from the corpus to %s." % (len(timeout_samples), timeout_dir)) return
def load_stats(fuzzer_dir): fuzzer_dir = os.path.abspath(os.path.expanduser(fuzzer_dir)) if not os.path.isdir(fuzzer_dir): print_warn("Invalid fuzzing directory specified: " + clr.GRA + "%s" % fuzzer_dir + clr.RST) return None fuzzer_stats = [] if os.path.isfile(os.path.join(fuzzer_dir, "fuzzer_stats")): # single afl-fuzz job stats = parse_stat_file(os.path.join(fuzzer_dir, "fuzzer_stats")) if stats: fuzzer_stats.append(stats) else: fuzzer_inst = [] for fdir in os.listdir(fuzzer_dir): if os.path.isdir(os.path.join(fuzzer_dir, fdir)): fuzzer_inst.append(os.path.join(fuzzer_dir, fdir, "fuzzer_stats")) for stat_file in fuzzer_inst: stats = parse_stat_file(stat_file) if stats: fuzzer_stats.append(stats) return fuzzer_stats
def invoke_cmin(input_dir, output_dir, target_cmd, mem_limit=None, timeout=None, qemu=False): success = True cmin_cmd = "afl-cmin " if mem_limit is not None: cmin_cmd += "-m %s " % convert_mem_limit(mem_limit) if timeout is not None: cmin_cmd += "-t %d " % int(timeout) if qemu: cmin_cmd += "-Q " cmd = "-I %s-i %s -o %s -- %s" % (cmin_cmd, input_dir, output_dir, target_cmd) print_ok("Executing: %s" % cmd) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: print_warn("afl-cmin failed with exit code %d!" % e.returncode) success = False return success
def fetch_stats(config_settings, twitter_inst): doExit = False # { 'fuzzer_dir': (stat, old_stat) } stat_dict = dict() while not doExit: try: for fuzzer in config_settings['fuzz_dirs']: stats = load_stats(fuzzer) if not stats: continue sum_stats = summarize_stats(stats) try: # stat_dict has already been initialized for fuzzer # old_stat <- last_stat old_stats = stat_dict[fuzzer][0].copy() except KeyError: # stat_dict has not yet been initialized for fuzzer # old_stat <- cur_stat old_stats = sum_stats.copy() # initialize/update stat_dict stat_dict[fuzzer] = (sum_stats, old_stats) stat_change = diff_stats(sum_stats, old_stats) if not diff_stats: continue print(prettify_stat(sum_stats, stat_change, True)) tweet = prettify_stat(sum_stats, stat_change, False) l = len(tweet) c = clr.LRD if l > 140 else clr.LGN print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...") try: twitter_inst.statuses.update(status=shorten_tweet(tweet)) except (twitter.TwitterHTTPError, URLError): print_warn( "Problem connecting to Twitter! Tweet not sent!") except Exception as e: print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")") if float(config_settings['interval']) < 0: doExit = True else: time.sleep(float(config_settings['interval']) * 60) except KeyboardInterrupt: print("\b\b") print_ok("Aborted by user. Good bye!") doExit = True
def invoke_cmin(input_dir, output_dir, target_cmd): success = True cmd = "afl-cmin -i %s -o %s -- %s" % (input_dir, output_dir, target_cmd) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: print_warn("afl-cmin failed with exit code %d!" % e.returncode) success = False return success
def main(argv): show_info() parser = argparse.ArgumentParser(description='afl-sync synchronizes fuzzer state directories between different \ locations. Supported are remote transfers through rsync that may use transport compression.', usage='afl-sync [-S SESSION] <cmd> <src_sync_dir> <dst_storage_dir>') parser.add_argument('cmd', help='Command to perform: push, pull or sync. Push transmits the local state from ' '<src_sync_dir> to the destination <dst_storage_dir>. Pull fetches remote state(s) into ' 'the local synchronization dir appending the \'.sync\' extension. Sync performs a ' 'pull operation followed by a push.') parser.add_argument('src_sync_dir', help='Source afl synchronisation directory containing state directories of afl instances.') parser.add_argument('dst_storage_dir', help='Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!') parser.add_argument('-S', '--session', dest='session', default=None, help='Name of an afl-multicore session. If provided, only fuzzers belonging to ' 'the specified session will be synced with the destination. Otherwise state ' 'directories of all fuzzers inside the synchronisation dir will be exchanged. ' 'Directories ending on \'.sync\' will never be pushed back to the destination!') args = parser.parse_args(argv[1:]) args.cmd = args.cmd.lower() if not args.cmd in ['push', 'pull', 'sync']: print_err('Sorry, unknown command requested!') sys.exit(1) if not os.path.exists(args.src_sync_dir): if args.cmd in ['pull', 'sync']: print_warn('Local afl sync dir does not exist! Will create it for you!') os.makedirs(args.src_sync_dir) else: print_err('Local afl sync dir does not exist!') sys.exit(1) server_config = { 'remote_path': args.dst_storage_dir, } fuzzer_config = { 'sync_dir': args.src_sync_dir, 'session': args.session, 'exclude_crashes': False, 'exclude_hangs': False, } rsyncEngine = AflRsync(server_config, fuzzer_config) if args.cmd == 'push': rsyncEngine.push() elif args.cmd == 'pull': rsyncEngine.pull() elif args.cmd == 'sync': rsyncEngine.sync()
def fetch_stats(config_settings, twitter_inst): doExit = False # { 'fuzzer_dir': (stat, old_stat) } stat_dict = dict() while not doExit: try: for fuzzer in config_settings["fuzz_dirs"]: stats = load_stats(fuzzer) if not stats: continue sum_stats = summarize_stats(stats) try: # stat_dict has already been initialized for fuzzer # old_stat <- last_stat old_stats = stat_dict[fuzzer][0].copy() except KeyError: # stat_dict has not yet been initialized for fuzzer # old_stat <- cur_stat old_stats = sum_stats.copy() # initialize/update stat_dict stat_dict[fuzzer] = (sum_stats, old_stats) stat_change = diff_stats(sum_stats, old_stats) if not diff_stats: continue print(prettify_stat(sum_stats, stat_change, True)) tweet = prettify_stat(sum_stats, stat_change, False) l = len(tweet) c = clr.LRD if l > 140 else clr.LGN print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...") try: twitter_inst.statuses.update(status=shorten_tweet(tweet)) except (twitter.TwitterHTTPError, URLError): print_warn("Problem connecting to Twitter! Tweet not sent!") except Exception as e: print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")") if float(config_settings["interval"]) < 0: doExit = True else: time.sleep(float(config_settings["interval"]) * 60) except KeyboardInterrupt: print("\b\b") print_ok("Aborted by user. Good bye!") doExit = True
def kill_session(session): if os.path.isfile("/tmp/afl_multicore.PGID.%s" % session): f = open("/tmp/afl_multicore.PGID.%s" % session) pgids = f.readlines() for pgid in pgids: try: print_ok("Killing jobs with PGID %s" % pgid.strip("\r\n")) os.killpg(int(pgid), signal.SIGTERM) except ProcessLookupError: print_warn("No processes with PGID %s found!" % (pgid.strip("\r\n"))) f.close() os.remove("/tmp/afl_multicore.PGID.%s" % session) else: print_err("PGID file '/tmp/afl_multicore.PGID.%s' not found! Aborting!" % session)
def invoke_cmin(input_dir, output_dir, target_cmd, mem_limit=None, timeout=None): success = True cmin_cmd = "afl-cmin " if mem_limit is not None: cmin_cmd += "-m %d " % int(mem_limit) if timeout is not None: cmin_cmd += "-t %d " % int(timeout) cmd = "%s-i %s -o %s -- %s" % (cmin_cmd, input_dir, output_dir, target_cmd) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: print_warn("afl-cmin failed with exit code %d!" % e.returncode) success = False return success
def kill_session(session): if os.path.isfile("/tmp/afl_multicore.PGID.%s" % session): f = open("/tmp/afl_multicore.PGID.%s" % session) pgids = f.readlines() for pgid in pgids: try: print_ok("Killing jobs with PGID %s" % pgid.strip('\r\n')) os.killpg(int(pgid), signal.SIGTERM) except ProcessLookupError: print_warn("No processes with PGID %s found!" % (pgid.strip('\r\n'))) f.close() os.remove("/tmp/afl_multicore.PGID.%s" % session) else: print_err( "PGID file '/tmp/afl_multicore.PGID.%s' not found! Aborting!" % session) sys.exit(1)
def fetch_stats(config_settings, twitter_inst): stat_dict = dict() for fuzzer in config_settings['fuzz_dirs']: stats = load_stats(fuzzer) if not stats: continue sum_stats = summarize_stats(stats) try: with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'r') as f: old_stats = json.load(f) except FileNotFoundError: old_stats = sum_stats.copy() # initialize/update stat_dict stat_dict[fuzzer] = (sum_stats, old_stats) stat_change = diff_stats(sum_stats, old_stats) with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'w') as f: json.dump(sum_stats, f) print(prettify_stat(sum_stats, stat_change, True)) tweet = prettify_stat(sum_stats, stat_change, False) l = len(tweet) c = clr.LRD if l > 140 else clr.LGN if twitter_inst: print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...") try: twitter_inst.statuses.update(status=shorten_tweet(tweet)) except (twitter.TwitterHTTPError, URLError): print_warn("Problem connecting to Twitter! Tweet not sent!") except Exception as e: print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")")
def fetch_stats(config_settings, twitter_inst): stat_dict = dict() for fuzzer in config_settings['fuzz_dirs']: stats = load_stats(fuzzer) if not stats: continue sum_stats = summarize_stats(stats) try: with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'r') as f: old_stats = json.load(f) except FileNotFoundError: old_stats = sum_stats.copy() # initialize/update stat_dict stat_dict[fuzzer] = (sum_stats, old_stats) stat_change = diff_stats(sum_stats, old_stats) with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'w') as f: json.dump(sum_stats, f) print(prettify_stat(sum_stats, stat_change, True)) tweet = prettify_stat(sum_stats, stat_change, False) l = len(tweet) c = clr.LRD if l > 140 else clr.LGN print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...") try: twitter_inst.statuses.update(status=shorten_tweet(tweet)) except (twitter.TwitterHTTPError, URLError): print_warn("Problem connecting to Twitter! Tweet not sent!") except Exception as e: print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")")
def main(argv): show_info() parser = argparse.ArgumentParser(description="afl-minimize performs several optimization steps to reduce the size\n \ of an afl-fuzz corpus.", usage="afl-minimize [-c COLLECTION_DIR [--cmin [opts]] [--tmin [opts]]] [-d] [-h]\n \ [-j] sync_dir -- target_cmd\n") parser.add_argument("-c", "--collect", dest="collection_dir", help="Collect all samples from the synchronisation dir and store them in the collection dir. \ Existing files in the collection directory will be overwritten!", default=None) parser.add_argument("--cmin", dest="invoke_cmin", action="store_const", const=True, default=False, help="Run afl-cmin on collection dir. Has no effect without '-c'.") parser.add_argument("--cmin-mem-limit", dest="cmin_mem_limit", default=None, help="Set memory limit for afl-cmin.") parser.add_argument("--cmin-timeout", dest="cmin_timeout", default=None, help="Set timeout for afl-cmin.") parser.add_argument("--tmin", dest="invoke_tmin", action="store_const", const=True, default=False, help="Run afl-tmin on minimized collection dir if used together with '--cmin'\ or on unoptimized collection dir otherwise. Has no effect without '-c'.") parser.add_argument("--tmin-mem-limit", dest="tmin_mem_limit", default=None, help="Set memory limit for afl-tmin.") parser.add_argument("--tmin-timeout", dest="tmin_timeout", default=None, help="Set timeout for afl-tmin.") parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_const", const=True, default=False, help="Perform dry-run on collection dir, if '-c' is provided or on \ synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus.") parser.add_argument("-j", "--threads", dest="num_threads", default=1, help="Enable parallel dry-run and t-minimization step by specifying the number of threads \ afl-minimize will utilize.") parser.add_argument("sync_dir", help="afl synchronisation directory containing multiple fuzzers and their queues.") parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \ Use '@@' to specify crash sample input file position (see afl-fuzz usage).") args = parser.parse_args(argv[1:]) if not args.collection_dir and not args.dry_run: print_err("No operation requested. You should at least provide '-c'") print_err("for sample collection or '-d' for a dry-run. Use '--help' for") print_err("usage instructions or checkout README.md for details.") return sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir)) if not os.path.exists(sync_dir): print_err("No valid directory provided for <SYNC_DIR>!") return args.target_cmd = " ".join(args.target_cmd).split() args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0])) if not os.path.exists(args.target_cmd[0]): print_err("Target binary not found!") return args.target_cmd = " ".join(args.target_cmd) if not args.num_threads: threads = 1 else: threads = int(args.num_threads) if args.collection_dir: out_dir = os.path.abspath(os.path.expanduser(args.collection_dir)) if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0: os.makedirs(out_dir, exist_ok=True) print_ok("Looking for fuzzing queues in '%s'." % sync_dir) fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False) # collect samples from fuzzer queues print_ok("Found %d fuzzers, collecting samples." % len(fuzzers)) sample_index = afl_collect.build_sample_index(sync_dir, out_dir, fuzzers) print_ok("Successfully indexed %d samples." % len(sample_index.index)) print_ok("Copying %d samples into collection directory..." % len(sample_index.index)) afl_collect.copy_samples(sample_index) else: print_warn("Collection directory exists and is not empty!") print_warn("Skipping collection step...") if args.invoke_cmin: # invoke cmin on collection print_ok("Executing: afl-cmin -i %s -o %s.cmin -- %s" % (out_dir, out_dir, args.target_cmd)) invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd, mem_limit=args.cmin_mem_limit, timeout=args.cmin_timeout) if args.invoke_tmin: # invoke tmin on minimized collection print_ok("Executing: afl-tmin -i %s.cmin/* -o %s.cmin.tmin/* -- %s" % (out_dir, out_dir, args.target_cmd)) tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True) invoke_tmin(tmin_samples, "%s.cmin.tmin" % out_dir, args.target_cmd, num_threads=threads, mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout) elif args.invoke_tmin: # invoke tmin on collection print_ok("Executing: afl-tmin -i %s/* -o %s.tmin/* -- %s" % (out_dir, out_dir, args.target_cmd)) tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True) invoke_tmin(tmin_samples, "%s.tmin" % out_dir, args.target_cmd, num_threads=threads, mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout) if args.dry_run: # invoke dry-run on collected/minimized corpus if args.invoke_cmin and args.invoke_tmin: print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin.tmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.cmin.tmin.crashes" % out_dir, "%s.cmin.tmin.hangs" % out_dir, args.target_cmd, num_threads=threads) elif args.invoke_cmin: print_ok("Performing dry-run in %s.cmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.cmin.crashes" % out_dir, "%s.cmin.hangs" % out_dir, args.target_cmd, num_threads=threads) elif args.invoke_tmin: print_ok("Performing dry-run in %s.tmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.tmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.tmin.crashes" % out_dir, "%s.tmin.hangs" % out_dir, args.target_cmd, num_threads=threads) else: print_ok("Performing dry-run in %s..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.crashes" % out_dir, "%s.hangs" % out_dir, args.target_cmd, num_threads=threads) else: if args.dry_run: print_ok("Looking for fuzzing queues in '%s'." % sync_dir) fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False) print_ok("Found %d fuzzers, performing dry run." % len(fuzzers)) print_warn("Be patient! Depending on the corpus size this step can take hours...") # invoke dry-run on original corpus for f in fuzzers: for q_dir in f[1]: q_dir_complete = os.path.join(sync_dir, f[0], q_dir) print_ok("Processing %s..." % q_dir_complete) dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(q_dir_complete, abs_path=True) invoke_dryrun(dryrun_samples, os.path.join(sync_dir, f[0], "crashes"), os.path.join(sync_dir, f[0], "hangs"), args.target_cmd, num_threads=threads)
def main(argv): show_info() parser = argparse.ArgumentParser(description="Post selected contents of fuzzer_stats to Twitter.", usage="afl-stats [-h] [-c config]\n") parser.add_argument("-c", "--config", dest="config_file", help="afl-stats config file (Default: afl-stats.conf)!", default="afl-stats.conf") args = parser.parse_args(argv[1:]) config_settings = read_config(args.config_file) twitter_inst = twitter_init() doExit = False # { 'fuzzer_dir': (stat, old_stat) } stat_dict = dict() while not doExit: try: for fuzzer in config_settings['fuzz_dirs']: stats = load_stats(fuzzer) if not stats: continue sum_stats = summarize_stats(stats) try: # stat_dict has already been initialized for fuzzer # old_stat <- last_stat old_stats = stat_dict[fuzzer][0].copy() except KeyError: # stat_dict has not yet been initialized for fuzzer # old_stat <- cur_stat old_stats = sum_stats.copy() # initialize/update stat_dict stat_dict[fuzzer] = (sum_stats, old_stats) stat_change = diff_stats(sum_stats, old_stats) if not diff_stats: continue print(prettify_stat(sum_stats, stat_change, True)) tweet = prettify_stat(sum_stats, stat_change, False) l = len(tweet) c = clr.LRD if l>140 else clr.LGN print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...") try: twitter_inst.statuses.update(status=shorten_tweet(tweet)) except (twitter.TwitterHTTPError, URLError): print_warn("Problem connecting to Twitter! Tweet not sent!") except Exception as e: print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")") if float(config_settings['interval']) < 0: doExit = True else: time.sleep(float(config_settings['interval'])*60) except KeyboardInterrupt: print("\b\b") print_ok("Aborted by user. Good bye!") doExit = True
def main(argv): show_info() parser = argparse.ArgumentParser(description="afl-collect copies all crash sample files from an afl sync dir used \ by multiple fuzzers when fuzzing in parallel into a single location providing easy access for further crash analysis.", usage="afl-collect [-d DATABASE] [-e|-g GDB_EXPL_SCRIPT_FILE] [-f LIST_FILENAME]\n \ [-h] [-j THREADS] [-m] [-r] [-rr] sync_dir collection_dir -- target_cmd") parser.add_argument("sync_dir", help="afl synchronisation directory crash samples will be collected from.") parser.add_argument("collection_dir", help="Output directory that will hold a copy of all crash samples and other generated files. \ Existing files in the collection directory will be overwritten!") parser.add_argument("-d", "--database", dest="database_file", help="Submit sample data into an sqlite3 database (\ only when used together with '-e'). afl-collect skips processing of samples already found in existing database.", default=None) parser.add_argument("-e", "--execute-gdb-script", dest="gdb_expl_script_file", help="Generate and execute a gdb+exploitable script after crash sample collection for crash \ classification. (Like option '-g', plus script execution.)", default=None) parser.add_argument("-f", "--filelist", dest="list_filename", default=None, help="Writes all collected crash sample filenames into a file in the collection directory.") parser.add_argument("-g", "--generate-gdb-script", dest="gdb_script_file", help="Generate gdb script to run 'exploitable.py' on all collected crash samples. Generated \ script will be placed into collection directory.", default=None) parser.add_argument("-j", "--threads", dest="num_threads", default=1, help="Enable parallel analysis by specifying the number of threads afl-collect will utilize.") parser.add_argument("-m", "--minimize-filenames", dest="min_filename", action="store_const", const=True, default=False, help="Minimize crash sample file names by only keeping fuzzer name and ID.") parser.add_argument("-r", "--remove-invalid", dest="remove_invalid", action="store_const", const=True, default=False, help="Verify collected crash samples and remove samples that do not lead to \ crashes (runs 'afl-vcrash.py -r' on collection directory). This step is done prior to any script file \ or file list generation/execution.") parser.add_argument("-rr", "--remove-unexploitable", dest="remove_unexploitable", action="store_const", const=True, default=False, help="Remove crash samples that have an exploitable classification of \ 'NOT_EXPLOITABLE' or 'PROBABLY_NOT_EXPLOITABLE'. Sample file removal will take place after gdb+exploitable \ script execution. Has no effect without '-e'.") parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \ Use '@@' to specify crash sample input file position (see afl-fuzz usage).") args = parser.parse_args(argv[1:]) sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir)) if not os.path.exists(sync_dir): print_err("No valid directory provided for <SYNC_DIR>!") return if args.collection_dir: out_dir = os.path.abspath(os.path.expanduser(args.collection_dir)) if not os.path.exists(out_dir): os.makedirs(out_dir, exist_ok=True) else: print_err("No valid directory provided for <OUT_DIR>!") return args.target_cmd = " ".join(args.target_cmd).split() args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0])) if not os.path.exists(args.target_cmd[0]): print_err("Target binary not found!") return args.target_cmd = " ".join(args.target_cmd) if args.database_file: db_file = os.path.abspath(os.path.expanduser(args.database_file)) else: db_file = None print_ok("Going to collect crash samples from '%s'." % sync_dir) # initialize database if db_file: lite_db = con_sqlite.sqliteConnector(db_file) lite_db.init_database() else: lite_db = None fuzzers = get_fuzzer_instances(sync_dir) print_ok("Found %d fuzzers, collecting crash samples." % len(fuzzers)) sample_index = build_sample_index(sync_dir, out_dir, fuzzers, lite_db, args.min_filename) if len(sample_index.index) > 0: print_ok("Successfully indexed %d crash samples." % len(sample_index.index)) elif db_file: print_warn("No unseen samples found. Check your database for results!") return else: print_warn("No samples found. Check directory settings!") return if args.remove_invalid: from afl_utils import afl_vcrash invalid_samples = afl_vcrash.verify_samples(int(args.num_threads), sample_index.inputs(), args.target_cmd) # store invalid samples in db print_ok("Saving invalid sample info to database.") if args.gdb_expl_script_file and db_file: for sample in invalid_samples: sample_name = sample_index.outputs(input_file=sample) dataset = {'sample': sample_name[0]['output'], 'classification': 'INVALID', 'description': 'Sample does not cause a crash in the target.', 'hash': ''} if not lite_db.dataset_exists(dataset): lite_db.insert_dataset(dataset) # remove invalid samples from sample index sample_index.remove_inputs(invalid_samples) print_warn("Removed %d invalid crash samples from index." % len(invalid_samples)) # generate gdb+exploitable script if args.gdb_expl_script_file: divided_index = sample_index.divide(int(args.num_threads)) for i in range(0, int(args.num_threads), 1): generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_expl_script_file), divided_index[i], args.target_cmd, i, intermediate=True) # execute gdb+exploitable script classification_data = execute_gdb_script(out_dir, args.gdb_expl_script_file, len(sample_index.inputs()), int(args.num_threads)) # Submit crash classification data into database print_ok("Saving sample classification info to database.") if db_file: for dataset in classification_data: if not lite_db.dataset_exists(dataset): lite_db.insert_dataset(dataset) # de-dupe by exploitable hash seen = set() seen_add = seen.add classification_data_dedupe = [x for x in classification_data if x['hash'] not in seen and not seen_add(x['hash'])] # remove dupe samples identified by exploitable hash uninteresting_samples = [x['sample'] for x in classification_data if x not in classification_data_dedupe] sample_index.remove_outputs(uninteresting_samples) print_warn("Removed %d duplicate samples from index. Will continue with %d remaining samples." % (len(uninteresting_samples), len(sample_index.index))) # remove crash samples that are classified uninteresting if args.remove_unexploitable: classification_unexploitable = [ 'NOT_EXPLOITABLE', 'PROBABLY_NOT_EXPLOITABLE', ] uninteresting_samples = [] for c in classification_data_dedupe: if c['classification'] in classification_unexploitable: uninteresting_samples.append(c['sample']) sample_index.remove_outputs(uninteresting_samples) print_warn("Removed %d uninteresting crash samples from index." % len(uninteresting_samples)) # generate output gdb script generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_expl_script_file), sample_index, args.target_cmd, 0) elif args.gdb_script_file: generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_script_file), sample_index, args.target_cmd) print_ok("Copying %d samples into output directory..." % len(sample_index.index)) files_collected = copy_samples(sample_index) # generate filelist of collected crash samples if args.list_filename: generate_sample_list(os.path.abspath(os.path.expanduser(args.list_filename)), files_collected) print_ok("Generated crash sample list '%s'." % os.path.abspath(os.path.expanduser(args.list_filename)))
def main(argv): show_info() parser = argparse.ArgumentParser( description= 'afl-sync synchronizes fuzzer state directories between different \ locations. Supported are remote transfers through rsync that may use transport compression.', usage='afl-sync [-S SESSION] <cmd> <src_sync_dir> <dst_storage_dir>') parser.add_argument( 'cmd', help= 'Command to perform: push, pull or sync. Push transmits the local state from ' '<src_sync_dir> to the destination <dst_storage_dir>. Pull fetches remote state(s) into ' 'the local synchronization dir appending the \'.sync\' extension. Sync performs a ' 'pull operation followed by a push.') parser.add_argument( 'src_sync_dir', help= 'Source afl synchronisation directory containing state directories of afl instances.' ) parser.add_argument( 'dst_storage_dir', help= 'Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!' ) parser.add_argument( '--chmod', help= 'Affect destination\'s file and directory permissions, e.g. --chmod=g+rw to add ' 'read/write group permissions.', metavar='PERMS') parser.add_argument( '--chown', help= 'Affect destination\'s file and directory user and group, e.g. --chown=foo:bar to ' 'let the files be owned by user foo and group bar.', metavar='USER:GROUP') parser.add_argument( '-S', '--session', dest='session', default=None, help= 'Name of an afl-multicore session. If provided, only fuzzers belonging to ' 'the specified session will be synced with the destination. Otherwise state ' 'directories of all fuzzers inside the synchronisation dir will be exchanged. ' 'Directories ending on \'.sync\' will never be pushed back to the destination!' ) args = parser.parse_args(argv[1:]) args.cmd = args.cmd.lower() if not args.cmd in ['push', 'pull', 'sync']: print_err('Sorry, unknown command requested!') sys.exit(1) rsync_put_options = _rsync_default_options[:] rsync_get_options = _rsync_default_options[:] if args.chmod or args.chown: # these arguments are meaningless with pull since they should only # affect the remote side if args.cmd == 'pull': print_warn( '--chmod and --chown have no effect with pull and will be ignored.' ) if args.chmod: rsync_put_options.append('--chmod={}'.format(args.chmod)) if args.chown: rsync_put_options.append('--protect-args') rsync_put_options.append('--chown={}'.format(args.chown)) if not os.path.exists(args.src_sync_dir): if args.cmd in ['pull', 'sync']: print_warn( 'Local afl sync dir does not exist! Will create it for you!') os.makedirs(args.src_sync_dir) else: print_err('Local afl sync dir does not exist!') sys.exit(1) server_config = { 'remote_path': args.dst_storage_dir, } fuzzer_config = { 'sync_dir': args.src_sync_dir, 'session': args.session, 'exclude_crashes': False, 'exclude_hangs': False, } rsync_config = { 'get': rsync_get_options, 'put': rsync_put_options, } rsyncEngine = AflRsync(server_config, fuzzer_config, rsync_config) if args.cmd == 'push': rsyncEngine.push() elif args.cmd == 'pull': rsyncEngine.pull() elif args.cmd == 'sync': rsyncEngine.sync()
def main(argv): show_info() parser = argparse.ArgumentParser(description="afl-minimize performs several optimization steps to reduce the size\n \ of an afl-fuzz corpus.", usage="afl-minimize [-c COLLECTION_DIR [--cmin [opts]] [--tmin [opts]]] [--reseed]\n \ [-d] [-h] [-j] sync_dir -- target_cmd\n") parser.add_argument("-c", "--collect", dest="collection_dir", help="Collect all samples from the synchronisation dir and store them in the collection dir.", default=None) parser.add_argument("--cmin", dest="invoke_cmin", action="store_const", const=True, default=False, help="Run afl-cmin on collection dir. Has no effect without '-c'.") parser.add_argument("--cmin-mem-limit", dest="cmin_mem_limit", default=None, help="Set memory limit for afl-cmin.") parser.add_argument("--cmin-timeout", dest="cmin_timeout", default=None, help="Set timeout for afl-cmin.") parser.add_argument("--cmin-qemu", dest="cmin_qemu", default=False, action="store_const", const=True, help="Enable qemu mode afl-cmin.") parser.add_argument("--reseed", dest="reseed", default=False, action="store_const", const=True, help="Reseed afl-fuzz with the \ collected (and optimized) corpus. This replaces all sync_dir queues with the newly generated corpus.") parser.add_argument("--tmin", dest="invoke_tmin", action="store_const", const=True, default=False, help="Run afl-tmin on minimized collection dir if used together with '--cmin'\ or on unoptimized collection dir otherwise. Has no effect without '-c'.") parser.add_argument("--tmin-mem-limit", dest="tmin_mem_limit", default=None, help="Set memory limit for afl-tmin.") parser.add_argument("--tmin-timeout", dest="tmin_timeout", default=None, help="Set timeout for afl-tmin.") parser.add_argument("--tmin-qemu", dest="tmin_qemu", default=False, action="store_const", const=True, help="Enable qemu mode afl-tmin.") parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_const", const=True, default=False, help="Perform dry-run on collection dir, if '-c' is provided or on \ synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus.") parser.add_argument("-j", "--threads", dest="num_threads", default=1, help="Enable parallel dry-run and t-minimization step by specifying the number of threads \ afl-minimize will utilize.") parser.add_argument("sync_dir", help="afl synchronisation directory containing multiple fuzzers and their queues.") parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \ Use '@@' to specify crash sample input file position (see afl-fuzz usage).") args = parser.parse_args(argv[1:]) if not args.collection_dir and not args.dry_run: print_err("No operation requested. You should at least provide '-c'") print_err("for sample collection or '-d' for a dry-run. Use '--help' for") print_err("usage instructions or checkout README.md for details.") return sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir)) if not os.path.exists(sync_dir): print_err("No valid directory provided for <SYNC_DIR>!") return args.target_cmd = " ".join(args.target_cmd).split() args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0])) if not os.path.exists(args.target_cmd[0]): print_err("Target binary not found!") return args.target_cmd = " ".join(args.target_cmd) threads = int(args.num_threads) if args.collection_dir: out_dir = os.path.abspath(os.path.expanduser(args.collection_dir)) if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0: os.makedirs(out_dir, exist_ok=True) print_ok("Looking for fuzzing queues in '%s'." % sync_dir) fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False) # collect samples from fuzzer queues print_ok("Found %d fuzzers, collecting samples." % len(fuzzers)) sample_index = afl_collect.build_sample_index(sync_dir, out_dir, fuzzers, omit_fuzzer_name=True) print_ok("Successfully indexed %d samples." % len(sample_index.index)) print_ok("Copying %d samples into collection directory..." % len(sample_index.index)) afl_collect.copy_samples(sample_index) else: print_warn("Collection directory exists and is not empty!") print_warn("Skipping collection step...") if args.invoke_cmin: # invoke cmin on collection invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd, mem_limit=args.cmin_mem_limit, timeout=args.cmin_timeout, qemu=args.cmin_qemu) if args.invoke_tmin: # invoke tmin on minimized collection tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True) invoke_tmin(tmin_samples, "%s.cmin.tmin" % out_dir, args.target_cmd, num_threads=threads, mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout, qemu=args.tmin_qemu) elif args.invoke_tmin: # invoke tmin on collection tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True) invoke_tmin(tmin_samples, "%s.tmin" % out_dir, args.target_cmd, num_threads=threads, mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout, qemu=args.tmin_qemu) if args.dry_run: # invoke dry-run on collected/minimized corpus if args.invoke_cmin and args.invoke_tmin: print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin.tmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.cmin.tmin.crashes" % out_dir, "%s.cmin.tmin.hangs" % out_dir, args.target_cmd, num_threads=threads) elif args.invoke_cmin: print_ok("Performing dry-run in %s.cmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.cmin.crashes" % out_dir, "%s.cmin.hangs" % out_dir, args.target_cmd, num_threads=threads) elif args.invoke_tmin: print_ok("Performing dry-run in %s.tmin..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.tmin" % out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.tmin.crashes" % out_dir, "%s.tmin.hangs" % out_dir, args.target_cmd, num_threads=threads) else: print_ok("Performing dry-run in %s..." % out_dir) print_warn("Be patient! Depending on the corpus size this step can take hours...") dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True) invoke_dryrun(dryrun_samples, "%s.crashes" % out_dir, "%s.hangs" % out_dir, args.target_cmd, num_threads=threads) elif args.reseed: optimized_corpus = out_dir if args.invoke_cmin: optimized_corpus = optimized_corpus + ".cmin" if args.invoke_tmin: optimized_corpus = optimized_corpus + ".tmin" afl_reseed(sync_dir, optimized_corpus) else: if args.dry_run: print_ok("Looking for fuzzing queues in '%s'." % sync_dir) fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False) print_ok("Found %d fuzzers, performing dry run." % len(fuzzers)) print_warn("Be patient! Depending on the corpus size this step can take hours...") # invoke dry-run on original corpus for f in fuzzers: for q_dir in f[1]: q_dir_complete = os.path.join(sync_dir, f[0], q_dir) print_ok("Processing %s..." % q_dir_complete) dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(q_dir_complete, abs_path=True) invoke_dryrun(dryrun_samples, os.path.join(sync_dir, f[0], "crashes"), os.path.join(sync_dir, f[0], "hangs"), args.target_cmd, num_threads=threads)
def parse_stat_file(stat_file, summary=True): try: f = open(stat_file, "r") lines = f.readlines() f.close() summary_stats = { 'fuzzer_pid': None, 'execs_done': None, 'execs_per_sec': None, 'paths_total': None, 'paths_favored': None, 'pending_favs': None, 'pending_total': None, 'unique_crashes': None, 'unique_hangs': None, 'afl_banner': None } complete_stats = { 'last_update': '', 'start_time': '', 'fuzzer_pid': '', 'cycles_done': '', 'execs_done': '', 'execs_per_sec': '', 'paths_total': '', 'paths_favored': '', 'paths_found': '', 'paths_imported': '', 'max_depth': '', 'cur_path': '', 'pending_favs': '', 'pending_total': '', 'variable_paths': '', 'stability': '', 'bitmap_cvg': '', 'unique_crashes': '', 'unique_hangs': '', 'last_path': '', 'last_crash': '', 'last_hang': '', 'execs_since_crash': '', 'exec_timeout': '', 'afl_banner': '', 'afl_version': '', 'command_line': '' } for l in lines: if summary: stats = summary_stats for k in stats.keys(): if k != "fuzzer_pid": if k in l: stats[k] = l[19:].strip(": \r\n") else: if k in l: stats[k] = fuzzer_alive(int(l[19:].strip(": \r\n"))) else: stats = complete_stats for k in stats.keys(): if k in l: stats[k] = l[19:].strip(": %\r\n") return stats except FileNotFoundError as e: print_warn("Stat file " + clr.GRA + "%s" % e.filename + clr.RST + " not found!") return None
def main(argv): show_info() parser = argparse.ArgumentParser( description= "afl-collect copies all crash sample files from an afl sync dir used \ by multiple fuzzers when fuzzing in parallel into a single location providing easy access for further crash analysis.", usage= "afl-collect [-d DATABASE] [-e|-g GDB_EXPL_SCRIPT_FILE] [-f LIST_FILENAME]\n \ [-h] [-j THREADS] [-m] [-r] [-rr] sync_dir collection_dir -- target_cmd") parser.add_argument( "sync_dir", help= "afl synchronisation directory crash samples will be collected from.") parser.add_argument( "collection_dir", help= "Output directory that will hold a copy of all crash samples and other generated files. \ Existing files in the collection directory will be overwritten!") parser.add_argument("-d", "--database", dest="database_file", help="Submit sample data into an sqlite3 database (\ only when used together with '-e'). afl-collect skips processing of samples already found in existing database.", default=None) parser.add_argument( "-e", "--execute-gdb-script", dest="gdb_expl_script_file", help= "Generate and execute a gdb+exploitable script after crash sample collection for crash \ classification. (Like option '-g', plus script execution.)", default=None) parser.add_argument( "-f", "--filelist", dest="list_filename", default=None, help= "Writes all collected crash sample filenames into a file in the collection directory." ) parser.add_argument( "-g", "--generate-gdb-script", dest="gdb_script_file", help= "Generate gdb script to run 'exploitable.py' on all collected crash samples. Generated \ script will be placed into collection directory.", default=None) parser.add_argument( "-j", "--threads", dest="num_threads", default=1, help= "Enable parallel analysis by specifying the number of threads afl-collect will utilize." ) parser.add_argument( "-m", "--minimize-filenames", dest="min_filename", action="store_const", const=True, default=False, help= "Minimize crash sample file names by only keeping fuzzer name and ID.") parser.add_argument( "-r", "--remove-invalid", dest="remove_invalid", action="store_const", const=True, default=False, help= "Verify collected crash samples and remove samples that do not lead to \ crashes or cause timeouts (runs 'afl-vcrash.py -r' on collection directory). This step is done prior to any script \ file execution or file list generation.") parser.add_argument( "-rr", "--remove-unexploitable", dest="remove_unexploitable", action="store_const", const=True, default=False, help="Remove crash samples that have an exploitable classification of \ 'NOT_EXPLOITABLE' or 'PROBABLY_NOT_EXPLOITABLE'. Sample file removal will take place after gdb+exploitable \ script execution. Has no effect without '-e'.") parser.add_argument( "target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \ Use '@@' to specify crash sample input file position (see afl-fuzz usage).") args = parser.parse_args(argv[1:]) sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir)) if not os.path.exists(sync_dir): print_err("No valid directory provided for <SYNC_DIR>!") return out_dir = os.path.abspath(os.path.expanduser(args.collection_dir)) if not os.path.exists(out_dir): os.makedirs(out_dir, exist_ok=True) args.target_cmd = " ".join(args.target_cmd).split() args.target_cmd[0] = os.path.abspath(os.path.expanduser( args.target_cmd[0])) if not os.path.exists(args.target_cmd[0]): print_err("Target binary not found!") return args.target_cmd = " ".join(args.target_cmd) if args.database_file: db_file = os.path.abspath(os.path.expanduser(args.database_file)) else: db_file = None print_ok("Going to collect crash samples from '%s'." % sync_dir) # initialize database if db_file: lite_db = con_sqlite.sqliteConnector(db_file) lite_db.init_database('Data', db_table_spec) else: lite_db = None fuzzers = get_fuzzer_instances(sync_dir) print_ok("Found %d fuzzers, collecting crash samples." % len(fuzzers)) sample_index = build_sample_index(sync_dir, out_dir, fuzzers, lite_db, args.min_filename) if len(sample_index.index) > 0: print_ok("Successfully indexed %d crash samples." % len(sample_index.index)) elif db_file: print_warn("No unseen samples found. Check your database for results!") return else: print_warn("No samples found. Check directory settings!") return if args.remove_invalid: from afl_utils import afl_vcrash invalid_samples, timeout_samples = afl_vcrash.verify_samples( int(args.num_threads), sample_index.inputs(), args.target_cmd, timeout_secs=10) # store invalid samples in db if args.gdb_expl_script_file and db_file: print_ok("Saving invalid sample info to database.") for sample in invalid_samples: sample_name = sample_index.outputs(input_file=sample) dataset = { 'Sample': sample_name[0], 'Classification': 'INVALID', 'Classification_Description': 'Sample does not cause a crash in the target.', 'Hash': '', 'User_Comment': '' } if not lite_db.dataset_exists('Data', dataset, ['Sample']): lite_db.insert_dataset('Data', dataset) for sample in timeout_samples: sample_name = sample_index.outputs(input_file=sample) dataset = { 'Sample': sample_name[0], 'Classification': 'TIMEOUT', 'Classification_Description': 'Sample caused a target execution timeout.', 'Hash': '', 'User_Comment': '' } if not lite_db.dataset_exists('Data', dataset, ['Sample']): lite_db.insert_dataset('Data', dataset) # remove invalid samples from sample index sample_index.remove_inputs(invalid_samples + timeout_samples) print_warn("Removed %d invalid crash samples from index." % len(invalid_samples)) print_warn("Removed %d timed out samples from index." % len(timeout_samples)) # generate gdb+exploitable script if args.gdb_expl_script_file: divided_index = sample_index.divide(int(args.num_threads)) for i in range(0, int(args.num_threads), 1): generate_gdb_exploitable_script(os.path.join( out_dir, args.gdb_expl_script_file), divided_index[i], args.target_cmd, i, intermediate=True) # execute gdb+exploitable script classification_data = execute_gdb_script(out_dir, args.gdb_expl_script_file, len(sample_index.inputs()), int(args.num_threads)) # Submit crash classification data into database if db_file: print_ok("Saving sample classification info to database.") for dataset in classification_data: if not lite_db.dataset_exists('Data', dataset, ['Sample']): lite_db.insert_dataset('Data', dataset) # de-dupe by exploitable hash seen = set() seen_add = seen.add classification_data_dedupe = [ x for x in classification_data if x['Hash'] not in seen and not seen_add(x['Hash']) ] # remove dupe samples identified by exploitable hash uninteresting_samples = [ x['Sample'] for x in classification_data if x not in classification_data_dedupe ] sample_index.remove_outputs(uninteresting_samples) print_warn( "Removed %d duplicate samples from index. Will continue with %d remaining samples." % (len(uninteresting_samples), len(sample_index.index))) # remove crash samples that are classified uninteresting if args.remove_unexploitable: classification_unexploitable = [ 'NOT_EXPLOITABLE', 'PROBABLY_NOT_EXPLOITABLE', ] uninteresting_samples = [] for c in classification_data_dedupe: if c['Classification'] in classification_unexploitable: uninteresting_samples.append(c['Sample']) sample_index.remove_outputs(uninteresting_samples) print_warn("Removed %d uninteresting crash samples from index." % len(uninteresting_samples)) # generate output gdb script generate_gdb_exploitable_script( os.path.join(out_dir, args.gdb_expl_script_file), sample_index, args.target_cmd, 0) elif args.gdb_script_file: generate_gdb_exploitable_script( os.path.join(out_dir, args.gdb_script_file), sample_index, args.target_cmd) print_ok("Copying %d samples into output directory..." % len(sample_index.index)) files_collected = copy_samples(sample_index) # generate filelist of collected crash samples if args.list_filename: generate_sample_list( os.path.abspath(os.path.expanduser(args.list_filename)), files_collected) print_ok("Generated crash sample list '%s'." % os.path.abspath(os.path.expanduser(args.list_filename))) # write db contents to file and close db connection if db_file: lite_db.commit_close()