def diff_stats(sum_stats, old_stats):
if len(sum_stats) != len(old_stats):
print_warn("Stats corrupted for '" + clr.GRA + "%s" % sum_stats["afl_banner"] + clr.RST + "'!")
return None
diff_stat = {
"fuzzers": len(sum_stats),
"fuzzer_pid": 0,
"execs_done": 0,
"execs_per_sec": 0,
"paths_total": 0,
"paths_favored": 0,
"pending_favs": 0,
"pending_total": 0,
"unique_crashes": 0,
"unique_hangs": 0,
"afl_banner": 0,
"host": socket.gethostname()[:10],
}
for k in sum_stats.keys():
if k not in ["afl_banner", "host"]:
diff_stat[k] = sum_stats[k] - old_stats[k]
else:
diff_stat[k] = sum_stats[k]
return diff_stat
def diff_stats(sum_stats, old_stats):
if len(sum_stats) != len(old_stats):
print_warn("Stats corrupted for '" + clr.GRA + "%s" % sum_stats['afl_banner'] + clr.RST + "'!")
return None
diff_stat = {
'fuzzers': len(sum_stats),
'fuzzer_pid': 0,
'execs_done': 0,
'execs_per_sec': 0,
'paths_total': 0,
'paths_favored': 0,
'pending_favs': 0,
'pending_total': 0,
'unique_crashes': 0,
'unique_hangs': 0,
'afl_banner': 0,
'host': socket.gethostname()[:10]
}
for k in sum_stats.keys():
if k not in ['afl_banner', 'host']:
diff_stat[k] = sum_stats[k] - old_stats[k]
else:
diff_stat[k] = sum_stats[k]
return diff_stat
def diff_stats(sum_stats, old_stats):
if len(sum_stats) != len(old_stats):
print_warn("Stats corrupted for '" + clr.GRA + "%s" % sum_stats['afl_banner'] + clr.RST + "'!")
return None
diff_stat = {
'fuzzers': len(sum_stats),
'fuzzer_pid': 0,
'execs_done': 0,
'execs_per_sec': 0,
'paths_total': 0,
'paths_favored': 0,
'pending_favs': 0,
'pending_total': 0,
'unique_crashes': 0,
'unique_hangs': 0,
'afl_banner': 0,
'host': socket.gethostname()[:10]
}
for k in sum_stats.keys():
if k not in ['afl_banner', 'host']:
diff_stat[k] = sum_stats[k] - old_stats[k]
else:
diff_stat[k] = sum_stats[k]
return diff_stat
def parse_stat_file(stat_file):
try:
f = open(stat_file, "r")
lines = f.readlines()
f.close()
stats = {
'fuzzer_pid': None,
'execs_done': None,
'execs_per_sec': None,
'paths_total': None,
'paths_favored': None,
'pending_favs': None,
'pending_total': None,
'unique_crashes': None,
'unique_hangs': None,
'afl_banner': None
}
for l in lines:
for k in stats.keys():
if k != "fuzzer_pid":
if k in l:
stats[k] = l[16:].strip(" \r\n")
else:
if k in l:
stats[k] = fuzzer_alive(int(l[16:].strip(" \r\n")))
return stats
except FileNotFoundError as e:
print_warn("Stat file " + clr.GRA + "%s" % e.filename + clr.RST + " not found!")
return None
def invoke_dryrun(input_files, crash_dir, timeout_dir, target_cmd, timeout=60, num_threads=1):
invalid_samples, timeout_samples = afl_vcrash.verify_samples(num_threads, input_files, target_cmd,
timeout_secs=timeout)
invalid_sample_set = set(invalid_samples+timeout_samples)
input_sample_set = set(input_files)
crashes_set = input_sample_set - invalid_sample_set
crashes = list(crashes_set)
if len(crashes) > 0:
if not os.path.exists(crash_dir):
os.makedirs(crash_dir, exist_ok=True)
for c in crashes:
shutil.move(c, os.path.join(crash_dir, os.path.basename(c)))
print_warn("Moved %d crash samples from the corpus to %s." % (len(crashes), crash_dir))
if len(timeout_samples) > 0:
if not os.path.exists(timeout_dir):
os.makedirs(timeout_dir, exist_ok=True)
for t in timeout_samples:
shutil.move(t, os.path.join(timeout_dir, os.path.basename(t)))
print_warn("Moved %d timeouts from the corpus to %s." % (len(timeout_samples), timeout_dir))
return
def load_stats(fuzzer_dir, summary=True):
fuzzer_dir = os.path.abspath(os.path.expanduser(fuzzer_dir))
if not os.path.isdir(fuzzer_dir):
print_warn("Invalid fuzzing directory specified: " + clr.GRA + "%s" % fuzzer_dir + clr.RST)
return None
fuzzer_stats = []
if os.path.isfile(os.path.join(fuzzer_dir, "fuzzer_stats")):
# single afl-fuzz job
stats = parse_stat_file(os.path.join(fuzzer_dir, "fuzzer_stats"), summary)
if stats:
fuzzer_stats.append(stats)
else:
fuzzer_inst = []
for fdir in os.listdir(fuzzer_dir):
if os.path.isdir(os.path.join(fuzzer_dir, fdir)):
fuzzer_inst.append(os.path.join(fuzzer_dir, fdir, "fuzzer_stats"))
for stat_file in fuzzer_inst:
stats = parse_stat_file(stat_file, summary)
if stats:
fuzzer_stats.append(stats)
return fuzzer_stats
def parse_stat_file(stat_file):
try:
f = open(stat_file, "r")
lines = f.readlines()
f.close()
stats = {
'fuzzer_pid': None,
'execs_done': None,
'execs_per_sec': None,
'paths_total': None,
'paths_favored': None,
'pending_favs': None,
'pending_total': None,
'unique_crashes': None,
'unique_hangs': None,
'afl_banner': None
}
for l in lines:
for k in stats.keys():
if k != "fuzzer_pid":
if k in l:
stats[k] = l[16:].strip(" \r\n")
else:
if k in l:
stats[k] = fuzzer_alive(int(l[16:].strip(" \r\n")))
return stats
except FileNotFoundError as e:
print_warn("Stat file " + clr.GRA + "%s" % e.filename + clr.RST +
" not found!")
return None
def invoke_dryrun(input_files, crash_dir, timeout_dir, target_cmd, num_threads=1):
invalid_samples, timeout_samples = afl_vcrash.verify_samples(num_threads, input_files, target_cmd)
invalid_sample_set = set(invalid_samples+timeout_samples)
input_sample_set = set(input_files)
crashes_set = input_sample_set - invalid_sample_set
crashes = list(crashes_set)
if len(crashes) > 0:
if not os.path.exists(crash_dir):
os.makedirs(crash_dir, exist_ok=True)
for c in crashes:
shutil.move(c, os.path.join(crash_dir, os.path.basename(c)))
print_warn("Moved %d crash samples from the corpus to %s." % (len(crashes), crash_dir))
if len(timeout_samples) > 0:
if not os.path.exists(timeout_dir):
os.makedirs(timeout_dir, exist_ok=True)
for t in timeout_samples:
shutil.move(t, os.path.join(timeout_dir, os.path.basename(t)))
print_warn("Moved %d timeouts from the corpus to %s." % (len(timeout_samples), timeout_dir))
return
def load_stats(fuzzer_dir):
fuzzer_dir = os.path.abspath(os.path.expanduser(fuzzer_dir))
if not os.path.isdir(fuzzer_dir):
print_warn("Invalid fuzzing directory specified: " + clr.GRA + "%s" % fuzzer_dir + clr.RST)
return None
fuzzer_stats = []
if os.path.isfile(os.path.join(fuzzer_dir, "fuzzer_stats")):
# single afl-fuzz job
stats = parse_stat_file(os.path.join(fuzzer_dir, "fuzzer_stats"))
if stats:
fuzzer_stats.append(stats)
else:
fuzzer_inst = []
for fdir in os.listdir(fuzzer_dir):
if os.path.isdir(os.path.join(fuzzer_dir, fdir)):
fuzzer_inst.append(os.path.join(fuzzer_dir, fdir, "fuzzer_stats"))
for stat_file in fuzzer_inst:
stats = parse_stat_file(stat_file)
if stats:
fuzzer_stats.append(stats)
return fuzzer_stats
def invoke_cmin(input_dir,
output_dir,
target_cmd,
mem_limit=None,
timeout=None,
qemu=False):
success = True
cmin_cmd = "afl-cmin "
if mem_limit is not None:
cmin_cmd += "-m %s " % convert_mem_limit(mem_limit)
if timeout is not None:
cmin_cmd += "-t %d " % int(timeout)
if qemu:
cmin_cmd += "-Q "
cmd = "-I %s-i %s -o %s -- %s" % (cmin_cmd, input_dir, output_dir,
target_cmd)
print_ok("Executing: %s" % cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
print_warn("afl-cmin failed with exit code %d!" % e.returncode)
success = False
return success
def fetch_stats(config_settings, twitter_inst):
doExit = False
# { 'fuzzer_dir': (stat, old_stat) }
stat_dict = dict()
while not doExit:
try:
for fuzzer in config_settings['fuzz_dirs']:
stats = load_stats(fuzzer)
if not stats:
continue
sum_stats = summarize_stats(stats)
try:
# stat_dict has already been initialized for fuzzer
# old_stat <- last_stat
old_stats = stat_dict[fuzzer][0].copy()
except KeyError:
# stat_dict has not yet been initialized for fuzzer
# old_stat <- cur_stat
old_stats = sum_stats.copy()
# initialize/update stat_dict
stat_dict[fuzzer] = (sum_stats, old_stats)
stat_change = diff_stats(sum_stats, old_stats)
if not diff_stats:
continue
print(prettify_stat(sum_stats, stat_change, True))
tweet = prettify_stat(sum_stats, stat_change, False)
l = len(tweet)
c = clr.LRD if l > 140 else clr.LGN
print_ok("Tweeting status (%s%d" % (c, l) + clr.RST +
" chars)...")
try:
twitter_inst.statuses.update(status=shorten_tweet(tweet))
except (twitter.TwitterHTTPError, URLError):
print_warn(
"Problem connecting to Twitter! Tweet not sent!")
except Exception as e:
print_err("Sending tweet failed (Reason: " + clr.GRA +
"%s" % e.__cause__ + clr.RST + ")")
if float(config_settings['interval']) < 0:
doExit = True
else:
time.sleep(float(config_settings['interval']) * 60)
except KeyboardInterrupt:
print("\b\b")
print_ok("Aborted by user. Good bye!")
doExit = True
def invoke_cmin(input_dir, output_dir, target_cmd):
success = True
cmd = "afl-cmin -i %s -o %s -- %s" % (input_dir, output_dir, target_cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
print_warn("afl-cmin failed with exit code %d!" % e.returncode)
success = False
return success
def main(argv):
show_info()
parser = argparse.ArgumentParser(description='afl-sync synchronizes fuzzer state directories between different \
locations. Supported are remote transfers through rsync that may use transport compression.',
usage='afl-sync [-S SESSION] <cmd> <src_sync_dir> <dst_storage_dir>')
parser.add_argument('cmd',
help='Command to perform: push, pull or sync. Push transmits the local state from '
'<src_sync_dir> to the destination <dst_storage_dir>. Pull fetches remote state(s) into '
'the local synchronization dir appending the \'.sync\' extension. Sync performs a '
'pull operation followed by a push.')
parser.add_argument('src_sync_dir',
help='Source afl synchronisation directory containing state directories of afl instances.')
parser.add_argument('dst_storage_dir',
help='Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!')
parser.add_argument('-S', '--session', dest='session', default=None,
help='Name of an afl-multicore session. If provided, only fuzzers belonging to '
'the specified session will be synced with the destination. Otherwise state '
'directories of all fuzzers inside the synchronisation dir will be exchanged. '
'Directories ending on \'.sync\' will never be pushed back to the destination!')
args = parser.parse_args(argv[1:])
args.cmd = args.cmd.lower()
if not args.cmd in ['push', 'pull', 'sync']:
print_err('Sorry, unknown command requested!')
sys.exit(1)
if not os.path.exists(args.src_sync_dir):
if args.cmd in ['pull', 'sync']:
print_warn('Local afl sync dir does not exist! Will create it for you!')
os.makedirs(args.src_sync_dir)
else:
print_err('Local afl sync dir does not exist!')
sys.exit(1)
server_config = {
'remote_path': args.dst_storage_dir,
}
fuzzer_config = {
'sync_dir': args.src_sync_dir,
'session': args.session,
'exclude_crashes': False,
'exclude_hangs': False,
}
rsyncEngine = AflRsync(server_config, fuzzer_config)
if args.cmd == 'push':
rsyncEngine.push()
elif args.cmd == 'pull':
rsyncEngine.pull()
elif args.cmd == 'sync':
rsyncEngine.sync()
def main(argv):
show_info()
parser = argparse.ArgumentParser(description='afl-sync synchronizes fuzzer state directories between different \
locations. Supported are remote transfers through rsync that may use transport compression.',
usage='afl-sync [-S SESSION] <cmd> <src_sync_dir> <dst_storage_dir>')
parser.add_argument('cmd',
help='Command to perform: push, pull or sync. Push transmits the local state from '
'<src_sync_dir> to the destination <dst_storage_dir>. Pull fetches remote state(s) into '
'the local synchronization dir appending the \'.sync\' extension. Sync performs a '
'pull operation followed by a push.')
parser.add_argument('src_sync_dir',
help='Source afl synchronisation directory containing state directories of afl instances.')
parser.add_argument('dst_storage_dir',
help='Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!')
parser.add_argument('-S', '--session', dest='session', default=None,
help='Name of an afl-multicore session. If provided, only fuzzers belonging to '
'the specified session will be synced with the destination. Otherwise state '
'directories of all fuzzers inside the synchronisation dir will be exchanged. '
'Directories ending on \'.sync\' will never be pushed back to the destination!')
args = parser.parse_args(argv[1:])
args.cmd = args.cmd.lower()
if not args.cmd in ['push', 'pull', 'sync']:
print_err('Sorry, unknown command requested!')
sys.exit(1)
if not os.path.exists(args.src_sync_dir):
if args.cmd in ['pull', 'sync']:
print_warn('Local afl sync dir does not exist! Will create it for you!')
os.makedirs(args.src_sync_dir)
else:
print_err('Local afl sync dir does not exist!')
sys.exit(1)
server_config = {
'remote_path': args.dst_storage_dir,
}
fuzzer_config = {
'sync_dir': args.src_sync_dir,
'session': args.session,
'exclude_crashes': False,
'exclude_hangs': False,
}
rsyncEngine = AflRsync(server_config, fuzzer_config)
if args.cmd == 'push':
rsyncEngine.push()
elif args.cmd == 'pull':
rsyncEngine.pull()
elif args.cmd == 'sync':
rsyncEngine.sync()
def fetch_stats(config_settings, twitter_inst):
doExit = False
# { 'fuzzer_dir': (stat, old_stat) }
stat_dict = dict()
while not doExit:
try:
for fuzzer in config_settings["fuzz_dirs"]:
stats = load_stats(fuzzer)
if not stats:
continue
sum_stats = summarize_stats(stats)
try:
# stat_dict has already been initialized for fuzzer
# old_stat <- last_stat
old_stats = stat_dict[fuzzer][0].copy()
except KeyError:
# stat_dict has not yet been initialized for fuzzer
# old_stat <- cur_stat
old_stats = sum_stats.copy()
# initialize/update stat_dict
stat_dict[fuzzer] = (sum_stats, old_stats)
stat_change = diff_stats(sum_stats, old_stats)
if not diff_stats:
continue
print(prettify_stat(sum_stats, stat_change, True))
tweet = prettify_stat(sum_stats, stat_change, False)
l = len(tweet)
c = clr.LRD if l > 140 else clr.LGN
print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...")
try:
twitter_inst.statuses.update(status=shorten_tweet(tweet))
except (twitter.TwitterHTTPError, URLError):
print_warn("Problem connecting to Twitter! Tweet not sent!")
except Exception as e:
print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")")
if float(config_settings["interval"]) < 0:
doExit = True
else:
time.sleep(float(config_settings["interval"]) * 60)
except KeyboardInterrupt:
print("\b\b")
print_ok("Aborted by user. Good bye!")
doExit = True
def kill_session(session):
if os.path.isfile("/tmp/afl_multicore.PGID.%s" % session):
f = open("/tmp/afl_multicore.PGID.%s" % session)
pgids = f.readlines()
for pgid in pgids:
try:
print_ok("Killing jobs with PGID %s" % pgid.strip("\r\n"))
os.killpg(int(pgid), signal.SIGTERM)
except ProcessLookupError:
print_warn("No processes with PGID %s found!" % (pgid.strip("\r\n")))
f.close()
os.remove("/tmp/afl_multicore.PGID.%s" % session)
else:
print_err("PGID file '/tmp/afl_multicore.PGID.%s' not found! Aborting!" % session)
def invoke_cmin(input_dir, output_dir, target_cmd, mem_limit=None, timeout=None):
success = True
cmin_cmd = "afl-cmin "
if mem_limit is not None:
cmin_cmd += "-m %d " % int(mem_limit)
if timeout is not None:
cmin_cmd += "-t %d " % int(timeout)
cmd = "%s-i %s -o %s -- %s" % (cmin_cmd, input_dir, output_dir, target_cmd)
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError as e:
print_warn("afl-cmin failed with exit code %d!" % e.returncode)
success = False
return success
def kill_session(session):
if os.path.isfile("/tmp/afl_multicore.PGID.%s" % session):
f = open("/tmp/afl_multicore.PGID.%s" % session)
pgids = f.readlines()
for pgid in pgids:
try:
print_ok("Killing jobs with PGID %s" % pgid.strip('\r\n'))
os.killpg(int(pgid), signal.SIGTERM)
except ProcessLookupError:
print_warn("No processes with PGID %s found!" %
(pgid.strip('\r\n')))
f.close()
os.remove("/tmp/afl_multicore.PGID.%s" % session)
else:
print_err(
"PGID file '/tmp/afl_multicore.PGID.%s' not found! Aborting!" %
session)
sys.exit(1)
def fetch_stats(config_settings, twitter_inst):
stat_dict = dict()
for fuzzer in config_settings['fuzz_dirs']:
stats = load_stats(fuzzer)
if not stats:
continue
sum_stats = summarize_stats(stats)
try:
with open('.afl_stats.{}'.format(os.path.basename(fuzzer)),
'r') as f:
old_stats = json.load(f)
except FileNotFoundError:
old_stats = sum_stats.copy()
# initialize/update stat_dict
stat_dict[fuzzer] = (sum_stats, old_stats)
stat_change = diff_stats(sum_stats, old_stats)
with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'w') as f:
json.dump(sum_stats, f)
print(prettify_stat(sum_stats, stat_change, True))
tweet = prettify_stat(sum_stats, stat_change, False)
l = len(tweet)
c = clr.LRD if l > 140 else clr.LGN
if twitter_inst:
print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...")
try:
twitter_inst.statuses.update(status=shorten_tweet(tweet))
except (twitter.TwitterHTTPError, URLError):
print_warn("Problem connecting to Twitter! Tweet not sent!")
except Exception as e:
print_err("Sending tweet failed (Reason: " + clr.GRA +
"%s" % e.__cause__ + clr.RST + ")")
def fetch_stats(config_settings, twitter_inst):
stat_dict = dict()
for fuzzer in config_settings['fuzz_dirs']:
stats = load_stats(fuzzer)
if not stats:
continue
sum_stats = summarize_stats(stats)
try:
with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'r') as f:
old_stats = json.load(f)
except FileNotFoundError:
old_stats = sum_stats.copy()
# initialize/update stat_dict
stat_dict[fuzzer] = (sum_stats, old_stats)
stat_change = diff_stats(sum_stats, old_stats)
with open('.afl_stats.{}'.format(os.path.basename(fuzzer)), 'w') as f:
json.dump(sum_stats, f)
print(prettify_stat(sum_stats, stat_change, True))
tweet = prettify_stat(sum_stats, stat_change, False)
l = len(tweet)
c = clr.LRD if l > 140 else clr.LGN
print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...")
try:
twitter_inst.statuses.update(status=shorten_tweet(tweet))
except (twitter.TwitterHTTPError, URLError):
print_warn("Problem connecting to Twitter! Tweet not sent!")
except Exception as e:
print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")")
def main(argv):
show_info()
parser = argparse.ArgumentParser(description="afl-minimize performs several optimization steps to reduce the size\n \
of an afl-fuzz corpus.",
usage="afl-minimize [-c COLLECTION_DIR [--cmin [opts]] [--tmin [opts]]] [-d] [-h]\n \
[-j] sync_dir -- target_cmd\n")
parser.add_argument("-c", "--collect", dest="collection_dir",
help="Collect all samples from the synchronisation dir and store them in the collection dir. \
Existing files in the collection directory will be overwritten!", default=None)
parser.add_argument("--cmin", dest="invoke_cmin", action="store_const", const=True,
default=False, help="Run afl-cmin on collection dir. Has no effect without '-c'.")
parser.add_argument("--cmin-mem-limit", dest="cmin_mem_limit", default=None, help="Set memory limit for afl-cmin.")
parser.add_argument("--cmin-timeout", dest="cmin_timeout", default=None, help="Set timeout for afl-cmin.")
parser.add_argument("--tmin", dest="invoke_tmin", action="store_const", const=True,
default=False, help="Run afl-tmin on minimized collection dir if used together with '--cmin'\
or on unoptimized collection dir otherwise. Has no effect without '-c'.")
parser.add_argument("--tmin-mem-limit", dest="tmin_mem_limit", default=None, help="Set memory limit for afl-tmin.")
parser.add_argument("--tmin-timeout", dest="tmin_timeout", default=None, help="Set timeout for afl-tmin.")
parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_const", const=True,
default=False, help="Perform dry-run on collection dir, if '-c' is provided or on \
synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus.")
parser.add_argument("-j", "--threads", dest="num_threads", default=1,
help="Enable parallel dry-run and t-minimization step by specifying the number of threads \
afl-minimize will utilize.")
parser.add_argument("sync_dir", help="afl synchronisation directory containing multiple fuzzers and their queues.")
parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
if not args.collection_dir and not args.dry_run:
print_err("No operation requested. You should at least provide '-c'")
print_err("for sample collection or '-d' for a dry-run. Use '--help' for")
print_err("usage instructions or checkout README.md for details.")
return
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
if not args.num_threads:
threads = 1
else:
threads = int(args.num_threads)
if args.collection_dir:
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0:
os.makedirs(out_dir, exist_ok=True)
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
# collect samples from fuzzer queues
print_ok("Found %d fuzzers, collecting samples." % len(fuzzers))
sample_index = afl_collect.build_sample_index(sync_dir, out_dir, fuzzers)
print_ok("Successfully indexed %d samples." % len(sample_index.index))
print_ok("Copying %d samples into collection directory..." % len(sample_index.index))
afl_collect.copy_samples(sample_index)
else:
print_warn("Collection directory exists and is not empty!")
print_warn("Skipping collection step...")
if args.invoke_cmin:
# invoke cmin on collection
print_ok("Executing: afl-cmin -i %s -o %s.cmin -- %s" % (out_dir, out_dir, args.target_cmd))
invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd, mem_limit=args.cmin_mem_limit,
timeout=args.cmin_timeout)
if args.invoke_tmin:
# invoke tmin on minimized collection
print_ok("Executing: afl-tmin -i %s.cmin/* -o %s.cmin.tmin/* -- %s" % (out_dir, out_dir,
args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.cmin.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout)
elif args.invoke_tmin:
# invoke tmin on collection
print_ok("Executing: afl-tmin -i %s/* -o %s.tmin/* -- %s" % (out_dir, out_dir, args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout)
if args.dry_run:
# invoke dry-run on collected/minimized corpus
if args.invoke_cmin and args.invoke_tmin:
print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.tmin.crashes" % out_dir, "%s.cmin.tmin.hangs" % out_dir,
args.target_cmd, num_threads=threads)
elif args.invoke_cmin:
print_ok("Performing dry-run in %s.cmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.crashes" % out_dir, "%s.cmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
elif args.invoke_tmin:
print_ok("Performing dry-run in %s.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.tmin.crashes" % out_dir, "%s.tmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
else:
print_ok("Performing dry-run in %s..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_dryrun(dryrun_samples, "%s.crashes" % out_dir, "%s.hangs" % out_dir, args.target_cmd,
num_threads=threads)
else:
if args.dry_run:
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
print_ok("Found %d fuzzers, performing dry run." % len(fuzzers))
print_warn("Be patient! Depending on the corpus size this step can take hours...")
# invoke dry-run on original corpus
for f in fuzzers:
for q_dir in f[1]:
q_dir_complete = os.path.join(sync_dir, f[0], q_dir)
print_ok("Processing %s..." % q_dir_complete)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(q_dir_complete, abs_path=True)
invoke_dryrun(dryrun_samples, os.path.join(sync_dir, f[0], "crashes"),
os.path.join(sync_dir, f[0], "hangs"), args.target_cmd, num_threads=threads)
def main(argv):
show_info()
parser = argparse.ArgumentParser(description="Post selected contents of fuzzer_stats to Twitter.",
usage="afl-stats [-h] [-c config]\n")
parser.add_argument("-c", "--config", dest="config_file",
help="afl-stats config file (Default: afl-stats.conf)!", default="afl-stats.conf")
args = parser.parse_args(argv[1:])
config_settings = read_config(args.config_file)
twitter_inst = twitter_init()
doExit = False
# { 'fuzzer_dir': (stat, old_stat) }
stat_dict = dict()
while not doExit:
try:
for fuzzer in config_settings['fuzz_dirs']:
stats = load_stats(fuzzer)
if not stats:
continue
sum_stats = summarize_stats(stats)
try:
# stat_dict has already been initialized for fuzzer
# old_stat <- last_stat
old_stats = stat_dict[fuzzer][0].copy()
except KeyError:
# stat_dict has not yet been initialized for fuzzer
# old_stat <- cur_stat
old_stats = sum_stats.copy()
# initialize/update stat_dict
stat_dict[fuzzer] = (sum_stats, old_stats)
stat_change = diff_stats(sum_stats, old_stats)
if not diff_stats:
continue
print(prettify_stat(sum_stats, stat_change, True))
tweet = prettify_stat(sum_stats, stat_change, False)
l = len(tweet)
c = clr.LRD if l>140 else clr.LGN
print_ok("Tweeting status (%s%d" % (c, l) + clr.RST + " chars)...")
try:
twitter_inst.statuses.update(status=shorten_tweet(tweet))
except (twitter.TwitterHTTPError, URLError):
print_warn("Problem connecting to Twitter! Tweet not sent!")
except Exception as e:
print_err("Sending tweet failed (Reason: " + clr.GRA + "%s" % e.__cause__ + clr.RST + ")")
if float(config_settings['interval']) < 0:
doExit = True
else:
time.sleep(float(config_settings['interval'])*60)
except KeyboardInterrupt:
print("\b\b")
print_ok("Aborted by user. Good bye!")
doExit = True
def main(argv):
show_info()
parser = argparse.ArgumentParser(description="afl-collect copies all crash sample files from an afl sync dir used \
by multiple fuzzers when fuzzing in parallel into a single location providing easy access for further crash analysis.",
usage="afl-collect [-d DATABASE] [-e|-g GDB_EXPL_SCRIPT_FILE] [-f LIST_FILENAME]\n \
[-h] [-j THREADS] [-m] [-r] [-rr] sync_dir collection_dir -- target_cmd")
parser.add_argument("sync_dir", help="afl synchronisation directory crash samples will be collected from.")
parser.add_argument("collection_dir",
help="Output directory that will hold a copy of all crash samples and other generated files. \
Existing files in the collection directory will be overwritten!")
parser.add_argument("-d", "--database", dest="database_file", help="Submit sample data into an sqlite3 database (\
only when used together with '-e'). afl-collect skips processing of samples already found in existing database.",
default=None)
parser.add_argument("-e", "--execute-gdb-script", dest="gdb_expl_script_file",
help="Generate and execute a gdb+exploitable script after crash sample collection for crash \
classification. (Like option '-g', plus script execution.)",
default=None)
parser.add_argument("-f", "--filelist", dest="list_filename", default=None,
help="Writes all collected crash sample filenames into a file in the collection directory.")
parser.add_argument("-g", "--generate-gdb-script", dest="gdb_script_file",
help="Generate gdb script to run 'exploitable.py' on all collected crash samples. Generated \
script will be placed into collection directory.", default=None)
parser.add_argument("-j", "--threads", dest="num_threads", default=1,
help="Enable parallel analysis by specifying the number of threads afl-collect will utilize.")
parser.add_argument("-m", "--minimize-filenames", dest="min_filename", action="store_const", const=True,
default=False, help="Minimize crash sample file names by only keeping fuzzer name and ID.")
parser.add_argument("-r", "--remove-invalid", dest="remove_invalid", action="store_const", const=True,
default=False, help="Verify collected crash samples and remove samples that do not lead to \
crashes (runs 'afl-vcrash.py -r' on collection directory). This step is done prior to any script file \
or file list generation/execution.")
parser.add_argument("-rr", "--remove-unexploitable", dest="remove_unexploitable", action="store_const", const=True,
default=False, help="Remove crash samples that have an exploitable classification of \
'NOT_EXPLOITABLE' or 'PROBABLY_NOT_EXPLOITABLE'. Sample file removal will take place after gdb+exploitable \
script execution. Has no effect without '-e'.")
parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
if args.collection_dir:
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir):
os.makedirs(out_dir, exist_ok=True)
else:
print_err("No valid directory provided for <OUT_DIR>!")
return
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
if args.database_file:
db_file = os.path.abspath(os.path.expanduser(args.database_file))
else:
db_file = None
print_ok("Going to collect crash samples from '%s'." % sync_dir)
# initialize database
if db_file:
lite_db = con_sqlite.sqliteConnector(db_file)
lite_db.init_database()
else:
lite_db = None
fuzzers = get_fuzzer_instances(sync_dir)
print_ok("Found %d fuzzers, collecting crash samples." % len(fuzzers))
sample_index = build_sample_index(sync_dir, out_dir, fuzzers, lite_db, args.min_filename)
if len(sample_index.index) > 0:
print_ok("Successfully indexed %d crash samples." % len(sample_index.index))
elif db_file:
print_warn("No unseen samples found. Check your database for results!")
return
else:
print_warn("No samples found. Check directory settings!")
return
if args.remove_invalid:
from afl_utils import afl_vcrash
invalid_samples = afl_vcrash.verify_samples(int(args.num_threads), sample_index.inputs(), args.target_cmd)
# store invalid samples in db
print_ok("Saving invalid sample info to database.")
if args.gdb_expl_script_file and db_file:
for sample in invalid_samples:
sample_name = sample_index.outputs(input_file=sample)
dataset = {'sample': sample_name[0]['output'], 'classification': 'INVALID',
'description': 'Sample does not cause a crash in the target.', 'hash': ''}
if not lite_db.dataset_exists(dataset):
lite_db.insert_dataset(dataset)
# remove invalid samples from sample index
sample_index.remove_inputs(invalid_samples)
print_warn("Removed %d invalid crash samples from index." % len(invalid_samples))
# generate gdb+exploitable script
if args.gdb_expl_script_file:
divided_index = sample_index.divide(int(args.num_threads))
for i in range(0, int(args.num_threads), 1):
generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_expl_script_file), divided_index[i],
args.target_cmd, i, intermediate=True)
# execute gdb+exploitable script
classification_data = execute_gdb_script(out_dir, args.gdb_expl_script_file, len(sample_index.inputs()),
int(args.num_threads))
# Submit crash classification data into database
print_ok("Saving sample classification info to database.")
if db_file:
for dataset in classification_data:
if not lite_db.dataset_exists(dataset):
lite_db.insert_dataset(dataset)
# de-dupe by exploitable hash
seen = set()
seen_add = seen.add
classification_data_dedupe = [x for x in classification_data
if x['hash'] not in seen and not seen_add(x['hash'])]
# remove dupe samples identified by exploitable hash
uninteresting_samples = [x['sample'] for x in classification_data
if x not in classification_data_dedupe]
sample_index.remove_outputs(uninteresting_samples)
print_warn("Removed %d duplicate samples from index. Will continue with %d remaining samples." %
(len(uninteresting_samples), len(sample_index.index)))
# remove crash samples that are classified uninteresting
if args.remove_unexploitable:
classification_unexploitable = [
'NOT_EXPLOITABLE',
'PROBABLY_NOT_EXPLOITABLE',
]
uninteresting_samples = []
for c in classification_data_dedupe:
if c['classification'] in classification_unexploitable:
uninteresting_samples.append(c['sample'])
sample_index.remove_outputs(uninteresting_samples)
print_warn("Removed %d uninteresting crash samples from index." % len(uninteresting_samples))
# generate output gdb script
generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_expl_script_file), sample_index,
args.target_cmd, 0)
elif args.gdb_script_file:
generate_gdb_exploitable_script(os.path.join(out_dir, args.gdb_script_file), sample_index, args.target_cmd)
print_ok("Copying %d samples into output directory..." % len(sample_index.index))
files_collected = copy_samples(sample_index)
# generate filelist of collected crash samples
if args.list_filename:
generate_sample_list(os.path.abspath(os.path.expanduser(args.list_filename)), files_collected)
print_ok("Generated crash sample list '%s'." % os.path.abspath(os.path.expanduser(args.list_filename)))
def main(argv):
show_info()
parser = argparse.ArgumentParser(
description=
'afl-sync synchronizes fuzzer state directories between different \
locations. Supported are remote transfers through rsync that may use transport compression.',
usage='afl-sync [-S SESSION] <cmd> <src_sync_dir> <dst_storage_dir>')
parser.add_argument(
'cmd',
help=
'Command to perform: push, pull or sync. Push transmits the local state from '
'<src_sync_dir> to the destination <dst_storage_dir>. Pull fetches remote state(s) into '
'the local synchronization dir appending the \'.sync\' extension. Sync performs a '
'pull operation followed by a push.')
parser.add_argument(
'src_sync_dir',
help=
'Source afl synchronisation directory containing state directories of afl instances.'
)
parser.add_argument(
'dst_storage_dir',
help=
'Destination directory used as fuzzer state storage. This shouldn\'t be an afl sync dir!'
)
parser.add_argument(
'--chmod',
help=
'Affect destination\'s file and directory permissions, e.g. --chmod=g+rw to add '
'read/write group permissions.',
metavar='PERMS')
parser.add_argument(
'--chown',
help=
'Affect destination\'s file and directory user and group, e.g. --chown=foo:bar to '
'let the files be owned by user foo and group bar.',
metavar='USER:GROUP')
parser.add_argument(
'-S',
'--session',
dest='session',
default=None,
help=
'Name of an afl-multicore session. If provided, only fuzzers belonging to '
'the specified session will be synced with the destination. Otherwise state '
'directories of all fuzzers inside the synchronisation dir will be exchanged. '
'Directories ending on \'.sync\' will never be pushed back to the destination!'
)
args = parser.parse_args(argv[1:])
args.cmd = args.cmd.lower()
if not args.cmd in ['push', 'pull', 'sync']:
print_err('Sorry, unknown command requested!')
sys.exit(1)
rsync_put_options = _rsync_default_options[:]
rsync_get_options = _rsync_default_options[:]
if args.chmod or args.chown:
# these arguments are meaningless with pull since they should only
# affect the remote side
if args.cmd == 'pull':
print_warn(
'--chmod and --chown have no effect with pull and will be ignored.'
)
if args.chmod:
rsync_put_options.append('--chmod={}'.format(args.chmod))
if args.chown:
rsync_put_options.append('--protect-args')
rsync_put_options.append('--chown={}'.format(args.chown))
if not os.path.exists(args.src_sync_dir):
if args.cmd in ['pull', 'sync']:
print_warn(
'Local afl sync dir does not exist! Will create it for you!')
os.makedirs(args.src_sync_dir)
else:
print_err('Local afl sync dir does not exist!')
sys.exit(1)
server_config = {
'remote_path': args.dst_storage_dir,
}
fuzzer_config = {
'sync_dir': args.src_sync_dir,
'session': args.session,
'exclude_crashes': False,
'exclude_hangs': False,
}
rsync_config = {
'get': rsync_get_options,
'put': rsync_put_options,
}
rsyncEngine = AflRsync(server_config, fuzzer_config, rsync_config)
if args.cmd == 'push':
rsyncEngine.push()
elif args.cmd == 'pull':
rsyncEngine.pull()
elif args.cmd == 'sync':
rsyncEngine.sync()
def main(argv):
show_info()
parser = argparse.ArgumentParser(description="afl-minimize performs several optimization steps to reduce the size\n \
of an afl-fuzz corpus.",
usage="afl-minimize [-c COLLECTION_DIR [--cmin [opts]] [--tmin [opts]]] [--reseed]\n \
[-d] [-h] [-j] sync_dir -- target_cmd\n")
parser.add_argument("-c", "--collect", dest="collection_dir",
help="Collect all samples from the synchronisation dir and store them in the collection dir.",
default=None)
parser.add_argument("--cmin", dest="invoke_cmin", action="store_const", const=True,
default=False, help="Run afl-cmin on collection dir. Has no effect without '-c'.")
parser.add_argument("--cmin-mem-limit", dest="cmin_mem_limit", default=None, help="Set memory limit for afl-cmin.")
parser.add_argument("--cmin-timeout", dest="cmin_timeout", default=None, help="Set timeout for afl-cmin.")
parser.add_argument("--cmin-qemu", dest="cmin_qemu", default=False, action="store_const", const=True,
help="Enable qemu mode afl-cmin.")
parser.add_argument("--reseed", dest="reseed", default=False, action="store_const", const=True, help="Reseed afl-fuzz with the \
collected (and optimized) corpus. This replaces all sync_dir queues with the newly generated corpus.")
parser.add_argument("--tmin", dest="invoke_tmin", action="store_const", const=True,
default=False, help="Run afl-tmin on minimized collection dir if used together with '--cmin'\
or on unoptimized collection dir otherwise. Has no effect without '-c'.")
parser.add_argument("--tmin-mem-limit", dest="tmin_mem_limit", default=None, help="Set memory limit for afl-tmin.")
parser.add_argument("--tmin-timeout", dest="tmin_timeout", default=None, help="Set timeout for afl-tmin.")
parser.add_argument("--tmin-qemu", dest="tmin_qemu", default=False, action="store_const", const=True,
help="Enable qemu mode afl-tmin.")
parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_const", const=True,
default=False, help="Perform dry-run on collection dir, if '-c' is provided or on \
synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus.")
parser.add_argument("-j", "--threads", dest="num_threads", default=1,
help="Enable parallel dry-run and t-minimization step by specifying the number of threads \
afl-minimize will utilize.")
parser.add_argument("sync_dir", help="afl synchronisation directory containing multiple fuzzers and their queues.")
parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
if not args.collection_dir and not args.dry_run:
print_err("No operation requested. You should at least provide '-c'")
print_err("for sample collection or '-d' for a dry-run. Use '--help' for")
print_err("usage instructions or checkout README.md for details.")
return
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
threads = int(args.num_threads)
if args.collection_dir:
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0:
os.makedirs(out_dir, exist_ok=True)
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
# collect samples from fuzzer queues
print_ok("Found %d fuzzers, collecting samples." % len(fuzzers))
sample_index = afl_collect.build_sample_index(sync_dir, out_dir, fuzzers, omit_fuzzer_name=True)
print_ok("Successfully indexed %d samples." % len(sample_index.index))
print_ok("Copying %d samples into collection directory..." % len(sample_index.index))
afl_collect.copy_samples(sample_index)
else:
print_warn("Collection directory exists and is not empty!")
print_warn("Skipping collection step...")
if args.invoke_cmin:
# invoke cmin on collection
invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd, mem_limit=args.cmin_mem_limit,
timeout=args.cmin_timeout, qemu=args.cmin_qemu)
if args.invoke_tmin:
# invoke tmin on minimized collection
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.cmin.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout, qemu=args.tmin_qemu)
elif args.invoke_tmin:
# invoke tmin on collection
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout, qemu=args.tmin_qemu)
if args.dry_run:
# invoke dry-run on collected/minimized corpus
if args.invoke_cmin and args.invoke_tmin:
print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.tmin.crashes" % out_dir, "%s.cmin.tmin.hangs" % out_dir,
args.target_cmd, num_threads=threads)
elif args.invoke_cmin:
print_ok("Performing dry-run in %s.cmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.crashes" % out_dir, "%s.cmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
elif args.invoke_tmin:
print_ok("Performing dry-run in %s.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.tmin.crashes" % out_dir, "%s.tmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
else:
print_ok("Performing dry-run in %s..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_dryrun(dryrun_samples, "%s.crashes" % out_dir, "%s.hangs" % out_dir, args.target_cmd,
num_threads=threads)
elif args.reseed:
optimized_corpus = out_dir
if args.invoke_cmin:
optimized_corpus = optimized_corpus + ".cmin"
if args.invoke_tmin:
optimized_corpus = optimized_corpus + ".tmin"
afl_reseed(sync_dir, optimized_corpus)
else:
if args.dry_run:
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
print_ok("Found %d fuzzers, performing dry run." % len(fuzzers))
print_warn("Be patient! Depending on the corpus size this step can take hours...")
# invoke dry-run on original corpus
for f in fuzzers:
for q_dir in f[1]:
q_dir_complete = os.path.join(sync_dir, f[0], q_dir)
print_ok("Processing %s..." % q_dir_complete)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(q_dir_complete, abs_path=True)
invoke_dryrun(dryrun_samples, os.path.join(sync_dir, f[0], "crashes"),
os.path.join(sync_dir, f[0], "hangs"), args.target_cmd, num_threads=threads)
def parse_stat_file(stat_file, summary=True):
try:
f = open(stat_file, "r")
lines = f.readlines()
f.close()
summary_stats = {
'fuzzer_pid': None,
'execs_done': None,
'execs_per_sec': None,
'paths_total': None,
'paths_favored': None,
'pending_favs': None,
'pending_total': None,
'unique_crashes': None,
'unique_hangs': None,
'afl_banner': None
}
complete_stats = {
'last_update': '',
'start_time': '',
'fuzzer_pid': '',
'cycles_done': '',
'execs_done': '',
'execs_per_sec': '',
'paths_total': '',
'paths_favored': '',
'paths_found': '',
'paths_imported': '',
'max_depth': '',
'cur_path': '',
'pending_favs': '',
'pending_total': '',
'variable_paths': '',
'stability': '',
'bitmap_cvg': '',
'unique_crashes': '',
'unique_hangs': '',
'last_path': '',
'last_crash': '',
'last_hang': '',
'execs_since_crash': '',
'exec_timeout': '',
'afl_banner': '',
'afl_version': '',
'command_line': ''
}
for l in lines:
if summary:
stats = summary_stats
for k in stats.keys():
if k != "fuzzer_pid":
if k in l:
stats[k] = l[19:].strip(": \r\n")
else:
if k in l:
stats[k] = fuzzer_alive(int(l[19:].strip(": \r\n")))
else:
stats = complete_stats
for k in stats.keys():
if k in l:
stats[k] = l[19:].strip(": %\r\n")
return stats
except FileNotFoundError as e:
print_warn("Stat file " + clr.GRA + "%s" % e.filename + clr.RST + " not found!")
return None
def main(argv):
show_info()
parser = argparse.ArgumentParser(
description=
"afl-collect copies all crash sample files from an afl sync dir used \
by multiple fuzzers when fuzzing in parallel into a single location providing easy access for further crash analysis.",
usage=
"afl-collect [-d DATABASE] [-e|-g GDB_EXPL_SCRIPT_FILE] [-f LIST_FILENAME]\n \
[-h] [-j THREADS] [-m] [-r] [-rr] sync_dir collection_dir -- target_cmd")
parser.add_argument(
"sync_dir",
help=
"afl synchronisation directory crash samples will be collected from.")
parser.add_argument(
"collection_dir",
help=
"Output directory that will hold a copy of all crash samples and other generated files. \
Existing files in the collection directory will be overwritten!")
parser.add_argument("-d",
"--database",
dest="database_file",
help="Submit sample data into an sqlite3 database (\
only when used together with '-e'). afl-collect skips processing of samples already found in existing database.",
default=None)
parser.add_argument(
"-e",
"--execute-gdb-script",
dest="gdb_expl_script_file",
help=
"Generate and execute a gdb+exploitable script after crash sample collection for crash \
classification. (Like option '-g', plus script execution.)",
default=None)
parser.add_argument(
"-f",
"--filelist",
dest="list_filename",
default=None,
help=
"Writes all collected crash sample filenames into a file in the collection directory."
)
parser.add_argument(
"-g",
"--generate-gdb-script",
dest="gdb_script_file",
help=
"Generate gdb script to run 'exploitable.py' on all collected crash samples. Generated \
script will be placed into collection directory.",
default=None)
parser.add_argument(
"-j",
"--threads",
dest="num_threads",
default=1,
help=
"Enable parallel analysis by specifying the number of threads afl-collect will utilize."
)
parser.add_argument(
"-m",
"--minimize-filenames",
dest="min_filename",
action="store_const",
const=True,
default=False,
help=
"Minimize crash sample file names by only keeping fuzzer name and ID.")
parser.add_argument(
"-r",
"--remove-invalid",
dest="remove_invalid",
action="store_const",
const=True,
default=False,
help=
"Verify collected crash samples and remove samples that do not lead to \
crashes or cause timeouts (runs 'afl-vcrash.py -r' on collection directory). This step is done prior to any script \
file execution or file list generation.")
parser.add_argument(
"-rr",
"--remove-unexploitable",
dest="remove_unexploitable",
action="store_const",
const=True,
default=False,
help="Remove crash samples that have an exploitable classification of \
'NOT_EXPLOITABLE' or 'PROBABLY_NOT_EXPLOITABLE'. Sample file removal will take place after gdb+exploitable \
script execution. Has no effect without '-e'.")
parser.add_argument(
"target_cmd",
nargs="+",
help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir):
os.makedirs(out_dir, exist_ok=True)
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(
args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
if args.database_file:
db_file = os.path.abspath(os.path.expanduser(args.database_file))
else:
db_file = None
print_ok("Going to collect crash samples from '%s'." % sync_dir)
# initialize database
if db_file:
lite_db = con_sqlite.sqliteConnector(db_file)
lite_db.init_database('Data', db_table_spec)
else:
lite_db = None
fuzzers = get_fuzzer_instances(sync_dir)
print_ok("Found %d fuzzers, collecting crash samples." % len(fuzzers))
sample_index = build_sample_index(sync_dir, out_dir, fuzzers, lite_db,
args.min_filename)
if len(sample_index.index) > 0:
print_ok("Successfully indexed %d crash samples." %
len(sample_index.index))
elif db_file:
print_warn("No unseen samples found. Check your database for results!")
return
else:
print_warn("No samples found. Check directory settings!")
return
if args.remove_invalid:
from afl_utils import afl_vcrash
invalid_samples, timeout_samples = afl_vcrash.verify_samples(
int(args.num_threads),
sample_index.inputs(),
args.target_cmd,
timeout_secs=10)
# store invalid samples in db
if args.gdb_expl_script_file and db_file:
print_ok("Saving invalid sample info to database.")
for sample in invalid_samples:
sample_name = sample_index.outputs(input_file=sample)
dataset = {
'Sample': sample_name[0],
'Classification': 'INVALID',
'Classification_Description':
'Sample does not cause a crash in the target.',
'Hash': '',
'User_Comment': ''
}
if not lite_db.dataset_exists('Data', dataset, ['Sample']):
lite_db.insert_dataset('Data', dataset)
for sample in timeout_samples:
sample_name = sample_index.outputs(input_file=sample)
dataset = {
'Sample': sample_name[0],
'Classification': 'TIMEOUT',
'Classification_Description':
'Sample caused a target execution timeout.',
'Hash': '',
'User_Comment': ''
}
if not lite_db.dataset_exists('Data', dataset, ['Sample']):
lite_db.insert_dataset('Data', dataset)
# remove invalid samples from sample index
sample_index.remove_inputs(invalid_samples + timeout_samples)
print_warn("Removed %d invalid crash samples from index." %
len(invalid_samples))
print_warn("Removed %d timed out samples from index." %
len(timeout_samples))
# generate gdb+exploitable script
if args.gdb_expl_script_file:
divided_index = sample_index.divide(int(args.num_threads))
for i in range(0, int(args.num_threads), 1):
generate_gdb_exploitable_script(os.path.join(
out_dir, args.gdb_expl_script_file),
divided_index[i],
args.target_cmd,
i,
intermediate=True)
# execute gdb+exploitable script
classification_data = execute_gdb_script(out_dir,
args.gdb_expl_script_file,
len(sample_index.inputs()),
int(args.num_threads))
# Submit crash classification data into database
if db_file:
print_ok("Saving sample classification info to database.")
for dataset in classification_data:
if not lite_db.dataset_exists('Data', dataset, ['Sample']):
lite_db.insert_dataset('Data', dataset)
# de-dupe by exploitable hash
seen = set()
seen_add = seen.add
classification_data_dedupe = [
x for x in classification_data
if x['Hash'] not in seen and not seen_add(x['Hash'])
]
# remove dupe samples identified by exploitable hash
uninteresting_samples = [
x['Sample'] for x in classification_data
if x not in classification_data_dedupe
]
sample_index.remove_outputs(uninteresting_samples)
print_warn(
"Removed %d duplicate samples from index. Will continue with %d remaining samples."
% (len(uninteresting_samples), len(sample_index.index)))
# remove crash samples that are classified uninteresting
if args.remove_unexploitable:
classification_unexploitable = [
'NOT_EXPLOITABLE',
'PROBABLY_NOT_EXPLOITABLE',
]
uninteresting_samples = []
for c in classification_data_dedupe:
if c['Classification'] in classification_unexploitable:
uninteresting_samples.append(c['Sample'])
sample_index.remove_outputs(uninteresting_samples)
print_warn("Removed %d uninteresting crash samples from index." %
len(uninteresting_samples))
# generate output gdb script
generate_gdb_exploitable_script(
os.path.join(out_dir, args.gdb_expl_script_file), sample_index,
args.target_cmd, 0)
elif args.gdb_script_file:
generate_gdb_exploitable_script(
os.path.join(out_dir, args.gdb_script_file), sample_index,
args.target_cmd)
print_ok("Copying %d samples into output directory..." %
len(sample_index.index))
files_collected = copy_samples(sample_index)
# generate filelist of collected crash samples
if args.list_filename:
generate_sample_list(
os.path.abspath(os.path.expanduser(args.list_filename)),
files_collected)
print_ok("Generated crash sample list '%s'." %
os.path.abspath(os.path.expanduser(args.list_filename)))
# write db contents to file and close db connection
if db_file:
lite_db.commit_close()
