def test_build_sample_index(self):
sync_dir = 'testdata/sync'
out_dir = 'testdata/out'
fuzzer_inst = [
('fuzz000', ['queue']),
('fuzz001', ['queue'])
]
expected_index = [
{'input': os.path.abspath('testdata/sync/fuzz000/queue/sample0'), 'fuzzer': 'fuzz000',
'output': 'fuzz000:sample0'},
{'input': os.path.abspath('testdata/sync/fuzz000/queue/sample1'), 'fuzzer': 'fuzz000',
'output': 'fuzz000:sample1'},
{'input': os.path.abspath('testdata/sync/fuzz000/queue/sample2'), 'fuzzer': 'fuzz000',
'output': 'fuzz000:sample2'},
{'input': os.path.abspath('testdata/sync/fuzz000/queue/sample3'), 'fuzzer': 'fuzz000',
'output': 'fuzz000:sample3'},
{'input': os.path.abspath('testdata/sync/fuzz000/queue/sample4'), 'fuzzer': 'fuzz000',
'output': 'fuzz000:sample4'},
{'input': os.path.abspath('testdata/sync/fuzz001/queue/sample0'), 'fuzzer': 'fuzz001',
'output': 'fuzz001:sample0'},
{'input': os.path.abspath('testdata/sync/fuzz001/queue/sample1'), 'fuzzer': 'fuzz001',
'output': 'fuzz001:sample1'},
{'input': os.path.abspath('testdata/sync/fuzz001/queue/sample2'), 'fuzzer': 'fuzz001',
'output': 'fuzz001:sample2'},
{'input': os.path.abspath('testdata/sync/fuzz001/queue/sample3'), 'fuzzer': 'fuzz001',
'output': 'fuzz001:sample3'},
{'input': os.path.abspath('testdata/sync/fuzz001/queue/sample4'), 'fuzzer': 'fuzz001',
'output': 'fuzz001:sample4'},
]
result = afl_collect.build_sample_index(sync_dir, out_dir, fuzzer_inst)
self.assertListEqual(expected_index, result.index)
def main(argv):
show_info()
parser = argparse.ArgumentParser(description="afl-minimize performs several optimization steps to reduce the size\n \
of an afl-fuzz corpus.",
usage="afl-minimize [-c COLLECTION_DIR [--cmin [opts]] [--tmin [opts]]] [-d] [-h]\n \
[-j] sync_dir -- target_cmd\n")
parser.add_argument("-c", "--collect", dest="collection_dir",
help="Collect all samples from the synchronisation dir and store them in the collection dir. \
Existing files in the collection directory will be overwritten!", default=None)
parser.add_argument("--cmin", dest="invoke_cmin", action="store_const", const=True,
default=False, help="Run afl-cmin on collection dir. Has no effect without '-c'.")
parser.add_argument("--cmin-mem-limit", dest="cmin_mem_limit", default=None, help="Set memory limit for afl-cmin.")
parser.add_argument("--cmin-timeout", dest="cmin_timeout", default=None, help="Set timeout for afl-cmin.")
parser.add_argument("--tmin", dest="invoke_tmin", action="store_const", const=True,
default=False, help="Run afl-tmin on minimized collection dir if used together with '--cmin'\
or on unoptimized collection dir otherwise. Has no effect without '-c'.")
parser.add_argument("--tmin-mem-limit", dest="tmin_mem_limit", default=None, help="Set memory limit for afl-tmin.")
parser.add_argument("--tmin-timeout", dest="tmin_timeout", default=None, help="Set timeout for afl-tmin.")
parser.add_argument("-d", "--dry-run", dest="dry_run", action="store_const", const=True,
default=False, help="Perform dry-run on collection dir, if '-c' is provided or on \
synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus.")
parser.add_argument("-j", "--threads", dest="num_threads", default=1,
help="Enable parallel dry-run and t-minimization step by specifying the number of threads \
afl-minimize will utilize.")
parser.add_argument("sync_dir", help="afl synchronisation directory containing multiple fuzzers and their queues.")
parser.add_argument("target_cmd", nargs="+", help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
if not args.collection_dir and not args.dry_run:
print_err("No operation requested. You should at least provide '-c'")
print_err("for sample collection or '-d' for a dry-run. Use '--help' for")
print_err("usage instructions or checkout README.md for details.")
return
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
if not args.num_threads:
threads = 1
else:
threads = int(args.num_threads)
if args.collection_dir:
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0:
os.makedirs(out_dir, exist_ok=True)
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
# collect samples from fuzzer queues
print_ok("Found %d fuzzers, collecting samples." % len(fuzzers))
sample_index = afl_collect.build_sample_index(sync_dir, out_dir, fuzzers)
print_ok("Successfully indexed %d samples." % len(sample_index.index))
print_ok("Copying %d samples into collection directory..." % len(sample_index.index))
afl_collect.copy_samples(sample_index)
else:
print_warn("Collection directory exists and is not empty!")
print_warn("Skipping collection step...")
if args.invoke_cmin:
# invoke cmin on collection
print_ok("Executing: afl-cmin -i %s -o %s.cmin -- %s" % (out_dir, out_dir, args.target_cmd))
invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd, mem_limit=args.cmin_mem_limit,
timeout=args.cmin_timeout)
if args.invoke_tmin:
# invoke tmin on minimized collection
print_ok("Executing: afl-tmin -i %s.cmin/* -o %s.cmin.tmin/* -- %s" % (out_dir, out_dir,
args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.cmin.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout)
elif args.invoke_tmin:
# invoke tmin on collection
print_ok("Executing: afl-tmin -i %s/* -o %s.tmin/* -- %s" % (out_dir, out_dir, args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_tmin(tmin_samples, "%s.tmin" % out_dir, args.target_cmd, num_threads=threads,
mem_limit=args.tmin_mem_limit, timeout=args.tmin_timeout)
if args.dry_run:
# invoke dry-run on collected/minimized corpus
if args.invoke_cmin and args.invoke_tmin:
print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.tmin.crashes" % out_dir, "%s.cmin.tmin.hangs" % out_dir,
args.target_cmd, num_threads=threads)
elif args.invoke_cmin:
print_ok("Performing dry-run in %s.cmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.cmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.cmin.crashes" % out_dir, "%s.cmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
elif args.invoke_tmin:
print_ok("Performing dry-run in %s.tmin..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir("%s.tmin" % out_dir,
abs_path=True)
invoke_dryrun(dryrun_samples, "%s.tmin.crashes" % out_dir, "%s.tmin.hangs" % out_dir, args.target_cmd,
num_threads=threads)
else:
print_ok("Performing dry-run in %s..." % out_dir)
print_warn("Be patient! Depending on the corpus size this step can take hours...")
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(out_dir, abs_path=True)
invoke_dryrun(dryrun_samples, "%s.crashes" % out_dir, "%s.hangs" % out_dir, args.target_cmd,
num_threads=threads)
else:
if args.dry_run:
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir, crash_dirs=False)
print_ok("Found %d fuzzers, performing dry run." % len(fuzzers))
print_warn("Be patient! Depending on the corpus size this step can take hours...")
# invoke dry-run on original corpus
for f in fuzzers:
for q_dir in f[1]:
q_dir_complete = os.path.join(sync_dir, f[0], q_dir)
print_ok("Processing %s..." % q_dir_complete)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(q_dir_complete, abs_path=True)
invoke_dryrun(dryrun_samples, os.path.join(sync_dir, f[0], "crashes"),
os.path.join(sync_dir, f[0], "hangs"), args.target_cmd, num_threads=threads)
def main(argv):
show_info()
parser = argparse.ArgumentParser(
description=
"afl-minimize performs several optimization steps to reduce the size\n \
of an afl-fuzz corpus.",
usage=
"afl-minimize [-c COLLECTION_DIR [--cmin] [--tmin]] [-d] [-h] [-j] sync_dir \
-- target_cmd\n")
parser.add_argument(
"-c",
"--collect",
dest="collection_dir",
help=
"Collect all samples from the synchronisation dir and store them in the collection dir. \
Existing files in the collection directory will be overwritten!",
default=None)
parser.add_argument(
"--cmin",
dest="invoke_cmin",
action="store_const",
const=True,
default=False,
help="Run afl-cmin on collection dir. Has no effect without '-c'.")
parser.add_argument(
"--tmin",
dest="invoke_tmin",
action="store_const",
const=True,
default=False,
help=
"Run afl-tmin on minimized collection dir if used together with '--cmin'\
or on unoptimized collection dir otherwise. Has no effect without '-c'.")
parser.add_argument(
"-d",
"--dry-run",
dest="dry_run",
action="store_const",
const=True,
default=False,
help="Perform dry-run on collection dir, if '-c' is provided or on \
synchronisation dir otherwise. Dry-run will move intermittent crashes out of the corpus."
)
parser.add_argument(
"-j",
"--threads",
dest="num_threads",
default=1,
help=
"Enable parallel dry-run and t-minimization step by specifying the number of threads \
afl-minimize will utilize.")
parser.add_argument(
"sync_dir",
help=
"afl synchronisation directory containing multiple fuzzers and their queues."
)
parser.add_argument(
"target_cmd",
nargs="+",
help="Path to the target binary and its command line arguments. \
Use '@@' to specify crash sample input file position (see afl-fuzz usage).")
args = parser.parse_args(argv[1:])
if not args.collection_dir and not args.dry_run:
print_err("No operation requested. You should at least provide '-c'")
print_err(
"for sample collection or '-d' for a dry-run. Use '--help' for")
print_err("usage instructions or checkout README.md for details.")
return
sync_dir = os.path.abspath(os.path.expanduser(args.sync_dir))
if not os.path.exists(sync_dir):
print_err("No valid directory provided for <SYNC_DIR>!")
return
args.target_cmd = " ".join(args.target_cmd).split()
args.target_cmd[0] = os.path.abspath(os.path.expanduser(
args.target_cmd[0]))
if not os.path.exists(args.target_cmd[0]):
print_err("Target binary not found!")
return
args.target_cmd = " ".join(args.target_cmd)
if not args.num_threads:
threads = 1
else:
threads = int(args.num_threads)
if args.collection_dir:
out_dir = os.path.abspath(os.path.expanduser(args.collection_dir))
if not os.path.exists(out_dir) or len(os.listdir(out_dir)) == 0:
os.makedirs(out_dir, exist_ok=True)
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir,
crash_dirs=False)
# collect samples from fuzzer queues
print_ok("Found %d fuzzers, collecting samples." % len(fuzzers))
sample_index = afl_collect.build_sample_index(
sync_dir, out_dir, fuzzers)
print_ok("Successfully indexed %d samples." %
len(sample_index.index))
print_ok("Copying %d samples into collection directory..." %
len(sample_index.index))
afl_collect.copy_samples(sample_index)
else:
print_warn("Collection directory exists and is not empty!")
print_warn("Skipping collection step...")
if args.invoke_cmin:
# invoke cmin on collection
print_ok("Executing: afl-cmin -i %s -o %s.cmin -- %s" %
(out_dir, out_dir, args.target_cmd))
invoke_cmin(out_dir, "%s.cmin" % out_dir, args.target_cmd)
if args.invoke_tmin:
# invoke tmin on minimized collection
print_ok(
"Executing: afl-tmin -i %s.cmin/* -o %s.cmin.tmin/* -- %s"
% (out_dir, out_dir, args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(
"%s.cmin" % out_dir, abs_path=True)
tmin_num_samples_processed = invoke_tmin(tmin_samples,
"%s.cmin.tmin" %
out_dir,
args.target_cmd,
num_threads=threads)
elif args.invoke_tmin:
# invoke tmin on collection
print_ok("Executing: afl-tmin -i %s/* -o %s.tmin/* -- %s" %
(out_dir, out_dir, args.target_cmd))
tmin_num_samples, tmin_samples = afl_collect.get_samples_from_dir(
out_dir, abs_path=True)
tmin_num_samples_processed = invoke_tmin(tmin_samples,
"%s.tmin" % out_dir,
args.target_cmd,
num_threads=threads)
if args.dry_run:
# invoke dry-run on collected/minimized corpus
if args.invoke_cmin and args.invoke_tmin:
print_ok("Performing dry-run in %s.cmin.tmin..." % out_dir)
print_warn(
"Be patient! Depending on the corpus size this step can take hours..."
)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(
"%s.cmin.tmin" % out_dir, abs_path=True)
invoke_dryrun(dryrun_samples,
"%s.cmin.tmin.crashes" % out_dir,
args.target_cmd,
num_threads=threads)
elif args.invoke_cmin:
print_ok("Performing dry-run in %s.cmin..." % out_dir)
print_warn(
"Be patient! Depending on the corpus size this step can take hours..."
)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(
"%s.cmin" % out_dir, abs_path=True)
invoke_dryrun(dryrun_samples,
"%s.cmin.crashes" % out_dir,
args.target_cmd,
num_threads=threads)
elif args.invoke_tmin:
print_ok("Performing dry-run in %s.tmin..." % out_dir)
print_warn(
"Be patient! Depending on the corpus size this step can take hours..."
)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(
"%s.tmin" % out_dir, abs_path=True)
invoke_dryrun(dryrun_samples,
"%s.tmin.crashes" % out_dir,
args.target_cmd,
num_threads=threads)
else:
print_ok("Performing dry-run in %s..." % out_dir)
print_warn(
"Be patient! Depending on the corpus size this step can take hours..."
)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(
out_dir, abs_path=True)
invoke_dryrun(dryrun_samples,
out_dir,
args.target_cmd,
num_threads=threads)
else:
if args.dry_run:
print_ok("Looking for fuzzing queues in '%s'." % sync_dir)
fuzzers = afl_collect.get_fuzzer_instances(sync_dir,
crash_dirs=False)
print_ok("Found %d fuzzers, performing dry run." % len(fuzzers))
print_warn(
"Be patient! Depending on the corpus size this step can take hours..."
)
# invoke dry-run on original corpus
for f in fuzzers:
for q_dir in f[1]:
q_dir_complete = os.path.join(sync_dir, f[0], q_dir)
print_ok("Processing %s..." % q_dir_complete)
dryrun_num_samples, dryrun_samples = afl_collect.get_samples_from_dir(
q_dir_complete, abs_path=True)
invoke_dryrun(dryrun_samples,
os.path.join(sync_dir, f[0], "crashes"),
args.target_cmd,
num_threads=threads)
