def collect_results(self) -> None:
""" @brief Copies the results from the master fuzzer to local tc & img
directory.
Operations are performed in parallel, using self.cores number of jobs
at a time. Only collects the available queue output directories in the
.afl_results directory. Expects most of the testcases to be already
copied on termination. Present solely to collect testcases orphaned
on the previous instance of PMFuzz.
@return None """
printi('Collecting stage 2, iter ' + str(self.iter_id))
# Get all the output directories for all the testcases that were run
o_dirs = self.get_o_tc_dirs()
# Create a parallel object
prl = Parallel(self.collect_tc,
self.cores,
failure_mode=Parallel.FAILURE_EXIT,
transparent_io=False,
verbose=self.verbose)
cnt = 0
found_cases = os.listdir(self.tc_dir)
# Collect testcases from all of them
for o_dir in o_dirs:
gen_cases = [name for name in listdir(o_dir) \
if name.startswith('id') == True]
for gen_case in gen_cases:
# Parent name (name of the testcase that generated the image
# for this testcase)
parent_name = path.dirname(path.dirname(o_dir)) + ','
# Remove unnecessary information from testcase's name and check
# if this testcase is not already copied
clean_name = parent_name + nh.clean_tc_name(gen_case)
if not path.basename(clean_name) in found_cases:
prl.run([o_dir, gen_case, clean_name])
cnt += 1
prl.wait()
if self.verbose:
printv('%d cases processed (%d already exists).' \
% (cnt, len(found_cases)))
def tc_gen_crash_sites(self, raw_tcname):
""" Generates crash sites for a testcase
Testcases is read and crash sites are generated for each of the
failure point using the testcase's parent's image. These generated
results are then sent to the crashsite directory:
(e.g., <outdir>/stage=1,iter=1/crashsites/)
@see Stage2.tc_gen_crash_sites()
@param raw_tcname Path to a testcase to generate crash sites for
"""
printi('Generating crash images for testcase: ' + raw_tcname)
clean_name = nh.clean_tc_name(path.basename(raw_tcname))
# Create an empty image to run failure injection
crash_img_prefix = None
with TempEmptyImage(self.cfg, self.verbose) as tmp_img:
crash_img_prefix = tmp_img
if self.verbose:
printv('Generating crash images for image %s' %
crash_img_prefix)
finj.run_failure_inj(self.cfg,
self.cfg.tgtcmd,
tmp_img,
raw_tcname,
clean_name,
create=False,
verbose=self.verbose)
crash_imgs_pattern = crash_img_prefix.replace('.pm_pool', '') + '.' \
+ clean_name.replace('.testcase', '') + '.*'
crash_imgs = glob(crash_imgs_pattern)
if self.verbose:
printi('Total %d crash images generated.' % len(crash_imgs))
printv('Compressing all the crash sites')
self.compress_new_crash_sites(crash_img_prefix, clean_name)
self.add_cs_hash_lcl()
if self.verbose:
printv('Crash sites compressed')
def collect_results(self) -> None:
""" Copies the results from the master fuzzer to local tc & img
directory
@return None """
found_cases = os.listdir(self.tc_dir)
gen_cases = [name for name in listdir(self.o_tc_dir) \
if name.startswith('id') == True]
cnt = 0
for gen_case in gen_cases:
# Remove unnecessary information from testcase's name and check if
# this testcase is not already copied
clean_name = nh.clean_tc_name(gen_case)
if not clean_name in found_cases:
self.collect_tc(gen_case, clean_name)
cnt += 1
if self.verbose:
printv('%d cases processed (%d already exists).' \
% (cnt, len(found_cases)))
def tc_gen_crash_sites(self, raw_tcname):
""" Generates crash sites for a testcase
Testcases is read and crash sites are generated for each of the
failure point using the testcase's parent's image. These generated
results are then sent to the crashsite directory:
(e.g., <outdir>/stage=2,iter=1/crashsites/)
@param raw_tcname Path to a testcase to generate crash sites for
"""
printi('Generating crash images for testcase: ' + raw_tcname)
clean_name = nh.clean_tc_name(path.basename(raw_tcname))
tc_components = os.path.normpath(raw_tcname).split(os.sep)
parent_name = tc_components[-4] + '.testcase'
printi('Clean name: ' + clean_name)
printi('Parent name: ' + parent_name)
# NOTE:
# Create image names, *_uniq files are to provide each instance of
# tc_gen_crash_sites() with a unique image to work with
pm_dir = tempfile.mkdtemp(prefix='pmfuzz-cs-gen-st2-',
dir=self.cfg('pmfuzz.img_loc'))
parent_cmpr_img = nh.get_parent_img(parent_name,
self.dedup.dedup_dir_gbl,
'exists',
isparent=True,
verbose=self.verbose)
# Create image path
parent_img_name = ''
if parent_cmpr_img.endswith(nh.CMPR_PM_IMG_EXT):
parent_img_name = nh.get_metadata_files(parent_name)['pm_pool']
elif parent_cmpr_img.endswith(nh.CMPR_CRASH_SITE_EXT):
parent_img_name = nh.get_metadata_files(parent_name)['crash_site']
parent_img_name_uniq = nh.get_metadata_files(parent_name)\
['clean'] + '<pid=' + str(os.getpid()) + '>'\
+ '.' + nh.CRASH_SITE_EXT
parent_img = path.join(pm_dir, parent_img_name)
parent_img_uniq = path.join(pm_dir, parent_img_name_uniq)
# Decompres+Copy the image
if not os.path.isfile(parent_img):
decompress(parent_cmpr_img, parent_img, self.verbose)
printv('tempimg: %s -> %s' % (parent_cmpr_img, parent_img))
copypreserve(parent_img, parent_img_uniq)
printv('unique image: %s -> %s' % (parent_img, parent_img))
finj.run_failure_inj(self.cfg, self.cfg.tgtcmd, parent_img_uniq,
raw_tcname, clean_name, self.verbose)
crash_imgs_pattern = parent_img.replace('.pm_pool', '') + '.' \
+ clean_name.replace('.testcase', '') + '.*'
crash_imgs = glob(crash_imgs_pattern)
if self.verbose:
printi('Total %d crash images generated.' % len(crash_imgs))
printw('Deleting %s' % (parent_img))
os.remove(parent_img_uniq)
if self.verbose:
printv('Compressing all the crash sites')
self.process_new_crash_sites(parent_img_uniq, clean_name)
if self.verbose:
printv('Crash sites compressed')
def _terminate_cs(self, csname: str, timer):
""" @brief Terminates a crash site run
@return None """
if self.verbose and timer != None:
time_delta_str = timer.elapsed_hr()
printv("Time elapsed: {:0>8}".format(time_delta_str))
write_state(self.outdir, 'Collecting CS ' + csname)
# Read the pid
pid = None
q_dir = path.join(self.get_result_dir(csname), 'queue')
pid_f = path.join(self.get_result_dir(csname), 'pid')
# Kill only if the pid file exists (indicating a running AFL process)
if path.isfile(pid_f):
printi('Killing ' + pid_f)
with open(pid_f, 'r') as fobj:
pid = int(fobj.read().strip())
# Kill the running AFL instance
os.kill(pid, signal.SIGTERM) # Send signal 9
# Remove the pid file
remove(pid_f)
# Remove the image file
imgdir = self.cfg['pmfuzz']['img_loc'] + '/'
imgpm = path.join(imgdir, 'pmfuzz-cs-run-' + csname)
dir2del = glob(imgpm + '*')
abort_if(
len(dir2del) == 0,
'Unable to find anything with glob ' + imgpm + '*')
abort_if(
len(dir2del) > 1, 'Too many matches for glob ' + imgpm + '*')
if self.verbose:
printv('Removing image %s' % dir2del[0])
rmtree(dir2del[0])
# TODO: Move this to a single function for both _terminate_tc and
# _terminate_cs
printi('Collecting testcases for ' + csname)
core_count = self.cores // 2 if self.cores // 2 != 0 else 1
printi('Using %d cores for collecting results' % core_count)
# Create a parallel object for collecting testcases
prl_ct = Parallel(
self.collect_tc,
core_count,
transparent_io=False,
failure_mode=Parallel.FAILURE_EXIT,
name='Collect TC',
verbose=self.verbose,
)
# Create a parallel object for collecting crash sites
prl_gen_cs = Parallel(
self.tc_gen_crash_sites,
core_count,
transparent_io=False,
failure_mode=Parallel.FAILURE_EXIT,
name='Gen Crash Site',
verbose=self.verbose,
)
q_dir_contents = os.listdir(q_dir)
q_dir_contents = [f for f in q_dir_contents if f.startswith('id')]
# Collect the generated testcases
for childtc in q_dir_contents:
if childtc != '.state':
if self.verbose:
printv('Collecting %s from the queue directory' \
% childtc)
exp_img_path = nh.get_parent_img(csname + '.' + nh.TC_EXT,
self.dedup.dedup_dir_gbl,
get='exists',
isparent=True)
abort_if(not os.path.isfile(exp_img_path),
'Sanity check: Compressed image ' + exp_img_path + \
' should exist, but is missing.')
clean_name = csname + ',' + nh.clean_tc_name(childtc)
# Run collect_tc()
prl_ct.run([q_dir, childtc, clean_name])
# Generate crash sites by injecting failures
tcdir = path.join(self.afl_dir, path.basename(csname))
if self.cfg['pmfuzz']['failure_injection']['enable']:
tcdir_path = path.join(tcdir, 'master_fuzzer', 'queue',
childtc)
randval = randrange(100)
if randval < self.CS_GEN_THRESH:
prl_gen_cs.run([tcdir_path])
else:
self.printv('Skipping cs generation'\
+f' ({randval} < {self.CS_GEN_THRESH})')
prl_ct.wait()
prl_gen_cs.wait()
printi('Cleaning up local uncompressed images')
self.clean_up_uncmpr_lcl()
self.add_cs_hash_lcl()
# Deduplicate testcases with existing results
testcases_path = listdir(self.tc_dir)
testcases_path = [path.join(self.tc_dir, fname) \
for fname in testcases_path]
DedupEngine(testcases_path, self.verbose, checker=nh.is_map).run()
lcl_cfg = self.cfg['pmfuzz']['stage']['dedup']['local']
write_state(self.outdir, 'Minimizing local')
self.dedup.run(
fdedup=True,
min_tc=False, # TODO
min_corpus=lcl_cfg['minimize_corpus'],
gbl=False)
printi('Killed testcase %s (pid %d).' % (csname, pid))
def _terminate_testcase(self, testcasename: str, timer):
""" @brief Terminates a testcase
@return None """
if self.verbose and timer != None:
time_delta_str = timer.elapsed_hr()
printv("Time elapsed: {:0>8}".format(time_delta_str))
write_state(self.outdir, 'Collecting ' + testcasename)
# Read the pid
pid = None
q_dir = path.join(self.get_result_dir(testcasename), 'queue')
pid_f = path.join(self.get_result_dir(testcasename), 'pid')
# abort_if(not path.isfile(pid_f), 'PID file not found at ' + pid_f)
self.printv('Terminating ' + testcasename + ', checking for pid_f at '\
+ pid_f)
# Kill only if the pid file exists (indicating a running AFL process)
if path.isfile(pid_f):
printi('Killing ' + pid_f)
with open(pid_f, 'r') as fobj:
pid = int(fobj.read().strip())
# Kill the running AFL instance
os.kill(pid, signal.SIGTERM) # Send signal 9
# Remove the pid file
remove(pid_f)
# Remove the image file
self.printv('Removing directory %s' \
% self.get_img_dir(testcasename))
rmtree(self.get_img_dir(testcasename))
printi('Collecting testcases for ' + testcasename)
core_count = self.cores // 2 if self.cores // 2 != 0 else 1
printi('Using %d cores for collecting results' % core_count)
# Create a parallel object for collecting testcases
prl_ct = Parallel(
self.collect_tc,
core_count,
transparent_io=True,
failure_mode=Parallel.FAILURE_EXIT,
name='Collect TC',
verbose=self.verbose,
)
# Create a parallel object for collecting crash sites
prl_gen_cs = Parallel(
self.tc_gen_crash_sites,
core_count,
transparent_io=True,
failure_mode=Parallel.FAILURE_EXIT,
name='Gen Crash Site',
verbose=self.verbose,
)
q_dir_contents = os.listdir(q_dir)
q_dir_contents = [f for f in q_dir_contents if f.startswith('id')]
# Collect the generated testcases
for childtc in q_dir_contents:
if childtc != '.state':
if self.verbose:
printv('Collecting %s from the queue directory' \
% childtc)
exp_img_path = path.join(self.dedup.dedup_dir_gbl,
testcasename + '.pm_pool.tar.gz')
abort_if(not os.path.isfile(exp_img_path),
'Sanity check: Compressed image ' + exp_img_path + \
' should exist, but is missing.')
clean_name = testcasename + ',' + nh.clean_tc_name(childtc)
# Run collect_tc()
prl_ct.run([q_dir, childtc, clean_name])
# Generate crash sites by injecting failures
tcdir = path.join(self.afl_dir,
path.basename(testcasename))
if self.cfg['pmfuzz']['failure_injection']['enable']:
tcdir_path = path.join(tcdir, 'master_fuzzer', 'queue',
childtc)
randval = randrange(100)
if randval < self.CS_GEN_THRESH:
prl_gen_cs.run([tcdir_path])
else:
self.printv('Skipping cs generation'\
+f' ({randval} < {self.CS_GEN_THRESH})')
prl_ct.wait()
prl_gen_cs.wait()
printi('Cleaning up local uncompressed images')
self.clean_up_uncmpr_lcl()
self.add_cs_hash_lcl()
#! Disable this: We don't need to deduplicate the maps
# # Deduplicate testcases with existing results
# testcases_path = listdir(self.tc_dir)
# testcases_path = [path.join(self.tc_dir, fname) \
# for fname in testcases_path]
# DedupEngine(
# testcases_path,
# self.verbose,
# checker=nh.is_map
# ).run()
lcl_cfg = self.cfg['pmfuzz']['stage']['dedup']['local']
write_state(self.outdir, 'Minimizing local')
self.dedup.run(
fdedup=True,
min_tc=False, # TODO
min_corpus=lcl_cfg['minimize_corpus'],
gbl=False)
printi('Killed testcase %s (pid %d).' % (testcasename, pid))
else:
self.printv('Did not kill testcase %s.' % (testcasename))