def local_testcases_list(self):
""" @brief List of all the testcase and corresponding images in the local
tc/img directories
@return List of tuple with first entry the path to the testcase and
second entry a corresponding path to the image or None """
result = []
lcl_tc_files = listdir(self.tc_dir)
# Complete the path
lcl_tc_files = [path.join(self.tc_dir, fname) for fname in lcl_tc_files\
if nh.is_tc(fname)]
for filepath in lcl_tc_files:
# If this testcase has a corresponding pool image
imgname = path.basename(
filepath.replace(Dedup.EXT_TC, Dedup.EXT_PM_CMPR_POOL))
imgpath = path.join(self.img_dir, imgname)
abort_if(not path.isfile(imgpath),
'Image not found `%s\'' % imgpath)
# Create a tuple
entry = (filepath, imgpath)
# Add tuple to the results
result.append(entry)
return result
def local_dedup_list(self):
""" @brief Get all the testcase and corresponding images in the local
dedup store
@return List of tuple with first entry the path to the testcase and
second entry a corresponding path to the image or None """
result = []
loc_dedup_files = listdir(self.dedup_dir_loc)
for filepath in loc_dedup_files:
# Only process testcases
if nh.is_tc(filepath, all=True):
entry = [filepath, None]
# If this testcase has a corresponding pool image
imgpath = filepath.replace(Dedup.EXT_TC, Dedup.EXT_PM_POOL)
if imgpath in loc_dedup_files:
# Add it to the tuple
entry[1] = imgpath
# Add tuple to the results
result.append(tuple(entry))
return result
def minimize_testcases(self):
""" Minimize global testcases
@return None"""
abort_if(
len(self.global_dedup_list_tc) == 0,
'Empty global deduplication list')
testcases, _ = map(list, zip(*self.global_dedup_list_tc))
prl = parallel.Parallel(self.minimize_testcase, self.cores)
for testcase in testcases:
# Only process unminimized testcases, exclude anything that has
# .min in its name
if nh.is_tc(testcase):
# TODO: fix this
# Get the minimized name to check if this is already processed
testcase_min_name = nh.get_tc_min(testcase, Dedup.EXT_TC)
if not testcase_min_name in testcases:
# Only run if minimization is enabled
if self.cfg['pmfuzz']\
['stage']['dedup']['global']['minimize_tc']:
prl.run([testcase])
else: # else, just create a copy of the destcase
copypreserve(testcase, testcase_min_name)
prl.wait()
def global_dedup_list_tc(self):
""" @brief Get all testcase and corresponding images from global store.
Does not include crash sites, to list that, see @ref
global_dedup_list_cs
@return List of tuple with first entry the path to the testcase and
second entry a corresponding path to the image or None """
result = []
gbl_dedup_files = listdir(self.dedup_dir_gbl)
# Complete the path
gbl_dedup_files = [path.join(self.dedup_dir_gbl, fname) \
for fname in gbl_dedup_files]
for filepath in gbl_dedup_files:
# Only process testcases
if nh.is_tc(filepath):
entry = [filepath, None]
# If this testcase has a corresponding pool image
imgpath = nh.get_metadata_files(filepath)['pm_cmpr_pool']
if imgpath in gbl_dedup_files:
# Add it to the tuple
entry[1] = imgpath
else:
printw('No image found %s' % imgpath)
# Add tuple to the results
result.append(tuple(entry))
if self.verbose:
printv('Returning %d cases' % len(result))
return result
def minimize_corpus_lcl(self):
""" @brief Minimizes the local testcase directory by combining it with
the global testcases.
This method uses modified afl-cmin that supports providing before and
after shell scripts that provide the corresponding tgtcmd for each
testcase. Before and after shell scripts are passed to afl-cmin using
a config directory (tmp_cfgdir).
@returns None """
# Create temporary directories for managing corpus
tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir)
tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir)
# Copy all testcases from global dedup to tmp_indir to allow
# minimization on global corpus along with local
for f in os.listdir(self.dedup_dir_loc):
if nh.is_tc(f):
src = path.join(self.dedup_dir_loc, f)
dest = path.join(tmp_indir, f)
if self.verbose:
printv(f'Copying to testcase: {src} -> {dest}')
copypreserve(src, dest)
elif nh.is_map(f) or nh.is_pm_map(f):
src = path.join(self.dedup_dir_loc, f)
dest = path.join(tmp_mapdir, f)
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to map: {src} -> {dest}')
elif self.verbose:
printv(f'Did not find {src}')
# Copy all the test cases from self.tc_dir to temp directory for corpus
# minimization
lcl_tcs = [path.join(self.tc_dir, f) for f in os.listdir(self.tc_dir)]
for f in lcl_tcs:
if nh.is_tc(f):
src = f
dest = path.join(tmp_indir, path.basename(f))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to testcase: {src} -> {dest}')
elif nh.is_map(f) or nh.is_pm_map(f):
src = f
dest = path.join(tmp_mapdir, path.basename(f))
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to map: {src} -> {dest}')
elif self.verbose:
printv(f'Did not find {src}')
total_tcs = len(os.listdir(tmp_indir))
# Create a temp image for cases that don't have any parent
_, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-',
dir=self.tempdir)
_, tgtcmd_loc \
= nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg)
gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose)
# Run the actual thing
outdir = run_afl_cmin(
indir=tmp_indir,
pmfuzzdir=self.outdir,
tgtcmd=self.cfg.tgtcmd,
cfg=self.cfg,
# cfgdir=tmp_cfgdir,
verbose=self.verbose,
mapdir=tmp_mapdir,
dry_run=False,
)
dropped_files = []
for f in os.listdir(tmp_indir):
if f not in os.listdir(outdir):
dropped_files.append(f)
if self.verbose:
printv('Dropping %d of %d testcases after cmin' \
% (len(dropped_files), total_tcs))
# Remove the dropped testcases and associated metadata files from the
# local testcase directory
for f in dropped_files:
metadata_files = nh.get_metadata_files(f)
for metadata_file_type in metadata_files:
file_to_delete = path.join(self.tc_dir,
metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
if self.verbose:
printv('Removed ' + file_to_delete)
except FileNotFoundError:
pass
# Remove the dropped testcases and associated metadata files from the
# global dedup directory
for f in dropped_files:
metadata_files = nh.get_metadata_files(f)
types_to_delete \
= ['testcase', 'min_testcase', 'pm_map', 'map']
for metadata_file_type in types_to_delete:
file_to_delete = path.join(self.dedup_dir_gbl,
metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
placeholder_f = path.join(
self.dedup_dir_gbl,
nh.get_metadata_files(f, True)['deleted'])
with open(placeholder_f, 'w') as obj:
obj.write('Deleted at epoch = ' +
str(int(time.time())))
if self.verbose:
printv('Writing to ' + placeholder_f)
printv('Removed ' + file_to_delete)
except FileNotFoundError:
pass
# TODO: Clean up file descriptors
if self.verbose:
printv('Removing dir %s' % tmp_indir)
printv('Removing dir %s' % tmp_mapdir)
printv('Removing %s' % temp_img)
rmtree(tmp_indir)
rmtree(tmp_mapdir)
os.remove(temp_img)
return
def minimize_corpus_gbl(self):
""" @brief Minimizes the global dedup directory.
This method uses modified afl-cmin that supports providing before and
after shell scripts that provide the corresponding tgtcmd for each
testcase. Before and after shell scripts are passed to afl-cmin using
a config directory (tmp_cfgdir).
@todo Replace call to copypreserve with a softlink
@todo Cleanup indir and cfgdir after completion
@returns None """
write_state(self.outdir, 'Minimizing global corpus')
# Create temporary directories for managing corpus
tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir)
tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir)
# Copy all the test cases from self.tc_dir to temp directory for corpus
# minimization
total_global_count = 0
for tc, _ in self.global_dedup_list_tc:
if nh.is_tc(tc):
total_global_count += 1
src = tc
dest = path.join(tmp_indir, path.basename(tc))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying testcase {src} -> {dest}')
# Copy the map
src = path.join(path.dirname(tc), 'map_' + path.basename(tc))
dest = path.join(tmp_mapdir, 'map_' + path.basename(tc))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying map {src} -> {dest}')
# Copy the pm map
src = path.join(path.dirname(tc),
'pm_map_' + path.basename(tc))
dest = path.join(tmp_mapdir, 'pm_map_' + path.basename(tc))
if path.isfile(src):
copypreserve(src, dest)
self.printv(f'Copying map {src} -> {dest}')
# Create a temp image for cases that don't have any parent
fd, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-',
dir=self.tempdir)
# We don't actually need this file, todo: do better
os.close(fd)
os.remove(temp_img)
_, tgtcmd_loc = \
nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg)
gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose)
# Run the actual thing
outdir = run_afl_cmin(
indir=tmp_indir,
pmfuzzdir=self.outdir,
tgtcmd=tgtcmd_loc,
cfg=self.cfg,
# cfgdir=tmp_cfgdir,
verbose=self.verbose,
dry_run=False,
mapdir=tmp_mapdir,
)
dropped_files = []
for f in os.listdir(tmp_indir):
if f not in os.listdir(outdir):
dropped_files.append(f)
if self.verbose:
printv('Dropping %d of %d testcases after cmin' \
% (len(dropped_files), total_global_count))
# Remove the dropped testcases and associated metadata files
for f in dropped_files:
metadata_files = nh.get_metadata_files(f, deleted=False)
for metadata_file_type in metadata_files:
if not metadata_files[metadata_file_type].endswith('.tar.gz'):
file_to_delete = path.join(
self.dedup_dir_gbl, metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
if self.verbose:
printv('Removing ' + file_to_delete)
except FileNotFoundError:
pass
# Create a placeholder .deleted file to avoid stage1 from
# repopulating that testcase
placeholder_f = path.join(
self.dedup_dir_gbl,
nh.get_metadata_files(f, deleted=True)['deleted'])
with open(placeholder_f, 'w') as obj:
obj.write('deleted at epoch=%d' % int(time.time()))
if self.verbose:
printv('Removing dir %s' % tmp_indir)
printv('Removing dir %s' % tmp_mapdir)
printv('Removing %s' % temp_img)
rmtree(tmp_indir)
rmtree(tmp_mapdir)
os.remove(temp_img)
return