def execute_tcs(self, tc_dir, img_dir):
tcs = [path.join(tc_dir, fn) for fn in os.listdir(tc_dir)]
imgs = [path.join(img_dir, fn) for fn in os.listdir(img_dir)]
iter = 0
for tc in filter(nh.is_tc, tcs):
# print(tc)
# iter += 1
# if iter > 5:
# return
img = self.empty_img
if nh.ancestor_cnt(path.basename(tc)) > 0:
parent = nh.get_testcase_parent(path.basename(tc))
img = nh.get_metadata_files(parent)['pm_cmpr_pool']
img = path.join(path.dirname(img_dir), '@dedup', img)
abort_if(
path.basename(img) not in os.listdir(img_dir),
'Img %s not found in %s' % (img, img_dir))
printi('Running %s with image %s' % (tc, img))
img_cmpr = img.endswith('.tar.gz')
if img_cmpr:
decompress(img, '/mnt/pmem0/', verbose=self.verbose)
img = path.join(
'/mnt/pmem0/',
path.basename(nh.get_metadata_files(img)['pm_pool']))
self.run_tgt(tc, img)
if img_cmpr:
os.remove(img)
def _delete_tcs(self, testcases):
""" @brief deletes a testcase and all metadata files with it,
including images and maps.
@param testcases List of testcase name or complete path
@return None"""
delete_q = []
# Find all the metadata files associated with all the testcases
for testcase in testcases:
metadata_files = nh.get_metadata_files(testcase)
delete_q += metadata_files.values()
# Write the placeholder file to indicate that this file is deleted
placeholder_f \
= nh.get_metadata_files(testcase, deleted=True)['deleted']
with open(placeholder_f, 'w') as obj:
obj.write('Deleted at epoch=%d' % int(time.time()))
remove_files(delete_q, self.verbose, warn=True, force=True)
def global_dedup_list_tc(self):
""" @brief Get all testcase and corresponding images from global store.
Does not include crash sites, to list that, see @ref
global_dedup_list_cs
@return List of tuple with first entry the path to the testcase and
second entry a corresponding path to the image or None """
result = []
gbl_dedup_files = listdir(self.dedup_dir_gbl)
# Complete the path
gbl_dedup_files = [path.join(self.dedup_dir_gbl, fname) \
for fname in gbl_dedup_files]
for filepath in gbl_dedup_files:
# Only process testcases
if nh.is_tc(filepath):
entry = [filepath, None]
# If this testcase has a corresponding pool image
imgpath = nh.get_metadata_files(filepath)['pm_cmpr_pool']
if imgpath in gbl_dedup_files:
# Add it to the tuple
entry[1] = imgpath
else:
printw('No image found %s' % imgpath)
# Add tuple to the results
result.append(tuple(entry))
if self.verbose:
printv('Returning %d cases' % len(result))
return result
def gen_img(self, dest_dir, compress_img=True, img_path=None):
""" @brief Generate image for a testcase file
@param testcase_f str that points to the testcase to use for generation
@param dest_dir str Directory to store the resulting image
@param tgtcmd List of str for the command for running the target prog
@param cfg Config for setting up the target process' environment
@return None """
testcase_f = self.testcase_f
cfg = self.cfg
verbose = self.verbose
tgtcmd = cfg.tgtcmd
tgtcmd_loc = None
if img_path == None:
img_name = path.basename(nh.get_metadata_files(testcase_f)['pm_pool'])
# Get the command for generating the image
img_path, tgtcmd_loc = nh.set_img_name(tgtcmd, img_name, cfg)
abort_if(img_path=='', 'Unable to parse image path, tgtcmd=' + \
str(tgtcmd) + ', img_name=' + img_name)
else:
img_name = path.basename(img_path)
# Get the command for generating the image
img_path, tgtcmd_loc = nh.set_img_path(tgtcmd, img_path, cfg)
abort_if(img_path=='', 'Unable to parse image path, tgtcmd=' + \
str(tgtcmd) + ', img_name=' + img_name)
# Enable persistence for image in the environment
env = cfg.get_env(persist=True)
if verbose:
printv('cmd: %s' % (' '.join(tgtcmd_loc)))
printv('env: %s' % (str(env)))
printv('stdin: %s' % testcase_f)
# Create a tempfile for writing run output
fd, tmp = tempfile.mkstemp(prefix='pmfuzz-img-gen-output-')
os.close(fd)
if verbose:
printv('Writing image generation output to ' + tmp)
with open(tmp, 'w') as out:
with open(testcase_f, 'r') as testcase_obj:
exit_code = exec_shell(
cmd=tgtcmd_loc,
stdin=testcase_obj,
stdout=out,
stderr=subprocess.STDOUT,
env=env,
wait=True,
timeout = 30 # Set a generous timeout of 30 seconds so things don't crash
)
code_desc, success = translate_exit_code(exit_code)
if not success:
abort('Image generation failed: ' + code_desc)
if not path.isfile(img_path):
abort('Image generation failed (%s).' % img_path)
# Copy the file back to the pmfuzz result directory
if compress_img:
src = img_path
dest = path.join(dest_dir, path.basename(img_path))
dest = nh.get_metadata_files(dest)['pm_cmpr_pool']
compress(src, dest, verbose)
remove(src)
if verbose:
printw('Deleting: ' + src)
def update_local(self):
""" @brief Copies testcases and images from global dedup store to local
dedup store
Local is updated in two steps:
1. The testcases and corresponding images are copied
2. Any crash site generated in the previous interation is copied
@return None """
printi('Updating local')
loc_tc_files = listdir(self.dedup_dir_loc)
loc_tc_files = [f.replace('.min', '') for f in loc_tc_files]
# 1. Copy testcases
for testcase, img in self.global_dedup_list_tc:
# A testcase might not have an associated image
if img == None:
img = ''
dest_name_tc = path.basename(testcase).replace('.min', '')
dest_name_img = path.basename(img)
#* Note: This piece of code assumes that dedup would only run
#* update_local on stage 2
abort_if(self.stage != 2, 'update_local() is only supported on ' \
+ 'stage 2')
is_min_tc = True #re.search(nh.TC_MIN_REGEX, path.basename(testcase))
# Make sure this testcase was generated from the immediate parent
# of this stage and not earlier than that
iter_id = nh.iter_cnt(path.basename(testcase))
is_parent = iter_id == self.iter_id
# Copy only if destination does not exists and the testcase is a
# minimum testcase
if dest_name_tc not in loc_tc_files and is_min_tc and is_parent:
src = testcase
dest = path.join(self.dedup_dir_loc, dest_name_tc)
# Remove 'min' from testestcase name
dest = dest.replace('.min', '')
if self.verbose:
printv(f'Copying to local dedup: {src} -> {dest}')
copypreserve(src, dest)
if img != '':
# Copy image
src = img
dest = path.join(self.dedup_dir_loc, dest_name_img)
# Remove 'min' from testestcase name
dest = dest.replace('.min', '')
if self.verbose:
printv(f'Copying to local dedup: {src} -> {dest}')
copypreserve(src, dest)
abort_if(not os.path.isfile(dest), 'Cannot copy')
else:
abort('Image not found for %s' % dest_name_tc)
# Copy map
testcasebasename = os.path.basename(testcase)
metadata_files = nh.get_metadata_files(testcasebasename)
src = os.path.join(self.dedup_dir_gbl, metadata_files['map'])
dest = os.path.join(self.dedup_dir_loc, metadata_files['map'])
if self.verbose:
printv(f'Copying map to local dedup {src} -> {dest}')
copypreserve(src, dest)
# Copy PM map
testcasebasename = os.path.basename(testcase)
metadata_files = nh.get_metadata_files(testcasebasename)
src = os.path.join(self.dedup_dir_gbl,
metadata_files['pm_map'])
dest = os.path.join(self.dedup_dir_loc,
metadata_files['pm_map'])
copied = False
if os.path.isfile(src):
copied = True
copypreserve(src, dest)
if copied:
self.printv(f'Copying map to local dedup {src} -> {dest}')
else:
self.printv(f'Did not find {src}')
if len(self.global_dedup_list_cs) == 0:
self.printv('Did not find any crash site in global dedup')
else:
self.printv('=> ' + str(self.global_dedup_list_cs))
# 2. Copy crash images
for cs in self.global_dedup_list_cs:
if path.basename(cs) not in os.listdir(self.dedup_dir_loc):
src = cs
dest = path.join(self.dedup_dir_loc, path.basename(src))
self.printv('Copying cs: %s -> %s' % (src, dest))
copypreserve(src, dest)
def update_global(self):
""" @brief Copies testcases and images from local tc/img store to global
store
Global is updated in two steps:
1. The testcases and corresponding images are copied
2. Any crash site generated is copied
@return None """
printi('Updating global')
gbl_tc_files = listdir(self.dedup_dir_gbl)
# 1. Copy testcases and corresponding images
for testcase, img in self.local_testcases_list:
dest_name_tc = path.basename(testcase)
dest_name_img = path.basename(img)
# for deleted cases
dest_name_placeholder \
= nh.get_metadata_files(dest_name_tc, deleted=True)['deleted']
# Copy only if destination does not exists
if (dest_name_tc not in gbl_tc_files) \
and (dest_name_placeholder not in gbl_tc_files):
printi('Copying files related to ' \
+ path.basename(dest_name_tc))
# Copy testcase
src = testcase
dest = path.join(self.dedup_dir_gbl, dest_name_tc)
copypreserve(src, dest)
if self.verbose:
printv('Copying to global dedup: %s -> %s' % (src, dest))
# Copy exec map
src = testcase
src = path.join(path.dirname(src), 'map_' + path.basename(src))
dest = path.join(self.dedup_dir_gbl, 'map_' + dest_name_tc)
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv('Copying to exec map: %s -> %s' % (src, dest))
else:
if self.verbose:
printv('Unable to find exec map: ' + src)
# Copy PM map
src = testcase.replace(Dedup.TESTCASE_DIR, Dedup.TESTCASE_DIR)
src = path.join(path.dirname(src),
'pm_map_' + path.basename(src))
dest = path.join(self.dedup_dir_gbl, 'pm_map_' + dest_name_tc)
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv('Copying to PM map: %s -> %s' % (src, dest))
else:
if self.verbose:
printv('Unable to find PM map: ' + src)
# Copy image
src = img
dest = path.join(self.dedup_dir_gbl, dest_name_img)
copypreserve(src, dest)
abort_if(not os.path.isfile(dest), 'Cannot copy')
if self.verbose:
printv('Copying to global dedup: %s -> %s' % (src, dest))
# 2. Copy crash sites
for fname in os.listdir(self.img_dir):
should_copy = fname not in os.listdir(self.dedup_dir_gbl)
if nh.is_cmpr_crash_site(fname) and should_copy:
src = path.join(self.img_dir, fname)
dest = path.join(self.dedup_dir_gbl, fname)
if self.verbose:
printv('Copying cs %s -> %s' % (src, dest))
copypreserve(src, dest)
def minimize_corpus_lcl(self):
""" @brief Minimizes the local testcase directory by combining it with
the global testcases.
This method uses modified afl-cmin that supports providing before and
after shell scripts that provide the corresponding tgtcmd for each
testcase. Before and after shell scripts are passed to afl-cmin using
a config directory (tmp_cfgdir).
@returns None """
# Create temporary directories for managing corpus
tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir)
tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir)
# Copy all testcases from global dedup to tmp_indir to allow
# minimization on global corpus along with local
for f in os.listdir(self.dedup_dir_loc):
if nh.is_tc(f):
src = path.join(self.dedup_dir_loc, f)
dest = path.join(tmp_indir, f)
if self.verbose:
printv(f'Copying to testcase: {src} -> {dest}')
copypreserve(src, dest)
elif nh.is_map(f) or nh.is_pm_map(f):
src = path.join(self.dedup_dir_loc, f)
dest = path.join(tmp_mapdir, f)
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to map: {src} -> {dest}')
elif self.verbose:
printv(f'Did not find {src}')
# Copy all the test cases from self.tc_dir to temp directory for corpus
# minimization
lcl_tcs = [path.join(self.tc_dir, f) for f in os.listdir(self.tc_dir)]
for f in lcl_tcs:
if nh.is_tc(f):
src = f
dest = path.join(tmp_indir, path.basename(f))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to testcase: {src} -> {dest}')
elif nh.is_map(f) or nh.is_pm_map(f):
src = f
dest = path.join(tmp_mapdir, path.basename(f))
if path.isfile(src):
copypreserve(src, dest)
if self.verbose:
printv(f'Copying to map: {src} -> {dest}')
elif self.verbose:
printv(f'Did not find {src}')
total_tcs = len(os.listdir(tmp_indir))
# Create a temp image for cases that don't have any parent
_, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-',
dir=self.tempdir)
_, tgtcmd_loc \
= nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg)
gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose)
# Run the actual thing
outdir = run_afl_cmin(
indir=tmp_indir,
pmfuzzdir=self.outdir,
tgtcmd=self.cfg.tgtcmd,
cfg=self.cfg,
# cfgdir=tmp_cfgdir,
verbose=self.verbose,
mapdir=tmp_mapdir,
dry_run=False,
)
dropped_files = []
for f in os.listdir(tmp_indir):
if f not in os.listdir(outdir):
dropped_files.append(f)
if self.verbose:
printv('Dropping %d of %d testcases after cmin' \
% (len(dropped_files), total_tcs))
# Remove the dropped testcases and associated metadata files from the
# local testcase directory
for f in dropped_files:
metadata_files = nh.get_metadata_files(f)
for metadata_file_type in metadata_files:
file_to_delete = path.join(self.tc_dir,
metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
if self.verbose:
printv('Removed ' + file_to_delete)
except FileNotFoundError:
pass
# Remove the dropped testcases and associated metadata files from the
# global dedup directory
for f in dropped_files:
metadata_files = nh.get_metadata_files(f)
types_to_delete \
= ['testcase', 'min_testcase', 'pm_map', 'map']
for metadata_file_type in types_to_delete:
file_to_delete = path.join(self.dedup_dir_gbl,
metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
placeholder_f = path.join(
self.dedup_dir_gbl,
nh.get_metadata_files(f, True)['deleted'])
with open(placeholder_f, 'w') as obj:
obj.write('Deleted at epoch = ' +
str(int(time.time())))
if self.verbose:
printv('Writing to ' + placeholder_f)
printv('Removed ' + file_to_delete)
except FileNotFoundError:
pass
# TODO: Clean up file descriptors
if self.verbose:
printv('Removing dir %s' % tmp_indir)
printv('Removing dir %s' % tmp_mapdir)
printv('Removing %s' % temp_img)
rmtree(tmp_indir)
rmtree(tmp_mapdir)
os.remove(temp_img)
return
def minimize_corpus_gbl(self):
""" @brief Minimizes the global dedup directory.
This method uses modified afl-cmin that supports providing before and
after shell scripts that provide the corresponding tgtcmd for each
testcase. Before and after shell scripts are passed to afl-cmin using
a config directory (tmp_cfgdir).
@todo Replace call to copypreserve with a softlink
@todo Cleanup indir and cfgdir after completion
@returns None """
write_state(self.outdir, 'Minimizing global corpus')
# Create temporary directories for managing corpus
tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir)
tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir)
# Copy all the test cases from self.tc_dir to temp directory for corpus
# minimization
total_global_count = 0
for tc, _ in self.global_dedup_list_tc:
if nh.is_tc(tc):
total_global_count += 1
src = tc
dest = path.join(tmp_indir, path.basename(tc))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying testcase {src} -> {dest}')
# Copy the map
src = path.join(path.dirname(tc), 'map_' + path.basename(tc))
dest = path.join(tmp_mapdir, 'map_' + path.basename(tc))
copypreserve(src, dest)
if self.verbose:
printv(f'Copying map {src} -> {dest}')
# Copy the pm map
src = path.join(path.dirname(tc),
'pm_map_' + path.basename(tc))
dest = path.join(tmp_mapdir, 'pm_map_' + path.basename(tc))
if path.isfile(src):
copypreserve(src, dest)
self.printv(f'Copying map {src} -> {dest}')
# Create a temp image for cases that don't have any parent
fd, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-',
dir=self.tempdir)
# We don't actually need this file, todo: do better
os.close(fd)
os.remove(temp_img)
_, tgtcmd_loc = \
nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg)
gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose)
# Run the actual thing
outdir = run_afl_cmin(
indir=tmp_indir,
pmfuzzdir=self.outdir,
tgtcmd=tgtcmd_loc,
cfg=self.cfg,
# cfgdir=tmp_cfgdir,
verbose=self.verbose,
dry_run=False,
mapdir=tmp_mapdir,
)
dropped_files = []
for f in os.listdir(tmp_indir):
if f not in os.listdir(outdir):
dropped_files.append(f)
if self.verbose:
printv('Dropping %d of %d testcases after cmin' \
% (len(dropped_files), total_global_count))
# Remove the dropped testcases and associated metadata files
for f in dropped_files:
metadata_files = nh.get_metadata_files(f, deleted=False)
for metadata_file_type in metadata_files:
if not metadata_files[metadata_file_type].endswith('.tar.gz'):
file_to_delete = path.join(
self.dedup_dir_gbl, metadata_files[metadata_file_type])
try:
os.remove(file_to_delete)
if self.verbose:
printv('Removing ' + file_to_delete)
except FileNotFoundError:
pass
# Create a placeholder .deleted file to avoid stage1 from
# repopulating that testcase
placeholder_f = path.join(
self.dedup_dir_gbl,
nh.get_metadata_files(f, deleted=True)['deleted'])
with open(placeholder_f, 'w') as obj:
obj.write('deleted at epoch=%d' % int(time.time()))
if self.verbose:
printv('Removing dir %s' % tmp_indir)
printv('Removing dir %s' % tmp_mapdir)
printv('Removing %s' % temp_img)
rmtree(tmp_indir)
rmtree(tmp_mapdir)
os.remove(temp_img)
return
def construct_cmin_sh_files(self, srcdir, scriptsdir, imgdirs, tmp_img):
""" @brief Constructs the .before.sh and .after.sh for testcases
@param srcdir Path to directory containing the testcases
@param scriptsdir Path to directory to write the scripts in
@param imgdirs List of path to the directory that contains all the
compressed img
@param Path to an empty image for testcases with no parents
@return None
"""
# Create config directory containing the command for each testcase
for f in os.listdir(srcdir):
tc_name = path.basename(f)
parent = nh.get_testcase_parent(tc_name)
# Results of the first stage don't have any parents and have to be
# run with empty images
has_parent = nh.iter_cnt(tc_name) > 1
with open(path.join(scriptsdir, tc_name + '.before.sh'),
'w') as obj:
# Generate a decompress commmand to decompress the images only
# if the testcase is not from the first stage
if has_parent:
# Src
img_src_fname = nh.get_metadata_files(
parent)['pm_cmpr_pool']
img_src = None
for imgdir in imgdirs:
img_src = path.join(imgdir, img_src_fname)
if os.path.isfile(img_src):
break
else:
abort('No candidate for parent image found in %d dirs'\
% len(imgdirs))
# Dest
img_dest_fname = nh.get_metadata_files(parent)['pm_pool']
img_dest = path.join(scriptsdir, img_dest_fname)
decompress_cmd = \
get_decompress_cmd(img_src, img_dest, self.verbose)
# Generate an echo command with the target command
tgtcmd_loc = list(self.cfg.tgtcmd)
_, tgtcmd_loc \
= nh.set_img_path(tgtcmd_loc, img_dest, self.cfg)
echo_cmd = 'echo "%s"' % (' '.join(tgtcmd_loc))
obj.write('%s;\n%s;\n' %
(' '.join(decompress_cmd), echo_cmd))
else:
# 1. Set the temp image in tgt command
tgtcmd_loc = list(self.cfg.tgtcmd)
_, tgtcmd_loc \
= nh.set_img_path(tgtcmd_loc, tmp_img, self.cfg)
# 2. Generate an echo command with the target command
echo_cmd = 'echo "%s"' % (' '.join(tgtcmd_loc))
# 3. Construct the shell script and write it
obj.write('%s;\n' % (echo_cmd))
with open(path.join(scriptsdir, tc_name + '.after.sh'),
'w') as obj:
if has_parent:
# Generate command to remove the image
rm_img_cmd = 'rm %s' % (img_dest)
obj.write('%s;\n' % (rm_img_cmd))
else:
# Don't do anything, ':' character is NOP for shell
obj.write(':;\n')
def tc_gen_crash_sites(self, raw_tcname):
""" Generates crash sites for a testcase
Testcases is read and crash sites are generated for each of the
failure point using the testcase's parent's image. These generated
results are then sent to the crashsite directory:
(e.g., <outdir>/stage=2,iter=1/crashsites/)
@param raw_tcname Path to a testcase to generate crash sites for
"""
printi('Generating crash images for testcase: ' + raw_tcname)
clean_name = nh.clean_tc_name(path.basename(raw_tcname))
tc_components = os.path.normpath(raw_tcname).split(os.sep)
parent_name = tc_components[-4] + '.testcase'
printi('Clean name: ' + clean_name)
printi('Parent name: ' + parent_name)
# NOTE:
# Create image names, *_uniq files are to provide each instance of
# tc_gen_crash_sites() with a unique image to work with
pm_dir = tempfile.mkdtemp(prefix='pmfuzz-cs-gen-st2-',
dir=self.cfg('pmfuzz.img_loc'))
parent_cmpr_img = nh.get_parent_img(parent_name,
self.dedup.dedup_dir_gbl,
'exists',
isparent=True,
verbose=self.verbose)
# Create image path
parent_img_name = ''
if parent_cmpr_img.endswith(nh.CMPR_PM_IMG_EXT):
parent_img_name = nh.get_metadata_files(parent_name)['pm_pool']
elif parent_cmpr_img.endswith(nh.CMPR_CRASH_SITE_EXT):
parent_img_name = nh.get_metadata_files(parent_name)['crash_site']
parent_img_name_uniq = nh.get_metadata_files(parent_name)\
['clean'] + '<pid=' + str(os.getpid()) + '>'\
+ '.' + nh.CRASH_SITE_EXT
parent_img = path.join(pm_dir, parent_img_name)
parent_img_uniq = path.join(pm_dir, parent_img_name_uniq)
# Decompres+Copy the image
if not os.path.isfile(parent_img):
decompress(parent_cmpr_img, parent_img, self.verbose)
printv('tempimg: %s -> %s' % (parent_cmpr_img, parent_img))
copypreserve(parent_img, parent_img_uniq)
printv('unique image: %s -> %s' % (parent_img, parent_img))
finj.run_failure_inj(self.cfg, self.cfg.tgtcmd, parent_img_uniq,
raw_tcname, clean_name, self.verbose)
crash_imgs_pattern = parent_img.replace('.pm_pool', '') + '.' \
+ clean_name.replace('.testcase', '') + '.*'
crash_imgs = glob(crash_imgs_pattern)
if self.verbose:
printi('Total %d crash images generated.' % len(crash_imgs))
printw('Deleting %s' % (parent_img))
os.remove(parent_img_uniq)
if self.verbose:
printv('Compressing all the crash sites')
self.process_new_crash_sites(parent_img_uniq, clean_name)
if self.verbose:
printv('Crash sites compressed')