def local_testcases_list(self): """ @brief List of all the testcase and corresponding images in the local tc/img directories @return List of tuple with first entry the path to the testcase and second entry a corresponding path to the image or None """ result = [] lcl_tc_files = listdir(self.tc_dir) # Complete the path lcl_tc_files = [path.join(self.tc_dir, fname) for fname in lcl_tc_files\ if nh.is_tc(fname)] for filepath in lcl_tc_files: # If this testcase has a corresponding pool image imgname = path.basename( filepath.replace(Dedup.EXT_TC, Dedup.EXT_PM_CMPR_POOL)) imgpath = path.join(self.img_dir, imgname) abort_if(not path.isfile(imgpath), 'Image not found `%s\'' % imgpath) # Create a tuple entry = (filepath, imgpath) # Add tuple to the results result.append(entry) return result
def local_dedup_list(self): """ @brief Get all the testcase and corresponding images in the local dedup store @return List of tuple with first entry the path to the testcase and second entry a corresponding path to the image or None """ result = [] loc_dedup_files = listdir(self.dedup_dir_loc) for filepath in loc_dedup_files: # Only process testcases if nh.is_tc(filepath, all=True): entry = [filepath, None] # If this testcase has a corresponding pool image imgpath = filepath.replace(Dedup.EXT_TC, Dedup.EXT_PM_POOL) if imgpath in loc_dedup_files: # Add it to the tuple entry[1] = imgpath # Add tuple to the results result.append(tuple(entry)) return result
def minimize_testcases(self): """ Minimize global testcases @return None""" abort_if( len(self.global_dedup_list_tc) == 0, 'Empty global deduplication list') testcases, _ = map(list, zip(*self.global_dedup_list_tc)) prl = parallel.Parallel(self.minimize_testcase, self.cores) for testcase in testcases: # Only process unminimized testcases, exclude anything that has # .min in its name if nh.is_tc(testcase): # TODO: fix this # Get the minimized name to check if this is already processed testcase_min_name = nh.get_tc_min(testcase, Dedup.EXT_TC) if not testcase_min_name in testcases: # Only run if minimization is enabled if self.cfg['pmfuzz']\ ['stage']['dedup']['global']['minimize_tc']: prl.run([testcase]) else: # else, just create a copy of the destcase copypreserve(testcase, testcase_min_name) prl.wait()
def global_dedup_list_tc(self): """ @brief Get all testcase and corresponding images from global store. Does not include crash sites, to list that, see @ref global_dedup_list_cs @return List of tuple with first entry the path to the testcase and second entry a corresponding path to the image or None """ result = [] gbl_dedup_files = listdir(self.dedup_dir_gbl) # Complete the path gbl_dedup_files = [path.join(self.dedup_dir_gbl, fname) \ for fname in gbl_dedup_files] for filepath in gbl_dedup_files: # Only process testcases if nh.is_tc(filepath): entry = [filepath, None] # If this testcase has a corresponding pool image imgpath = nh.get_metadata_files(filepath)['pm_cmpr_pool'] if imgpath in gbl_dedup_files: # Add it to the tuple entry[1] = imgpath else: printw('No image found %s' % imgpath) # Add tuple to the results result.append(tuple(entry)) if self.verbose: printv('Returning %d cases' % len(result)) return result
def minimize_corpus_lcl(self): """ @brief Minimizes the local testcase directory by combining it with the global testcases. This method uses modified afl-cmin that supports providing before and after shell scripts that provide the corresponding tgtcmd for each testcase. Before and after shell scripts are passed to afl-cmin using a config directory (tmp_cfgdir). @returns None """ # Create temporary directories for managing corpus tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir) tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir) # Copy all testcases from global dedup to tmp_indir to allow # minimization on global corpus along with local for f in os.listdir(self.dedup_dir_loc): if nh.is_tc(f): src = path.join(self.dedup_dir_loc, f) dest = path.join(tmp_indir, f) if self.verbose: printv(f'Copying to testcase: {src} -> {dest}') copypreserve(src, dest) elif nh.is_map(f) or nh.is_pm_map(f): src = path.join(self.dedup_dir_loc, f) dest = path.join(tmp_mapdir, f) if path.isfile(src): copypreserve(src, dest) if self.verbose: printv(f'Copying to map: {src} -> {dest}') elif self.verbose: printv(f'Did not find {src}') # Copy all the test cases from self.tc_dir to temp directory for corpus # minimization lcl_tcs = [path.join(self.tc_dir, f) for f in os.listdir(self.tc_dir)] for f in lcl_tcs: if nh.is_tc(f): src = f dest = path.join(tmp_indir, path.basename(f)) copypreserve(src, dest) if self.verbose: printv(f'Copying to testcase: {src} -> {dest}') elif nh.is_map(f) or nh.is_pm_map(f): src = f dest = path.join(tmp_mapdir, path.basename(f)) if path.isfile(src): copypreserve(src, dest) if self.verbose: printv(f'Copying to map: {src} -> {dest}') elif self.verbose: printv(f'Did not find {src}') total_tcs = len(os.listdir(tmp_indir)) # Create a temp image for cases that don't have any parent _, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-', dir=self.tempdir) _, tgtcmd_loc \ = nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg) gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose) # Run the actual thing outdir = run_afl_cmin( indir=tmp_indir, pmfuzzdir=self.outdir, tgtcmd=self.cfg.tgtcmd, cfg=self.cfg, # cfgdir=tmp_cfgdir, verbose=self.verbose, mapdir=tmp_mapdir, dry_run=False, ) dropped_files = [] for f in os.listdir(tmp_indir): if f not in os.listdir(outdir): dropped_files.append(f) if self.verbose: printv('Dropping %d of %d testcases after cmin' \ % (len(dropped_files), total_tcs)) # Remove the dropped testcases and associated metadata files from the # local testcase directory for f in dropped_files: metadata_files = nh.get_metadata_files(f) for metadata_file_type in metadata_files: file_to_delete = path.join(self.tc_dir, metadata_files[metadata_file_type]) try: os.remove(file_to_delete) if self.verbose: printv('Removed ' + file_to_delete) except FileNotFoundError: pass # Remove the dropped testcases and associated metadata files from the # global dedup directory for f in dropped_files: metadata_files = nh.get_metadata_files(f) types_to_delete \ = ['testcase', 'min_testcase', 'pm_map', 'map'] for metadata_file_type in types_to_delete: file_to_delete = path.join(self.dedup_dir_gbl, metadata_files[metadata_file_type]) try: os.remove(file_to_delete) placeholder_f = path.join( self.dedup_dir_gbl, nh.get_metadata_files(f, True)['deleted']) with open(placeholder_f, 'w') as obj: obj.write('Deleted at epoch = ' + str(int(time.time()))) if self.verbose: printv('Writing to ' + placeholder_f) printv('Removed ' + file_to_delete) except FileNotFoundError: pass # TODO: Clean up file descriptors if self.verbose: printv('Removing dir %s' % tmp_indir) printv('Removing dir %s' % tmp_mapdir) printv('Removing %s' % temp_img) rmtree(tmp_indir) rmtree(tmp_mapdir) os.remove(temp_img) return
def minimize_corpus_gbl(self): """ @brief Minimizes the global dedup directory. This method uses modified afl-cmin that supports providing before and after shell scripts that provide the corresponding tgtcmd for each testcase. Before and after shell scripts are passed to afl-cmin using a config directory (tmp_cfgdir). @todo Replace call to copypreserve with a softlink @todo Cleanup indir and cfgdir after completion @returns None """ write_state(self.outdir, 'Minimizing global corpus') # Create temporary directories for managing corpus tmp_indir = tempfile.mkdtemp(prefix='cmin-in-', dir=self.tempdir) tmp_mapdir = tempfile.mkdtemp(prefix='cmin-map-dir-', dir=self.tempdir) # Copy all the test cases from self.tc_dir to temp directory for corpus # minimization total_global_count = 0 for tc, _ in self.global_dedup_list_tc: if nh.is_tc(tc): total_global_count += 1 src = tc dest = path.join(tmp_indir, path.basename(tc)) copypreserve(src, dest) if self.verbose: printv(f'Copying testcase {src} -> {dest}') # Copy the map src = path.join(path.dirname(tc), 'map_' + path.basename(tc)) dest = path.join(tmp_mapdir, 'map_' + path.basename(tc)) copypreserve(src, dest) if self.verbose: printv(f'Copying map {src} -> {dest}') # Copy the pm map src = path.join(path.dirname(tc), 'pm_map_' + path.basename(tc)) dest = path.join(tmp_mapdir, 'pm_map_' + path.basename(tc)) if path.isfile(src): copypreserve(src, dest) self.printv(f'Copying map {src} -> {dest}') # Create a temp image for cases that don't have any parent fd, temp_img = tempfile.mkstemp(prefix='pmfuzz-tmp-img-', dir=self.tempdir) # We don't actually need this file, todo: do better os.close(fd) os.remove(temp_img) _, tgtcmd_loc = \ nh.set_img_path(list(self.cfg.tgtcmd), temp_img, self.cfg) gen_tgt_img(tgtcmd_loc, self.cfg, verbose=self.verbose) # Run the actual thing outdir = run_afl_cmin( indir=tmp_indir, pmfuzzdir=self.outdir, tgtcmd=tgtcmd_loc, cfg=self.cfg, # cfgdir=tmp_cfgdir, verbose=self.verbose, dry_run=False, mapdir=tmp_mapdir, ) dropped_files = [] for f in os.listdir(tmp_indir): if f not in os.listdir(outdir): dropped_files.append(f) if self.verbose: printv('Dropping %d of %d testcases after cmin' \ % (len(dropped_files), total_global_count)) # Remove the dropped testcases and associated metadata files for f in dropped_files: metadata_files = nh.get_metadata_files(f, deleted=False) for metadata_file_type in metadata_files: if not metadata_files[metadata_file_type].endswith('.tar.gz'): file_to_delete = path.join( self.dedup_dir_gbl, metadata_files[metadata_file_type]) try: os.remove(file_to_delete) if self.verbose: printv('Removing ' + file_to_delete) except FileNotFoundError: pass # Create a placeholder .deleted file to avoid stage1 from # repopulating that testcase placeholder_f = path.join( self.dedup_dir_gbl, nh.get_metadata_files(f, deleted=True)['deleted']) with open(placeholder_f, 'w') as obj: obj.write('deleted at epoch=%d' % int(time.time())) if self.verbose: printv('Removing dir %s' % tmp_indir) printv('Removing dir %s' % tmp_mapdir) printv('Removing %s' % temp_img) rmtree(tmp_indir) rmtree(tmp_mapdir) os.remove(temp_img) return