def collect_results(self) -> None: """ @brief Copies the results from the master fuzzer to local tc & img directory. Operations are performed in parallel, using self.cores number of jobs at a time. Only collects the available queue output directories in the .afl_results directory. Expects most of the testcases to be already copied on termination. Present solely to collect testcases orphaned on the previous instance of PMFuzz. @return None """ printi('Collecting stage 2, iter ' + str(self.iter_id)) # Get all the output directories for all the testcases that were run o_dirs = self.get_o_tc_dirs() # Create a parallel object prl = Parallel(self.collect_tc, self.cores, failure_mode=Parallel.FAILURE_EXIT, transparent_io=False, verbose=self.verbose) cnt = 0 found_cases = os.listdir(self.tc_dir) # Collect testcases from all of them for o_dir in o_dirs: gen_cases = [name for name in listdir(o_dir) \ if name.startswith('id') == True] for gen_case in gen_cases: # Parent name (name of the testcase that generated the image # for this testcase) parent_name = path.dirname(path.dirname(o_dir)) + ',' # Remove unnecessary information from testcase's name and check # if this testcase is not already copied clean_name = parent_name + nh.clean_tc_name(gen_case) if not path.basename(clean_name) in found_cases: prl.run([o_dir, gen_case, clean_name]) cnt += 1 prl.wait() if self.verbose: printv('%d cases processed (%d already exists).' \ % (cnt, len(found_cases)))
def tc_gen_crash_sites(self, raw_tcname): """ Generates crash sites for a testcase Testcases is read and crash sites are generated for each of the failure point using the testcase's parent's image. These generated results are then sent to the crashsite directory: (e.g., <outdir>/stage=1,iter=1/crashsites/) @see Stage2.tc_gen_crash_sites() @param raw_tcname Path to a testcase to generate crash sites for """ printi('Generating crash images for testcase: ' + raw_tcname) clean_name = nh.clean_tc_name(path.basename(raw_tcname)) # Create an empty image to run failure injection crash_img_prefix = None with TempEmptyImage(self.cfg, self.verbose) as tmp_img: crash_img_prefix = tmp_img if self.verbose: printv('Generating crash images for image %s' % crash_img_prefix) finj.run_failure_inj(self.cfg, self.cfg.tgtcmd, tmp_img, raw_tcname, clean_name, create=False, verbose=self.verbose) crash_imgs_pattern = crash_img_prefix.replace('.pm_pool', '') + '.' \ + clean_name.replace('.testcase', '') + '.*' crash_imgs = glob(crash_imgs_pattern) if self.verbose: printi('Total %d crash images generated.' % len(crash_imgs)) printv('Compressing all the crash sites') self.compress_new_crash_sites(crash_img_prefix, clean_name) self.add_cs_hash_lcl() if self.verbose: printv('Crash sites compressed')
def collect_results(self) -> None: """ Copies the results from the master fuzzer to local tc & img directory @return None """ found_cases = os.listdir(self.tc_dir) gen_cases = [name for name in listdir(self.o_tc_dir) \ if name.startswith('id') == True] cnt = 0 for gen_case in gen_cases: # Remove unnecessary information from testcase's name and check if # this testcase is not already copied clean_name = nh.clean_tc_name(gen_case) if not clean_name in found_cases: self.collect_tc(gen_case, clean_name) cnt += 1 if self.verbose: printv('%d cases processed (%d already exists).' \ % (cnt, len(found_cases)))
def tc_gen_crash_sites(self, raw_tcname): """ Generates crash sites for a testcase Testcases is read and crash sites are generated for each of the failure point using the testcase's parent's image. These generated results are then sent to the crashsite directory: (e.g., <outdir>/stage=2,iter=1/crashsites/) @param raw_tcname Path to a testcase to generate crash sites for """ printi('Generating crash images for testcase: ' + raw_tcname) clean_name = nh.clean_tc_name(path.basename(raw_tcname)) tc_components = os.path.normpath(raw_tcname).split(os.sep) parent_name = tc_components[-4] + '.testcase' printi('Clean name: ' + clean_name) printi('Parent name: ' + parent_name) # NOTE: # Create image names, *_uniq files are to provide each instance of # tc_gen_crash_sites() with a unique image to work with pm_dir = tempfile.mkdtemp(prefix='pmfuzz-cs-gen-st2-', dir=self.cfg('pmfuzz.img_loc')) parent_cmpr_img = nh.get_parent_img(parent_name, self.dedup.dedup_dir_gbl, 'exists', isparent=True, verbose=self.verbose) # Create image path parent_img_name = '' if parent_cmpr_img.endswith(nh.CMPR_PM_IMG_EXT): parent_img_name = nh.get_metadata_files(parent_name)['pm_pool'] elif parent_cmpr_img.endswith(nh.CMPR_CRASH_SITE_EXT): parent_img_name = nh.get_metadata_files(parent_name)['crash_site'] parent_img_name_uniq = nh.get_metadata_files(parent_name)\ ['clean'] + '<pid=' + str(os.getpid()) + '>'\ + '.' + nh.CRASH_SITE_EXT parent_img = path.join(pm_dir, parent_img_name) parent_img_uniq = path.join(pm_dir, parent_img_name_uniq) # Decompres+Copy the image if not os.path.isfile(parent_img): decompress(parent_cmpr_img, parent_img, self.verbose) printv('tempimg: %s -> %s' % (parent_cmpr_img, parent_img)) copypreserve(parent_img, parent_img_uniq) printv('unique image: %s -> %s' % (parent_img, parent_img)) finj.run_failure_inj(self.cfg, self.cfg.tgtcmd, parent_img_uniq, raw_tcname, clean_name, self.verbose) crash_imgs_pattern = parent_img.replace('.pm_pool', '') + '.' \ + clean_name.replace('.testcase', '') + '.*' crash_imgs = glob(crash_imgs_pattern) if self.verbose: printi('Total %d crash images generated.' % len(crash_imgs)) printw('Deleting %s' % (parent_img)) os.remove(parent_img_uniq) if self.verbose: printv('Compressing all the crash sites') self.process_new_crash_sites(parent_img_uniq, clean_name) if self.verbose: printv('Crash sites compressed')
def _terminate_cs(self, csname: str, timer): """ @brief Terminates a crash site run @return None """ if self.verbose and timer != None: time_delta_str = timer.elapsed_hr() printv("Time elapsed: {:0>8}".format(time_delta_str)) write_state(self.outdir, 'Collecting CS ' + csname) # Read the pid pid = None q_dir = path.join(self.get_result_dir(csname), 'queue') pid_f = path.join(self.get_result_dir(csname), 'pid') # Kill only if the pid file exists (indicating a running AFL process) if path.isfile(pid_f): printi('Killing ' + pid_f) with open(pid_f, 'r') as fobj: pid = int(fobj.read().strip()) # Kill the running AFL instance os.kill(pid, signal.SIGTERM) # Send signal 9 # Remove the pid file remove(pid_f) # Remove the image file imgdir = self.cfg['pmfuzz']['img_loc'] + '/' imgpm = path.join(imgdir, 'pmfuzz-cs-run-' + csname) dir2del = glob(imgpm + '*') abort_if( len(dir2del) == 0, 'Unable to find anything with glob ' + imgpm + '*') abort_if( len(dir2del) > 1, 'Too many matches for glob ' + imgpm + '*') if self.verbose: printv('Removing image %s' % dir2del[0]) rmtree(dir2del[0]) # TODO: Move this to a single function for both _terminate_tc and # _terminate_cs printi('Collecting testcases for ' + csname) core_count = self.cores // 2 if self.cores // 2 != 0 else 1 printi('Using %d cores for collecting results' % core_count) # Create a parallel object for collecting testcases prl_ct = Parallel( self.collect_tc, core_count, transparent_io=False, failure_mode=Parallel.FAILURE_EXIT, name='Collect TC', verbose=self.verbose, ) # Create a parallel object for collecting crash sites prl_gen_cs = Parallel( self.tc_gen_crash_sites, core_count, transparent_io=False, failure_mode=Parallel.FAILURE_EXIT, name='Gen Crash Site', verbose=self.verbose, ) q_dir_contents = os.listdir(q_dir) q_dir_contents = [f for f in q_dir_contents if f.startswith('id')] # Collect the generated testcases for childtc in q_dir_contents: if childtc != '.state': if self.verbose: printv('Collecting %s from the queue directory' \ % childtc) exp_img_path = nh.get_parent_img(csname + '.' + nh.TC_EXT, self.dedup.dedup_dir_gbl, get='exists', isparent=True) abort_if(not os.path.isfile(exp_img_path), 'Sanity check: Compressed image ' + exp_img_path + \ ' should exist, but is missing.') clean_name = csname + ',' + nh.clean_tc_name(childtc) # Run collect_tc() prl_ct.run([q_dir, childtc, clean_name]) # Generate crash sites by injecting failures tcdir = path.join(self.afl_dir, path.basename(csname)) if self.cfg['pmfuzz']['failure_injection']['enable']: tcdir_path = path.join(tcdir, 'master_fuzzer', 'queue', childtc) randval = randrange(100) if randval < self.CS_GEN_THRESH: prl_gen_cs.run([tcdir_path]) else: self.printv('Skipping cs generation'\ +f' ({randval} < {self.CS_GEN_THRESH})') prl_ct.wait() prl_gen_cs.wait() printi('Cleaning up local uncompressed images') self.clean_up_uncmpr_lcl() self.add_cs_hash_lcl() # Deduplicate testcases with existing results testcases_path = listdir(self.tc_dir) testcases_path = [path.join(self.tc_dir, fname) \ for fname in testcases_path] DedupEngine(testcases_path, self.verbose, checker=nh.is_map).run() lcl_cfg = self.cfg['pmfuzz']['stage']['dedup']['local'] write_state(self.outdir, 'Minimizing local') self.dedup.run( fdedup=True, min_tc=False, # TODO min_corpus=lcl_cfg['minimize_corpus'], gbl=False) printi('Killed testcase %s (pid %d).' % (csname, pid))
def _terminate_testcase(self, testcasename: str, timer): """ @brief Terminates a testcase @return None """ if self.verbose and timer != None: time_delta_str = timer.elapsed_hr() printv("Time elapsed: {:0>8}".format(time_delta_str)) write_state(self.outdir, 'Collecting ' + testcasename) # Read the pid pid = None q_dir = path.join(self.get_result_dir(testcasename), 'queue') pid_f = path.join(self.get_result_dir(testcasename), 'pid') # abort_if(not path.isfile(pid_f), 'PID file not found at ' + pid_f) self.printv('Terminating ' + testcasename + ', checking for pid_f at '\ + pid_f) # Kill only if the pid file exists (indicating a running AFL process) if path.isfile(pid_f): printi('Killing ' + pid_f) with open(pid_f, 'r') as fobj: pid = int(fobj.read().strip()) # Kill the running AFL instance os.kill(pid, signal.SIGTERM) # Send signal 9 # Remove the pid file remove(pid_f) # Remove the image file self.printv('Removing directory %s' \ % self.get_img_dir(testcasename)) rmtree(self.get_img_dir(testcasename)) printi('Collecting testcases for ' + testcasename) core_count = self.cores // 2 if self.cores // 2 != 0 else 1 printi('Using %d cores for collecting results' % core_count) # Create a parallel object for collecting testcases prl_ct = Parallel( self.collect_tc, core_count, transparent_io=True, failure_mode=Parallel.FAILURE_EXIT, name='Collect TC', verbose=self.verbose, ) # Create a parallel object for collecting crash sites prl_gen_cs = Parallel( self.tc_gen_crash_sites, core_count, transparent_io=True, failure_mode=Parallel.FAILURE_EXIT, name='Gen Crash Site', verbose=self.verbose, ) q_dir_contents = os.listdir(q_dir) q_dir_contents = [f for f in q_dir_contents if f.startswith('id')] # Collect the generated testcases for childtc in q_dir_contents: if childtc != '.state': if self.verbose: printv('Collecting %s from the queue directory' \ % childtc) exp_img_path = path.join(self.dedup.dedup_dir_gbl, testcasename + '.pm_pool.tar.gz') abort_if(not os.path.isfile(exp_img_path), 'Sanity check: Compressed image ' + exp_img_path + \ ' should exist, but is missing.') clean_name = testcasename + ',' + nh.clean_tc_name(childtc) # Run collect_tc() prl_ct.run([q_dir, childtc, clean_name]) # Generate crash sites by injecting failures tcdir = path.join(self.afl_dir, path.basename(testcasename)) if self.cfg['pmfuzz']['failure_injection']['enable']: tcdir_path = path.join(tcdir, 'master_fuzzer', 'queue', childtc) randval = randrange(100) if randval < self.CS_GEN_THRESH: prl_gen_cs.run([tcdir_path]) else: self.printv('Skipping cs generation'\ +f' ({randval} < {self.CS_GEN_THRESH})') prl_ct.wait() prl_gen_cs.wait() printi('Cleaning up local uncompressed images') self.clean_up_uncmpr_lcl() self.add_cs_hash_lcl() #! Disable this: We don't need to deduplicate the maps # # Deduplicate testcases with existing results # testcases_path = listdir(self.tc_dir) # testcases_path = [path.join(self.tc_dir, fname) \ # for fname in testcases_path] # DedupEngine( # testcases_path, # self.verbose, # checker=nh.is_map # ).run() lcl_cfg = self.cfg['pmfuzz']['stage']['dedup']['local'] write_state(self.outdir, 'Minimizing local') self.dedup.run( fdedup=True, min_tc=False, # TODO min_corpus=lcl_cfg['minimize_corpus'], gbl=False) printi('Killed testcase %s (pid %d).' % (testcasename, pid)) else: self.printv('Did not kill testcase %s.' % (testcasename))