def run(self, input_id_map_list, cov_file_list): """ For each input, -convert ktest -create sync dir -build cmd -create new process job """ se_info("{0} activated. input list : {1}".format(self, input_id_map_list)) for input_id_map in input_id_map_list: pid = self.get_new_pid() input_idx = 0 #--generate klee seed ktest # print input_id_map afl_input = input_id_map['input'] klee_seed = self.seed_dir+"/klee_instance_sym_"+str(pid).zfill(6)+".ktest" # print "before calling converter" # print afl_input self.call_converter("a2k", afl_input, klee_seed, self.bitmodel, self.input_type) if not os.path.exists(klee_seed): print "no seed" + klee_seed continue #--create sync_dir for new klee instance new_sync_dir = self.sync_dir_base+"/klee_instance_sym_"+str(pid).zfill(6)+"/queue" utils.mkdir_force(new_sync_dir) #--build klee instance cmd edge_ids = [x for x in input_id_map['interesting_edges']] stdin_len = os.path.getsize(afl_input) klee_cmd = self.build_cmd(klee_seed, edge_ids, new_sync_dir, stdin_len, afl_input, cov_file_list[input_idx]) print ' '.join(klee_cmd) #--construct process meta data, add to jobs list kw = {'mock_eof':True,'mem_cap': self.max_mem} p = multiprocessing.Process(target=utils.exec_async, args=[klee_cmd], kwargs=kw) p.daemon = True task_st = {} task_st['instance'] = p task_st['sync_dir'] = new_sync_dir task_st['seed'] = klee_seed task_st['cmd'] = klee_cmd if "AFLUnCovSearcher" in self.get_search_heuristics(): task_st['afl_cov'] = self.fuzzer_cov_file self.jobs[pid] = task_st input_idx = input_idx + 1 for pid, task in self.jobs.iteritems(): try: if pid not in self.started_jobs: task['instance'].start() task['real_pid'] = task['instance'].pid # print "starting klee process: ", task['real_pid'] self.started_jobs.add(pid) else: se_info("WTF the process {0} is already started".format(pid)) except Exception: pass
def __init__(self, config, target): self.jobs = {} self.started_jobs = set() self.config = config self.target = target self.get_config() utils.mkdir_force(self.seed_dir) self.pid_ctr = 0 se_info("Concolic Explorer using searcher[{0}]".format(''.join(self.get_search_heuristics())))
def get_moriarty_config(self): config = ConfigParser.ConfigParser() config.read(self.config) if "moriarty" not in config.sections() or "afl" not in config.sections( ): moriarty_info("Config file read error") sys.exit() try: self.compile_script = config.get("moriarty", "compile_script") except Exception: self.compile_script = None self.target_bin = config.get("moriarty", "target_bin").replace( "@target", self.proj_dir) self.target_bc = config.get("moriarty", "target_bc").replace( "@target", self.proj_dir) self.cov_file_base = config.get("moriarty", "sync_dir").replace( "@target", self.proj_dir) self.switch_heuristic = config.get("switch oracle", "strategy") self.max_allow_se_num = int( config.get("moriarty", "max_explorer_instance")) self.num_of_fuzzers = int(config.get("afl", "slave_num")) + 1 self.num_of_explorers = self.se_factory.get_se_size( ) * self.max_allow_se_num self.is_sym_explorer_activated = True if config.has_option( "sym_explorer", "bin") else False self.is_conc_explorer_activated = True if config.has_option( "conc_explorer", "bin") else False try: self.batch_run_seed_num = int( config.get("moriarty", "batch_run_input_num")) except Exception: self.batch_run_seed_num = 1 moriarty_info( "Number of explorers: {0}, each batch run {1} inputs".format( self.num_of_explorers, self.batch_run_seed_num)) try: self.only_count_se_cov = True if "true" in config.get( "moriarty", "only_count_se_cov") else False except Exception: self.only_count_se_cov = True try: self.log_explored_seed_dir = config.get( "moriarty", "explored_seed_dir").replace("@target", self.proj_dir) utils.mkdir_force(self.log_explored_seed_dir) moriarty_info("Saving explored inputs to {0}".format( self.log_explored_seed_dir)) except Exception: moriarty_info("Will not save explored inputs") self.log_explored_seed_dir = None for explorer_id in xrange(self.num_of_explorers): cov = self.cov_file_base cov_suffix = "/coverage.csv" cov = cov + "/.tmp_se_" + str(explorer_id) + ".cov" self.explorer_cov_file_list.append(cov) for slave_id in xrange(self.num_of_fuzzers): #constructing coverage file list #constructing sanitizer_edge file list cov_suffix = "/coverage.csv" san_suffix = "/edge_sanitizer.csv" cov = self.cov_file_base san = self.cov_file_base if slave_id == 0: cov = cov + "/master" + cov_suffix san = san + "/master" + san_suffix else: cov = cov + "/slave_" + str(slave_id).zfill(6) + cov_suffix san = san + "/slave_" + str(slave_id).zfill(6) + san_suffix self.cov_file_list.append(cov) self.san_file_list.append(san) self.clean_up_last_session() #collecting the recommended edges for reasoning try: self.loc_map = self.proj_dir + '/locmap.csv' self.find_loc_script = config.get("afl", "root") + "/find_source_loc.sh" self.recommend_edge_log = config.get("moriarty", "recommend_edge_log").replace( "@target", self.proj_dir) except Exception: self.recommend_edge_log = None self.loc_map = None self.find_loc_script = None
def make_dirs(self): mkdir_force(self.seed_dir)
def run(self, input_id_map_list, cov_file): """ -create seed-out-dir For each input, -convert ktest move to seed-out-dir -create sync dir -build cmd -create new process job """ pid = self.get_new_pid() qsym_seed_dir = self.my_in_dir(str(pid)) mkdir_force(qsym_seed_dir) se_info("{0} activated. input list : {1}".format( self, [x['input'] for x in input_id_map_list])) se_info("{0} activated. input score : {1}".format( self, [x['score'] for x in input_id_map_list])) se_info("{0} activated. input size: {1}".format( self, [x['size'] for x in input_id_map_list])) # sync previousley generated seeds self.sync_gen_seeds( ) #Redundant if the QSYM explorer cycle is always running shorter than the main cycle # launch qsym for each inputs in my_in_dir for input_id_map in input_id_map_list: #QSYM does not support batch mode assert len(input_id_map_list) <= 1 # print input_id_map afl_input = input_id_map['input'] qsym_seed = os.path.join(qsym_seed_dir, afl_input.split("/")[-1]) shutil.copy2(afl_input, qsym_seed) if not os.path.exists(qsym_seed): se_info("no seed created: " + qsym_seed) continue #--create sync_dir for new qsym instance key = "qsym_instance_conc_" + str(pid).zfill(6) new_sync_dir = self.my_sync_dir(key) mkdir_force(new_sync_dir) # temp dir to store tmp genearated seeds # filtered seeds will be transfer to new_sync_dir tmp_dir = tempfile.mkdtemp() mkdir_force(tmp_dir) #--build qsym instance cmd q, qsym_cmd = self.build_cmd(qsym_seed, tmp_dir, self.bitmap) print ' '.join(qsym_cmd) # q.run(self.max_time_per_seed) #--construct process meta data, add to jobs list kw = { 'stdin': q.stdin, 'mem_cap': self.max_mem, 'use_shell': True, 'testcase_dir': q.testcase_dir, 'target_base_path': self.base_dir_from_afl_input(afl_input) } p = multiprocessing.Process( target=utils.qsym_exec_async, args=[qsym_cmd], kwargs=kw) # Needs docker implementation p.daemon = True task_st = {} task_st['instance'] = p task_st['sync_dir'] = new_sync_dir task_st['cmd'] = qsym_cmd task_st['tmp_dir'] = tmp_dir task_st['qsym'] = q task_st['seed_index'] = 0 task_st['synced'] = False task_st['key'] = key task_st['processed'] = False self.jobs[pid] = task_st for pid, task in self.jobs.iteritems(): try: if pid not in self.started_jobs: task['instance'].start() task['real_pid'] = task['instance'].pid self.started_jobs.add(pid) except Exception: pass return (key, [x['input'] for x in input_id_map_list])
def run(self, input_id_map_list, cov_file): """ -create seed-out-dir For each input, -convert ktest move to seed-out-dir -create sync dir -build cmd -create new process job """ pid = self.get_new_pid() klee_seed_dir = self.seed_dir + "/klee_instance_conc_" + str(pid) utils.mkdir_force(klee_seed_dir) input_counter = 0 max_input_size = 0 se_info("{0} activated. input list : {1}".format( self, [x['input'] for x in input_id_map_list])) se_info("{0} activated. input score : {1}".format( self, [x['score'] for x in input_id_map_list])) try: se_info("{0} activated. input size: {1}".format( self, [x['size'] for x in input_id_map_list])) except Exception: pass for input_id_map in input_id_map_list: #--generate klee seed ktest # print input_id_map afl_input = utils.from_simple_to_afl_name(input_id_map['input']) if not afl_input: continue if max_input_size < os.path.getsize(afl_input): max_input_size = os.path.getsize(afl_input) klee_seed = klee_seed_dir + "/" + str(input_counter).zfill( 6) + ".ktest" # print "before calling converter" self.call_converter("a2k", afl_input, klee_seed, self.bitmodel, self.input_type) input_counter += 1 if not os.path.exists(klee_seed): print "no seed" + klee_seed continue #--create sync_dir for new klee instance new_sync_dir = self.sync_dir_base + "/klee_instance_conc_" + str( pid).zfill(6) + "/queue" utils.mkdir_force(new_sync_dir) #--build klee instance cmd edge_ids = [x for x in input_id_map['interesting_edges']] klee_cmd = self.build_cmd(klee_seed_dir, edge_ids, new_sync_dir, max_input_size, afl_input, cov_file) print ' '.join(klee_cmd) #--construct process meta data, add to jobs list kw = {'mock_eof': True, 'mem_cap': self.max_mem, 'use_shell': True} p = multiprocessing.Process(target=utils.exec_async, args=[klee_cmd], kwargs=kw) p.daemon = True task_st = {} task_st['instance'] = p task_st['sync_dir'] = new_sync_dir task_st['seed'] = klee_seed task_st['cmd'] = klee_cmd if "AFLUnCovSearcher" in self.get_search_heuristics(): task_st['afl_cov'] = self.fuzzer_cov_file self.jobs[pid] = task_st for pid, task in self.jobs.iteritems(): try: if pid not in self.started_jobs: task['instance'].start() task['real_pid'] = task['instance'].pid # print "starting klee process: ", task['real_pid'] self.started_jobs.add(pid) else: se_info( "WTF the process {0} is already started".format(pid)) except Exception: pass