def __pre_sync_handler(self): log_mapserver("__pre_sync_handler: " + str(self.round_counter_master_pre ) + " / " + str(self.round_counter)) if (self.round_counter_master_pre == self.round_counter):# or self.abortion_alredy_sent: send_msg(KAFL_TAG_UNTOUCHED_NODES, self.treemap.get_num_of_untouched_nodes(), self.comm.to_master_from_mapserver_queue) return True return False
def __respond_redqueen_req(self, response): payload = response.data[0] self.q.set_payload(payload) if not self.q.execute_in_redqueen_mode(se_mode=False): self.__restart_vm() send_msg(KAFL_TAG_REQ_REDQUEEN, True, self.comm.to_master_from_slave_queue, source=self.slave_id) return send_msg(KAFL_TAG_REQ_REDQUEEN, True, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __result_tag_handler(self, request): self.comm.slave_locks_B[request.source].acquire() results = request.data payloads = [] bitmaps = [] payload_shm = self.comm.get_mapserver_payload_shm(request.source) bitmap_shm = self.comm.get_bitmap_shm(request.source) for result in results: if result.new_bits: bitmap_shm.seek(result.pos * self.comm.get_bitmap_shm_size()) payload_shm.seek(result.pos * self.comm.get_mapserver_payload_shm_size()) length = payload_shm.read(4) data_len = (ord(length[3]) << 24) + (ord(length[2]) << 16) + (ord(length[1]) << 8) + \ (ord(length[0])) payloads.append(payload_shm.read(data_len)) bitmaps.append(bitmap_shm.read( self.comm.get_bitmap_shm_size())) else: payloads.append(None) bitmaps.append(None) #log_mapserver("[MAPS]\t\ SKIP") self.comm.slave_locks_A[request.source].release() for i in range(len(results)): if results[i].reloaded: self.abortion_counter += 1 if results[i].new_bits: if results[i].timeout: self.mapserver_state_obj.timeout += 1 new_hash = mmh3.hash64(bitmaps[i]) self.__check_hash(new_hash, bitmaps[i], payloads[i], results[i].crash, results[i].timeout, results[i].kasan, results[i].slave_id, results[i].reloaded, results[i].performance, results[i].qid, results[i].pos) self.last_hash = new_hash self.round_counter += 1 if self.effector_initial_bitmap: if self.effector_initial_bitmap != new_hash: for j in results[i].affected_bytes: if not self.effector_map[j]: self.effector_map[j] = True else: self.round_counter += 1 # TODO: Replace const value by performance*(1/50)s if self.abortion_counter >= self.abortion_threshold: if not self.abortion_alredy_sent: log_mapserver("Stage abortion limit (" + str(self.abortion_threshold) + ") reached!") send_msg(KAFL_TAG_ABORT_REQ, self.mapserver_state_obj, self.comm.to_master_queue) self.abortion_alredy_sent = True self.comm.stage_abortion_notifier.value = True
def __respond_benchmark_req(self, response): payload = response.data[0] benchmark_rate = response.data[1] for i in range(benchmark_rate): self.q.set_payload(payload) self.q.send_payload() if self.q.crashed or self.q.timeout or self.q.kasan: self.__restart_vm() send_msg(KAFL_TAG_REQ_BENCHMARK, None, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __verification_sync_handler(self): log_mapserver("__verificatiom_sync_handler: " + str(self.round_counter_verification_sync) + " / " + str(self.round_counter)) if (self.round_counter_verification_sync == self.round_counter): send_msg(KAFL_TAG_REQ_VERIFY_SYNC, 0, self.comm.to_master_from_mapserver_queue) return True return False
def __effector_sync_handler(self): log_mapserver("__effector_sync_handler: " + str(self.round_counter_effector_sync) + " / " + str(self.round_counter)) if (self.round_counter_effector_sync == self.round_counter) or self.abortion_alredy_sent: send_msg(KAFL_TAG_GET_EFFECTOR, self.effector_map, self.comm.to_master_from_mapserver_queue) return True return False
def __respond_bitmap_req(self, response): self.q.set_payload(response.data) while True: try: bitmap = self.q.send_payload() break except: log_slave("__respond_bitmap_req failed...\n%s"%(traceback.format_exc()), self.slave_id) self.__restart_vm() send_msg(KAFL_TAG_REQ_BITMAP, bitmap, self.comm.to_master_from_slave_queue, source=self.slave_id)
def loop(self): self.comm.reload_semaphore.acquire() self.q.start() self.comm.reload_semaphore.release() send_msg(KAFL_TAG_START, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) while True: if self.comm.slave_termination.value: return self.interprocess_proto_handler()
def __fin_preliminary_tag_handler(self, request): # Todo flush shadow map if self.preliminary_mode != request.data: self.preliminary_mode = request.data if self.preliminary_mode: self.state["preliminary"] = 0 self.last_hash = "" log_mapserver("Preliminary Mode: " + str(self.preliminary_mode)) send_msg(KAFL_TAG_REQ_PRELIMINARY, self.treemap.toggle_preliminary_mode(request.data), self.comm.to_master_from_mapserver_queue)
def __respond_verification(self, response): jobs = response.data[0] methods = response.data[1] results = [] i = 0 self.comm.slave_locks_A[self.slave_id].acquire() while True: payload, payload_shm_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i, self.comm.get_master_payload_shm_size()) payload_content_len_init = struct.unpack("I", payload[0:4])[0] payload_content_len = perform_trim(payload_content_len_init, self.q.send_payload, self.q.modify_payload_size, self.error_handler) if payload_content_len_init != payload_content_len: log_slave("TRIM: " + "{0:.2f}".format(((payload_content_len*1.0)/(payload_content_len_init*1.0))*100.0) + "% (" + str(payload_content_len) + "/" + str(payload_content_len_init) + ")", self.slave_id) patches = jobs[0] if len(patches) > 0: log_slave("Got payload to fix with size: %d and patches %s"%( payload_content_len, patches), self.slave_id ) if len(patches): log_redq("Slave "+str(self.slave_id)+" Orig Payload: " + repr(payload[4:4+payload_content_len])) hash = HashFixer(self.q, self.redqueen_state) new_payload = hash.try_fix_data(payload[4:4+payload_content_len]) if new_payload: log_redq("Slave "+str(self.slave_id)+"Fixed Payload: " + repr("".join(map(chr,new_payload)))) payload = payload[:4]+"".join(map(chr,new_payload)) self.q.set_payload(new_payload) start_time = time.time() bitmap = self.q.send_payload(apply_patches=False) performance = time.time() - start_time log_slave("performance: " + str(1.0/performance) + " -> " + str(performance), self.slave_id) break if not bitmap: log_slave("SHM ERROR....", self.slave_id) new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(), bitmap, payload, payload_shm_size, effector_mode_hash=None, apply_patches = False) if new_bits: self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size()) results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i], self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id)) self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
def __post_sync_handler(self): if self.round_counter_master_post == self.round_counter: #or self.abortion_alredy_sent: if self.post_sync_master_tag == KAFL_TAG_NXT_UNFIN: data = self.treemap.get_next(self.performance, finished=False) else: data = self.treemap.get_next(self.performance, finished=True) self.__update_state() self.mapserver_state_obj.level = data.level + 1 state = data.node_state #data = data.load_payload() #data = state if state == KaflNodeState.in_progress or state == KaflNodeState.finished: send_msg(KAFL_TAG_NXT_UNFIN, data, self.comm.to_master_from_mapserver_queue) else: send_msg(KAFL_TAG_NXT_FIN, data, self.comm.to_master_from_mapserver_queue) #if len(self.shadow_map) > 1024: # self.shadow_map = set() return True return False
def interprocess_proto_handler(self): response = recv_msg(self.comm.to_slave_queues[self.slave_id]) if response.tag == KAFL_TAG_JOB: self.__respond_job_req(response) send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) elif response.tag == KAFL_TAG_REQ_BITMAP: self.__respond_bitmap_req(response) elif response.tag == KAFL_TAG_REQ_SAMPLING: self.__respond_sampling_req(response) elif response.tag == KAFL_TAG_REQ_BENCHMARK: self.__respond_benchmark_req(response) else: log_slave("Received TAG: " + str(response.tag), self.slave_id)
def __post_sync_handler(self): if self.round_counter_master_post == self.round_counter: self.treemap.resort_favs() if self.post_sync_master_tag == KAFL_TAG_NXT_UNFIN: data = self.treemap.get_next(self.performance, finished=False) else: data = self.treemap.get_next(self.performance, finished=True) self.__update_state() self.state["level"] = data.level + 1 if self.state["level"] > self.state["max_level"]: self.state["max_level"] = self.state["level"] state = data.node_state if state == KaflNodeState.in_progress or state == KaflNodeState.finished: send_msg(KAFL_TAG_NXT_UNFIN, data, self.comm.to_master_from_mapserver_queue) else: send_msg(KAFL_TAG_NXT_FIN, data, self.comm.to_master_from_mapserver_queue) self.round_counter = 0 return True return False
def interprocess_proto_handler(self): response = recv_msg(self.comm.to_slave_queues[self.slave_id]) if response.tag == KAFL_TAG_JOB: self.__respond_job_req(response) send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) elif response.tag == KAFL_TAG_REQ_PING: send_msg(KAFL_TAG_REQ_PING, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) elif response.tag == KAFL_TAG_UPDATE_REDQUEEN: self.redqueen_state.update_redqueen_patches(self.q.redqueen_workdir) self.q.send_payload(apply_patches=False) self.q.send_payload(apply_patches=True) send_msg(KAFL_TAG_UPDATE_REDQUEEN, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) elif response.tag == KAFL_TAG_REQ_BITMAP: self.__respond_bitmap_req(response) elif response.tag == KAFL_TAG_REQ_BITMAP_HASH: self.__respond_bitmap_hash_req(response) elif response.tag == KAFL_TAG_REQ_SAMPLING: self.__respond_sampling_req(response) elif response.tag == KAFL_TAG_REQ_BENCHMARK: self.__respond_benchmark_req(response) elif response.tag == KAFL_TAG_REQ_REDQUEEN: self.__respond_redqueen_req(response) elif response.tag == KAFL_TAG_REQ_VERIFY: self.__respond_verification(response) send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id) else: log_slave("Received TAG: " + str(response.tag), self.slave_id)
def __respond_job_req(self, response): results = [] performance = 0.0 counter = 0 jobs = response.data[0] methods = response.data[1] self.q.get_bb_delta() self.comm.slave_locks_A[self.slave_id].acquire() if self.comm.effector_mode.value: effector_mode_hash = (self.comm.effector_mode_hash_a.value, self.comm.effector_mode_hash_b.value) else: effector_mode_hash = None for i in range(len(jobs)): if not self.comm.stage_abortion_notifier.value: new_bits = True vm_reloaded = False self.reloaded = False bitmap = "" payload = "" payload_size = 0 if self.comm.slave_termination.value: self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return while True: while True: try: payload, payload_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i, self.comm.get_master_payload_shm_size()) start_time = time.time() bitmap = self.q.send_payload() performance = time.time() - start_time methods[i].bb_delta = self.q.get_bb_delta() if methods[i].bb_delta != 0: log_slave("FML", self.slave_id) break except Exception as e: log_slave(str(e), self.slave_id) log_slave("%s"%traceback.format_exc(), self.slave_id) if not self.__restart_vm(): return self.reloaded = True if not bitmap: log_slave("SHM ERROR....", self.slave_id) if not self.__restart_vm(): self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return else: break new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(), bitmap, payload, payload_size, effector_mode_hash=effector_mode_hash) if new_bits: self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size()) results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i], self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id)) self.soft_reload_counter += 1 if self.soft_reload_counter >= 10000: self.q.soft_reload() self.soft_reload_counter = 0 else: results.append(FuzzingResult(i, False, False, False, jobs[i], self.slave_id, 0.0, methods[i], None, reloaded=False, new_bits=False, qid=self.slave_id)) if self.comm.slave_termination.value: self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
def __effector_sync_handler(self): if (self.round_counter_effector_sync == self.round_counter): send_msg(KAFL_TAG_GET_EFFECTOR, self.effector_map, self.comm.to_master_from_mapserver_queue) return True return False
def __respond_sampling_req(self, response): payload = response.data[0] sampling_rate = response.data[1] self.stage_tick_treshold = 0 sampling_counter = 0 sampling_ticks = 0 error_counter = 0 round_checker = 0 self.__restart_vm() self.q.set_payload(payload) filter_hash = self.__check_filter_bitmaps() while True: error = False while True: try: self.q.enable_sampling_mode() bitmap = self.q.send_payload() break except: log_slave("Sampling fail...", self.slave_id) log_slave("%s"%traceback.format_exc(), self.slave_id) if not self.__restart_vm(): return for i in range(5): try: if error_counter >= 2: log_slave("Abort sampling...", self.slave_id) error = False break new_bitmap = self.q.send_payload() if self.q.crashed or self.q.timeout or self.q.kasan: log_slave("Sampling timeout...", self.slave_id) error_counter += 1 if not self.__restart_vm(): error = False break else: self.q.submit_sampling_run() sampling_counter += 1 sampling_ticks = self.q.end_ticks - self.q.start_ticks except: log_slave("Sampling wtf??!", self.slave_id) log_slave("%s"%traceback.format_exc(), self.slave_id) if not self.__restart_vm(): return while True: try: self.q.disable_sampling_mode() break except: log_slave("%s"%traceback.format_exc(), self.slave_id) if not self.__restart_vm(): return tmp_hash = self.__check_filter_bitmaps() if tmp_hash == filter_hash: round_checker += 1 else: round_checker = 0 filter_hash = tmp_hash if round_checker == 5: break log_slave("Sampling findished!", self.slave_id) if sampling_counter == 0: sampling_counter = 1 self.stage_tick_treshold = sampling_ticks / sampling_counter log_slave("sampling_ticks: " + str(sampling_ticks), self.slave_id) log_slave("sampling_counter: " + str(sampling_counter), self.slave_id) log_slave("STAGE_TICK_TRESHOLD: " + str(self.stage_tick_treshold), self.slave_id) if self.stage_tick_treshold == 0.0: self.stage_tick_treshold = 1.0 self.q.set_tick_timeout_treshold(3 * self.stage_tick_treshold * self.timeout_tick_factor) send_msg(KAFL_TAG_REQ_SAMPLING, bitmap, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __map_info_tag_handler(self, request): send_msg(KAFL_TAG_MAP_INFO, self.mapserver_state_obj, self.comm.to_master_queue)
def __redqueen_sync_handler(self): if (self.round_counter_redqueen_sync == self.round_counter): send_msg(KAFL_TAG_REDQUEEN_SYNC, 0, self.comm.to_master_from_mapserver_queue) return True return False
def __respond_job_req(self, response): results = [] performance = 0.0 counter = 0 self.comm.slave_locks_A[self.slave_id].acquire() for i in range(len(response.data)): if not self.comm.stage_abortion_notifier.value: new_bits = True vm_reloaded = False self.reloaded = False bitmap = "" payload = "" payload_size = 0 if self.comm.slave_termination.value: self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return while True: while True: try: payload, payload_size = self.q.copy_master_payload( self.comm.get_master_payload_shm( self.slave_id), i, self.comm.get_master_payload_shm_size()) start_time = time.time() bitmap = self.q.send_payload() performance = time.time() - start_time break except: if not self.__restart_vm(): return self.reloaded = True if not bitmap: log_slave("SHM ERROR....", self.slave_id) if not self.__restart_vm(): self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return else: break self.q.finalize_iteration() new_bits = self.q.copy_bitmap( self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(), bitmap, payload, payload_size, effector_mode=self.comm.effector_mode.value) if new_bits: self.q.copy_mapserver_payload( self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size()) if self.comm.slave_termination.value: self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return if self.q.timeout and not (self.q.crashed or self.q.kasan): vm_reloaded = True if mmh3.hash64(bitmap) not in self.false_positiv_map: while True: try: if not self.__restart_vm(): return start_time = time.time() bitmap = self.q.send_payload() performance = time.time() - start_time break except: pass if not self.q.timeout: #false positiv timeout self.false_positiv_map.add(mmh3.hash64(bitmap)) self.reloaded = False else: #false positiv timeout (already seen) self.reloaded = False counter += 1 if self.q.crashed or self.q.timeout or self.q.kasan or self.reloaded: vm_reloaded = True results.append( FuzzingResult(i, self.q.crashed, (self.q.timeout or self.reloaded), self.q.kasan, response.data[i], self.slave_id, performance, reloaded=vm_reloaded, qid=self.slave_id)) if not self.__restart_vm(): self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return self.reloaded = True else: results.append( FuzzingResult(i, self.q.crashed, (self.q.timeout or self.reloaded), self.q.kasan, response.data[i], self.slave_id, performance, reloaded=vm_reloaded, new_bits=new_bits, qid=self.slave_id)) if new_bits and self.auto_reload: self.__restart_vm() self.reloaded = False else: results.append( FuzzingResult(i, False, False, False, response.data[i], self.slave_id, 0.0, reloaded=False, new_bits=False, qid=self.slave_id)) if self.comm.slave_termination.value: self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id) return self.comm.slave_locks_B[self.slave_id].release() send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)