def __pre_sync_handler(self):
log_mapserver("__pre_sync_handler: " + str(self.round_counter_master_pre ) + " / " + str(self.round_counter))
if (self.round_counter_master_pre == self.round_counter):# or self.abortion_alredy_sent:
send_msg(KAFL_TAG_UNTOUCHED_NODES, self.treemap.get_num_of_untouched_nodes(),
self.comm.to_master_from_mapserver_queue)
return True
return False
def __respond_redqueen_req(self, response):
payload = response.data[0]
self.q.set_payload(payload)
if not self.q.execute_in_redqueen_mode(se_mode=False):
self.__restart_vm()
send_msg(KAFL_TAG_REQ_REDQUEEN, True, self.comm.to_master_from_slave_queue, source=self.slave_id)
return
send_msg(KAFL_TAG_REQ_REDQUEEN, True, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __result_tag_handler(self, request):
self.comm.slave_locks_B[request.source].acquire()
results = request.data
payloads = []
bitmaps = []
payload_shm = self.comm.get_mapserver_payload_shm(request.source)
bitmap_shm = self.comm.get_bitmap_shm(request.source)
for result in results:
if result.new_bits:
bitmap_shm.seek(result.pos * self.comm.get_bitmap_shm_size())
payload_shm.seek(result.pos *
self.comm.get_mapserver_payload_shm_size())
length = payload_shm.read(4)
data_len = (ord(length[3]) << 24) + (ord(length[2]) << 16) + (ord(length[1]) << 8) + \
(ord(length[0]))
payloads.append(payload_shm.read(data_len))
bitmaps.append(bitmap_shm.read(
self.comm.get_bitmap_shm_size()))
else:
payloads.append(None)
bitmaps.append(None)
#log_mapserver("[MAPS]\t\ SKIP")
self.comm.slave_locks_A[request.source].release()
for i in range(len(results)):
if results[i].reloaded:
self.abortion_counter += 1
if results[i].new_bits:
if results[i].timeout:
self.mapserver_state_obj.timeout += 1
new_hash = mmh3.hash64(bitmaps[i])
self.__check_hash(new_hash, bitmaps[i], payloads[i],
results[i].crash, results[i].timeout,
results[i].kasan, results[i].slave_id,
results[i].reloaded, results[i].performance,
results[i].qid, results[i].pos)
self.last_hash = new_hash
self.round_counter += 1
if self.effector_initial_bitmap:
if self.effector_initial_bitmap != new_hash:
for j in results[i].affected_bytes:
if not self.effector_map[j]:
self.effector_map[j] = True
else:
self.round_counter += 1
# TODO: Replace const value by performance*(1/50)s
if self.abortion_counter >= self.abortion_threshold:
if not self.abortion_alredy_sent:
log_mapserver("Stage abortion limit (" +
str(self.abortion_threshold) + ") reached!")
send_msg(KAFL_TAG_ABORT_REQ, self.mapserver_state_obj,
self.comm.to_master_queue)
self.abortion_alredy_sent = True
self.comm.stage_abortion_notifier.value = True
def __respond_benchmark_req(self, response):
payload = response.data[0]
benchmark_rate = response.data[1]
for i in range(benchmark_rate):
self.q.set_payload(payload)
self.q.send_payload()
if self.q.crashed or self.q.timeout or self.q.kasan:
self.__restart_vm()
send_msg(KAFL_TAG_REQ_BENCHMARK, None, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __verification_sync_handler(self):
log_mapserver("__verificatiom_sync_handler: " +
str(self.round_counter_verification_sync) + " / " +
str(self.round_counter))
if (self.round_counter_verification_sync == self.round_counter):
send_msg(KAFL_TAG_REQ_VERIFY_SYNC, 0,
self.comm.to_master_from_mapserver_queue)
return True
return False
def __effector_sync_handler(self):
log_mapserver("__effector_sync_handler: " +
str(self.round_counter_effector_sync) + " / " +
str(self.round_counter))
if (self.round_counter_effector_sync
== self.round_counter) or self.abortion_alredy_sent:
send_msg(KAFL_TAG_GET_EFFECTOR, self.effector_map,
self.comm.to_master_from_mapserver_queue)
return True
return False
def __respond_bitmap_req(self, response):
self.q.set_payload(response.data)
while True:
try:
bitmap = self.q.send_payload()
break
except:
log_slave("__respond_bitmap_req failed...\n%s"%(traceback.format_exc()), self.slave_id)
self.__restart_vm()
send_msg(KAFL_TAG_REQ_BITMAP, bitmap, self.comm.to_master_from_slave_queue, source=self.slave_id)
def loop(self):
self.comm.reload_semaphore.acquire()
self.q.start()
self.comm.reload_semaphore.release()
send_msg(KAFL_TAG_START, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
while True:
if self.comm.slave_termination.value:
return
self.interprocess_proto_handler()
def __fin_preliminary_tag_handler(self, request):
# Todo flush shadow map
if self.preliminary_mode != request.data:
self.preliminary_mode = request.data
if self.preliminary_mode:
self.state["preliminary"] = 0
self.last_hash = ""
log_mapserver("Preliminary Mode: " + str(self.preliminary_mode))
send_msg(KAFL_TAG_REQ_PRELIMINARY,
self.treemap.toggle_preliminary_mode(request.data),
self.comm.to_master_from_mapserver_queue)
def __respond_verification(self, response):
jobs = response.data[0]
methods = response.data[1]
results = []
i = 0
self.comm.slave_locks_A[self.slave_id].acquire()
while True:
payload, payload_shm_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i, self.comm.get_master_payload_shm_size())
payload_content_len_init = struct.unpack("I", payload[0:4])[0]
payload_content_len = perform_trim(payload_content_len_init, self.q.send_payload, self.q.modify_payload_size, self.error_handler)
if payload_content_len_init != payload_content_len:
log_slave("TRIM: " + "{0:.2f}".format(((payload_content_len*1.0)/(payload_content_len_init*1.0))*100.0) + "% (" + str(payload_content_len) + "/" + str(payload_content_len_init) + ")", self.slave_id)
patches = jobs[0]
if len(patches) > 0:
log_slave("Got payload to fix with size: %d and patches %s"%( payload_content_len, patches), self.slave_id )
if len(patches):
log_redq("Slave "+str(self.slave_id)+" Orig Payload: " + repr(payload[4:4+payload_content_len]))
hash = HashFixer(self.q, self.redqueen_state)
new_payload = hash.try_fix_data(payload[4:4+payload_content_len])
if new_payload:
log_redq("Slave "+str(self.slave_id)+"Fixed Payload: " + repr("".join(map(chr,new_payload))))
payload = payload[:4]+"".join(map(chr,new_payload))
self.q.set_payload(new_payload)
start_time = time.time()
bitmap = self.q.send_payload(apply_patches=False)
performance = time.time() - start_time
log_slave("performance: " + str(1.0/performance) + " -> " + str(performance), self.slave_id)
break
if not bitmap:
log_slave("SHM ERROR....", self.slave_id)
new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(),
bitmap, payload, payload_shm_size, effector_mode_hash=None, apply_patches = False)
if new_bits:
self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size())
results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i], self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id))
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
def __post_sync_handler(self):
if self.round_counter_master_post == self.round_counter: #or self.abortion_alredy_sent:
if self.post_sync_master_tag == KAFL_TAG_NXT_UNFIN:
data = self.treemap.get_next(self.performance, finished=False)
else:
data = self.treemap.get_next(self.performance, finished=True)
self.__update_state()
self.mapserver_state_obj.level = data.level + 1
state = data.node_state
#data = data.load_payload()
#data = state
if state == KaflNodeState.in_progress or state == KaflNodeState.finished:
send_msg(KAFL_TAG_NXT_UNFIN, data, self.comm.to_master_from_mapserver_queue)
else:
send_msg(KAFL_TAG_NXT_FIN, data, self.comm.to_master_from_mapserver_queue)
#if len(self.shadow_map) > 1024:
# self.shadow_map = set()
return True
return False
def interprocess_proto_handler(self):
response = recv_msg(self.comm.to_slave_queues[self.slave_id])
if response.tag == KAFL_TAG_JOB:
self.__respond_job_req(response)
send_msg(KAFL_TAG_REQ,
self.q.qemu_id,
self.comm.to_master_queue,
source=self.slave_id)
elif response.tag == KAFL_TAG_REQ_BITMAP:
self.__respond_bitmap_req(response)
elif response.tag == KAFL_TAG_REQ_SAMPLING:
self.__respond_sampling_req(response)
elif response.tag == KAFL_TAG_REQ_BENCHMARK:
self.__respond_benchmark_req(response)
else:
log_slave("Received TAG: " + str(response.tag), self.slave_id)
def __post_sync_handler(self):
if self.round_counter_master_post == self.round_counter:
self.treemap.resort_favs()
if self.post_sync_master_tag == KAFL_TAG_NXT_UNFIN:
data = self.treemap.get_next(self.performance, finished=False)
else:
data = self.treemap.get_next(self.performance, finished=True)
self.__update_state()
self.state["level"] = data.level + 1
if self.state["level"] > self.state["max_level"]:
self.state["max_level"] = self.state["level"]
state = data.node_state
if state == KaflNodeState.in_progress or state == KaflNodeState.finished:
send_msg(KAFL_TAG_NXT_UNFIN, data,
self.comm.to_master_from_mapserver_queue)
else:
send_msg(KAFL_TAG_NXT_FIN, data,
self.comm.to_master_from_mapserver_queue)
self.round_counter = 0
return True
return False
def interprocess_proto_handler(self):
response = recv_msg(self.comm.to_slave_queues[self.slave_id])
if response.tag == KAFL_TAG_JOB:
self.__respond_job_req(response)
send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
elif response.tag == KAFL_TAG_REQ_PING:
send_msg(KAFL_TAG_REQ_PING, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
elif response.tag == KAFL_TAG_UPDATE_REDQUEEN:
self.redqueen_state.update_redqueen_patches(self.q.redqueen_workdir)
self.q.send_payload(apply_patches=False)
self.q.send_payload(apply_patches=True)
send_msg(KAFL_TAG_UPDATE_REDQUEEN, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
elif response.tag == KAFL_TAG_REQ_BITMAP:
self.__respond_bitmap_req(response)
elif response.tag == KAFL_TAG_REQ_BITMAP_HASH:
self.__respond_bitmap_hash_req(response)
elif response.tag == KAFL_TAG_REQ_SAMPLING:
self.__respond_sampling_req(response)
elif response.tag == KAFL_TAG_REQ_BENCHMARK:
self.__respond_benchmark_req(response)
elif response.tag == KAFL_TAG_REQ_REDQUEEN:
self.__respond_redqueen_req(response)
elif response.tag == KAFL_TAG_REQ_VERIFY:
self.__respond_verification(response)
send_msg(KAFL_TAG_REQ, self.q.qemu_id, self.comm.to_master_queue, source=self.slave_id)
else:
log_slave("Received TAG: " + str(response.tag), self.slave_id)
def __respond_job_req(self, response):
results = []
performance = 0.0
counter = 0
jobs = response.data[0]
methods = response.data[1]
self.q.get_bb_delta()
self.comm.slave_locks_A[self.slave_id].acquire()
if self.comm.effector_mode.value:
effector_mode_hash = (self.comm.effector_mode_hash_a.value, self.comm.effector_mode_hash_b.value)
else:
effector_mode_hash = None
for i in range(len(jobs)):
if not self.comm.stage_abortion_notifier.value:
new_bits = True
vm_reloaded = False
self.reloaded = False
bitmap = ""
payload = ""
payload_size = 0
if self.comm.slave_termination.value:
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
return
while True:
while True:
try:
payload, payload_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i,
self.comm.get_master_payload_shm_size())
start_time = time.time()
bitmap = self.q.send_payload()
performance = time.time() - start_time
methods[i].bb_delta = self.q.get_bb_delta()
if methods[i].bb_delta != 0:
log_slave("FML", self.slave_id)
break
except Exception as e:
log_slave(str(e), self.slave_id)
log_slave("%s"%traceback.format_exc(), self.slave_id)
if not self.__restart_vm():
return
self.reloaded = True
if not bitmap:
log_slave("SHM ERROR....", self.slave_id)
if not self.__restart_vm():
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
return
else:
break
new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(), bitmap, payload, payload_size, effector_mode_hash=effector_mode_hash)
if new_bits:
self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size())
results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i],
self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id))
self.soft_reload_counter += 1
if self.soft_reload_counter >= 10000:
self.q.soft_reload()
self.soft_reload_counter = 0
else:
results.append(FuzzingResult(i, False, False, False, jobs[i], self.slave_id, 0.0, methods[i], None, reloaded=False, new_bits=False, qid=self.slave_id))
if self.comm.slave_termination.value:
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
return
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
def __effector_sync_handler(self):
if (self.round_counter_effector_sync == self.round_counter):
send_msg(KAFL_TAG_GET_EFFECTOR, self.effector_map,
self.comm.to_master_from_mapserver_queue)
return True
return False
def __respond_sampling_req(self, response):
payload = response.data[0]
sampling_rate = response.data[1]
self.stage_tick_treshold = 0
sampling_counter = 0
sampling_ticks = 0
error_counter = 0
round_checker = 0
self.__restart_vm()
self.q.set_payload(payload)
filter_hash = self.__check_filter_bitmaps()
while True:
error = False
while True:
try:
self.q.enable_sampling_mode()
bitmap = self.q.send_payload()
break
except:
log_slave("Sampling fail...", self.slave_id)
log_slave("%s"%traceback.format_exc(), self.slave_id)
if not self.__restart_vm():
return
for i in range(5):
try:
if error_counter >= 2:
log_slave("Abort sampling...", self.slave_id)
error = False
break
new_bitmap = self.q.send_payload()
if self.q.crashed or self.q.timeout or self.q.kasan:
log_slave("Sampling timeout...", self.slave_id)
error_counter += 1
if not self.__restart_vm():
error = False
break
else:
self.q.submit_sampling_run()
sampling_counter += 1
sampling_ticks = self.q.end_ticks - self.q.start_ticks
except:
log_slave("Sampling wtf??!", self.slave_id)
log_slave("%s"%traceback.format_exc(), self.slave_id)
if not self.__restart_vm():
return
while True:
try:
self.q.disable_sampling_mode()
break
except:
log_slave("%s"%traceback.format_exc(), self.slave_id)
if not self.__restart_vm():
return
tmp_hash = self.__check_filter_bitmaps()
if tmp_hash == filter_hash:
round_checker += 1
else:
round_checker = 0
filter_hash = tmp_hash
if round_checker == 5:
break
log_slave("Sampling findished!", self.slave_id)
if sampling_counter == 0:
sampling_counter = 1
self.stage_tick_treshold = sampling_ticks / sampling_counter
log_slave("sampling_ticks: " + str(sampling_ticks), self.slave_id)
log_slave("sampling_counter: " + str(sampling_counter), self.slave_id)
log_slave("STAGE_TICK_TRESHOLD: " + str(self.stage_tick_treshold), self.slave_id)
if self.stage_tick_treshold == 0.0:
self.stage_tick_treshold = 1.0
self.q.set_tick_timeout_treshold(3 * self.stage_tick_treshold * self.timeout_tick_factor)
send_msg(KAFL_TAG_REQ_SAMPLING, bitmap, self.comm.to_master_from_slave_queue, source=self.slave_id)
def __map_info_tag_handler(self, request):
send_msg(KAFL_TAG_MAP_INFO, self.mapserver_state_obj, self.comm.to_master_queue)
def __redqueen_sync_handler(self):
if (self.round_counter_redqueen_sync == self.round_counter):
send_msg(KAFL_TAG_REDQUEEN_SYNC, 0,
self.comm.to_master_from_mapserver_queue)
return True
return False
def __respond_job_req(self, response):
results = []
performance = 0.0
counter = 0
self.comm.slave_locks_A[self.slave_id].acquire()
for i in range(len(response.data)):
if not self.comm.stage_abortion_notifier.value:
new_bits = True
vm_reloaded = False
self.reloaded = False
bitmap = ""
payload = ""
payload_size = 0
if self.comm.slave_termination.value:
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
return
while True:
while True:
try:
payload, payload_size = self.q.copy_master_payload(
self.comm.get_master_payload_shm(
self.slave_id), i,
self.comm.get_master_payload_shm_size())
start_time = time.time()
bitmap = self.q.send_payload()
performance = time.time() - start_time
break
except:
if not self.__restart_vm():
return
self.reloaded = True
if not bitmap:
log_slave("SHM ERROR....", self.slave_id)
if not self.__restart_vm():
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
return
else:
break
self.q.finalize_iteration()
new_bits = self.q.copy_bitmap(
self.comm.get_bitmap_shm(self.slave_id),
i,
self.comm.get_bitmap_shm_size(),
bitmap,
payload,
payload_size,
effector_mode=self.comm.effector_mode.value)
if new_bits:
self.q.copy_mapserver_payload(
self.comm.get_mapserver_payload_shm(self.slave_id), i,
self.comm.get_mapserver_payload_shm_size())
if self.comm.slave_termination.value:
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
return
if self.q.timeout and not (self.q.crashed or self.q.kasan):
vm_reloaded = True
if mmh3.hash64(bitmap) not in self.false_positiv_map:
while True:
try:
if not self.__restart_vm():
return
start_time = time.time()
bitmap = self.q.send_payload()
performance = time.time() - start_time
break
except:
pass
if not self.q.timeout:
#false positiv timeout
self.false_positiv_map.add(mmh3.hash64(bitmap))
self.reloaded = False
else:
#false positiv timeout (already seen)
self.reloaded = False
counter += 1
if self.q.crashed or self.q.timeout or self.q.kasan or self.reloaded:
vm_reloaded = True
results.append(
FuzzingResult(i,
self.q.crashed,
(self.q.timeout or self.reloaded),
self.q.kasan,
response.data[i],
self.slave_id,
performance,
reloaded=vm_reloaded,
qid=self.slave_id))
if not self.__restart_vm():
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
return
self.reloaded = True
else:
results.append(
FuzzingResult(i,
self.q.crashed,
(self.q.timeout or self.reloaded),
self.q.kasan,
response.data[i],
self.slave_id,
performance,
reloaded=vm_reloaded,
new_bits=new_bits,
qid=self.slave_id))
if new_bits and self.auto_reload:
self.__restart_vm()
self.reloaded = False
else:
results.append(
FuzzingResult(i,
False,
False,
False,
response.data[i],
self.slave_id,
0.0,
reloaded=False,
new_bits=False,
qid=self.slave_id))
if self.comm.slave_termination.value:
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
return
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT,
results,
self.comm.to_mapserver_queue,
source=self.slave_id)
