def kafl_workdir_iterator(work_dir): input_id_time = list() start_time = time.time() for stats_file in glob.glob(work_dir + "/slave_stats_*"): if not stats_file: return None slave_stats = msgpack.unpackb(read_binary_file(stats_file), raw=False, strict_map_key=False) start_time = min(start_time, slave_stats['start_time']) # enumerate inputs from corpus/ and match against metainfo in metadata/ for input_file in glob.glob(work_dir + "/corpus/*/*"): if not input_file: return None input_id = os.path.basename(input_file).replace("payload_", "") meta_file = work_dir + "/metadata/node_{}".format(input_id) metadata = msgpack.unpackb(read_binary_file(meta_file), raw=False, strict_map_key=False) seconds = metadata["info"]["time"] - start_time nid = metadata["id"] #print("%s;%d" % (input_file, timestamp)) input_id_time.append([input_file, nid, seconds]) #yield (input_file, nid, timestamp) return input_id_time
def kafl_workdir_iterator(work_dir): input_id_time = list() start_time = time.time() for stats_file in glob.glob(work_dir + "/slave_stats_*"): if not stats_file: return None slave_stats = msgpack.unpackb(read_binary_file(stats_file), raw=False, strict_map_key=False) start_time = min(start_time, slave_stats['start_time']) # enumerate inputs from corpus/ and match against metainfo in metadata/ # TODO: Tracing crashes/timeouts has minimal overall improvement ~1-2% # Probably want to make this optional, and only trace a small sample # of non-regular payloads by default? for input_file in glob.glob(work_dir + "/corpus/[rckt]*/*"): if not input_file: return None input_id = os.path.basename(input_file).replace("payload_", "") meta_file = work_dir + "/metadata/node_{}".format(input_id) metadata = msgpack.unpackb(read_binary_file(meta_file), raw=False, strict_map_key=False) seconds = metadata["info"]["time"] - start_time nid = metadata["id"] input_id_time.append([input_file, nid, seconds]) return input_id_time
def benchmark(config): log_debug("Starting benchmark...") payload_file = config.argument_values["input"] q = qemu(1337, config, debug_mode=False) q.start() q.set_payload(read_binary_file(payload_file)) log_debug("Hash: " + str(q.send_payload().hash())) try: while True: start = time.time() execs = 0 # for i in range(execs): while (time.time() - start < REFRESH): q.set_payload(read_binary_file(payload_file)) q.send_payload() execs += 1 end = time.time() # print("Performance: " + str(execs/(end - start)) + "t/s") stdout.write(common.color.FLUSH_LINE + "Performance: " + str(execs / (end - start)) + "t/s") stdout.flush() except KeyboardInterrupt: stdout.write("\n") q.shutdown() return 0
def __set_agent_and_driver(self): driver_bin = read_binary_file(self.config.argument_values['driver']) bin = p32(len(driver_bin)) + driver_bin agent_bin = read_binary_file(self.config.argument_values['agent']) bin += p32(len(agent_bin)) + agent_bin assert (len(bin) <= self.agent_size) atomic_write(self.binary_filename, bin)
def shutdown(self): log_qemu("Shutting down Qemu after %d execs.." % self.persistent_runs, self.qemu_id) if not self.process: # start() has never been called, all files/shm are closed. return 0 # If Qemu exists, try to graciously read its I/O and SIGTERM it. # If still alive, attempt SIGKILL or loop-wait on kill -9. output = "<no output received>\n" try: self.process.terminate() output = strdump(self.process.communicate(timeout=1)[0], verbatim=True) except: pass if self.process.returncode is None: try: self.process.kill() except: pass log_qemu("Qemu exit code: %s" % str(self.process.returncode), self.qemu_id) header = "\n=================<Qemu %s Console Output>==================\n" % self.qemu_id footer = "====================</Console Output>======================\n" log_qemu(header + output + footer, self.qemu_id) header = "\n=================<Qemu %s Serial Output>==================\n" % self.qemu_id footer = "====================</Serial Output>======================\n" serial_out = strdump(read_binary_file(self.qemu_serial_log), verbatim=True) log_qemu(header + serial_out + footer, self.qemu_id) try: self.kafl_shm.close() except: pass try: self.fs_shm.close() except: pass try: os.close(self.kafl_shm_f) except: pass try: os.close(self.fs_shm_f) except: pass try: if self.stat_fd: self.stat_fd.close() except: pass return self.process.returncode
def send_next_task(self, conn): # Inputs placed to imports/ folder have priority. # This can also be used to inject additional seeds at runtime. imports = glob.glob(self.config.argument_values['work_dir'] + "/imports/*") if imports: path = imports.pop() print("Importing payload from %s" % path) seed = read_binary_file(path) os.remove(path) return self.comm.send_import(conn, { "type": "import", "payload": seed }) # Process items from queue.. node = self.queue.get_next() if node: return self.comm.send_node(conn, { "type": "node", "nid": node.get_id() }) # No work in queue. Tell slave to wait a little or attempt blind fuzzing. # If we see a lot of busy events, check the bitmap and warn on coverage issues. self.comm.send_busy(conn) self.busy_events += 1 if self.busy_events >= 10: self.busy_events = 0 main_bitmap = self.bitmap_storage.get_bitmap_for_node_type( "regular").c_bitmap if mmh3.hash(main_bitmap) == self.empty_hash: print_note( "Coverage bitmap is empty?! Check -ip0 or try better seeds." )
def check_funkyness_and_store_trace(self, data): global num_funky exec_res = self.q.send_payload() hash = exec_res.hash() trace1 = read_binary_file(self.config.argument_values['work_dir'] + "/pt_trace_dump_%d" % self.slave_id) exec_res = self.q.send_payload() if (hash != exec_res.hash()): print_warning("Validation identified funky bits, dumping!") num_funky += 1 trace_folder = self.config.argument_values['work_dir'] + "/traces/funky_%d_%d" % (num_funky, self.slave_id); os.makedirs(trace_folder) atomic_write(trace_folder + "/input", data) atomic_write(trace_folder + "/trace_a", trace1) trace2 = read_binary_file(self.config.argument_values["work_dir"] + "/pt_trace_dump_%d" % self.slave_id) atomic_write(trace_folder + "/trace_b", trace2) return exec_res
def main(): cfg = FuzzerConfiguration(IRPT_CONFIG) payload = read_binary_file(cfg.argument_values['payload']) q = qemu(0, cfg, debug_mode=0) if not q.start(): return i = 0 while i < len(payload): iocode = u32(payload[i:i + 4]) inlength = u32(payload[i + 4:i + 8]) outlength = u32(payload[i + 8:i + 12]) inbuffer = str(payload[i + 12:i + 12 + (inlength & 0xFFFFFF)]) log("[+] IoControlCode(%x) InBufferLength(%d)" % (iocode, inlength)) exec_res = q.send_irp(IRP(iocode, inlength, outlength, inbuffer)) if exec_res.is_crash(): if not exec_res.is_timeout(): log("Crashed!!") else: log("Timeout!!") q.shutdown() return i = i + 12 + inlength q.shutdown()
def debug_execution(config, execs, qemu_verbose=False, notifiers=True): log_debug("Starting debug execution...(%d rounds)" % execs) payload_file = config.argument_values["input"] zero_hash = mmh3.hash(("\x00" * config.config_values['BITMAP_SHM_SIZE']), signed=False) q = qemu(1337, config, debug_mode=True, notifiers=notifiers) assert q.start(), "Failed to start Qemu?" start = time.time() for i in range(execs): log_debug("Launching payload %d/%d.." % (i + 1, execs)) if i % 3 == 0: q.set_payload(read_binary_file(payload_file)) # time.sleep(0.01 * rand.int(0, 9)) # a = str(q.send_payload()) # hexdump(a) result = q.send_payload() current_hash = result.hash() if zero_hash == current_hash: log_debug("Feedback Hash: " + str(current_hash) + common.color.WARNING + " (WARNING: Zero hash found!)" + common.color.ENDC) else: log_debug("Feedback Hash: " + str(current_hash)) #log_debug("Full hexdump:\n" + hexdump(result.copy_to_array())) if result.is_crash(): q.restart() q.shutdown() end = time.time() print("Performance: " + str(execs / (end - start)) + "t/s") return 0
def start(config): prepare_working_dir(config.argument_values['work_dir']) if not post_self_check(config): return -1 # kAFL debug output is redirected to logs as part of -v mode. stdout will only print test/debug results. if config.argument_values['v']: enable_logging(config.argument_values["work_dir"]) # Without -ip0, Qemu will not active PT tracing and Redqueen will not # attempt to handle debug traps. This is a requirement for modes like gdb. if not config.argument_values['ip0']: print_warning("No trace region configured! Intel PT disabled!") try: # TODO: noise, benchmark, trace are working, others untested mode = config.argument_values['action'] if (mode == "noise"): payload_file = config.argument_values["input"] debug_non_det(config, read_binary_file(payload_file)) elif (mode == "benchmark"): benchmark(config) elif (mode == "gdb"): gdb_session(config, qemu_verbose=True) elif (mode == "trace"): debug_execution(config, config.argument_values['n']) elif (mode == "trace-qemu"): debug_execution(config, config.argument_values['n'], qemu_verbose=True) elif (mode == "printk"): debug_execution(config, 1, qemu_verbose=True, notifiers=False) elif (mode == "redqueen"): redqueen_dbg(config, qemu_verbose=False) elif (mode == "redqueen-qemu"): redqueen_dbg(config, qemu_verbose=True) elif (mode == "verify"): verify_dbg(config, qemu_verbose=True) else: print("Unknown debug mode. Exit") except Exception as e: raise finally: # cleanup os.system("stty sane") for i in range(512): if os.path.exists("/tmp/kAFL_printf.txt." + str(i)): os.remove("/tmp/kAFL_printf.txt." + str(i)) else: break print( "\nDone. Check logs for details.\nAny remaining qemu instances should be GC'ed on exit:" ) os.system("pgrep qemu-system") return 0
def load(self, f): i = 0 program_data = read_binary_file(f) while i < len(program_data): iocode = u32(program_data[i:i + 4]) inlength = u32(program_data[i + 4:i + 8]) outlength = u32(program_data[i + 8:i + 12]) inbuffer = str(program_data[i + 12:i + 12 + inlength]) self.irps.append(IRP(iocode, inlength, outlength, inbuffer)) i = i + 12 + inlength
def get_task(self): imports = glob.glob(self.config.argument_values['work_dir'] + "/imports/*") if imports: path = imports.pop() payload = read_binary_file(path) os.remove(path) return {"payload": payload, "type": "import"} elif self.queue.has_inputs(): node = self.queue.get_next() return {"type": "node", "nid": node.get_id()} else: return {"payload": random_string(), "type": "import"}
def read_file(self, name): retry = 4 data = None while retry > 0: try: data = read_binary_file(self.workdir + "/" + name) break except: retry -= 1 if data: return msgpack.unpackb(data, raw=False, strict_map_key=False) else: return None
def gdb_session(config, qemu_verbose=True, notifiers=True): import common.qemu_protocol as qemu_protocol payload_file = config.argument_values["input"] config.argument_values["gdbserver"] = True q = qemu(1337, config, notifiers=notifiers) print("Starting Qemu + GDB with payload %s" % payload_file) print("Connect with gdb to release guest from reset (localhost:1234)") if q.start(): q.set_payload(read_binary_file(payload_file)) result = q.debug_payload(apply_patches=False) print("Payload result: %s. Thank you for playing.." % qemu_protocol.CMDS[result]) q.shutdown()
def __perform_import(self): import_count = len( glob.glob(self.config.argument_values['work_dir'] + "/imports/*")) if import_count == 0: return self.kafl_state["technique"] = "IMPORT" log_master("Sync...(" + str(self.round_counter) + " inputs)") self.__sync_verification(self.round_counter) log_master("Importing...(" + str(import_count) + " inputs)") i = 0 for path in glob.glob(self.config.argument_values['work_dir'] + "/imports/*"): if (time.time() - self.start) >= self.refresh_rate: end = time.time() self.kafl_state.update_performance( int(((self.counter * 1.0) / (end - self.start)))) self.start = time.time() self.counter = 0 while True: msg = recv_msg(self.comm.to_master_queue) if msg.tag == KAFL_TAG_REQ: payload = read_binary_file(path) self.__task_send( [payload[:(64 << 10)]], [self.redqueen_state.get_candidate_hash_addrs()], msg.data, self.comm.to_slave_queues[int(msg.data)], [fuzz_methode(METHODE_IMPORT)], tag=KAFL_TAG_REQ_VERIFY) os.remove(path) i += 1 self.counter += 1 self.round_counter += 1 break else: log_master("Unknown Tag (" + str(msg.tag) + ") received during verification...") log_master("Sync...(" + str(self.round_counter) + " inputs)") self.__sync_verification(self.round_counter) log_master("Import done!")
def __perform_verification(self, input_count): self.kafl_state["technique"] = "VERIFICATION" log_master("Sync...(" + str(self.round_counter) + " inputs)") self.__sync_verification(self.round_counter) log_master("Verification...(" + str(input_count) + " inputs)") i = 0 for path in glob.glob(self.config.argument_values['work_dir'] + "/preliminary/preliminary_*"): if (time.time() - self.start) >= self.refresh_rate: end = time.time() self.kafl_state.update_performance( int(((self.counter * 1.0) / (end - self.start)))) self.start = time.time() self.counter = 0 while True: msg = recv_msg(self.comm.to_master_queue) if msg.tag == KAFL_TAG_REQ: payload = read_binary_file(path) methode = fuzz_methode() methode.read_from_file( self.config.argument_values['work_dir'], i + 1, preliminary=True) self.__task_send( [payload[:(64 << 10)]], [self.redqueen_state.get_candidate_hash_addrs()], msg.data, self.comm.to_slave_queues[int(msg.data)], [methode], tag=KAFL_TAG_REQ_VERIFY) i += 1 self.counter += 1 self.round_counter += 1 break else: log_master("Unknown Tag (" + str(msg.tag) + ") received during verification...") log_master("Sync...(" + str(self.round_counter) + " inputs)") self.__sync_verification(self.round_counter) log_master("Verification done!")
def havoc_splicing(data, files=None): if len(data) >= 2: for file in files: file_data = read_binary_file(file) if len(file_data) < 2: continue first_diff, last_diff = find_diffs(data, file_data) if last_diff < 2 or first_diff == last_diff: continue split_location = first_diff + RAND(last_diff - first_diff) data = array('B', data.tobytes()[:split_location] + file_data[split_location:len(data)]) #func(data.tobytes()) break return data
def redqueen_dbg(config, qemu_verbose=False): global thread_done log_debug("Starting Redqueen debug...") q = qemu(1337, config, debug_mode=True) q.start() payload = read_binary_file(config.argument_values["input"]) # q.set_payload(payload) if os.path.exists("patches"): shutil.copyfile("patches", "/tmp/redqueen_workdir_1337/redqueen_patches.txt") start = time.time() thread = Thread(target=lambda: redqueen_dbg_thread(q)) thread.start() result = q.execute_in_redqueen_mode(payload, debug_mode=True) thread_done = True thread.join() requeen_print_state(q) end = time.time() if result: print(common.color.OKGREEN + "Execution succeded!" + common.color.ENDC) else: print(common.color.FLUSH_LINE + common.color.FAIL + "Execution failed!" + common.color.ENDC) print("Time: " + str(end - start) + "t/s") num_muts, muts = parser.parse_rq_data( open( "/tmp/kafl_debug_workdir/redqueen_workdir_1337/redqueen_results.txt" ).read(), payload) count = 0 for offset in muts: for lhs in muts[offset]: for rhs in muts[offset][lhs]: count += 1 print(offset, lhs, rhs) print(count) return 0
def havoc_splicing(data, files): if len(data) < 2 or files is None: return data rand.shuffle(files) retry_limit = 64 for file in files[:retry_limit]: file_data = read_binary_file(file) if len(file_data) < 2: continue first_diff, last_diff = find_diffs(data, file_data) if last_diff < 2 or first_diff == last_diff: continue split_location = first_diff + rand.int(last_diff - first_diff) return data[:split_location] + file_data[split_location:] # none of the files are suitable return None
def mutate_seq_radamsa_array(data, func, max_iterations): global corpus_dir global input_dir global radamsa_path log_radamsa("Radamsa amount: %d" % max_iterations) if max_iterations == 0: return last_n = 5 rand_n = 10 files = sorted(glob.glob(corpus_dir + "/*/payload_*")) samples = files[-last_n:] + random.sample( files[:-last_n], max(0, min(rand_n, len(files) - last_n))) if not samples: return radamsa_cmd = [ radamsa_path, "-o", input_dir + "input_%05n", "-n", str(max_iterations) ] + samples #log_radamsa("Radamsa cmd: " + repr(radamsa_cmd)) p = subprocess.Popen(radamsa_cmd, stdin=subprocess.PIPE, shell=False) try: p.communicate(timeout=10) except subprocess.SubprocessError as e: log_radamsa("Radamsa exception %s" % str(e)) p.kill() p.communicate() for path in os.listdir(input_dir): #log_radamsa("Radamsa input %s" % path) func(read_binary_file(input_dir + path)) os.remove(input_dir + path)
def execute_once(config, qemu_verbose=False, notifiers=True): payload_file = config.argument_values["input"] log_debug("Execute payload %s.. " % payload_file) zero_hash = mmh3.hash(("\x00" * config.config_values['BITMAP_SHM_SIZE']), signed=False) q = qemu(1337, config, debug_mode=False, notifiers=notifiers) assert q.start(), "Failed to start Qemu?" q.set_payload(read_binary_file(payload_file)) result = q.send_payload() current_hash = result.hash() if zero_hash == current_hash: log_debug("Feedback Hash: " + str(current_hash) + common.color.WARNING + " (WARNING: Zero hash found!)" + common.color.ENDC) else: log_debug("Feedback Hash: " + str(current_hash)) q.shutdown() return 0
def funky_trace_run(q, input_path, retry=1): validations = 12 confirmations = 0 payload = read_binary_file(input_path) hashes = dict() for _ in range(validations): res = simple_trace_run(q, payload) if not res: return None # skip crahses and timeouts as they tend to be slow if res.is_crash(): return res h = res.hash() if h == null_hash: continue if h in hashes: hashes[h] += 1 else: hashes[h] = 1 # break early if we have a winner, with trace stored to temp file if hashes[h] >= 0.5 * validations: return res #print("Failed to get majority trace (retry=%d)\nHashes: %s\n" % (retry, str(hashes))) if retry > 0: q.restart() time.sleep(1) return funky_trace_run(q, input_path, retry=retry - 1) return None
def __read_msgpack(self, name): return msgpack.unpackb(read_binary_file(name), raw=False, strict_map_key=False)
def node_payload(self, nid): exit_reason = self.nodes[nid]["info"]["exit_reason"] filename = self.workdir + "/corpus/%s/payload_%05d" % (exit_reason, nid) return read_binary_file(filename)[0:1024] # TODO remove path traversal vuln
#!/usr/bin/env python """ Copyright (C) 2019 Sergej Schumilo, Cornelius Aschermann, Tim Blazytko This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. """ import msgpack import os import sys from pprint import pprint sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/../") from common.util import read_binary_file for arg in sys.argv[1:]: pprint(msgpack.unpackb(read_binary_file(arg)))
#!/usr/bin/env python3 # # Copyright 2019 Sergej Schumilo, Cornelius Aschermann, Tim Blazytko # Copyright 2020 Intel Corporation # # SPDX-License-Identifier: AGPL-3.0-or-later """ Pretty-Pring msgpack files produced by kAFL """ import os import sys import msgpack from pprint import pprint sys.path.insert( 0, os.path.dirname(os.path.realpath(__file__)) + "/../kAFL-Fuzzer/") from common.util import read_binary_file for arg in sys.argv[1:]: pprint( msgpack.unpackb(read_binary_file(arg), raw=False, strict_map_key=False))
def get_payload(exitreason, id): return read_binary_file( QueueNode.__get_payload_filename(exitreason, id))
def get_metadata(id): return msgpack.unpackb(read_binary_file( QueueNode.__get_metadata_filename(id)), raw=False, strict_map_key=False)
def load(self, input_id, was_colored, path): hook_info = read_file("%s/redqueen_result_%d.txt" % (path, input_id)) bin_info = read_binary_file("%s/input_%d.bin" % (path, input_id)) return self.load_data(input_id, was_colored, hook_info, bin_info)
def __read_payload(self, node_id, exit_reason): payload_file = self.workdir + "/corpus/" + exit_reason + "/payload_%05d" % node_id return read_binary_file(payload_file)