def kafl_workdir_iterator(work_dir):
input_id_time = list()
start_time = time.time()
for stats_file in glob.glob(work_dir + "/slave_stats_*"):
if not stats_file:
return None
slave_stats = msgpack.unpackb(read_binary_file(stats_file),
raw=False,
strict_map_key=False)
start_time = min(start_time, slave_stats['start_time'])
# enumerate inputs from corpus/ and match against metainfo in metadata/
for input_file in glob.glob(work_dir + "/corpus/*/*"):
if not input_file:
return None
input_id = os.path.basename(input_file).replace("payload_", "")
meta_file = work_dir + "/metadata/node_{}".format(input_id)
metadata = msgpack.unpackb(read_binary_file(meta_file),
raw=False,
strict_map_key=False)
seconds = metadata["info"]["time"] - start_time
nid = metadata["id"]
#print("%s;%d" % (input_file, timestamp))
input_id_time.append([input_file, nid, seconds])
#yield (input_file, nid, timestamp)
return input_id_time
def kafl_workdir_iterator(work_dir):
input_id_time = list()
start_time = time.time()
for stats_file in glob.glob(work_dir + "/slave_stats_*"):
if not stats_file:
return None
slave_stats = msgpack.unpackb(read_binary_file(stats_file),
raw=False,
strict_map_key=False)
start_time = min(start_time, slave_stats['start_time'])
# enumerate inputs from corpus/ and match against metainfo in metadata/
# TODO: Tracing crashes/timeouts has minimal overall improvement ~1-2%
# Probably want to make this optional, and only trace a small sample
# of non-regular payloads by default?
for input_file in glob.glob(work_dir + "/corpus/[rckt]*/*"):
if not input_file:
return None
input_id = os.path.basename(input_file).replace("payload_", "")
meta_file = work_dir + "/metadata/node_{}".format(input_id)
metadata = msgpack.unpackb(read_binary_file(meta_file),
raw=False,
strict_map_key=False)
seconds = metadata["info"]["time"] - start_time
nid = metadata["id"]
input_id_time.append([input_file, nid, seconds])
return input_id_time
def benchmark(config):
log_debug("Starting benchmark...")
payload_file = config.argument_values["input"]
q = qemu(1337, config, debug_mode=False)
q.start()
q.set_payload(read_binary_file(payload_file))
log_debug("Hash: " + str(q.send_payload().hash()))
try:
while True:
start = time.time()
execs = 0
# for i in range(execs):
while (time.time() - start < REFRESH):
q.set_payload(read_binary_file(payload_file))
q.send_payload()
execs += 1
end = time.time()
# print("Performance: " + str(execs/(end - start)) + "t/s")
stdout.write(common.color.FLUSH_LINE + "Performance: " +
str(execs / (end - start)) + "t/s")
stdout.flush()
except KeyboardInterrupt:
stdout.write("\n")
q.shutdown()
return 0
def __set_agent_and_driver(self):
driver_bin = read_binary_file(self.config.argument_values['driver'])
bin = p32(len(driver_bin)) + driver_bin
agent_bin = read_binary_file(self.config.argument_values['agent'])
bin += p32(len(agent_bin)) + agent_bin
assert (len(bin) <= self.agent_size)
atomic_write(self.binary_filename, bin)
def shutdown(self):
log_qemu("Shutting down Qemu after %d execs.." % self.persistent_runs, self.qemu_id)
if not self.process:
# start() has never been called, all files/shm are closed.
return 0
# If Qemu exists, try to graciously read its I/O and SIGTERM it.
# If still alive, attempt SIGKILL or loop-wait on kill -9.
output = "<no output received>\n"
try:
self.process.terminate()
output = strdump(self.process.communicate(timeout=1)[0], verbatim=True)
except:
pass
if self.process.returncode is None:
try:
self.process.kill()
except:
pass
log_qemu("Qemu exit code: %s" % str(self.process.returncode), self.qemu_id)
header = "\n=================<Qemu %s Console Output>==================\n" % self.qemu_id
footer = "====================</Console Output>======================\n"
log_qemu(header + output + footer, self.qemu_id)
header = "\n=================<Qemu %s Serial Output>==================\n" % self.qemu_id
footer = "====================</Serial Output>======================\n"
serial_out = strdump(read_binary_file(self.qemu_serial_log), verbatim=True)
log_qemu(header + serial_out + footer, self.qemu_id)
try:
self.kafl_shm.close()
except:
pass
try:
self.fs_shm.close()
except:
pass
try:
os.close(self.kafl_shm_f)
except:
pass
try:
os.close(self.fs_shm_f)
except:
pass
try:
if self.stat_fd:
self.stat_fd.close()
except:
pass
return self.process.returncode
def send_next_task(self, conn):
# Inputs placed to imports/ folder have priority.
# This can also be used to inject additional seeds at runtime.
imports = glob.glob(self.config.argument_values['work_dir'] +
"/imports/*")
if imports:
path = imports.pop()
print("Importing payload from %s" % path)
seed = read_binary_file(path)
os.remove(path)
return self.comm.send_import(conn, {
"type": "import",
"payload": seed
})
# Process items from queue..
node = self.queue.get_next()
if node:
return self.comm.send_node(conn, {
"type": "node",
"nid": node.get_id()
})
# No work in queue. Tell slave to wait a little or attempt blind fuzzing.
# If we see a lot of busy events, check the bitmap and warn on coverage issues.
self.comm.send_busy(conn)
self.busy_events += 1
if self.busy_events >= 10:
self.busy_events = 0
main_bitmap = self.bitmap_storage.get_bitmap_for_node_type(
"regular").c_bitmap
if mmh3.hash(main_bitmap) == self.empty_hash:
print_note(
"Coverage bitmap is empty?! Check -ip0 or try better seeds."
)
def check_funkyness_and_store_trace(self, data):
global num_funky
exec_res = self.q.send_payload()
hash = exec_res.hash()
trace1 = read_binary_file(self.config.argument_values['work_dir'] + "/pt_trace_dump_%d" % self.slave_id)
exec_res = self.q.send_payload()
if (hash != exec_res.hash()):
print_warning("Validation identified funky bits, dumping!")
num_funky += 1
trace_folder = self.config.argument_values['work_dir'] + "/traces/funky_%d_%d" % (num_funky, self.slave_id);
os.makedirs(trace_folder)
atomic_write(trace_folder + "/input", data)
atomic_write(trace_folder + "/trace_a", trace1)
trace2 = read_binary_file(self.config.argument_values["work_dir"] + "/pt_trace_dump_%d" % self.slave_id)
atomic_write(trace_folder + "/trace_b", trace2)
return exec_res
def main():
cfg = FuzzerConfiguration(IRPT_CONFIG)
payload = read_binary_file(cfg.argument_values['payload'])
q = qemu(0, cfg, debug_mode=0)
if not q.start():
return
i = 0
while i < len(payload):
iocode = u32(payload[i:i + 4])
inlength = u32(payload[i + 4:i + 8])
outlength = u32(payload[i + 8:i + 12])
inbuffer = str(payload[i + 12:i + 12 + (inlength & 0xFFFFFF)])
log("[+] IoControlCode(%x) InBufferLength(%d)" % (iocode, inlength))
exec_res = q.send_irp(IRP(iocode, inlength, outlength, inbuffer))
if exec_res.is_crash():
if not exec_res.is_timeout():
log("Crashed!!")
else:
log("Timeout!!")
q.shutdown()
return
i = i + 12 + inlength
q.shutdown()
def debug_execution(config, execs, qemu_verbose=False, notifiers=True):
log_debug("Starting debug execution...(%d rounds)" % execs)
payload_file = config.argument_values["input"]
zero_hash = mmh3.hash(("\x00" * config.config_values['BITMAP_SHM_SIZE']),
signed=False)
q = qemu(1337, config, debug_mode=True, notifiers=notifiers)
assert q.start(), "Failed to start Qemu?"
start = time.time()
for i in range(execs):
log_debug("Launching payload %d/%d.." % (i + 1, execs))
if i % 3 == 0:
q.set_payload(read_binary_file(payload_file))
# time.sleep(0.01 * rand.int(0, 9))
# a = str(q.send_payload())
# hexdump(a)
result = q.send_payload()
current_hash = result.hash()
if zero_hash == current_hash:
log_debug("Feedback Hash: " + str(current_hash) +
common.color.WARNING + " (WARNING: Zero hash found!)" +
common.color.ENDC)
else:
log_debug("Feedback Hash: " + str(current_hash))
#log_debug("Full hexdump:\n" + hexdump(result.copy_to_array()))
if result.is_crash():
q.restart()
q.shutdown()
end = time.time()
print("Performance: " + str(execs / (end - start)) + "t/s")
return 0
def start(config):
prepare_working_dir(config.argument_values['work_dir'])
if not post_self_check(config):
return -1
# kAFL debug output is redirected to logs as part of -v mode. stdout will only print test/debug results.
if config.argument_values['v']:
enable_logging(config.argument_values["work_dir"])
# Without -ip0, Qemu will not active PT tracing and Redqueen will not
# attempt to handle debug traps. This is a requirement for modes like gdb.
if not config.argument_values['ip0']:
print_warning("No trace region configured! Intel PT disabled!")
try:
# TODO: noise, benchmark, trace are working, others untested
mode = config.argument_values['action']
if (mode == "noise"):
payload_file = config.argument_values["input"]
debug_non_det(config, read_binary_file(payload_file))
elif (mode == "benchmark"):
benchmark(config)
elif (mode == "gdb"):
gdb_session(config, qemu_verbose=True)
elif (mode == "trace"):
debug_execution(config, config.argument_values['n'])
elif (mode == "trace-qemu"):
debug_execution(config,
config.argument_values['n'],
qemu_verbose=True)
elif (mode == "printk"):
debug_execution(config, 1, qemu_verbose=True, notifiers=False)
elif (mode == "redqueen"):
redqueen_dbg(config, qemu_verbose=False)
elif (mode == "redqueen-qemu"):
redqueen_dbg(config, qemu_verbose=True)
elif (mode == "verify"):
verify_dbg(config, qemu_verbose=True)
else:
print("Unknown debug mode. Exit")
except Exception as e:
raise
finally:
# cleanup
os.system("stty sane")
for i in range(512):
if os.path.exists("/tmp/kAFL_printf.txt." + str(i)):
os.remove("/tmp/kAFL_printf.txt." + str(i))
else:
break
print(
"\nDone. Check logs for details.\nAny remaining qemu instances should be GC'ed on exit:"
)
os.system("pgrep qemu-system")
return 0
def load(self, f):
i = 0
program_data = read_binary_file(f)
while i < len(program_data):
iocode = u32(program_data[i:i + 4])
inlength = u32(program_data[i + 4:i + 8])
outlength = u32(program_data[i + 8:i + 12])
inbuffer = str(program_data[i + 12:i + 12 + inlength])
self.irps.append(IRP(iocode, inlength, outlength, inbuffer))
i = i + 12 + inlength
def get_task(self):
imports = glob.glob(self.config.argument_values['work_dir'] +
"/imports/*")
if imports:
path = imports.pop()
payload = read_binary_file(path)
os.remove(path)
return {"payload": payload, "type": "import"}
elif self.queue.has_inputs():
node = self.queue.get_next()
return {"type": "node", "nid": node.get_id()}
else:
return {"payload": random_string(), "type": "import"}
def read_file(self, name):
retry = 4
data = None
while retry > 0:
try:
data = read_binary_file(self.workdir + "/" + name)
break
except:
retry -= 1
if data:
return msgpack.unpackb(data, raw=False, strict_map_key=False)
else:
return None
def gdb_session(config, qemu_verbose=True, notifiers=True):
import common.qemu_protocol as qemu_protocol
payload_file = config.argument_values["input"]
config.argument_values["gdbserver"] = True
q = qemu(1337, config, notifiers=notifiers)
print("Starting Qemu + GDB with payload %s" % payload_file)
print("Connect with gdb to release guest from reset (localhost:1234)")
if q.start():
q.set_payload(read_binary_file(payload_file))
result = q.debug_payload(apply_patches=False)
print("Payload result: %s. Thank you for playing.." % qemu_protocol.CMDS[result])
q.shutdown()
def __perform_import(self):
import_count = len(
glob.glob(self.config.argument_values['work_dir'] + "/imports/*"))
if import_count == 0:
return
self.kafl_state["technique"] = "IMPORT"
log_master("Sync...(" + str(self.round_counter) + " inputs)")
self.__sync_verification(self.round_counter)
log_master("Importing...(" + str(import_count) + " inputs)")
i = 0
for path in glob.glob(self.config.argument_values['work_dir'] +
"/imports/*"):
if (time.time() - self.start) >= self.refresh_rate:
end = time.time()
self.kafl_state.update_performance(
int(((self.counter * 1.0) / (end - self.start))))
self.start = time.time()
self.counter = 0
while True:
msg = recv_msg(self.comm.to_master_queue)
if msg.tag == KAFL_TAG_REQ:
payload = read_binary_file(path)
self.__task_send(
[payload[:(64 << 10)]],
[self.redqueen_state.get_candidate_hash_addrs()],
msg.data,
self.comm.to_slave_queues[int(msg.data)],
[fuzz_methode(METHODE_IMPORT)],
tag=KAFL_TAG_REQ_VERIFY)
os.remove(path)
i += 1
self.counter += 1
self.round_counter += 1
break
else:
log_master("Unknown Tag (" + str(msg.tag) +
") received during verification...")
log_master("Sync...(" + str(self.round_counter) + " inputs)")
self.__sync_verification(self.round_counter)
log_master("Import done!")
def __perform_verification(self, input_count):
self.kafl_state["technique"] = "VERIFICATION"
log_master("Sync...(" + str(self.round_counter) + " inputs)")
self.__sync_verification(self.round_counter)
log_master("Verification...(" + str(input_count) + " inputs)")
i = 0
for path in glob.glob(self.config.argument_values['work_dir'] +
"/preliminary/preliminary_*"):
if (time.time() - self.start) >= self.refresh_rate:
end = time.time()
self.kafl_state.update_performance(
int(((self.counter * 1.0) / (end - self.start))))
self.start = time.time()
self.counter = 0
while True:
msg = recv_msg(self.comm.to_master_queue)
if msg.tag == KAFL_TAG_REQ:
payload = read_binary_file(path)
methode = fuzz_methode()
methode.read_from_file(
self.config.argument_values['work_dir'],
i + 1,
preliminary=True)
self.__task_send(
[payload[:(64 << 10)]],
[self.redqueen_state.get_candidate_hash_addrs()],
msg.data,
self.comm.to_slave_queues[int(msg.data)], [methode],
tag=KAFL_TAG_REQ_VERIFY)
i += 1
self.counter += 1
self.round_counter += 1
break
else:
log_master("Unknown Tag (" + str(msg.tag) +
") received during verification...")
log_master("Sync...(" + str(self.round_counter) + " inputs)")
self.__sync_verification(self.round_counter)
log_master("Verification done!")
def havoc_splicing(data, files=None):
if len(data) >= 2:
for file in files:
file_data = read_binary_file(file)
if len(file_data) < 2:
continue
first_diff, last_diff = find_diffs(data, file_data)
if last_diff < 2 or first_diff == last_diff:
continue
split_location = first_diff + RAND(last_diff - first_diff)
data = array('B', data.tobytes()[:split_location] + file_data[split_location:len(data)])
#func(data.tobytes())
break
return data
def redqueen_dbg(config, qemu_verbose=False):
global thread_done
log_debug("Starting Redqueen debug...")
q = qemu(1337, config, debug_mode=True)
q.start()
payload = read_binary_file(config.argument_values["input"])
# q.set_payload(payload)
if os.path.exists("patches"):
shutil.copyfile("patches",
"/tmp/redqueen_workdir_1337/redqueen_patches.txt")
start = time.time()
thread = Thread(target=lambda: redqueen_dbg_thread(q))
thread.start()
result = q.execute_in_redqueen_mode(payload, debug_mode=True)
thread_done = True
thread.join()
requeen_print_state(q)
end = time.time()
if result:
print(common.color.OKGREEN + "Execution succeded!" + common.color.ENDC)
else:
print(common.color.FLUSH_LINE + common.color.FAIL +
"Execution failed!" + common.color.ENDC)
print("Time: " + str(end - start) + "t/s")
num_muts, muts = parser.parse_rq_data(
open(
"/tmp/kafl_debug_workdir/redqueen_workdir_1337/redqueen_results.txt"
).read(), payload)
count = 0
for offset in muts:
for lhs in muts[offset]:
for rhs in muts[offset][lhs]:
count += 1
print(offset, lhs, rhs)
print(count)
return 0
def havoc_splicing(data, files):
if len(data) < 2 or files is None:
return data
rand.shuffle(files)
retry_limit = 64
for file in files[:retry_limit]:
file_data = read_binary_file(file)
if len(file_data) < 2:
continue
first_diff, last_diff = find_diffs(data, file_data)
if last_diff < 2 or first_diff == last_diff:
continue
split_location = first_diff + rand.int(last_diff - first_diff)
return data[:split_location] + file_data[split_location:]
# none of the files are suitable
return None
def mutate_seq_radamsa_array(data, func, max_iterations):
global corpus_dir
global input_dir
global radamsa_path
log_radamsa("Radamsa amount: %d" % max_iterations)
if max_iterations == 0:
return
last_n = 5
rand_n = 10
files = sorted(glob.glob(corpus_dir + "/*/payload_*"))
samples = files[-last_n:] + random.sample(
files[:-last_n], max(0, min(rand_n,
len(files) - last_n)))
if not samples:
return
radamsa_cmd = [
radamsa_path, "-o", input_dir + "input_%05n", "-n",
str(max_iterations)
] + samples
#log_radamsa("Radamsa cmd: " + repr(radamsa_cmd))
p = subprocess.Popen(radamsa_cmd, stdin=subprocess.PIPE, shell=False)
try:
p.communicate(timeout=10)
except subprocess.SubprocessError as e:
log_radamsa("Radamsa exception %s" % str(e))
p.kill()
p.communicate()
for path in os.listdir(input_dir):
#log_radamsa("Radamsa input %s" % path)
func(read_binary_file(input_dir + path))
os.remove(input_dir + path)
def execute_once(config, qemu_verbose=False, notifiers=True):
payload_file = config.argument_values["input"]
log_debug("Execute payload %s.. " % payload_file)
zero_hash = mmh3.hash(("\x00" * config.config_values['BITMAP_SHM_SIZE']),
signed=False)
q = qemu(1337, config, debug_mode=False, notifiers=notifiers)
assert q.start(), "Failed to start Qemu?"
q.set_payload(read_binary_file(payload_file))
result = q.send_payload()
current_hash = result.hash()
if zero_hash == current_hash:
log_debug("Feedback Hash: " + str(current_hash) +
common.color.WARNING + " (WARNING: Zero hash found!)" +
common.color.ENDC)
else:
log_debug("Feedback Hash: " + str(current_hash))
q.shutdown()
return 0
def funky_trace_run(q, input_path, retry=1):
validations = 12
confirmations = 0
payload = read_binary_file(input_path)
hashes = dict()
for _ in range(validations):
res = simple_trace_run(q, payload)
if not res:
return None
# skip crahses and timeouts as they tend to be slow
if res.is_crash():
return res
h = res.hash()
if h == null_hash:
continue
if h in hashes:
hashes[h] += 1
else:
hashes[h] = 1
# break early if we have a winner, with trace stored to temp file
if hashes[h] >= 0.5 * validations:
return res
#print("Failed to get majority trace (retry=%d)\nHashes: %s\n" % (retry, str(hashes)))
if retry > 0:
q.restart()
time.sleep(1)
return funky_trace_run(q, input_path, retry=retry - 1)
return None
def __read_msgpack(self, name):
return msgpack.unpackb(read_binary_file(name), raw=False, strict_map_key=False)
def node_payload(self, nid):
exit_reason = self.nodes[nid]["info"]["exit_reason"]
filename = self.workdir + "/corpus/%s/payload_%05d" % (exit_reason, nid)
return read_binary_file(filename)[0:1024] # TODO remove path traversal vuln
#!/usr/bin/env python
"""
Copyright (C) 2019 Sergej Schumilo, Cornelius Aschermann, Tim Blazytko
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import msgpack
import os
import sys
from pprint import pprint
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + "/../")
from common.util import read_binary_file
for arg in sys.argv[1:]:
pprint(msgpack.unpackb(read_binary_file(arg)))
#!/usr/bin/env python3
#
# Copyright 2019 Sergej Schumilo, Cornelius Aschermann, Tim Blazytko
# Copyright 2020 Intel Corporation
#
# SPDX-License-Identifier: AGPL-3.0-or-later
"""
Pretty-Pring msgpack files produced by kAFL
"""
import os
import sys
import msgpack
from pprint import pprint
sys.path.insert(
0,
os.path.dirname(os.path.realpath(__file__)) + "/../kAFL-Fuzzer/")
from common.util import read_binary_file
for arg in sys.argv[1:]:
pprint(
msgpack.unpackb(read_binary_file(arg), raw=False,
strict_map_key=False))
def get_payload(exitreason, id):
return read_binary_file(
QueueNode.__get_payload_filename(exitreason, id))
def get_metadata(id):
return msgpack.unpackb(read_binary_file(
QueueNode.__get_metadata_filename(id)),
raw=False,
strict_map_key=False)
def load(self, input_id, was_colored, path):
hook_info = read_file("%s/redqueen_result_%d.txt" % (path, input_id))
bin_info = read_binary_file("%s/input_%d.bin" % (path, input_id))
return self.load_data(input_id, was_colored, hook_info, bin_info)
def __read_payload(self, node_id, exit_reason):
payload_file = self.workdir + "/corpus/" + exit_reason + "/payload_%05d" % node_id
return read_binary_file(payload_file)
