def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.debug_mode = config.argument_values['debug']
self.task_count = 0
self.task_paused = False
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queues = CycleQueue(self.config)
for _ in range(interface_manager.size()):
self.queues.append(InputQueue(self.config, self.statistics))
#self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
redqueen_global_config(
redq_hammering=self.config.argument_values['hammer_jmp_tables'],
redq_do_simple=self.config.argument_values['redq_do_simple'],
afl_arith_max=self.config.config_values['ARITHMETIC_MAX'])
self.imports = list(interface_manager.to_corpus())
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def __init__(self, slave_id, config, connection, auto_reload=False):
self.config = config
self.slave_id = slave_id
self.q = qemu(self.slave_id, self.config)
self.statistics = SlaveStatistics(self.slave_id, self.config)
self.logic = FuzzingStateLogic(self, self.config)
self.conn = connection
self.bitmap_storage = BitmapStorage(self.config, self.config.config_values['BITMAP_SHM_SIZE'], "master")
def __init__(self, slave_id, config, connection, auto_reload=False):
self.config = config
self.slave_id = slave_id
self.q = qemu(self.slave_id, self.config)
self.q.start(verbose=False)
print
"started qemu"
self.statistics = SlaveStatistics(self.slave_id, self.config)
self.logic = FuzzingStateLogic(self, self.config)
self.conn = connection
self.bitmap_storage = BitmapStorage(
self.config, self.config.config_values['BITMAP_SHM_SIZE'],
"master")
configure_log_prefix("%.2d" % slave_id)
def __init__(self, config, scheduler, statistics):
self.config = config
self.scheduler = scheduler
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
self.id_to_node = {}
self.current_cycle = []
self.bitmap_index_to_fav_node = {}
self.num_cycles = 0
self.pending_favorites = True
self.statistics = statistics
self.grimoire_scheduler = GrimoireScheduler()
def __init__(self, config, pid=0):
self.config = config
self.debug_mode = config.argument_values['debug']
self.statistics = ProcessStatistics(self.config)
self.bitmap_storage = BitmapStorage(config, config.config_values['BITMAP_SHM_SIZE'], "process", read_only=False)
log_process("Starting (pid: %d)" % os.getpid())
log_process("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
self.q = qemu(pid, self.config,
debug_mode=self.debug_mode)
self.optimizer = Optimizer(self.q, self.statistics)
self.reproducer = Reproducer(self.q, self.statistics)
self.database = Database(self.statistics) # load interface
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']))
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
class MasterProcess:
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.debug_mode = config.argument_values['debug']
self.task_count = 0
self.task_paused = False
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queues = CycleQueue(self.config)
for _ in range(interface_manager.size()):
self.queues.append(InputQueue(self.config, self.statistics))
#self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
redqueen_global_config(
redq_hammering=self.config.argument_values['hammer_jmp_tables'],
redq_do_simple=self.config.argument_values['redq_do_simple'],
afl_arith_max=self.config.config_values['ARITHMETIC_MAX'])
self.imports = list(interface_manager.to_corpus())
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def send_next_task(self, conn):
# for the switching queue.
if self.task_paused:
return
# Process items from queue..
node = self.queues.get_next_node()
if node:
self.task_count += 1
return self.comm.send_node(conn, {
"type": "node",
"nid": node.get_id()
})
# No work in queue. Tell slave to wait a little or attempt blind fuzzing.
# If we see a lot of busy events, check the bitmap and warn on coverage issues.
self.comm.send_busy(conn)
self.busy_events += 1
if self.busy_events >= 10:
self.busy_events = 0
main_bitmap = self.bitmap_storage.get_bitmap_for_node_type(
"regular").c_bitmap
if mmh3.hash(main_bitmap) == self.empty_hash:
print_note(
"Coverage bitmap is empty?! Check -ip0 or try better seeds."
)
def send_import(self, conn):
(iocode, payload) = self.imports.pop()
print("Importing payload from %s" % hex(iocode))
self.comm.send_import(conn, {
"type": "import",
"IoControlCode": iocode,
"payload": payload
})
def handle_msg(self, conn, msg):
if msg["type"] == MSG_NODE_DONE:
# Slave execution done, update queue item + send new task
self.task_count -= 1
log_master("Received results, sending next task..")
if msg["node_id"]:
self.queues.update_node_results(msg["node_id"], msg["results"],
msg["new_payload"])
self.send_next_task(conn)
elif msg["type"] == MSG_NEW_INPUT:
# Slave reports new interesting input
if self.debug_mode:
log_master("Received new input (exit=%s): %s" %
(msg["input"]["info"]["exit_reason"],
repr(msg["input"]["payload"][:24])))
node_struct = {
"info": msg["input"]["info"],
"state": {
"name": "initial"
}
}
self.maybe_insert_node(msg["input"]["payload"],
msg["input"]["bitmap"], node_struct)
elif msg["type"] == MSG_READY:
# Initial slave hello, send first task...
# log_master("Slave is ready..")
self.send_next_task(conn)
elif msg["type"] == MSG_NEXT_QUEUE:
# Flush slave message to switch queue.
self.task_paused = True
while self.task_count:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
self.handle_msg(conn, msg)
self.task_paused = False
# Switch to next queue.
self.queues.next()
# Inputs placed to imports/ folder have priority.
if self.imports:
self.send_import(conn)
self.send_next_task(conn)
else:
raise ValueError("unknown message type {}".format(msg))
def loop(self):
# Import a seed.
imported = True
while imported:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
assert msg["type"] == MSG_READY
if self.imports:
self.send_import(conn)
imported = False
while True:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
self.handle_msg(conn, msg)
self.statistics.event_slave_poll()
self.statistics.maybe_write_stats()
self.check_abort_condition()
def check_abort_condition(self):
import time
import datetime
t_limit = self.config.argument_values['abort_time']
n_limit = self.config.argument_values['abort_exec']
if t_limit:
if t_limit * 3600 < time.time(
) - self.statistics.data['start_time']:
raise SystemExit("Exit on timeout.")
if n_limit:
if n_limit < self.statistics.data['total_execs']:
raise SystemExit("Exit on max execs.")
def maybe_insert_node(self, payload, bitmap_array, node_struct):
print("New payload(0x%x) : " % node_struct["info"]["IoControlCode"],
payload[:16])
bitmap = ExecutionResult.bitmap_from_bytearray(
bitmap_array, node_struct["info"]["exit_reason"],
node_struct["info"]["performance"])
bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied
backup_data = bitmap.copy_to_array()
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(
bitmap)
new_data = bitmap.copy_to_array()
if should_store:
node = QueueNode(payload, bitmap_array, node_struct, write=False)
node.set_new_bytes(new_bytes, write=False)
node.set_new_bits(new_bits, write=False)
self.queues.insert_input(node, bitmap)
elif self.debug_mode:
if node_struct["info"]["exit_reason"] != "regular":
log_master("Payload found to be boring, not saved (exit=%s)" %
node_struct["info"]["exit_reason"])
for i in range(len(bitmap_array)):
if backup_data[i] != new_data[i]:
assert (False), "Bitmap mangled at {} {} {}".format(
i, repr(backup_data[i]), repr(new_data[i]))
class SlaveProcess:
def __init__(self, slave_id, config, connection, auto_reload=False):
self.config = config
self.slave_id = slave_id
self.q = qemu(self.slave_id,
self.config,
debug_mode=config.argument_values['debug'])
self.statistics = SlaveStatistics(self.slave_id, self.config)
self.logic = FuzzingStateLogic(self, self.config)
self.conn = connection
self.bitmap_storage = BitmapStorage(
self.config, self.config.config_values['BITMAP_SHM_SIZE'],
"master")
self.exec_count = 0
def handle_import(self, msg):
meta_data = {
"state": {
"name": "import"
},
"id": 0,
"info": {
"IoControlCode": msg["task"]["IoControlCode"]
}
}
payload = msg["task"]["payload"]
self.logic.process_node(payload, meta_data)
self.conn.send_ready()
def handle_busy(self):
busy_timeout = 1
kickstart = False
if kickstart: # spend busy cycle by feeding random strings?
log_slave("No ready work items, attempting random..",
self.slave_id)
start_time = time.time()
while (time.time() - start_time) < busy_timeout:
meta_data = {"state": {"name": "import"}, "id": 0}
payload = rand.bytes(rand.int(32))
self.logic.process_node(payload, meta_data)
else:
log_slave("No ready work items, waiting...", self.slave_id)
time.sleep(busy_timeout)
self.conn.send_ready()
def handle_node(self, msg):
meta_data = QueueNode.get_metadata(msg["task"]["nid"])
payload = QueueNode.get_payload(meta_data["info"]["exit_reason"],
meta_data["id"])
results, new_payload = self.logic.process_node(payload, meta_data)
if new_payload:
default_info = {
"method": "validate_bits",
"parent": meta_data["id"],
"IoControlCode": meta_data["info"]["IoControlCode"]
}
if self.validate_bits(new_payload, meta_data, default_info):
log_slave(
"Stage %s found alternative payload for node %d" %
(meta_data["state"]["name"], meta_data["id"]),
self.slave_id)
else:
log_slave(
"Provided alternative payload found invalid - bug in stage %s?"
% meta_data["state"]["name"], self.slave_id)
if self.exec_count > EXEC_LIMIT: # Fuzzing next queue
self.exec_count = 0
self.conn.send_next_queue()
self.conn.send_node_done(meta_data["id"], results, new_payload)
def handle_msg(self, msg):
if msg["type"] == MSG_RUN_NODE:
self.handle_node(msg)
elif msg["type"] == MSG_IMPORT:
self.handle_import(msg)
elif msg["type"] == MSG_BUSY:
self.handle_busy()
else:
raise ValueError("Unknown message type {}".format(msg))
return True
def loop(self):
if not self.q.start():
return
log_slave("Started qemu", self.slave_id)
while True:
try:
msg = self.conn.recv()
except ConnectionResetError:
log_slave("Lost connection to master. Shutting down.",
self.slave_id)
return False
self.handle_msg(msg)
def quick_validate(self, data, old_res, quiet=False):
# Validate in persistent mode. Faster but problematic for very funky targets
self.statistics.event_exec()
old_array = old_res.copy_to_array()
new_res = self.__execute(data).apply_lut()
new_array = new_res.copy_to_array()
if new_array == old_array:
return True
if not quiet:
log_slave("Input validation failed! Target is funky?..",
self.slave_id)
return False
def funky_validate(self, data, old_res):
# Validate in persistent mode with stochastic prop of funky results
validations = 8
confirmations = 0
for _ in range(validations):
if self.quick_validate(data, old_res, quiet=True):
confirmations += 1
if confirmations >= 0.8 * validations:
return True
log_slave(
"Funky input received %d/%d confirmations. Rejecting.." %
(confirmations, validations), self.slave_id)
if self.config.argument_values['v']:
self.store_funky(data)
return False
def store_funky(self, data):
global num_funky
num_funky += 1
# store funky input for further analysis
funky_folder = self.config.argument_values['work_dir'] + "/funky/"
atomic_write(
funky_folder + "input_%02d_%05d" % (self.slave_id, num_funky),
data)
def validate_bits(self, data, old_node, default_info):
new_bitmap, _ = self.execute(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
old_bits.update(old_node["new_bits"])
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def validate_bytes(self, data, old_node, default_info):
new_bitmap, _ = self.execute(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def execute_redqueen(self, data):
self.statistics.event_exec_redqueen()
return self.q.execute_in_redqueen_mode(data)
def __send_to_master(self, data, execution_res, info):
info["time"] = time.time()
info["exit_reason"] = execution_res.exit_reason
info["performance"] = execution_res.performance
if self.conn is not None:
self.conn.send_new_input(data, execution_res.copy_to_array(), info)
def trace_payload(self, data, info):
trace_file_in = self.config.argument_values[
'work_dir'] + "/redqueen_workdir_%d/pt_trace_results.txt" % self.slave_id
trace_folder = self.config.argument_values['work_dir'] + "/traces/"
trace_file_out = trace_folder + "payload_%05d" % info['id']
log_slave("Tracing payload_%05d.." % info['id'], self.slave_id)
try:
self.q.set_payload(data)
exec_res = self.q.execute_in_trace_mode(timeout_detection=False)
with open(trace_file_in, 'rb') as f_in:
with lz4.LZ4FrameFile(
trace_file_out + ".lz4",
'wb',
compression_level=lz4.COMPRESSIONLEVEL_MINHC) as f_out:
shutil.copyfileobj(f_in, f_out)
if not exec_res.is_regular():
self.statistics.event_reload()
self.q.reload()
except Exception as e:
log_slave(
"Failed to produce trace %s: %s (skipping..)" %
(trace_file_out, e), self.slave_id)
return None
return exec_res
def __execute(self, data, retry=0):
try:
self.q.set_payload(data)
return self.q.send_payload()
except (ValueError, BrokenPipeError):
if retry > 2:
# TODO if it reliably kills qemu, perhaps log to master for harvesting..
print_fail(
"Slave %d aborting due to repeated SHM/socket error. Check logs."
% self.slave_id)
log_slave(
"Aborting due to repeated SHM/socket error. Payload: %s" %
repr(data), self.slave_id)
raise
print_warning("SHM/socket error on Slave %d (retry %d)" %
(self.slave_id, retry))
log_slave("SHM/socket error, trying to restart qemu...",
self.slave_id)
self.statistics.event_reload()
if not self.q.restart():
raise
return self.__execute(data, retry=retry + 1)
def execute(self, data, info):
self.exec_count += 1
self.statistics.event_exec()
payload = p32(info["IoControlCode"]) + data
exec_res = self.__execute(payload)
is_new_input = self.bitmap_storage.should_send_to_master(exec_res)
crash = exec_res.is_crash()
stable = False
# store crashes and any validated new behavior
# do not validate timeouts and crashes at this point as they tend to be nondeterministic
if is_new_input:
if not crash:
assert exec_res.is_lut_applied()
if info["method"] == "import": # import don't need to check
stable = True
else:
if self.config.argument_values[
"funky"]: # TODO - Needed new determine algorihtm
stable = self.funky_validate(payload, exec_res)
else:
stable = self.quick_validate(payload, exec_res)
if not stable:
# TODO: auto-throttle persistent runs based on funky rate?
self.statistics.event_funky()
if crash or stable:
self.__send_to_master(data, exec_res, info)
else:
if crash:
log_slave(
"Crashing input found (%s), but not new (discarding)" %
(exec_res.exit_reason), self.slave_id)
# restart Qemu on crash
if crash:
self.statistics.event_reload()
self.q.reload()
return exec_res, is_new_input
class Process:
def __init__(self, config, pid=0):
self.config = config
self.debug_mode = config.argument_values['debug']
self.statistics = ProcessStatistics(self.config)
self.bitmap_storage = BitmapStorage(config, config.config_values['BITMAP_SHM_SIZE'], "process", read_only=False)
log_process("Starting (pid: %d)" % os.getpid())
log_process("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
self.q = qemu(pid, self.config,
debug_mode=self.debug_mode)
self.optimizer = Optimizer(self.q, self.statistics)
self.reproducer = Reproducer(self.q, self.statistics)
self.database = Database(self.statistics) # load interface
def __set_current_program(self, program):
self.cur_program = program
def maybe_insert_program(self, program, exec_res):
bitmap_array = exec_res.copy_to_array()
bitmap = ExecutionResult.bitmap_from_bytearray(bitmap_array, exec_res.exit_reason,
exec_res.performance)
bitmap.lut_applied = True # since we received the bitmap from the should_send_to_master, the lut was already applied
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(bitmap)
if should_store and not exec_res.is_crash():
program.set_new_bits(new_bits)
program.set_new_bytes(new_bytes)
self.optimizer.add(program, bitmap, new_bytes, new_bits)
def execute_irp(self, index):
"""
Send IRP to qemu agent and receive a coverage.
returns True if qemu has crashed.
"""
# send irp request.
irp = self.cur_program.irps[index]
exec_res = self.q.send_irp(irp)
self.statistics.event_exec(self.cur_program.get_state())
is_new_input = self.bitmap_storage.should_send_to_master(exec_res)
if is_new_input:
new_program = self.cur_program.clone_with_irps(self.cur_program.irps[:index+1])
self.maybe_insert_program(new_program, exec_res)
# restart Qemu on crash
if exec_res.is_crash():
if exec_res.is_timeout():
log("Timeouted maybe? (%x)" % irp.IoControlCode, "CRASH")
else:
log("Crashed maybe? (%x)" % irp.IoControlCode, "CRASH")
self.cur_program.save_to_file("unreproduced")
self.q.reload()
self.statistics.event_reload()
self.reproducer.add(self.cur_program.clone_with_irps(self.cur_program.irps[:index+1]))
return True
return False
def execute_program(self, program):
self.__set_current_program(program)
self.q.reload_driver()
for i in range(len(self.cur_program.irps)):
if self.execute_irp(i):
return True
def __execute_dependency(self, length):
self.q.reload_driver()
for i in range(length):
exec_res = self.q.send_irp(self.cur_program.irps[i])
if exec_res.is_crash():
return True
def execute_deterministic(self, program):
self.__set_current_program(program)
irps = self.cur_program.irps
for index in range(len(irps)):
if irps[index].InBufferLength == 0:
continue
# deterministic logic
# Walking bitfilps
if self.__execute_dependency(index) or bitflip.mutate_seq_walking_bit(self, index):
return
if bitflip.mutate_seq_two_walking_bits(self, index):
return
if bitflip.mutate_seq_four_walking_bits(self, index):
return
# Walking byte sets
if bitflip.mutate_seq_walking_byte(self, index):
return
if bitflip.mutate_seq_two_walking_bytes(self, index):
return
if bitflip.mutate_seq_four_walking_bytes(self, index):
return
# Arithmetic mutations
if arithmetic.mutate_seq_8_bit_arithmetic(self, index):
return
if arithmetic.mutate_seq_16_bit_arithmetic(self, index):
return
if arithmetic.mutate_seq_32_bit_arithmetic(self, index):
return
# Interesting value mutations
if interesting_values.mutate_seq_8_bit_interesting(self, index):
return
if interesting_values.mutate_seq_16_bit_interesting(self, index):
return
if interesting_values.mutate_seq_32_bit_interesting(self, index):
return
# Scan non paged area fault.
if wdmstyle.scan_page_fault(self, index):
return
# Resolve IOCTL dependency.
if wdmstyle.resolve_dependency(self):
return
self.cur_program.set_dirty(False)
def execute_havoc(self, program):
self.__set_current_program(program)
irps = self.cur_program.irps
for index in range(len(irps)):
if irps[index].InBufferLength == 0:
continue
# Random value mutations
if self.__execute_dependency(index) or havoc.mutate_seq_8_bit_rand8bit(self, index):
return
if havoc.mutate_seq_16_bit_rand16bit(self, index):
return
if havoc.mutate_seq_32_bit_rand32bit(self, index):
return
if havoc.mutate_seq_32_bit_rand16bit(self, index):
return
if havoc.mutate_seq_64_bit_rand8bit(self, index):
return
# InBufferLength mutation
if havoc.mutate_buffer_length(self, index):
return
if havoc.bruteforce_irps(self):
return
def loop(self):
# Start the QEMU
if not self.q.start():
return
# Start logging.
t = threading.Thread(target=self.log_current_state, args=())
t.start()
# basic coverage program.
program = self.database.get_next()
self.execute_program(program)
program.irps = program.irps[::-1]
self.execute_program(program)
while self.optimizer.optimizable():
new_programs = self.optimizer.optimize()
self.database.add(new_programs)
# Import seeds.
seed_directory = self.config.argument_values['seed_dir']
if len(os.listdir(seed_directory)):
for (directory, _, files) in os.walk(seed_directory):
for f in files:
path = os.path.join(directory, f)
log("Importing seed (%s)" % path)
if os.path.exists(path):
program = Program()
program.load(path)
# If a crash(or timeout) occurs, retry an execution.
while True:
if not self.execute_program(program):
log("[!] Imported seed crashed!")
break
self.reproducer.clear()
self.optimizer.clear()
while self.optimizer.optimizable():
new_programs = self.optimizer.optimize()
if new_programs:
self.database.add(new_programs)
log("[+] Count of initial unique programs : %d" % len(self.database.unique_programs))
if interface_manager.count() != len(self.database.unique_programs):
log("[!] Some IOCTL code were ignored maybe")
while True:
program = self.database.get_next()
programCopyed = copy.deepcopy(program)
for _ in range(1):
programCopyed.mutate(corpus_programs=self.database.get_programs())
# Execution
if programCopyed.get_dirty():
self.execute_deterministic(programCopyed)
else:
self.execute_havoc(programCopyed)
# Get new interesting corpus
while self.optimizer.optimizable():
new_programs = self.optimizer.optimize()
if new_programs:
self.database.add(new_programs)
# Start deterministic execution.
for prog in new_programs:
prog.set_state("AFLdetermin")
self.execute_deterministic(copy.deepcopy(prog))
# crash reproduction
self.reproducer.reproduce()
# synchronization
program.program_struct["dirty"] = programCopyed.program_struct["dirty"]
program.program_struct["exec_count"] = programCopyed.program_struct["exec_count"]
# Update update_probability_map of the corpus database.
self.database.update_probability_map()
def log_current_state(self):
while True:
time.sleep(3)
log('', label='')
log("---- Corpus Database -----" )
self.database.dump()
log("---- Current state (id=%d) ----" % self.cur_program.get_id())
log("exec_speed=%ds, state=%s" % (self.statistics.data["total_execs"] / (time.time() - self.statistics.data["start_time"]), self.cur_program.get_state()))
log("total_paths=%d, unique=%d, pending=%d" % (self.statistics.data["paths_total"], len(self.database.get_unique_programs()), self.statistics.data["paths_pending"]))
log("total_crash=%d, unique=%d, timeout=%d" % (self.statistics.data["findings"]["crash"], self.statistics.data['unique_findings']['crash'], self.statistics.data["findings"]["timeout"]))
def shutdown(self):
self.q.shutdown()
class SlaveProcess:
def __init__(self, slave_id, config, connection, auto_reload=False):
self.config = config
self.slave_id = slave_id
self.q = qemu(self.slave_id, self.config)
self.statistics = SlaveStatistics(self.slave_id, self.config)
self.logic = FuzzingStateLogic(self, self.config)
self.conn = connection
self.bitmap_storage = BitmapStorage(
self.config, self.config.config_values['BITMAP_SHM_SIZE'],
"master")
def handle_import(self, msg):
meta_data = {"state": {"name": "import"}, "id": 0}
payload = msg["task"]["payload"]
self.logic.process_node(payload, meta_data)
self.conn.send_ready()
def handle_busy(self):
busy_timeout = 1
kickstart = False
if kickstart: # spend busy cycle by feeding random strings?
log_slave("No ready work items, attempting random..",
self.slave_id)
start_time = time.time()
while (time.time() - start_time) < busy_timeout:
meta_data = {"state": {"name": "import"}, "id": 0}
payload = rand.bytes(rand.int(32))
self.logic.process_node(payload, meta_data)
else:
log_slave("No ready work items, waiting...", self.slave_id)
time.sleep(busy_timeout)
self.conn.send_ready()
def handle_node(self, msg):
meta_data = QueueNode.get_metadata(msg["task"]["nid"])
payload = QueueNode.get_payload(meta_data["info"]["exit_reason"],
meta_data["id"])
results, new_payload = self.logic.process_node(payload, meta_data)
if new_payload:
default_info = {
"method": "validate_bits",
"parent": meta_data["id"]
}
if self.validate_bits(new_payload, meta_data, default_info):
log_slave(
"Stage %s found alternative payload for node %d" %
(meta_data["state"]["name"], meta_data["id"]),
self.slave_id)
else:
log_slave(
"Provided alternative payload found invalid - bug in stage %s?"
% meta_data["state"]["name"], self.slave_id)
self.conn.send_node_done(meta_data["id"], results, new_payload)
def loop(self):
if not self.q.start():
return
log_slave("Started qemu", self.slave_id)
while True:
try:
msg = self.conn.recv()
except ConnectionResetError:
log_slave("Lost connection to master. Shutting down.",
self.slave_id)
return
if msg["type"] == MSG_RUN_NODE:
self.handle_node(msg)
elif msg["type"] == MSG_IMPORT:
self.handle_import(msg)
elif msg["type"] == MSG_BUSY:
self.handle_busy()
else:
raise ValueError("Unknown message type {}".format(msg))
def validate(self, data, old_array):
self.q.set_payload(data)
self.statistics.event_exec()
new_bitmap = self.q.send_payload().apply_lut()
new_array = new_bitmap.copy_to_array()
if new_array == old_array:
return True, new_bitmap
log_slave("Validation failed, ignoring this input", self.slave_id)
if False: # activate detailed logging of funky bitmaps
for i in range(new_bitmap.bitmap_size):
if old_array[i] != new_array[i]:
log_slave(
"Funky bit in validation bitmap: %d (%d vs %d)" %
(i, old_array[i], new_array[i]), self.slave_id)
return False, None
def validate_bits(self, data, old_node, default_info):
new_bitmap, _ = self.execute(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
old_bits.update(old_node["new_bits"])
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def validate_bytes(self, data, old_node, default_info):
new_bitmap, _ = self.execute(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def execute_redqueen(self, data):
self.statistics.event_exec_redqueen()
return self.q.execute_in_redqueen_mode(data)
def __execute(self, data, retry=0):
try:
self.q.set_payload(data)
if False: # activate detailed comparison of execution traces?
return self.check_funkyness_and_store_trace(data)
else:
return self.q.send_payload()
except BrokenPipeError:
if retry > 2:
# TODO if it reliably kills qemu, perhaps log to master for harvesting..
log_slave(
"Fatal: Repeated BrokenPipeError on input: %s" %
repr(data), self.slave_id)
raise
else:
log_slave("BrokenPipeError, trying to restart qemu...",
self.slave_id)
self.q.shutdown()
time.sleep(1)
self.q.start()
return self.__execute(data, retry=retry + 1)
assert False
def __send_to_master(self, data, execution_res, info):
info["time"] = time.time()
info["exit_reason"] = execution_res.exit_reason
info["performance"] = execution_res.performance
if self.conn is not None:
self.conn.send_new_input(data, execution_res.copy_to_array(), info)
def check_funkyness_and_store_trace(self, data):
global num_funky
exec_res = self.q.send_payload()
hash = exec_res.hash()
trace1 = read_binary_file(self.config.argument_values['work_dir'] +
"/pt_trace_dump_%d" % self.slave_id)
exec_res = self.q.send_payload()
if (hash != exec_res.hash()):
print_warning("Validation identified funky bits, dumping!")
num_funky += 1
trace_folder = self.config.argument_values[
'work_dir'] + "/traces/funky_%d_%d" % (num_funky,
self.slave_id)
os.makedirs(trace_folder)
atomic_write(trace_folder + "/input", data)
atomic_write(trace_folder + "/trace_a", trace1)
trace2 = read_binary_file(self.config.argument_values["work_dir"] +
"/pt_trace_dump_%d" % self.slave_id)
atomic_write(trace_folder + "/trace_b", trace2)
return exec_res
def execute(self, data, info):
self.statistics.event_exec()
exec_res = self.__execute(data)
is_new_input = self.bitmap_storage.should_send_to_master(exec_res)
crash = self.execution_exited_abnormally()
# store crashes and any validated new behavior
# do validate timeouts and crashes at this point as they tend to be nondeterministic
if is_new_input:
if not crash:
assert exec_res.is_lut_applied()
bitmap_array = exec_res.copy_to_array()
valid, exec_res = self.validate(data, bitmap_array)
if not valid:
self.statistics.event_funky()
log_slave("Input validation failed, throttling N/A",
self.slave_id)
# TODO: the funky event is already over at this point and the error may indeed not be deterministic
# how about we store some $num funky payloads for more focused exploration rather than discarding them?
#exec_res.exit_reason = 'funky'
#self.__send_to_master(data, exec_res, info)
if crash or valid:
self.__send_to_master(data, exec_res, info)
else:
if crash:
log_slave(
"Crashing input found (%s), but not new (discarding)" %
(exec_res.exit_reason), self.slave_id)
# restart Qemu on crash
if crash:
self.statistics.event_reload()
self.q.restart()
return exec_res, is_new_input
def execution_exited_abnormally(self):
return self.q.crashed or self.q.timeout or self.q.kasan
class SlaveProcess:
def __init__(self, slave_id, config, connection, auto_reload=False):
self.config = config
self.slave_id = slave_id
self.q = qemu(self.slave_id, self.config)
self.q.start(verbose=False)
print
"started qemu"
self.statistics = SlaveStatistics(self.slave_id, self.config)
self.logic = FuzzingStateLogic(self, self.config)
self.conn = connection
self.bitmap_storage = BitmapStorage(
self.config, self.config.config_values['BITMAP_SHM_SIZE'],
"master")
configure_log_prefix("%.2d" % slave_id)
def handle_server_msg(self, msg):
if msg["type"] == MSG_NEW_TASK:
return self.handle_task(msg)
if msg["type"] == MSG_QUEUE_STATUS:
return self.handle_queue_status(msg)
raise "unknown message type {}".format(msg)
def handle_task(self, msg):
if msg["task"]["type"] == "import":
meta_data = {"state": {"name": "import"}}
payload = msg["task"]["payload"]
elif msg["task"]["type"] == "node":
meta_data = QueueNode.get_metadata(msg["task"]["nid"])
payload = QueueNode.get_payload(meta_data["info"]["exit_reason"],
meta_data["id"])
print
"slave %d got task %d %s" % (self.slave_id, meta_data.get(
"node", {}).get("id", -1), repr(meta_data))
self.statistics.event_task(msg["task"])
results, new_payload = self.logic.process(payload, meta_data)
node_id = None
if new_payload != payload:
default_info = {
"method": "validate_bits",
"parent": meta_data["id"]
}
if self.validate_bits(new_payload, meta_data, default_info):
print("VALIDATE BITS OK")
else:
print(
"VALIDATE BITS FAILED BUG IN TRANSFORMATION!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
# assert False
if results:
node_id = meta_data["id"]
self.conn.send_task_performed(node_id, results, new_payload)
# print "performed task"
def handle_queue_status(self, msg):
pass
def loop(self):
while True:
# print "client waiting...."
msg = self.conn.recv()
# print "got %s"%repr(msg)
self.handle_server_msg(msg)
def validate(self, data, old_array):
self.q.set_payload(data)
self.statistics.event_exec()
new_bitmap = self.q.send_payload().apply_lut()
new_array = new_bitmap.copy_to_array()
if new_array == old_array:
print("Validate OK")
return True, new_bitmap
else:
for i in xrange(new_bitmap.bitmap_size):
if old_array[i] != new_array[i]:
safe_print("found fucky bit %d (%d vs %d)" %
(i, old_array[i], new_array[i]))
# assert(False)
print(
"VALIDATE FAILED, Not returning a bitmap!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
return False, None
def validate_bits(self, data, old_node, default_info):
new_bitmap, _ = self.execute_with_bitmap(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
old_bits.update(old_node["new_bits"])
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def validate_bytes(self, data, old_node, default_info):
new_bitmap, _ = self.execute_with_bitmap(data, default_info)
# handle non-det inputs
if new_bitmap is None:
return False
old_bits = old_node["new_bytes"].copy()
return GlobalBitmap.all_new_bits_still_set(old_bits, new_bitmap)
def execute_redqueen(self, data):
self.statistics.event_exec_redqueen()
return self.q.execute_in_redqueen_mode(data, debug_mode=False)
def execute_with_bitmap(self, data, info):
bitmap, new_input = self.__execute(data, info)
return bitmap, new_input
def execute(self, data, info):
bitmap, new_input = self.__execute(data, info)
return new_input
def __send_to_master(self, data, execution_res, info):
info["time"] = time.time()
info["exit_reason"] = execution_res.exit_reason
info["performance"] = execution_res.performance
if self.conn is not None:
self.conn.send_new_input(data, execution_res.copy_to_array(), info)
def check_fuckyness_and_store_trace(self, data):
global num_fucky
exec_res = self.q.send_payload()
hash = exec_res.hash()
trace1 = read_binary_file(self.config.argument_values['work_dir'] +
"/pt_trace_dump_%d" % self.slave_id)
exec_res = self.q.send_payload()
if (hash != exec_res.hash()):
safe_print(
"found fucky bits, dumping!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
)
num_fucky += 1
trace_folder = self.config.argument_values[
'work_dir'] + "/traces/fucky_%d_%d" % (num_fucky,
self.slave_id)
os.makedirs(trace_folder)
atomic_write(trace_folder + "/input", data)
atomic_write(trace_folder + "/trace_a", trace1)
trace2 = read_binary_file(self.config.argument_values["work_dir"] +
"/pt_trace_dump_%d" % self.slave_id)
atomic_write(trace_folder + "/trace_b", trace2)
return exec_res
def __execute(self, data, info):
self.statistics.event_exec()
self.q.set_payload(data)
if False: # Do not emit tracefiles on broken executions
exec_res = self.check_fuckyness_and_store_trace(data)
else:
exec_res = self.q.send_payload()
is_new_input = self.bitmap_storage.should_send_to_master(exec_res)
crash = self.execution_exited_abnormally(
) # we do not want to validate timeouts and crashes as they tend to be nondeterministic
if is_new_input:
if not crash:
assert exec_res.is_lut_applied()
bitmap_array = exec_res.copy_to_array()
valid, exec_res = self.validate(data, bitmap_array)
if crash or valid:
self.__send_to_master(data, exec_res, info)
return exec_res, is_new_input
def execution_exited_abnormally(self):
return self.q.crashed or self.q.timeout or self.q.kasan
# Todo: Fixme
def __restart_vm(self):
return True
if self.comm.slave_termination.value:
return False
self.comm.reload_semaphore.acquire()
try:
# raise Exception("!")
# QEMU is full of memory leaks...fixing it that way...
if self.soft_reload_counter >= 32:
self.soft_reload_counter = 0
raise Exception("...")
self.q.soft_reload()
self.soft_reload_counter += 1
except:
log_slave("restart failed %s" % traceback.format_exc(),
self.slave_id)
while True:
self.q.__del__()
self.q = qemu(self.slave_id, self.config)
if self.q.start():
break
else:
time.sleep(0.5)
log_slave("Fail Reload", self.slave_id)
self.comm.reload_semaphore.release()
self.q.set_tick_timeout_treshold(self.stage_tick_treshold *
self.timeout_tick_factor)
if self.comm.slave_termination.value:
return False
return True
class MasterProcess:
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def send_next_task(self, conn):
# Inputs placed to imports/ folder have priority.
# This can also be used to inject additional seeds at runtime.
imports = glob.glob(self.config.argument_values['work_dir'] +
"/imports/*")
if imports:
path = imports.pop()
print("Importing payload from %s" % path)
seed = read_binary_file(path)
os.remove(path)
return self.comm.send_import(conn, {
"type": "import",
"payload": seed
})
# Process items from queue..
node = self.queue.get_next()
if node:
return self.comm.send_node(conn, {
"type": "node",
"nid": node.get_id()
})
# No work in queue. Tell slave to wait a little or attempt blind fuzzing.
# If we see a lot of busy events, check the bitmap and warn on coverage issues.
self.comm.send_busy(conn)
self.busy_events += 1
if self.busy_events >= 10:
self.busy_events = 0
main_bitmap = self.bitmap_storage.get_bitmap_for_node_type(
"regular").c_bitmap
if mmh3.hash(main_bitmap) == self.empty_hash:
print_note(
"Coverage bitmap is empty?! Check -ip0 or try better seeds."
)
def loop(self):
while True:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
if msg["type"] == MSG_NODE_DONE:
# Slave execution done, update queue item + send new task
log_master("Received results, sending next task..")
if msg["node_id"]:
self.queue.update_node_results(msg["node_id"],
msg["results"],
msg["new_payload"])
self.send_next_task(conn)
elif msg["type"] == MSG_NEW_INPUT:
# Slave reports new interesting input
log_master("Received new input (exit=%s): %s" %
(msg["input"]["info"]["exit_reason"],
repr(msg["input"]["payload"][:24])))
node_struct = {
"info": msg["input"]["info"],
"state": {
"name": "initial"
}
}
self.maybe_insert_node(msg["input"]["payload"],
msg["input"]["bitmap"], node_struct)
elif msg["type"] == MSG_READY:
# Initial slave hello, send first task...
# log_master("Slave is ready..")
self.send_next_task(conn)
else:
raise ValueError("unknown message type {}".format(msg))
self.statistics.event_slave_poll()
self.statistics.maybe_write_stats()
def maybe_insert_node(self, payload, bitmap_array, node_struct):
bitmap = ExecutionResult.bitmap_from_bytearray(
bitmap_array, node_struct["info"]["exit_reason"],
node_struct["info"]["performance"])
bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied
backup_data = bitmap.copy_to_array()
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(
bitmap)
new_data = bitmap.copy_to_array()
if should_store:
node = QueueNode(payload, bitmap_array, node_struct, write=False)
node.set_new_bytes(new_bytes, write=False)
node.set_new_bits(new_bits, write=False)
self.queue.insert_input(node, bitmap)
else:
if node_struct["info"]["exit_reason"] != "regular":
log_master("Payload found to be boring, not saved (exit=%s)" %
node_struct["info"]["exit_reason"])
for i in range(len(bitmap_array)):
if backup_data[i] != new_data[i]:
assert (False), "Bitmap mangled at {} {} {}".format(
i, repr(backup_data[i]), repr(new_data[i]))
class InputQueue:
def __init__(self, config, scheduler, statistics):
self.config = config
self.scheduler = scheduler
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
self.id_to_node = {}
self.current_cycle = []
self.bitmap_index_to_fav_node = {}
self.num_cycles = 0
self.pending_favorites = True
self.statistics = statistics
self.grimoire_scheduler = GrimoireScheduler()
def get_next(self):
assert self.id_to_node
gram = self.grimoire_scheduler.get_next()
if gram:
return gram
node = self.current_cycle.pop() if self.current_cycle else None
while node:
if self.scheduler.should_be_scheduled(self, node):
return node
node = self.current_cycle.pop() if self.current_cycle else None
self.update_current_cycle()
return self.get_next()
def has_inputs(self):
return len(self.id_to_node) > 0
def update_current_cycle(self):
self.num_cycles += 1
self.current_cycle = list(self.id_to_node.values())
self.cull_queue(self.current_cycle)
self.sort_queue(self.current_cycle)
self.statistics.event_queue_cycle(self)
def cull_queue(self, nodes):
self.pending_favorites = False
for (index, (node, _)) in self.bitmap_index_to_fav_node.iteritems():
if node.get_state() != "finished":
self.pending_favorites = True
# TODO implement queue culling like afl?
def sort_queue(self, nodes):
nodes.sort(key=lambda n: self.scheduler.score_priority(n))
def get_node_by_id(self, id):
return self.id_to_node[id]
def num_inputs(self):
len(self.id_to_node)
def construct_node(
self,
payload,
bitmap,
new_bytes,
new_bits,
node_struct,
):
assert "fav_bits" not in node_struct
assert "level" not in node_struct
assert "new_bytes" not in node_struct
assert "new_bits" not in node_struct
node_struct["new_bytes"] = new_bytes
node_struct["new_bits"] = new_bits
node = QueueNode(payload, bitmap, node_struct, write=False)
node.clear_fav_bits(write=False)
parent = node_struct["info"]["parent"]
node.set_level(self.get_node_by_id(parent).get_level() +
1 if parent else 0,
write=False)
return node
def maybe_insert_node(self, payload, bitmap_array, node_struct):
bitmap = ExecutionResult.bitmap_from_bytearray(
bitmap_array, node_struct["info"]["exit_reason"],
node_struct["info"]["performance"])
bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied
backup_data = bitmap.copy_to_array()
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(
bitmap)
new_data = bitmap.copy_to_array()
if should_store:
self.insert_input(
self.construct_node(payload, bitmap_array, new_bytes, new_bits,
node_struct), bitmap)
else:
for i in xrange(len(bitmap_array)):
if backup_data[i] != new_data[i]:
print("diffing at {} {} {}".format(i, repr(backup_data[i]),
repr(new_data[i])))
safe_print("RECIEVED BORING INPUT, NOT SAVING..")
# assert(False)
def insert_input(self, node, bitmap):
safe_print(repr(node.node_struct))
self.grimoire_scheduler.insert_input(node)
node.set_fav_factor(self.scheduler.score_fav(node), write=True)
self.id_to_node[node.get_id()] = node
self.current_cycle.append(node)
print("saving input")
self.update_best_input_for_bitmap_entry(
node, bitmap) # TODO improve performance!
print("done saving input")
self.sort_queue(self.current_cycle)
self.statistics.event_new_node_found(node)
def should_overwrite_old_entry(self, index, val, node):
entry = self.bitmap_index_to_fav_node.get(index)
if not entry:
return True, None
old_node, old_val = entry
more_bits = val > old_val
better_score = (val == old_val
and node.get_fav_factor() < old_node.get_fav_factor())
if more_bits or better_score:
return True, old_node
return False, None
def update_best_input_for_bitmap_entry(self, node, bitmap):
changed_nodes = set()
for (index, val) in enumerate(bitmap.cbuffer):
if val == 0x0:
continue
overwrite, old_node = self.should_overwrite_old_entry(
index, val, node)
if overwrite:
self.bitmap_index_to_fav_node[index] = (node, val)
node.add_fav_bit(index, write=False)
changed_nodes.add(node)
if old_node:
old_node.remove_fav_bit(index, write=False)
changed_nodes.add(old_node)
for node in changed_nodes:
node.write_metadata()