def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.debug_mode = config.argument_values['debug']
self.task_count = 0
self.task_paused = False
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queues = CycleQueue(self.config)
for _ in range(interface_manager.size()):
self.queues.append(InputQueue(self.config, self.statistics))
#self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
redqueen_global_config(
redq_hammering=self.config.argument_values['hammer_jmp_tables'],
redq_do_simple=self.config.argument_values['redq_do_simple'],
afl_arith_max=self.config.config_values['ARITHMETIC_MAX'])
self.imports = list(interface_manager.to_corpus())
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']))
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.scheduler = Scheduler()
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.scheduler, self.statistics)
self.skip_zero = self.config.argument_values['s']
self.refresh_rate = self.config.config_values['UI_REFRESH_RATE']
self.use_effector_map = self.config.argument_values['d']
self.arith_max = self.config.config_values["ARITHMETIC_MAX"]
self.mode_fix_checksum = self.config.argument_values["fix_hashes"]
if not self.config.argument_values['D']:
self.use_effector_map = False
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
print("Master PID: %d\n", os.getpid())
log_master("Use effector maps: " + str(self.use_effector_map))
class MasterProcess:
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.debug_mode = config.argument_values['debug']
self.task_count = 0
self.task_paused = False
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queues = CycleQueue(self.config)
for _ in range(interface_manager.size()):
self.queues.append(InputQueue(self.config, self.statistics))
#self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
redqueen_global_config(
redq_hammering=self.config.argument_values['hammer_jmp_tables'],
redq_do_simple=self.config.argument_values['redq_do_simple'],
afl_arith_max=self.config.config_values['ARITHMETIC_MAX'])
self.imports = list(interface_manager.to_corpus())
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def send_next_task(self, conn):
# for the switching queue.
if self.task_paused:
return
# Process items from queue..
node = self.queues.get_next_node()
if node:
self.task_count += 1
return self.comm.send_node(conn, {
"type": "node",
"nid": node.get_id()
})
# No work in queue. Tell slave to wait a little or attempt blind fuzzing.
# If we see a lot of busy events, check the bitmap and warn on coverage issues.
self.comm.send_busy(conn)
self.busy_events += 1
if self.busy_events >= 10:
self.busy_events = 0
main_bitmap = self.bitmap_storage.get_bitmap_for_node_type(
"regular").c_bitmap
if mmh3.hash(main_bitmap) == self.empty_hash:
print_note(
"Coverage bitmap is empty?! Check -ip0 or try better seeds."
)
def send_import(self, conn):
(iocode, payload) = self.imports.pop()
print("Importing payload from %s" % hex(iocode))
self.comm.send_import(conn, {
"type": "import",
"IoControlCode": iocode,
"payload": payload
})
def handle_msg(self, conn, msg):
if msg["type"] == MSG_NODE_DONE:
# Slave execution done, update queue item + send new task
self.task_count -= 1
log_master("Received results, sending next task..")
if msg["node_id"]:
self.queues.update_node_results(msg["node_id"], msg["results"],
msg["new_payload"])
self.send_next_task(conn)
elif msg["type"] == MSG_NEW_INPUT:
# Slave reports new interesting input
if self.debug_mode:
log_master("Received new input (exit=%s): %s" %
(msg["input"]["info"]["exit_reason"],
repr(msg["input"]["payload"][:24])))
node_struct = {
"info": msg["input"]["info"],
"state": {
"name": "initial"
}
}
self.maybe_insert_node(msg["input"]["payload"],
msg["input"]["bitmap"], node_struct)
elif msg["type"] == MSG_READY:
# Initial slave hello, send first task...
# log_master("Slave is ready..")
self.send_next_task(conn)
elif msg["type"] == MSG_NEXT_QUEUE:
# Flush slave message to switch queue.
self.task_paused = True
while self.task_count:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
self.handle_msg(conn, msg)
self.task_paused = False
# Switch to next queue.
self.queues.next()
# Inputs placed to imports/ folder have priority.
if self.imports:
self.send_import(conn)
self.send_next_task(conn)
else:
raise ValueError("unknown message type {}".format(msg))
def loop(self):
# Import a seed.
imported = True
while imported:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
assert msg["type"] == MSG_READY
if self.imports:
self.send_import(conn)
imported = False
while True:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
self.handle_msg(conn, msg)
self.statistics.event_slave_poll()
self.statistics.maybe_write_stats()
self.check_abort_condition()
def check_abort_condition(self):
import time
import datetime
t_limit = self.config.argument_values['abort_time']
n_limit = self.config.argument_values['abort_exec']
if t_limit:
if t_limit * 3600 < time.time(
) - self.statistics.data['start_time']:
raise SystemExit("Exit on timeout.")
if n_limit:
if n_limit < self.statistics.data['total_execs']:
raise SystemExit("Exit on max execs.")
def maybe_insert_node(self, payload, bitmap_array, node_struct):
print("New payload(0x%x) : " % node_struct["info"]["IoControlCode"],
payload[:16])
bitmap = ExecutionResult.bitmap_from_bytearray(
bitmap_array, node_struct["info"]["exit_reason"],
node_struct["info"]["performance"])
bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied
backup_data = bitmap.copy_to_array()
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(
bitmap)
new_data = bitmap.copy_to_array()
if should_store:
node = QueueNode(payload, bitmap_array, node_struct, write=False)
node.set_new_bytes(new_bytes, write=False)
node.set_new_bits(new_bits, write=False)
self.queues.insert_input(node, bitmap)
elif self.debug_mode:
if node_struct["info"]["exit_reason"] != "regular":
log_master("Payload found to be boring, not saved (exit=%s)" %
node_struct["info"]["exit_reason"])
for i in range(len(bitmap_array)):
if backup_data[i] != new_data[i]:
assert (False), "Bitmap mangled at {} {} {}".format(
i, repr(backup_data[i]), repr(new_data[i]))
class MasterProcess:
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.busy_events = 0
self.empty_hash = mmh3.hash(
("\x00" * self.config.config_values['BITMAP_SHM_SIZE']),
signed=False)
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.statistics)
self.bitmap_storage = BitmapStorage(
config,
config.config_values['BITMAP_SHM_SIZE'],
"master",
read_only=False)
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
log_master("Starting (pid: %d)" % os.getpid())
log_master("Configuration dump:\n%s" %
pformat(config.argument_values, indent=4, compact=True))
def send_next_task(self, conn):
# Inputs placed to imports/ folder have priority.
# This can also be used to inject additional seeds at runtime.
imports = glob.glob(self.config.argument_values['work_dir'] +
"/imports/*")
if imports:
path = imports.pop()
print("Importing payload from %s" % path)
seed = read_binary_file(path)
os.remove(path)
return self.comm.send_import(conn, {
"type": "import",
"payload": seed
})
# Process items from queue..
node = self.queue.get_next()
if node:
return self.comm.send_node(conn, {
"type": "node",
"nid": node.get_id()
})
# No work in queue. Tell slave to wait a little or attempt blind fuzzing.
# If we see a lot of busy events, check the bitmap and warn on coverage issues.
self.comm.send_busy(conn)
self.busy_events += 1
if self.busy_events >= 10:
self.busy_events = 0
main_bitmap = self.bitmap_storage.get_bitmap_for_node_type(
"regular").c_bitmap
if mmh3.hash(main_bitmap) == self.empty_hash:
print_note(
"Coverage bitmap is empty?! Check -ip0 or try better seeds."
)
def loop(self):
while True:
for conn, msg in self.comm.wait(self.statistics.plot_thres):
if msg["type"] == MSG_NODE_DONE:
# Slave execution done, update queue item + send new task
log_master("Received results, sending next task..")
if msg["node_id"]:
self.queue.update_node_results(msg["node_id"],
msg["results"],
msg["new_payload"])
self.send_next_task(conn)
elif msg["type"] == MSG_NEW_INPUT:
# Slave reports new interesting input
log_master("Received new input (exit=%s): %s" %
(msg["input"]["info"]["exit_reason"],
repr(msg["input"]["payload"][:24])))
node_struct = {
"info": msg["input"]["info"],
"state": {
"name": "initial"
}
}
self.maybe_insert_node(msg["input"]["payload"],
msg["input"]["bitmap"], node_struct)
elif msg["type"] == MSG_READY:
# Initial slave hello, send first task...
# log_master("Slave is ready..")
self.send_next_task(conn)
else:
raise ValueError("unknown message type {}".format(msg))
self.statistics.event_slave_poll()
self.statistics.maybe_write_stats()
def maybe_insert_node(self, payload, bitmap_array, node_struct):
bitmap = ExecutionResult.bitmap_from_bytearray(
bitmap_array, node_struct["info"]["exit_reason"],
node_struct["info"]["performance"])
bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied
backup_data = bitmap.copy_to_array()
should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue(
bitmap)
new_data = bitmap.copy_to_array()
if should_store:
node = QueueNode(payload, bitmap_array, node_struct, write=False)
node.set_new_bytes(new_bytes, write=False)
node.set_new_bits(new_bits, write=False)
self.queue.insert_input(node, bitmap)
else:
if node_struct["info"]["exit_reason"] != "regular":
log_master("Payload found to be boring, not saved (exit=%s)" %
node_struct["info"]["exit_reason"])
for i in range(len(bitmap_array)):
if backup_data[i] != new_data[i]:
assert (False), "Bitmap mangled at {} {} {}".format(
i, repr(backup_data[i]), repr(new_data[i]))
class MasterProcess:
def __init__(self, config):
self.config = config
self.comm = ServerConnection(self.config)
self.scheduler = Scheduler()
self.statistics = MasterStatistics(self.config)
self.queue = InputQueue(self.config, self.scheduler, self.statistics)
self.skip_zero = self.config.argument_values['s']
self.refresh_rate = self.config.config_values['UI_REFRESH_RATE']
self.use_effector_map = self.config.argument_values['d']
self.arith_max = self.config.config_values["ARITHMETIC_MAX"]
self.mode_fix_checksum = self.config.argument_values["fix_hashes"]
if not self.config.argument_values['D']:
self.use_effector_map = False
if self.config.argument_values['hammer_jmp_tables']:
enable_hammering()
print("Master PID: %d\n", os.getpid())
log_master("Use effector maps: " + str(self.use_effector_map))
def get_task(self):
imports = glob.glob(self.config.argument_values['work_dir'] +
"/imports/*")
if imports:
path = imports.pop()
payload = read_binary_file(path)
os.remove(path)
return {"payload": payload, "type": "import"}
elif self.queue.has_inputs():
node = self.queue.get_next()
return {"type": "node", "nid": node.get_id()}
else:
return {"payload": random_string(), "type": "import"}
def loop(self):
while True:
for conn, msg in self.comm.wait():
if msg["type"] == MSG_TASK_RESULTS:
# print repr(msg)
if msg["node_id"]:
results = msg["results"]
if results:
node = self.queue.get_node_by_id(msg["node_id"])
node.update_metadata(results)
new_payload = msg["new_payload"]
if new_payload:
node.set_payload(new_payload)
self.comm.send_task(conn, self.get_task())
elif msg["type"] == MSG_NEW_INPUT:
node_struct = {
"info": msg["input"]["info"],
"state": {
"name": "initial"
}
}
self.queue.maybe_insert_node(msg["input"]["payload"],
msg["input"]["bitmap"],
node_struct)
# print "new input: {}".format(repr(msg["input"]["payload"]))
elif msg["type"] == MSG_HELLO:
print
"got CLIENT_HELLO"
self.comm.send_task(conn, self.get_task())
else:
raise "unknown message type {}".format(msg)