def __init__(self, config): self.config = config self.comm = ServerConnection(self.config) self.debug_mode = config.argument_values['debug'] self.task_count = 0 self.task_paused = False self.busy_events = 0 self.empty_hash = mmh3.hash( ("\x00" * self.config.config_values['BITMAP_SHM_SIZE']), signed=False) self.statistics = MasterStatistics(self.config) self.queues = CycleQueue(self.config) for _ in range(interface_manager.size()): self.queues.append(InputQueue(self.config, self.statistics)) #self.queue = InputQueue(self.config, self.statistics) self.bitmap_storage = BitmapStorage( config, config.config_values['BITMAP_SHM_SIZE'], "master", read_only=False) redqueen_global_config( redq_hammering=self.config.argument_values['hammer_jmp_tables'], redq_do_simple=self.config.argument_values['redq_do_simple'], afl_arith_max=self.config.config_values['ARITHMETIC_MAX']) self.imports = list(interface_manager.to_corpus()) log_master("Starting (pid: %d)" % os.getpid()) log_master("Configuration dump:\n%s" % pformat(config.argument_values, indent=4, compact=True))
def __init__(self, config): self.config = config self.comm = ServerConnection(self.config) self.busy_events = 0 self.empty_hash = mmh3.hash( ("\x00" * self.config.config_values['BITMAP_SHM_SIZE'])) self.statistics = MasterStatistics(self.config) self.queue = InputQueue(self.config, self.statistics) self.bitmap_storage = BitmapStorage( config, config.config_values['BITMAP_SHM_SIZE'], "master", read_only=False) if self.config.argument_values['hammer_jmp_tables']: enable_hammering() log_master("Starting (pid: %d)" % os.getpid()) log_master("Configuration dump:\n%s" % pformat(config.argument_values, indent=4, compact=True))
def __init__(self, config): self.config = config self.comm = ServerConnection(self.config) self.scheduler = Scheduler() self.statistics = MasterStatistics(self.config) self.queue = InputQueue(self.config, self.scheduler, self.statistics) self.skip_zero = self.config.argument_values['s'] self.refresh_rate = self.config.config_values['UI_REFRESH_RATE'] self.use_effector_map = self.config.argument_values['d'] self.arith_max = self.config.config_values["ARITHMETIC_MAX"] self.mode_fix_checksum = self.config.argument_values["fix_hashes"] if not self.config.argument_values['D']: self.use_effector_map = False if self.config.argument_values['hammer_jmp_tables']: enable_hammering() print("Master PID: %d\n", os.getpid()) log_master("Use effector maps: " + str(self.use_effector_map))
class MasterProcess: def __init__(self, config): self.config = config self.comm = ServerConnection(self.config) self.debug_mode = config.argument_values['debug'] self.task_count = 0 self.task_paused = False self.busy_events = 0 self.empty_hash = mmh3.hash( ("\x00" * self.config.config_values['BITMAP_SHM_SIZE']), signed=False) self.statistics = MasterStatistics(self.config) self.queues = CycleQueue(self.config) for _ in range(interface_manager.size()): self.queues.append(InputQueue(self.config, self.statistics)) #self.queue = InputQueue(self.config, self.statistics) self.bitmap_storage = BitmapStorage( config, config.config_values['BITMAP_SHM_SIZE'], "master", read_only=False) redqueen_global_config( redq_hammering=self.config.argument_values['hammer_jmp_tables'], redq_do_simple=self.config.argument_values['redq_do_simple'], afl_arith_max=self.config.config_values['ARITHMETIC_MAX']) self.imports = list(interface_manager.to_corpus()) log_master("Starting (pid: %d)" % os.getpid()) log_master("Configuration dump:\n%s" % pformat(config.argument_values, indent=4, compact=True)) def send_next_task(self, conn): # for the switching queue. if self.task_paused: return # Process items from queue.. node = self.queues.get_next_node() if node: self.task_count += 1 return self.comm.send_node(conn, { "type": "node", "nid": node.get_id() }) # No work in queue. Tell slave to wait a little or attempt blind fuzzing. # If we see a lot of busy events, check the bitmap and warn on coverage issues. self.comm.send_busy(conn) self.busy_events += 1 if self.busy_events >= 10: self.busy_events = 0 main_bitmap = self.bitmap_storage.get_bitmap_for_node_type( "regular").c_bitmap if mmh3.hash(main_bitmap) == self.empty_hash: print_note( "Coverage bitmap is empty?! Check -ip0 or try better seeds." ) def send_import(self, conn): (iocode, payload) = self.imports.pop() print("Importing payload from %s" % hex(iocode)) self.comm.send_import(conn, { "type": "import", "IoControlCode": iocode, "payload": payload }) def handle_msg(self, conn, msg): if msg["type"] == MSG_NODE_DONE: # Slave execution done, update queue item + send new task self.task_count -= 1 log_master("Received results, sending next task..") if msg["node_id"]: self.queues.update_node_results(msg["node_id"], msg["results"], msg["new_payload"]) self.send_next_task(conn) elif msg["type"] == MSG_NEW_INPUT: # Slave reports new interesting input if self.debug_mode: log_master("Received new input (exit=%s): %s" % (msg["input"]["info"]["exit_reason"], repr(msg["input"]["payload"][:24]))) node_struct = { "info": msg["input"]["info"], "state": { "name": "initial" } } self.maybe_insert_node(msg["input"]["payload"], msg["input"]["bitmap"], node_struct) elif msg["type"] == MSG_READY: # Initial slave hello, send first task... # log_master("Slave is ready..") self.send_next_task(conn) elif msg["type"] == MSG_NEXT_QUEUE: # Flush slave message to switch queue. self.task_paused = True while self.task_count: for conn, msg in self.comm.wait(self.statistics.plot_thres): self.handle_msg(conn, msg) self.task_paused = False # Switch to next queue. self.queues.next() # Inputs placed to imports/ folder have priority. if self.imports: self.send_import(conn) self.send_next_task(conn) else: raise ValueError("unknown message type {}".format(msg)) def loop(self): # Import a seed. imported = True while imported: for conn, msg in self.comm.wait(self.statistics.plot_thres): assert msg["type"] == MSG_READY if self.imports: self.send_import(conn) imported = False while True: for conn, msg in self.comm.wait(self.statistics.plot_thres): self.handle_msg(conn, msg) self.statistics.event_slave_poll() self.statistics.maybe_write_stats() self.check_abort_condition() def check_abort_condition(self): import time import datetime t_limit = self.config.argument_values['abort_time'] n_limit = self.config.argument_values['abort_exec'] if t_limit: if t_limit * 3600 < time.time( ) - self.statistics.data['start_time']: raise SystemExit("Exit on timeout.") if n_limit: if n_limit < self.statistics.data['total_execs']: raise SystemExit("Exit on max execs.") def maybe_insert_node(self, payload, bitmap_array, node_struct): print("New payload(0x%x) : " % node_struct["info"]["IoControlCode"], payload[:16]) bitmap = ExecutionResult.bitmap_from_bytearray( bitmap_array, node_struct["info"]["exit_reason"], node_struct["info"]["performance"]) bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied backup_data = bitmap.copy_to_array() should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue( bitmap) new_data = bitmap.copy_to_array() if should_store: node = QueueNode(payload, bitmap_array, node_struct, write=False) node.set_new_bytes(new_bytes, write=False) node.set_new_bits(new_bits, write=False) self.queues.insert_input(node, bitmap) elif self.debug_mode: if node_struct["info"]["exit_reason"] != "regular": log_master("Payload found to be boring, not saved (exit=%s)" % node_struct["info"]["exit_reason"]) for i in range(len(bitmap_array)): if backup_data[i] != new_data[i]: assert (False), "Bitmap mangled at {} {} {}".format( i, repr(backup_data[i]), repr(new_data[i]))
class MasterProcess: def __init__(self, config): self.config = config self.comm = ServerConnection(self.config) self.busy_events = 0 self.empty_hash = mmh3.hash( ("\x00" * self.config.config_values['BITMAP_SHM_SIZE']), signed=False) self.statistics = MasterStatistics(self.config) self.queue = InputQueue(self.config, self.statistics) self.bitmap_storage = BitmapStorage( config, config.config_values['BITMAP_SHM_SIZE'], "master", read_only=False) if self.config.argument_values['hammer_jmp_tables']: enable_hammering() log_master("Starting (pid: %d)" % os.getpid()) log_master("Configuration dump:\n%s" % pformat(config.argument_values, indent=4, compact=True)) def send_next_task(self, conn): # Inputs placed to imports/ folder have priority. # This can also be used to inject additional seeds at runtime. imports = glob.glob(self.config.argument_values['work_dir'] + "/imports/*") if imports: path = imports.pop() print("Importing payload from %s" % path) seed = read_binary_file(path) os.remove(path) return self.comm.send_import(conn, { "type": "import", "payload": seed }) # Process items from queue.. node = self.queue.get_next() if node: return self.comm.send_node(conn, { "type": "node", "nid": node.get_id() }) # No work in queue. Tell slave to wait a little or attempt blind fuzzing. # If we see a lot of busy events, check the bitmap and warn on coverage issues. self.comm.send_busy(conn) self.busy_events += 1 if self.busy_events >= 10: self.busy_events = 0 main_bitmap = self.bitmap_storage.get_bitmap_for_node_type( "regular").c_bitmap if mmh3.hash(main_bitmap) == self.empty_hash: print_note( "Coverage bitmap is empty?! Check -ip0 or try better seeds." ) def loop(self): while True: for conn, msg in self.comm.wait(self.statistics.plot_thres): if msg["type"] == MSG_NODE_DONE: # Slave execution done, update queue item + send new task log_master("Received results, sending next task..") if msg["node_id"]: self.queue.update_node_results(msg["node_id"], msg["results"], msg["new_payload"]) self.send_next_task(conn) elif msg["type"] == MSG_NEW_INPUT: # Slave reports new interesting input log_master("Received new input (exit=%s): %s" % (msg["input"]["info"]["exit_reason"], repr(msg["input"]["payload"][:24]))) node_struct = { "info": msg["input"]["info"], "state": { "name": "initial" } } self.maybe_insert_node(msg["input"]["payload"], msg["input"]["bitmap"], node_struct) elif msg["type"] == MSG_READY: # Initial slave hello, send first task... # log_master("Slave is ready..") self.send_next_task(conn) else: raise ValueError("unknown message type {}".format(msg)) self.statistics.event_slave_poll() self.statistics.maybe_write_stats() def maybe_insert_node(self, payload, bitmap_array, node_struct): bitmap = ExecutionResult.bitmap_from_bytearray( bitmap_array, node_struct["info"]["exit_reason"], node_struct["info"]["performance"]) bitmap.lut_applied = True # since we received the bitmap from the slave, the lut was already applied backup_data = bitmap.copy_to_array() should_store, new_bytes, new_bits = self.bitmap_storage.should_store_in_queue( bitmap) new_data = bitmap.copy_to_array() if should_store: node = QueueNode(payload, bitmap_array, node_struct, write=False) node.set_new_bytes(new_bytes, write=False) node.set_new_bits(new_bits, write=False) self.queue.insert_input(node, bitmap) else: if node_struct["info"]["exit_reason"] != "regular": log_master("Payload found to be boring, not saved (exit=%s)" % node_struct["info"]["exit_reason"]) for i in range(len(bitmap_array)): if backup_data[i] != new_data[i]: assert (False), "Bitmap mangled at {} {} {}".format( i, repr(backup_data[i]), repr(new_data[i]))