def get_all_mutations(self):
self.boring_cmps = set()
offsets_to_lhs_to_rhs_to_info = {}
num_mut = 0
orig_run_info = [r for r in self.run_infos if not r.was_colored]
#assert (len(orig_run_info) == 1)
if len(orig_run_info) != 1:
log_redq("Warning: Could not find canonical orig. run info! len(info)=%d" % len(orig_run_info))
return num_mut, offsets_to_lhs_to_rhs_to_info
orig_run_info = orig_run_info[0]
for addr_to_cmp in [self.addr_to_cmp, self.addr_to_inv_cmp]:
for addr in addr_to_cmp:
cmp = addr_to_cmp[addr]
was_cmp_interessting = False
if len(cmp.run_info_to_pairs) == len(self.run_infos):
for (offsets, lhs, rhs, encoding) in cmp.calc_mutations(orig_run_info, len(self.run_infos)):
offsets, lhs, rhs = self.strip_unchanged_bytes_from_mutation_values(offsets, lhs, rhs)
was_cmp_interessting = True
offsets_to_lhs_to_rhs_to_info[offsets] = offsets_to_lhs_to_rhs_to_info.get(offsets, {})
offsets_to_lhs_to_rhs_to_info[offsets][lhs] = offsets_to_lhs_to_rhs_to_info[offsets].get(lhs,{})
if not rhs in offsets_to_lhs_to_rhs_to_info[offsets][lhs]:
num_mut += 1
offsets_to_lhs_to_rhs_to_info[offsets][lhs][rhs] = offsets_to_lhs_to_rhs_to_info[offsets][lhs].get(rhs, MutInfo())
offsets_to_lhs_to_rhs_to_info[offsets][lhs][rhs].add_info(addr, encoding)
if not was_cmp_interessting:
self.boring_cmps.add(cmp.addr)
return num_mut, offsets_to_lhs_to_rhs_to_info
def __perform_rq_dict(self, payload_array, metadata):
rq_dict = havoc.get_redqueen_dict()
counter = 0
seen_addr_to_value = havoc.get_redqueen_seen_addr_to_value()
if len(payload_array) < 256:
for addr in rq_dict:
for repl in rq_dict[addr]:
if addr in seen_addr_to_value and (
len(seen_addr_to_value[addr]) > 32
or repl in seen_addr_to_value[addr]):
continue
if not addr in seen_addr_to_value:
seen_addr_to_value[addr] = set()
seen_addr_to_value[addr].add(repl)
log_redq("Attempting %s " % repr(repl))
for apply_dict in [
havoc.dict_insert_sequence,
havoc.dict_replace_sequence
]:
for i in range(len(payload_array) - len(repl)):
counter += 1
mutated = apply_dict(payload_array, repl, i)
# log_redq("dict_bf %d %s %s"%(i,repr(repl),repr(mutated)))
self.execute(mutated, label="rq_dict")
log_redq("RQ-Dict: Have performed %d iters" % counter)
def __perform_redqueen(self, payload, metadata):
self.stage_update_label("redq_coloring")
orig_hash = self.__get_bitmap_hash_robust(payload)
extension = bytes([207, 117, 130, 107, 183, 200, 143, 154])
appended_hash = self.__get_bitmap_hash_robust(payload + extension)
if orig_hash and orig_hash == appended_hash:
log_slave("Redqueen: input can be extended", self.slave.slave_id)
payload_array = bytearray(payload + extension)
else:
payload_array = bytearray(payload)
colored_alternatives = self.__perform_coloring(payload_array)
if colored_alternatives:
payload_array = colored_alternatives[0]
assert isinstance(colored_alternatives[0],
bytearray), print("!! ColoredAlternatives:",
repr(colored_alternatives[0]),
type(colored_alternatives[0]))
else:
log_redq("Input is not stable, skipping..")
return
rq_info = RedqueenInfoGatherer()
rq_info.make_paths(RedqueenWorkdir(self.slave.slave_id, self.config))
rq_info.verbose = False
for pld in colored_alternatives:
if self.execute_redqueen(pld):
rq_info.get_info(pld)
rq_info.get_proposals()
self.stage_update_label("redq_mutate")
rq_info.run_mutate_redqueen(payload_array, self.execute)
def get_info(self, input_data):
self.num_alternative_inputs += 1
self.save_rq_data(self.num_alternative_inputs, input_data)
log_redq("redqueen saving new input %d" % self.num_alternative_inputs)
with open(
self.collected_infos_path + "/input_%d.bin" %
(self.num_alternative_inputs), "wb") as f:
f.write(input_data)
def blacklist_hash_addr(self, addr):
self.lock.acquire()
if not addr in self.blacklisted_hashes:
log_redq("Blacklist Hash: %x" % addr)
self.blacklisted_hashes[addr] = 1
if addr in self.candidate_hashes:
del self.candidate_hashes[addr]
self.lock.release()
def add_candidate_hash_addr(self, addr):
self.lock.acquire()
if not addr in self.blacklisted_hashes:
if not addr in self.candidate_hashes:
log_redq("Hash candidate: %x" % addr)
self.candidate_hashes[addr] = 1
self.candidate_file_offsets[addr] = {}
self.lock.release()
def update_redqueen_blacklist(self, workdir):
log_redq("update blacklist with: %s" %
repr(self.blacklisted_compares.keys()))
with open(workdir.blacklist(), "w") as f:
for addr in self.blacklisted_compares.keys():
hexaddr = hex(addr).rstrip("L").lstrip("0x")
if hexaddr:
f.write(hexaddr + "\n")
f.flush()
os.fsync(f.fileno())
def redqueen_global_config(redq_hammering, redq_do_simple, afl_arith_max):
global HAMMER_LEA
global SKIP_SIMPLE
global AFL_ARITH_MAX
HAMMER_LEA = redq_hammering
SKIP_SIMPLE = not redq_do_simple
AFL_ARITH_MAX = afl_arith_max
log_redq("Config: hammer=%s, skip_simple=%s, arith_max=%s" %
(HAMMER_LEA, SKIP_SIMPLE, AFL_ARITH_MAX))
def __respond_verification(self, response):
jobs = response.data[0]
methods = response.data[1]
results = []
i = 0
self.comm.slave_locks_A[self.slave_id].acquire()
while True:
payload, payload_shm_size = self.q.copy_master_payload(self.comm.get_master_payload_shm(self.slave_id), i, self.comm.get_master_payload_shm_size())
payload_content_len_init = struct.unpack("I", payload[0:4])[0]
payload_content_len = perform_trim(payload_content_len_init, self.q.send_payload, self.q.modify_payload_size, self.error_handler)
if payload_content_len_init != payload_content_len:
log_slave("TRIM: " + "{0:.2f}".format(((payload_content_len*1.0)/(payload_content_len_init*1.0))*100.0) + "% (" + str(payload_content_len) + "/" + str(payload_content_len_init) + ")", self.slave_id)
patches = jobs[0]
if len(patches) > 0:
log_slave("Got payload to fix with size: %d and patches %s"%( payload_content_len, patches), self.slave_id )
if len(patches):
log_redq("Slave "+str(self.slave_id)+" Orig Payload: " + repr(payload[4:4+payload_content_len]))
hash = HashFixer(self.q, self.redqueen_state)
new_payload = hash.try_fix_data(payload[4:4+payload_content_len])
if new_payload:
log_redq("Slave "+str(self.slave_id)+"Fixed Payload: " + repr("".join(map(chr,new_payload))))
payload = payload[:4]+"".join(map(chr,new_payload))
self.q.set_payload(new_payload)
start_time = time.time()
bitmap = self.q.send_payload(apply_patches=False)
performance = time.time() - start_time
log_slave("performance: " + str(1.0/performance) + " -> " + str(performance), self.slave_id)
break
if not bitmap:
log_slave("SHM ERROR....", self.slave_id)
new_bits = self.q.copy_bitmap(self.comm.get_bitmap_shm(self.slave_id), i, self.comm.get_bitmap_shm_size(),
bitmap, payload, payload_shm_size, effector_mode_hash=None, apply_patches = False)
if new_bits:
self.q.copy_mapserver_payload(self.comm.get_mapserver_payload_shm(self.slave_id), i, self.comm.get_mapserver_payload_shm_size())
results.append(FuzzingResult(i, self.q.crashed, self.q.timeout, self.q.kasan, jobs[i], self.slave_id, performance, methods[i], mmh3.hash64(bitmap), reloaded=(self.q.timeout or self.q.crashed or self.q.kasan), new_bits=new_bits, qid=self.slave_id))
self.comm.slave_locks_B[self.slave_id].release()
send_msg(KAFL_TAG_RESULT, results, self.comm.to_mapserver_queue, source=self.slave_id)
def run_mutate_redqueen(self, payload_array, func, default_info):
for (offset, lhs, rhs, info) in self.enumerate_mutations():
if self.verbose:
log_redq("redqueen fuzz data %s" % repr(
(offset, lhs, rhs, info)))
def run(data):
default_info["redqueen"] = [repr(lhs), repr(rhs)] + list(
info.infos)
func(data, default_info)
RedqueenInfoGatherer.fuzz_data(payload_array, run, offset, lhs,
rhs)
def run_mutate_redqueen(self,
payload_array,
func,
kafl_state,
skip_null=True):
if kafl_state:
kafl_state.technique = "REDQUEEN"
for (offset, lhs, rhs, info) in self.enumerate_mutations():
if self.verbose:
log_redq("%s" % repr((offset, lhs, rhs, info)))
RedqueenInfoGatherer.fuzz_data(
payload_array, lambda data: func(data, [repr(
lhs), repr(rhs)] + list(info.infos), offset), offset, lhs,
rhs)
def try_fix_cmp_offset(self, shape, data, cmp, offsets, repls):
# try:
backup = {}
for i, repl in zip(offsets, repls):
backup[i] = data[i:i + len(repl)]
HashFixer.replace_data(data, i, array('B', repl))
if self.does_data_fix_cmp(shape, data, cmp):
log_redq("found candidate offset %x %s" %
(cmp.addr, repr(offsets)))
self.redqueen_state.add_candidate_file_offset(
cmp.addr, tuple(offsets))
return True
for i in offsets:
HashFixer.replace_data(data, i, backup[i])
return False
def try_fix_cmp(self, shape, fixed_data, run_info, cmp):
known_offsets = self.redqueen_state.get_candidate_file_offsets(
cmp.addr)
log_redq("known offsets for: %x = %s" % (cmp.addr, known_offsets))
mutations = [x for x in cmp.calc_mutations(run_info, 1)]
for (offsets, lhs, rhs, enc) in cmp.calc_mutations(run_info, 1):
if offsets in known_offsets:
if self.try_fix_cmp_with(shape, fixed_data, cmp, offsets, lhs,
rhs, enc):
return True
for (offsets, lhs, rhs, enc) in cmp.calc_mutations(run_info, 1):
if not offsets in known_offsets:
if self.try_fix_cmp_with(shape, fixed_data, cmp, offsets, lhs,
rhs, enc):
return True
return False
class RedqueenInfoGatherer:
def __init__(self):
self.num_alternative_inputs = 0
self.se_path = None
self.workdir = None
self.num_mutations = 0
self.verbose = False
def make_paths(self, workdir):
global se_saved_counter
se_saved_counter += 1
self.se_path = workdir.base_path + "/se_%d" % (se_saved_counter)
self.workdir = workdir
os.mkdir(self.se_path)
def get_info(self, input_data):
self.num_alternative_inputs += 1
self.save_rq_data(self.num_alternative_inputs, input_data)
with open(
self.se_path + "/input_%d.bin" % (self.num_alternative_inputs),
"wb") as f:
f.write(input_data)
def save_rq_data(self, id, data):
if os.path.exists(self.workdir.redqueen()):
copyfile(self.workdir.redqueen(),
"%s/redqueen_result_%d.txt" % (self.se_path, id))
if os.path.exists(self.workdir.symbolic()):
copyfile(self.workdir.symbolic(),
"%s/symbolic_result_%d.txt" % (self.se_path, id))
if os.path.exists(self.workdir.pt_trace()):
copyfile(self.workdir.pt_trace(),
"%s/trace_result_%d.txt" % (self.se_path, id))
if os.path.exists(self.workdir.code_dump()):
copyfile(self.workdir.code_dump(),
"%s/redqueen_vm.img" % (self.se_path))
with open("%s/fin_%d.txt" % (self.se_path, id), "w") as f:
f.write("OK")
def get_symbolic_proposals(self):
proposals = set()
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
kafl_sock = '/tmp/rq_se_sock'
try:
sock.connect(kafl_sock)
sock.sendall(self.se_path + "\n")
buffer = ""
while True:
res = sock.recv(16)
buffer += res
if "\n" in res:
break
except socket.error, msg:
log_redq("SE failed to get SE response, maybe no demon running?")
log_redq("SE %s" % msg)
return []
results = json.loads(buffer)
log_redq("SE JSON:" + str(results))
orig_id = self.num_alternative_inputs
return parser.parse_se(self.se_path, results, orig_id)
def get_symbolic_proposals(self):
proposals = set()
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
kafl_sock = '/tmp/rq_se_sock'
try:
sock.connect(kafl_sock)
sock.sendall(self.se_path + "\n")
buffer = ""
while True:
res = sock.recv(16)
buffer += res
if "\n" in res:
break
except socket.error, msg:
log_redq("SE failed to get SE response, maybe no demon running?")
log_redq("SE %s" % msg)
return []
def run_mutate_redqueen(self, payload_array, func):
for (offset, lhs, rhs, info) in self.enumerate_mutations():
if self.verbose:
log_redq("redqueen fuzz data %s" % repr(
(offset, lhs, rhs, info)))
def run(data):
extra_info = {
"redqueen": [repr(lhs), repr(rhs)] + list(info.infos)
}
func(data, None, extra_info)
assert isinstance(payload_array,
bytearray), print("fuzz_data:",
type(payload_array),
type(lhs[0]), type(rhs[0]))
RedqueenInfoGatherer.fuzz_data(payload_array, run, offset, lhs,
rhs)
def __perform_redqueen(self, payload_array, colored_alternatives):
if self.config.argument_values['r']:
t = time.time()
rq_info = RedqueenInfoGatherer()
rq_info.make_paths(RedqueenWorkdir(0))
rq_info.verbose = False
for payload in colored_alternatives:
if self.__redqueen(payload):
self.__sync_redqueen(0)
self.kafl_state["technique"] = "REDQUEEN"
rq_info.get_info(payload)
if not self.mode_se_only:
rq_info.get_proposals()
self.kafl_state[
"progress_requeen_amount"] = rq_info.get_num_mutations()
log_master("Redqueen Stage...(" +
str(self.kafl_state["progress_requeen_amount"]) +
")")
rq_info.run_mutate_redqueen(payload_array,
self.__redqueen_handler,
kafl_state=self.kafl_state,
skip_null=True)
if self.mode_fix_checksum:
for addr in rq_info.get_hash_candidates():
self.redqueen_state.add_candidate_hash_addr(addr)
self.__buffered_handler(None, last_payload=True)
log_master("Redqueen Sync...")
tmp_progress_redqueen = self.kafl_state["progress_redqueen"]
self.kafl_state["progress_redqueen"] = self.kafl_state[
"progress_requeen_amount"]
self.__sync_redqueen(tmp_progress_redqueen)
self.__update_redqueen_slaves()
duration = time.time() - t
self.kafl_state[
"time_redqueen"] = self.kafl_state["time_redqueen"] + duration
log_redq("TIME IN REDQUEEN: %fs" %
self.kafl_state["time_redqueen"])
def get_cmps(self, data):
# log_redq("runnning on %s"%repr("".join( map(chr, data) )) )
self.qemu.set_payload(data)
# self.qemu.send_enable_patches()
log_redq("hashfix run in rq mode")
self.qemu.send_rq_set_whitelist_instrumentation()
self.qemu.send_enable_redqueen()
self.qemu.send_payload(timeout_detection=True, apply_patches=True)
log_redq("hashfix run in non rq mode")
self.qemu.send_disable_redqueen()
self.qemu.send_payload(timeout_detection=True, apply_patches=True)
log_redq("hashfix done running, now parsing")
res = self.parse_redqueen_results(data)
log_redq("hashfix done parsing")
return res
def try_fix_cmp_with(self, shape, fixed_data, cmp, offsets, lhs, rhs, enc):
log_redq("Trying mutation %s" % (repr((offsets, lhs, rhs, enc))))
if list(map(len, lhs)) != list(map(len, rhs)):
return False
self.redqueen_state.update_redqueen_whitelist(
self.qemu.redqueen_workdir,
self.redqueen_state.get_candidate_hash_addrs())
try:
if self.try_fix_cmp_offset(shape, fixed_data, cmp, offsets, rhs):
log_redq("Mutation fixed it")
return True
log_redq("Mutation didn't Fix it")
return False
except Exception as e:
log_redq("fixing hash failed %s" % traceback.format_exc())
raise e
def __perform_dict(self, payload_array, payload):
self.kafl_state["technique"] = "DICT-BF"
log_master("Dict on %s" % repr(payload_array.tostring()))
dict = havoc.get_redqueen_dict()
log_redq("using %s" % repr(dict))
counter = 0
if len(payload_array) < 256:
for addr in dict:
for repl in dict[addr]:
if addr in self.seen_addr_to_value and (
len(self.seen_addr_to_value[addr]) > 32
or repl in self.seen_addr_to_value[addr]):
continue
if not addr in self.seen_addr_to_value:
self.seen_addr_to_value[addr] = set()
self.seen_addr_to_value[addr].add(repl)
for i in range(len(payload_array)):
counter += 1
mutated = havoc.apply_dict_to_data(
payload_array, repl, i).tostring()
self.__dict_bf_handler(mutated)
log_redq("have performed %d iters" % counter)
self.__buffered_handler(None, last_payload=True)
def clear_redqueen_dict():
global redqueen_dict, redqueen_addr_list
log_redq("clearing dict %s" % repr(redqueen_dict))
redqueen_dict = {}
redqueen_addr_list = []
def enable_hammering():
global HAMMER_LEA
log_redq("Hammering enabled!")
HAMMER_LEA = True
def try_fix_data(self, data):
self.qemu.send_payload(timeout_detection=True, apply_patches=False)
self.qemu.send_payload(timeout_detection=True, apply_patches=True)
log_redq("PATCHES %s\n" % repr(
list(map(hex, self.redqueen_state.get_candidate_hash_addrs()))))
log_redq("BLACKLIST %s\n" % repr(
list(map(hex, self.redqueen_state.get_blacklisted_hash_addrs()))))
self.redqueen_state.update_redqueen_patches(self.qemu.redqueen_workdir)
self.redqueen_state.update_redqueen_whitelist(
self.qemu.redqueen_workdir,
self.redqueen_state.get_candidate_hash_addrs())
fixed_data = array('B', data)
orig_cmps, _ = self.get_cmps(fixed_data)
shape = self.get_shape(orig_cmps)
log_redq("shape of hashes: ")
for addr in shape:
log_redq("\t%x: %d" % (addr, shape[addr]))
if len(shape) == 0:
return fixed_data
num_iters = min(len(orig_cmps)**2 + 1, len(orig_cmps) * 3 + 1)
num_cmps = sum(shape.values()) + 1
if num_iters < num_cmps:
num_iters = num_cmps
log_redq("try fixing for %d iters" % num_iters)
for i in range(num_iters):
broken_checks, run_info = self.get_broken_cmps(fixed_data)
log_redq("got %d broken checks\n" % len(broken_checks))
if not broken_checks:
return fixed_data
cmp = broken_checks.pop(-1)
if not self.try_fix_cmp(shape, fixed_data, run_info, cmp):
log_redq("cmp at %x unfixable:" % cmp.addr)
self.mark_unfixable(cmp)
broken_checks, run_info = self.get_broken_cmps(fixed_data)
for cmp in broken_checks:
self.mark_unfixable(cmp)
return False
def mark_unfixable(self, cmp):
log_redq("Unfixable cmp at: %x" % cmp.addr)
self.blacklisted_addrs.add(cmp.addr)
self.redqueen_state.blacklist_hash_addr(cmp.addr)
