def resolve_windbg_path(self):
try:
reg = ConnectRegistry(None,HKEY_LOCAL_MACHINE)
key = OpenKey(reg, r"SOFTWARE\Microsoft\Microsoft SDKs\Windows")
if key:
for i in range(QueryInfoKey(key)[0]):
value = EnumKey(key, i)
if value:
full_key = r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\\" + value
key2 = OpenKey(reg, full_key)
if key2:
name = QueryValueEx(key2, "ProductName")
name = name[0]
if name and name.startswith("Microsoft Windows SDK for Windows"):
vals = QueryValueEx(key2, "InstallationFolder")
val = vals[0]
if val is not None:
log("Found installation path at %s" % val)
self.windbg_path = val
break
CloseKey(key2)
CloseKey(key)
except WindowsError:
print "Cannot resolve Windows SDKs path:", sys.exc_info()[1]
print "Did you install Windows SDKs for Windows?"
except:
print "Cannot resolve Windows SDKs path:", sys.exc_info()[1]
def reload_statistics(self):
if self.state_file is not None:
with open(self.state_file, "rb") as f:
self.stats = json.loads(f.read())
line = "Reloaded statistics: Min %d, Max %d, Avg %f"
line = line % (self.stats["min"], self.stats["max"], self.stats["avg"])
log(line)
def run(self, timeout=60):
def target():
debug('Thread started')
self.process = subprocess.Popen("exec %s" % self.cmd, shell=True)
self.process.communicate()
debug('Thread finished')
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
log('Terminating process after timeout (%d)' % timeout)
self.process.terminate()
self.process.terminate()
self.process.kill()
self.process.wait()
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
ret = abs(ret) + 128
return ret
def run(self, timeout=60):
def target():
debug('Thread started')
self.process = subprocess.Popen("exec %s" % self.cmd, shell=True)
self.process.communicate()
debug('Thread finished')
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
log('Terminating process after timeout (%d)' % timeout)
try:
self.process.terminate()
self.process.terminate()
self.process.kill()
self.process.wait()
except:
log("Error killing process: %s" % str(sys.exc_info()[1]))
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
ret = abs(ret) + 128
return ret
def reload_statistics(self):
if self.state_file is not None:
with open(self.state_file, "rb") as f:
self.stats = json.loads(f.read())
line = "Reloaded statistics: Min %d, Max %d, Avg %f"
line = line % (self.stats["min"], self.stats["max"], self.stats["avg"])
log(line)
def debug_server(self, shared_queue):
self.read_configuration()
uid = int(self.server_uid)
if os.getuid() != uid:
os.setresuid(uid, uid, uid)
gid = int(self.server_gid)
if os.getgid() != gid:
os.setresgid(gid, gid, gid)
for key in self.env:
debug("Setting environment variable %s=%s" % (key, self.env[key]))
os.putenv(key, self.env[key])
if self.pre_command is not None:
os.system(self.pre_command)
crash = None
for i in range(0, 3):
try:
crash = self.launch_debugger(self.timeout, self.command, "")
break
except:
log("Exception: %s" % sys.exc_info()[1])
continue
if self.post_command is not None:
os.system(self.post_command)
if crash is not None:
self.crash_info = crash
shared_queue.put(crash)
return True
return False
def check_cpu(self):
while True:
try:
if self.pid is None:
time.sleep(0.2)
continue
proc = psutil.Process(self.pid)
if proc is None:
break
cpu = 0
l = []
for x in xrange(20):
tmp = int(proc.cpu_percent(interval=0.1))
cpu += tmp
l.append(tmp)
if cpu is not None and (cpu <= 100 or l.count(0) > 10):
log("CPU at 0%, killing")
self.do_stop = True
pykd.breakin()
break
else:
time.sleep(0.5)
except psutil.NoSuchProcess:
self.do_stop = True
break
def resolve_windbg_path(self):
try:
reg = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
key = OpenKey(reg, r"SOFTWARE\Microsoft\Microsoft SDKs\Windows")
if key:
for i in range(QueryInfoKey(key)[0]):
value = EnumKey(key, i)
if value:
full_key = r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\\" + value
key2 = OpenKey(reg, full_key)
if key2:
name = QueryValueEx(key2, "ProductName")
name = name[0]
if name and name.startswith(
"Microsoft Windows SDK for Windows"):
vals = QueryValueEx(key2, "InstallationFolder")
val = vals[0]
if val is not None:
log("Found installation path at %s" % val)
self.windbg_path = val
break
CloseKey(key2)
CloseKey(key)
except WindowsError:
print "Cannot resolve Windows SDKs path:", sys.exc_info()[1]
print "Did you install Windows SDKs for Windows?"
except:
print "Cannot resolve Windows SDKs path:", sys.exc_info()[1]
def debug_server(self, shared_queue):
self.read_configuration()
uid = int(self.server_uid)
if os.getuid() != uid:
os.setresuid(uid, uid, uid)
gid = int(self.server_gid)
if os.getgid() != gid:
os.setresgid(gid, gid, gid)
for key in self.env:
debug("Setting environment variable %s=%s" % (key, self.env[key]))
os.putenv(key, self.env[key])
if self.pre_command is not None:
os.system(self.pre_command)
crash = None
for i in range(0,3):
try:
crash = self.launch_debugger(self.timeout, self.command, "")
break
except:
log("Exception: %s" % sys.exc_info()[1])
continue
if self.post_command is not None:
os.system(self.post_command)
if crash is not None:
self.crash_info = crash
shared_queue.put(crash)
return True
return False
def __init__(self, tube_prefix):
if tube_prefix.endswith("-samples"):
log("Notice: Removing '-samples' suffix from the queue name")
tube_prefix = tube_prefix.replace("-samples", "")
self.tube_prefix = tube_prefix
self.q = get_queue(watch=False, name="%s-samples" % tube_prefix)
def apply_bytes(self, offset, size, buf):
debug("Acquiring lock...")
self.lock.acquire()
try:
debug("Saving old generation (%s)" %
sha1(self.template).hexdigest())
if len(self.generations) >= self.max_generations:
del self.generations[0]
self.generations.append([
bytearray(self.template),
dict(self.stats), self.generation_value
])
if self.save_generations and buf != "":
file_hash = sha1(buf).hexdigest()
ext = os.path.splitext(self.input_file)[1]
filename = "generation_%s%s" % (file_hash, ext)
filename = os.path.join(self.output, filename)
log("Writing discovered generation file %s (%s)" %
(file_hash, filename))
with open(filename, "wb") as f:
f.write(buf)
if not self.radamsa:
debug("Applying patch at offset %d of size %d" %
(offset, size))
else:
debug("Replacing old buffer")
self.template = buf
"""
if self.skip_bytes > 0:
header = self.template[0:self.skip_bytes]
if len(buf) > len(self.template):
self.template = bytearray(buf)
else:
for i in range(size):
self.template[offset+i] = buf[i]
if self.skip_bytes > 0:
self.template[0:self.skip_bytes] = header
"""
if self.current_state is not None:
ext = os.path.splitext(self.input_file)[1]
filename = "%s%s" % (self.current_state, ext)
filename = os.path.join(self.output, filename)
file_hash = sha1(self.template).hexdigest()
debug("Creating or updating current state file %s (%s)" %
(filename, file_hash))
with open(filename, "wb") as f:
f.write(self.template)
finally:
debug("Releasing lock...")
self.lock.release()
def do_fuzz(cfg, section):
try:
fuzzer = CGenericFuzzer(cfg, section)
fuzzer.fuzz()
except KeyboardInterrupt:
log("Aborted")
except:
log("Error: %s" % str(sys.exc_info()[1]))
def do_fuzz(cfg, section):
try:
fuzzer = CGenericFuzzer(cfg, section)
fuzzer.fuzz()
except KeyboardInterrupt:
log("Aborted")
except:
log("Error: %s" % str(sys.exc_info()[1]))
def timeout_func(self):
log("Timeout (%d seconds), killing the target..." % self.timeout)
self.do_stop = True
try:
pykd.breakin()
except:
# A race condition might happen in the timeout function and in
# such cases we must ignore the error.
pass
def timeout_func(self):
log("Timeout (%d seconds), killing the target..." % self.timeout)
self.do_stop = True
try:
pykd.breakin()
except:
# A race condition might happen in the timeout function and in
# such cases we must ignore the error.
pass
def get_html_buffer(self, lines, skip_tags):
log("Rewriting HTML...")
rewriter = CHTMLRewriter()
ret = rewriter.rewrite("\n".join(lines), skip_tags)
if not ret:
return None, None
lines = ret
log("Total line(s) %d" % len(lines))
return "".join(lines), lines
def run(self, timeout=60, get_output=False):
def target():
debug('Thread started')
if os.name == "nt":
line = self.cmd
shell = False
else: # Unix based
line = "exec %s" % self.cmd
shell = True
if get_output:
self.process = subprocess.Popen(line, stdout=subprocess.PIPE,\
stderr=subprocess.PIPE, shell=shell)
self.pid = self.process.pid
out, err = self.process.communicate()
self.stdout = out[:8192]
self.stderr = err[:8192]
else:
self.process = subprocess.Popen(line, shell=shell)
self.pid = self.process.pid
self.process.communicate()
debug('Thread finished')
thread = threading.Thread(target=target)
thread.start()
if str(timeout).lower() == "auto":
self.thread = threading.Thread(target=self.check_cpu)
self.thread.start()
thread.join(self.default_timeout)
else:
thread.join(timeout)
if thread.is_alive():
log('Terminating process after timeout (%s)' % str(timeout))
try:
self.do_kill()
except:
log("Error killing process: %s" % str(sys.exc_info()[1]))
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
if os.name == "nt":
ret = ret & 0xFFFFFFFF
else:
ret = abs(ret) + 128
return ret
def get_html_buffer(self, lines, skip_tags):
log("Rewriting HTML...")
rewriter = CHTMLRewriter()
ret = rewriter.rewrite("\n".join(lines), skip_tags)
if not ret:
return None, None
lines = ret
log("Total line(s) %d" % len(lines))
return "".join(lines), lines
def launch_debugger(self, timeout, command, filename):
self.iface.timeout = int(timeout)
if command.find("@@") > -1:
cmd = [command.replace("@@", filename), ]
else:
cmd = [command, filename]
log("Launching debugger with command %s" % " ".join(cmd))
crash = self.iface.main(cmd)
return crash
def run(self, timeout=60, get_output=False):
def target():
debug('Thread started')
if os.name == "nt":
line = self.cmd
shell = False
else: # Unix based
line = "exec %s" % self.cmd
shell = True
if get_output:
self.process = subprocess.Popen(line, stdout=subprocess.PIPE,\
stderr=subprocess.PIPE, shell=shell)
self.pid = self.process.pid
out, err = self.process.communicate()
self.stdout = out[:8192]
self.stderr = err[:8192]
else:
self.process = subprocess.Popen(line, shell=shell)
self.pid = self.process.pid
self.process.communicate()
debug('Thread finished')
thread = threading.Thread(target=target)
thread.start()
if str(timeout).lower() == "auto":
self.thread = threading.Thread(target=self.check_cpu)
self.thread.start()
thread.join(self.default_timeout)
else:
thread.join(timeout)
if thread.is_alive():
log('Terminating process after timeout (%s)' % str(timeout))
try:
self.do_kill()
except:
log("Error killing process: %s" % str(sys.exc_info()[1]))
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
if os.name == "nt":
ret = ret & 0xFFFFFFFF
else:
ret = abs(ret) + 128
return ret
def minimize(self, template, outdir):
self.read_template(template)
log("Performing line-level test case minimization")
start_at = os.getenv("NIGHTMARE_ITERATION")
if start_at is not None:
start_at = int(start_at)
log("Starting from iteration %d\n" % start_at)
else:
start_at = 0
self.do_try(outdir, start_at)
def minimize(self, template, outdir):
self.read_template(template)
log("Performing line-level test case minimization")
start_at = os.getenv("NIGHTMARE_ITERATION")
if start_at is not None:
start_at = int(start_at)
log("Starting from iteration %d\n" % start_at)
else:
start_at = 0
self.do_try(outdir, start_at)
def recalculate_statistics(self, old_stats, bbs):
self.stats["max"] = bbs
self.stats["min"] = old_stats["max"]
self.stats["avg"] = (self.stats["max"] + self.stats["min"]) / 2.
#self.stats = self.mgr.dict(self.stats)
line = "New statistics: Min %d, Max %d, Avg %f"
line = line % (self.stats["min"], self.stats["max"], self.stats["avg"])
log(line)
if self.state_file is not None:
with open(self.state_file, "wb") as f:
f.write(json.dumps(dict(self.stats)))
def recalculate_statistics(self, old_stats, bbs):
self.stats["max"] = bbs
self.stats["min"] = old_stats["max"]
self.stats["avg"] = (self.stats["max"] + self.stats["min"]) / 2.
#self.stats = self.mgr.dict(self.stats)
line = "New statistics: Min %d, Max %d, Avg %f"
line = line % (self.stats["min"], self.stats["max"], self.stats["avg"])
log(line)
if self.state_file is not None:
with open(self.state_file, "wb") as f:
f.write(json.dumps(dict(self.stats)))
def apply_bytes(self, offset, size, buf):
debug("Acquiring lock...")
self.lock.acquire()
try:
debug("Saving old generation (%s)" % sha1(self.template).hexdigest())
if len(self.generations) >= self.max_generations:
del self.generations[0]
self.generations.append([bytearray(self.template), dict(self.stats), self.generation_value])
if self.save_generations:
file_hash = sha1(buf).hexdigest()
ext = os.path.splitext(self.input_file)[1]
filename = "generation_%s%s" % (file_hash, ext)
filename = os.path.join(self.output, filename)
log("Writing discovered generation file %s (%s)" % (file_hash, filename))
with open(filename, "wb") as f:
f.write(buf)
if not self.radamsa:
debug("Applying patch at offset %d of size %d" % (offset, size))
else:
debug("Replacing old buffer")
self.template = buf
"""
if self.skip_bytes > 0:
header = self.template[0:self.skip_bytes]
if len(buf) > len(self.template):
self.template = bytearray(buf)
else:
for i in range(size):
self.template[offset+i] = buf[i]
if self.skip_bytes > 0:
self.template[0:self.skip_bytes] = header
"""
if self.current_state is not None:
ext = os.path.splitext(self.input_file)[1]
filename = "%s%s" % (self.current_state, ext)
filename = os.path.join(self.output, filename)
file_hash = sha1(self.template).hexdigest()
debug("Creating or updating current state file %s (%s)" % (filename, file_hash))
with open(filename, "wb") as f:
f.write(self.template)
finally:
debug("Releasing lock...")
self.lock.release()
def disasm_around(self):
try:
lines = pykd.dbgCommand("u %s-c L12" % self.pc_register)
for line in lines.split("\n"):
tmp = re.findall("([a-f0-9]{1,}) ([a-f0-9]{2,}) (.*)", line)
if len(tmp) > 0:
line = tmp[0]
addr = line[0]
dis = line[2]
self.crash_data.add_data("disassembly", int(addr, 16), dis)
except:
log("Error in disasm_around: %s" % str(sys.exc_info()[1]))
def launch_target(self, temp_file, lines, outdir):
try:
crashed = False
for key in self.env:
os.putenv(key, self.env[key])
self.remove_crash_path()
if self.pre_command is not None:
log("Running pre-command %s" % self.pre_command)
os.system(self.pre_command)
if self.command.find("@@") == -1:
cmd = "%s %s" % (self.command, temp_file)
else:
cmd = self.command.replace("@@", temp_file)
ret = self.execute_command(cmd, self.timeout)
if self.post_command is not None:
log("Running post-command %s" % self.post_command)
os.system(self.post_command)
if ret in RETURN_SIGNALS or (self.signal is not None and ret == self.signal) or \
self.crash_file_exists():
crashed = True
self.template = lines
log("Process crashed as expected...")
buf = "\n".join(self.template)
if not os.path.exists(outdir):
log("Directory %s does not exists, creating it..." %
outdir)
os.mkdir(outdir)
filename = os.path.join(
outdir,
"last_minimized%d%s" % (os.getpid(), self.extension))
with open(filename, "wb") as f:
f.write(buf)
log("Last minimized test case %s written to disk." % filename)
if self.should_notify_crash():
# TODO: Write a temporary file and put an enqueue the crash
self.put_new_crash(buf)
self.remove_crash_path()
finally:
os.remove(temp_file)
return crashed
def fuzz(self):
log("Launching fuzzer, listening in tube %s" % self.tube_name)
while 1:
value = self.q.stats_tube(self.tube_name)["current-jobs-ready"]
debug("Total of %d job(s) in queue" % value)
job = self.q.reserve()
buf, temp_file = json.loads(job.body)
buf = base64.b64decode(buf)
debug("Launching sample %s..." % os.path.basename(temp_file))
if self.launch_sample(buf):
log("We have a crash, moving to %s queue..." % self.crash_tube)
crash = self.crash_info
d = {temp_file:self.crash_info}
self.crash_q.put(json.dumps(d))
self.crash_info = None
log("$PC 0x%08x Signal %s Exploitable %s " % (crash["pc"], crash["signal"], crash["exploitable"]))
if crash["disasm"] is not None:
log("%08x: %s" % (crash["disasm"][0], crash["disasm"][1]))
else:
file_delete = os.path.basename(temp_file)
self.delete_q.put(str(file_delete))
if self.cleanup is not None:
debug("Running clean-up command %s" % self.cleanup)
os.system(self.cleanup)
debug("Done")
job.delete()
if self.iface == gdb_iface:
break
def minimize(self, template, crash, diff, outdir):
self.read_diff(diff)
self.read_template(template)
self.read_crash(crash)
log("Performing test case minimization with a total of %d change(s)" % len(self.diff))
start_at = os.getenv("NIGHTMARE_ITERATION")
if start_at is not None:
start_at = int(start_at)
log("Starting from iteration %d\n" % start_at)
else:
start_at = 0
self.do_try(outdir, start_at)
def minimize(self, template, crash, diff, outdir):
self.read_diff(diff)
self.read_template(template)
self.read_crash(crash)
log("Performing test case minimization with a total of %d change(s)" % len(self.diff))
start_at = os.getenv("NIGHTMARE_ITERATION")
if start_at is not None:
start_at = int(start_at)
log("Starting from iteration %d\n" % start_at)
else:
start_at = 0
self.do_try(outdir, start_at)
def fuzz(self):
log("Launching fuzzer, listening in tube %s" % self.tube_name)
while 1:
value = self.q.stats_tube(self.tube_name)["current-jobs-ready"]
debug("Total of %d job(s) in queue" % value)
job = self.q.reserve()
buf, temp_file = json.loads(job.body)
buf = base64.b64decode(buf)
debug("Launching sample %s..." % os.path.basename(temp_file))
if self.launch_sample(buf):
log("We have a crash, moving to %s queue..." % self.crash_tube)
crash = self.crash_info
d = {temp_file:self.crash_info}
self.crash_q.put(json.dumps(d))
self.crash_info = None
log("$PC 0x%08x Signal %s Exploitable %s " % (crash["pc"], crash["signal"], crash["exploitable"]))
if crash["disasm"] is not None:
log("%08x: %s" % (crash["disasm"][0], crash["disasm"][1]))
else:
file_delete = os.path.basename(temp_file)
self.delete_q.put(str(file_delete))
if self.cleanup is not None:
debug("Running clean-up command %s" % self.cleanup)
os.system(self.cleanup)
debug("Done")
job.delete()
if self.iface == gdb_iface:
break
def launch_debugger(self, timeout, command, filename):
self.iface.timeout = int(timeout)
if command.find("@@") > -1:
cmd = [command.replace("@@", filename), ]
else:
cmd = [command, filename]
log("Launching debugger with command %s" % " ".join(cmd))
if self.iface != pykd_iface:
crash = self.iface.main(" ".join(cmd))
else:
reload(pykd_iface)
crash = pykd_iface.main(cmd, mode=self.mode, windbg_path=self.windbg_path, exploitable_path=self.exploitable_path)
return crash
def launch_target(self, temp_file, lines, outdir):
try:
crashed = False
for key in self.env:
os.putenv(key, self.env[key])
self.remove_crash_path()
if self.pre_command is not None:
log("Running pre-command %s" % self.pre_command)
os.system(self.pre_command)
if self.command.find("@@") == -1:
cmd = "%s %s" % (self.command, temp_file)
else:
cmd = self.command.replace("@@", temp_file)
ret = self.execute_command(cmd, self.timeout)
if self.post_command is not None:
log("Running post-command %s" % self.post_command)
os.system(self.post_command)
if ret in RETURN_SIGNALS or (self.signal is not None and ret == self.signal) or \
self.crash_file_exists():
crashed = True
self.template = lines
log("Process crashed as expected...")
buf = "\n".join(self.template)
if not os.path.exists(outdir):
log("Directory %s does not exists, creating it..." % outdir)
os.mkdir(outdir)
filename = os.path.join(outdir, "last_minimized%d%s" % (os.getpid(), self.extension))
with open(filename, "wb") as f:
f.write(buf)
log("Last minimized test case %s written to disk." % filename)
if self.should_notify_crash():
# TODO: Write a temporary file and put an enqueue the crash
self.put_new_crash(buf)
self.remove_crash_path()
finally:
os.remove(temp_file)
return crashed
def put(self, filename):
temp_file = tempfile.mktemp()
try:
buf = open(filename, "rb").read()
with open(temp_file, "wb") as f:
f.write(buf)
json_buf = json.dumps([base64.b64encode(buf), temp_file])
self.q.put(json_buf)
l = "File '%s' put in queue %s as temporary file '%s'"
log(l % (filename, self.tube_prefix, temp_file))
except:
raise
os.remove(temp_file)
def do_try(self, outdir, start_at=0):
# Try to minimize to just one change
current_change = 0
minimized = False
iteration = 0
for i in range(len(self.diff)):
for pos in self.diff:
if start_at <= iteration:
log("Minimizing, iteration %d (Max. %d)..." %
(iteration, (len(self.diff)) * len(self.diff)))
temp_file = tempfile.mktemp()
buf = bytearray(self.template)
if pos not in self.crash:
continue
buf[pos] = self.crash[pos]
with open(temp_file, "wb") as f:
f.write(buf)
try:
for key in self.env:
os.putenv(key, self.env[key])
cmd = "%s %s" % (self.command, temp_file)
cmd_obj = TimeoutCommand(cmd)
ret = cmd_obj.run(timeout=self.timeout)
if ret in RETURN_SIGNALS:
log("Successfully minimized, caught signal %d (%s)!"
% (ret, RETURN_SIGNALS[ret]))
filename = sha1(buf).hexdigest()
filename = os.path.join(
outdir, "%s%s" % (filename, self.extension))
shutil.copy(temp_file, filename)
log("Minized test case %s written to disk." %
filename)
minimized = True
break
finally:
os.remove(temp_file)
if minimized:
break
iteration += 1
if minimized:
break
value = self.diff.pop()
if value in self.crash:
self.template[value] = self.crash[value]
del self.crash[value]
if not minimized:
log("Sorry, could not minimize crashing file!")
def __init__(self, arch, cfg, section, metrics=10):
if int(arch) not in [32, 64]:
raise Exception("Invalid architecture %s" % str(arch))
self.arch = arch
self.cfg = cfg
self.section = section
self.metrics = 10
self.bininst_tool = None
self.mgr = Manager()
self.stats = self.mgr.dict()
self.read_configuration()
cpus = os.getenv("NIGHTMARE_PROCESSES")
if cpus is not None:
cpus = int(cpus)
else:
cpus = cpu_count()
self.procs = cpus
self.discard_data = self.mgr.list()
if self.procs > self.metrics:
log("The number of processes is bigger than the number of metrics, adjusting it to %d"
% self.procs)
self.metrics = self.procs
# Default output directory is current path
self.output = "."
self.input_file = None
self.is_dir = False
# Maximum number of bytes to mutate per each try
self.max_size = random.randint(1, 8)
log("Selected a maximum size of %d change(s) to apply" % self.max_size)
# Only for the iterative mutator
self.stats["iteration"] = 0
self.stats["iteration_char"] = 0
self.generations = self.mgr.list()
self.generation_value = 0
self.max_generations = 10
self.bugs = 0
def rewrite(self, buf, skip_tags=0):
fragments = self.fragment(buf)
l = []
tags = []
total_skipped = 0
removed = False
log("Skipping a total of %d tags" % skip_tags)
for tag in fragments:
if total_skipped >= skip_tags and not removed:
log("*** Removing tag %s" % tag.name)
removed = True
continue
total_skipped += 1
tag_src = "<%s" % tag.name
if tag.name != "script":
for attr in tag.attrs:
tag_src += ' %s="%s"' % (attr, tag.attrs[attr])
tag_src += ">"
l.append(tag_src)
if tag.text is not None and tag.text != "\n" and tag.name != "script":
for tmp_line in tag.text.split("\n"):
if tmp_line != "":
l.append(tmp_line)
elif tag.name == "script":
self.has_js = True
l.extend(self.add_javascript(tag))
if tag.name in ["script", "link"] or tag.closed:
l.append("</%s>" % tag.name)
else:
tags.append(tag)
while len(tags) > 0:
tag = tags.pop()
if not tag:
break
l.append("</%s>" % tag.name)
if not removed:
return False
return l
def launch_debugger(self, timeout, command, filename):
if command.find("@@") > -1:
cmd = [command.replace("@@", filename), ]
else:
cmd = [command, filename]
log("Launching debugger with command %s" % " ".join(cmd))
if not has_pykd or self.iface != pykd_iface:
self.iface.timeout = int(timeout)
if self.debugging_interface == "asan":
crash = self.iface.main(asan_symbolizer_path=self.asan_symbolizer_path, args=cmd)
else:
crash = self.iface.main(cmd)
else:
reload(pykd_iface)
crash = pykd_iface.main(cmd, self.timeout, mode=self.mode, windbg_path=self.windbg_path, exploitable_path=self.exploitable_path)
return crash
def __init__(self, arch, cfg, section, metrics=10):
if int(arch) not in [32, 64]:
raise Exception("Invalid architecture %s" % str(arch))
self.arch = arch
self.cfg = cfg
self.section = section
self.metrics = 10
self.bininst_tool=None
self.mgr = Manager()
self.stats = self.mgr.dict()
self.read_configuration()
cpus = os.getenv("NIGHTMARE_PROCESSES")
if cpus is not None:
cpus = int(cpus)
else:
cpus = cpu_count()
self.procs = cpus
self.discard_data = self.mgr.list()
if self.procs > self.metrics:
log("The number of processes is bigger than the number of metrics, adjusting it to %d" % self.procs)
self.metrics = self.procs
# Default output directory is current path
self.output = "."
self.input_file = None
self.is_dir = False
# Maximum number of bytes to mutate per each try
self.max_size = random.randint(1, 8)
log("Selected a maximum size of %d change(s) to apply" % self.max_size)
# Only for the iterative mutator
self.stats["iteration"] = 0
self.stats["iteration_char"] = 0
self.generations = self.mgr.list()
self.generation_value = 0
self.max_generations = 10
self.bugs = 0
def launch_debugger(self, timeout, command, filename):
if command.find("@@") > -1:
cmd = [command.replace("@@", filename), ]
else:
cmd = [command, filename]
log("Launching debugger with command %s" % " ".join(cmd))
if not has_pykd or self.iface != pykd_iface:
self.iface.timeout = int(timeout)
if self.debugging_interface == "asan":
crash = self.iface.main(asan_symbolizer_path=self.asan_symbolizer_path, args=cmd)
else:
crash = self.iface.main(" ".join(cmd))
else:
reload(pykd_iface)
crash = pykd_iface.main(cmd, self.timeout, mode=self.mode, windbg_path=self.windbg_path, exploitable_path=self.exploitable_path)
return crash
def rewrite(self, buf, skip_tags=0):
fragments = self.fragment(buf)
l = []
tags = []
total_skipped = 0
removed = False
log("Skipping a total of %d tags" % skip_tags)
for tag in fragments:
if total_skipped >= skip_tags and not removed:
log("*** Removing tag %s" % tag.name)
removed = True
continue
total_skipped += 1
tag_src = "<%s" % tag.name
if tag.name != "script":
for attr in tag.attrs:
tag_src += ' %s="%s"' % (attr, tag.attrs[attr])
tag_src += ">"
l.append(tag_src)
if tag.text is not None and tag.text != "\n" and tag.name != "script":
for tmp_line in tag.text.split("\n"):
if tmp_line != "":
l.append(tmp_line)
elif tag.name == "script":
self.has_js = True
l.extend(self.add_javascript(tag))
if tag.name in ["script", "link"] or tag.closed:
l.append("</%s>" % tag.name)
else:
tags.append(tag)
while len(tags) > 0:
tag = tags.pop()
if not tag:
break
l.append("</%s>" % tag.name)
if not removed:
return False
return l
def iterative_mutator(self, template):
debug("Acquiring lock")
self.lock.acquire()
try:
buf = bytearray(template)
buf[self.skip_bytes + self.stats["iteration"]] = chr(self.stats["iteration_char"])
ret = self.stats["iteration"], 1, buf
self.stats["iteration_char"] += 1
if self.stats["iteration_char"] > 255:
self.stats["iteration_char"] = 0
self.stats["iteration"] += 1
log("Current iteration %d" % self.stats["iteration"])
finally:
debug("Releasing lock")
self.lock.release()
return ret
def iterative_mutator(self, template):
debug("Acquiring lock")
self.lock.acquire()
try:
buf = bytearray(template)
buf[self.skip_bytes + self.stats["iteration"]] = chr(self.stats["iteration_char"])
ret = self.stats["iteration"], 1, buf
self.stats["iteration_char"] += 1
if self.stats["iteration_char"] > 255:
self.stats["iteration_char"] = 0
self.stats["iteration"] += 1
log("Current iteration %d" % self.stats["iteration"])
finally:
debug("Releasing lock")
self.lock.release()
return ret
def do_try(self, outdir, start_at=0):
# Try to minimize to just one change
current_change = 0
minimized = False
iteration = 0
for i in range(len(self.diff)):
for pos in self.diff:
if start_at <= iteration:
log("Minimizing, iteration %d (Max. %d)..." % (iteration, (len(self.diff)) * len(self.diff)))
temp_file = tempfile.mktemp()
buf = bytearray(self.template)
if pos not in self.crash:
continue
buf[pos] = self.crash[pos]
with open(temp_file, "wb") as f:
f.write(buf)
try:
for key in self.env:
os.putenv(key, self.env[key])
cmd = "%s %s" % (self.command, temp_file)
cmd_obj = TimeoutCommand(cmd)
ret = cmd_obj.run(timeout=self.timeout)
if ret in RETURN_SIGNALS:
log("Successfully minimized, caught signal %d (%s)!" % (ret, RETURN_SIGNALS[ret]))
filename = sha1(buf).hexdigest()
filename = os.path.join(outdir, "%s%s" % (filename, self.extension))
shutil.copy(temp_file, filename)
log("Minized test case %s written to disk." % filename)
minimized = True
break
finally:
os.remove(temp_file)
if minimized:
break
iteration += 1
if minimized:
break
value = self.diff.pop()
if value in self.crash:
self.template[value] = self.crash[value]
del self.crash[value]
if not minimized:
log("Sorry, could not minimize crashing file!")
def fuzz(self):
log("Launching client/server fuzzer, listening in tube %s" % self.tube_name)
self.shared_queue = Queue()
while 1:
self.crash_info = None
log("Launching server with command %s" % self.command)
self.p = Process(target=self.debug_server, args=(self.shared_queue,))
self.p.start()
self.p.join(10)
while self.p.is_alive():
log("Running client")
client = Process(target=self.launch_client, args=(self.shared_queue,))
client.start()
client.join()
if self.crash_info is not None:
log("Server crashed, yuppie!")
else:
log("Server exited...")
def launch_debugger(self, timeout, command, filename):
self.iface.timeout = int(timeout)
if command.find("@@") > -1:
cmd = [
command.replace("@@", filename),
]
else:
cmd = [command, filename]
log("Launching debugger with command %s" % " ".join(cmd))
if self.iface != pykd_iface:
crash = self.iface.main(" ".join(cmd))
else:
reload(pykd_iface)
crash = pykd_iface.main(cmd,
mode=self.mode,
windbg_path=self.windbg_path,
exploitable_path=self.exploitable_path)
return crash
def launch_sample(self, buf):
# Re-read configuration each time we're running the fuzzer so the
# new changes are immediately applied.
self.read_configuration()
filename = tempfile.mktemp(suffix=self.extension)
f = open(filename, "wb")
f.write(buf)
f.close()
file = filename.split('/')[-1]
#os.putenv("NIGHTMARE_TIMEOUT", str(self.timeout))
for key in self.env:
debug("Setting environment variable %s=%s" % (key, self.env[key]))
os.putenv(key, self.env[key])
if self.pre_command is not None:
if pre_command.find("@@") > -1 and pre_command.find("$$") > -1:
self.pre_command.replace('@@', filename)
self.pre_command.replace('$$', file)
log(self.pre_command)
os.system(self.pre_command)
crash = None
for i in range(0,3):
try:
crash = self.launch_debugger(self.timeout, self.command, filename)
break
except:
log("Exception: %s" % sys.exc_info()[1])
continue
if self.post_command is not None:
os.system(self.post_command)
if crash is not None:
self.crash_info = crash
return True
else:
os.remove(filename)
return False
def run(self, timeout=60):
def target():
debug('Thread started')
if os.name == "nt":
line = self.cmd
shell = False
else: # Unix based
line = "exec %s" % self.cmd
shell = True
self.process = subprocess.Popen(line, shell=shell)
self.process.communicate()
debug('Thread finished')
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
log('Terminating process after timeout (%d)' % timeout)
try:
self.process.terminate()
self.process.terminate()
self.process.kill()
self.process.wait()
except:
log("Error killing process: %s" % str(sys.exc_info()[1]))
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
if os.name == "nt":
ret = ret & 0xFFFFFFFF
else:
ret = abs(ret) + 128
return ret
def run(self, timeout=60):
def target():
debug("Thread started")
if os.name == "nt":
line = self.cmd
shell = False
else: # Unix based
line = "exec %s" % self.cmd
shell = True
self.process = subprocess.Popen(line, shell=shell)
self.process.communicate()
debug("Thread finished")
thread = threading.Thread(target=target)
thread.start()
thread.join(timeout)
if thread.is_alive():
log("Terminating process after timeout (%d)" % timeout)
try:
self.process.terminate()
self.process.terminate()
self.process.kill()
self.process.wait()
except:
log("Error killing process: %s" % str(sys.exc_info()[1]))
thread.join()
self.process.wait()
ret = self.process.returncode
# A negative return code means a signal was received and the return
# code is -1 * SIGNAL. Return the expected Unix return code.
if ret is not None and ret < 0:
if os.name == "nt":
ret = ret & 0xFFFFFFFF
else:
ret = abs(ret) + 128
return ret
def put_new_crash(self, buf):
try:
if self.local_files:
# We can live the files somewhere on disk, nfp_engine.py is
# running in the same machine
filename = tempfile.mktemp()
with open(filename, "wb") as f:
f.write(buf)
d = {os.path.abspath(filename):self.last_crash}
else:
# The minimizer is running in a box different to the one were
# nfp_engine.py is running, put the whole file in the queue
self.last_crash["has_file"] = True
zbuf = base64.b64encode(zlib.compress(buf))
d = {zbuf:self.last_crash}
log("Putting the new crash in the queue...")
self.crash_q.put(json.dumps(d))
except:
log("Error putting the new crash in the queue: %s" % (str(sys.exc_info()[1])))
if self.local_files:
os.remove(filename)
def put_new_crash(self, buf):
try:
if self.local_files:
# We can live the files somewhere on disk, nfp_engine.py is
# running in the same machine
filename = tempfile.mktemp()
with open(filename, "wb") as f:
f.write(buf)
d = {os.path.abspath(filename): self.last_crash}
else:
# The minimizer is running in a box different to the one were
# nfp_engine.py is running, put the whole file in the queue
self.last_crash["has_file"] = True
zbuf = base64.b64encode(zlib.compress(buf))
d = {zbuf: self.last_crash}
log("Putting the new crash in the queue...")
self.crash_q.put(json.dumps(d))
except:
log("Error putting the new crash in the queue: %s" %
(str(sys.exc_info()[1])))
if self.local_files:
os.remove(filename)
def fuzz(self):
log("Launching client/server fuzzer, listening in tube %s" %
self.tube_name)
self.shared_queue = Queue()
while 1:
self.crash_info = None
log("Launching server with command %s" % self.command)
self.p = Process(target=self.debug_server,
args=(self.shared_queue, ))
self.p.start()
self.p.join(10)
while self.p.is_alive():
log("Running client")
client = Process(target=self.launch_client,
args=(self.shared_queue, ))
client.start()
client.join()
if self.crash_info is not None:
log("Server crashed, yuppie!")
else:
log("Server exited...")
def launch_sample(self, buf):
# Re-read configuration each time we're running the fuzzer so the
# new changes are immediately applied.
self.read_configuration()
filename = tempfile.mktemp(suffix=self.extension)
f = open(filename, "wb")
f.write(buf)
f.close()
#os.putenv("NIGHTMARE_TIMEOUT", str(self.timeout))
for key in self.env:
debug("Setting environment variable %s=%s" % (key, self.env[key]))
os.putenv(key, self.env[key])
if self.pre_command is not None:
os.system(self.pre_command)
crash = None
for i in range(0,3):
try:
crash = self.launch_debugger(self.timeout, self.command, filename)
break
except:
log("Exception: %s" % sys.exc_info()[1])
raise
continue
if self.post_command is not None:
os.system(self.post_command)
if crash is not None:
self.crash_info = crash
return True
else:
os.remove(filename)
return False
def launch_client(self, shared_queue):
self.read_configuration()
gid = int(self.client_gid)
if gid != os.getgid():
os.setgid(gid)
uid = int(self.client_uid)
if uid != os.getuid():
os.setuid(uid)
value = self.q.stats_tube(self.tube_name)["current-jobs-ready"]
debug("Total of %d job(s) in queue" % value)
job = self.q.reserve()
buf, temp_file = json.loads(job.body)
buf = base64.b64decode(buf)
log("Launching sample %s..." % os.path.basename(temp_file))
cmd = "%s %s" % (self.client_command, temp_file)
ret = os.system(cmd)
crash_info = None
try:
crash_info = shared_queue.get(timeout=1)
except:
crash_info = None
if crash_info is not None:
log("We have a crash, moving to %s queue..." % self.crash_tube)
crash = crash_info
d = {temp_file: crash_info}
self.crash_q.put(json.dumps(d))
self.crash_info = None
log("$PC 0x%08x Signal %s Exploitable %s " %
(crash["pc"], crash["signal"], crash["exploitable"]))
if crash["disasm"] is not None:
log("%08x: %s" % (crash["disasm"][0], crash["disasm"][1]))
else:
file_delete = os.path.basename(temp_file)
self.delete_q.put(str(file_delete))
if self.cleanup is not None:
debug("Running clean-up command %s" % self.cleanup)
os.system(self.cleanup)
debug("Done")
job.delete()
def fuzz_one_internal(self, template):
# Get mutated data using @template as the template buffer.
offset, size, buf = self.mutate(template)
filename = mktemp(suffix = self.extension)
debug("Creating temporary file %s" % filename)
with open(filename, "wb") as f:
f.write(buf)
debug("Performing code coverage...")
metrics = []
self.record_metric(filename, metrics)
for metric in metrics:
bbs = int(metric.unique_bbs)
if bbs > self.stats["max"]:
if not self.radamsa:
log("GOOD! Found an interesting change at 0x%x! Covered basic blocks %d, original maximum %d" % (offset, bbs, self.stats["max"]))
else:
log("GOOD! Found an interesting change! Covered basic blocks %d, original maximum %d" % (bbs, self.stats["max"]))
if self.iterative:
self.stats["iteration_char"] = 0
self.stats["iteration"] += 1
increase = (bbs - self.stats["max"])
self.generation_value += increase
self.apply_bytes(offset, size, buf)
self.generation_value = 0
old_stats = self.mgr.dict(self.stats)
self.lock.acquire()
try:
debug("Recalculating statistics...")
self.recalculate_statistics(old_stats, bbs)
finally:
self.lock.release()
elif bbs < self.stats["min"]:
debug("Bad metric found: minimum basic block(s) %d, current test-case basic block(s) %d" % (self.stats["min"], bbs))
self.discard_bytes(offset, size, buf)
self.generation_value -= 3
else:
line = "Uninteresting data with current test-case: min %d, max %d, current %d"
line = line % (self.stats["min"], self.stats["max"], bbs)
debug(line)
self.discard_bytes(offset, size, buf)
self.generation_value -= 1
if metric.exit_code in RETURN_SIGNALS:
self.generation_value += abs(self.generation_bottom_level)
ret = metric.exit_code
log("*** Found a BUG, caught signal %d (%s), hurra!" % (ret, RETURN_SIGNALS[ret]))
self.dump_poc(filename, offset, size, buf)
self.bugs += 1
debug("Removing test-case %s" % filename)
os.remove(filename)
def fuzz_one_internal(self, template):
# Get mutated data using @template as the template buffer.
offset, size, buf = self.mutate(template)
filename = mktemp(suffix = self.extension)
debug("Creating temporary file %s" % filename)
with open(filename, "wb") as f:
f.write(buf)
debug("Performing code coverage...")
metrics = []
self.record_metric(filename, metrics)
for metric in metrics:
bbs = int(metric.unique_bbs)
if bbs > self.stats["max"]:
if not self.radamsa:
log("GOOD! Found an interesting change at 0x%x! Covered basic blocks %d, original maximum %d" % (offset, bbs, self.stats["max"]))
else:
log("GOOD! Found an interesting change! Covered basic blocks %d, original maximum %d" % (bbs, self.stats["max"]))
if self.iterative:
self.stats["iteration_char"] = 0
self.stats["iteration"] += 1
increase = (bbs - self.stats["max"])
self.generation_value += increase
self.apply_bytes(offset, size, buf)
self.generation_value = 0
old_stats = self.mgr.dict(self.stats)
self.lock.acquire()
try:
debug("Recalculating statistics...")
self.recalculate_statistics(old_stats, bbs)
finally:
self.lock.release()
elif bbs < self.stats["min"]:
debug("Bad metric found: minimum basic block(s) %d, current test-case basic block(s) %d" % (self.stats["min"], bbs))
self.discard_bytes(offset, size, buf)
self.generation_value -= 3
else:
line = "Uninteresting data with current test-case: min %d, max %d, current %d"
line = line % (self.stats["min"], self.stats["max"], bbs)
debug(line)
self.discard_bytes(offset, size, buf)
self.generation_value -= 1
if metric.exit_code in RETURN_SIGNALS:
self.generation_value += abs(self.generation_bottom_level)
ret = metric.exit_code
log("*** Found a BUG, caught signal %d (%s), hurra!" % (ret, RETURN_SIGNALS[ret]))
self.dump_poc(filename, offset, size, buf)
self.bugs += 1
debug("Removing test-case %s" % filename)
os.remove(filename)
