def run_test(test_func, args):
log_info("Start test: " + test_func.__module__)
module_path = os.path.dirname(test_func.__module__.replace('.', '/'))
previous_cwd = os.getcwd()
os.chdir(os.path.join(previous_cwd, module_path))
test_func(args)
os.chdir(previous_cwd)
def run_test(test_func, args):
log_info("Start test: "+test_func.__module__)
module_path = os.path.dirname(test_func.__module__.replace('.','/'))
previous_cwd = os.getcwd()
os.chdir(os.path.join(previous_cwd, module_path))
ret = test_func(args)
os.chdir(previous_cwd)
return ret
def test_learn(args):
machine = Machine("x86_64")
# Compil tests
log_info("Remove old files")
os.system("make clean")
log_info("Compile C files")
status = os.system("make")
assert status == 0
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x[:-2] for x in files if x.endswith(".c")]
for c_file in c_files:
cont = Container.from_stream(open(c_file))
func_name = c_file
main_addr = cont.symbol_pool["main"].offset
func_addr = cont.symbol_pool[func_name].offset
log_info("Learning " + func_name + " over " + func_name + ".c")
cmd = [
"sibyl", "learn", "-t", "miasm", "-m",
hex(main_addr), func_name, c_file
]
sibyl = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = sibyl.communicate()
assert sibyl.returncode == 0
log_info("Testing generated class")
mod = imp.new_module("testclass")
exec stdout in mod.__dict__
classTest = getattr(mod, "Test" + c_file)
tl = TestLauncher(c_file, machine, ABI_AMD64_SYSTEMV, [classTest],
"gcc")
possible_funcs = tl.run(func_addr)
if tl.possible_funcs:
log_success("Generated class recognize the function " + func_name)
else:
log_error("Generated class failed to recognize the function " +
func_name)
log_info("Remove old files")
os.system("make clean")
def test_learn(args):
machine = Machine("x86_64")
# Compil tests
log_info("Remove old files")
os.system("make clean")
log_info("Compile C files")
status = os.system("make")
assert status == 0
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x[:-2] for x in files if x.endswith(".c")]
# Ways to invoke
to_invoke = {
"Miasm": invoke_miasm,
}
if args.pin_tracer:
to_invoke["PIN"] = invoke_pin
# Learn + test
fail = False
for filename in c_files:
if filename in unsupported:
log_error("Skip %s (unsupported)" % filename)
continue
with open(filename) as fdesc:
cont = Container.from_stream(fdesc)
func_name = filename
func_addr = cont.loc_db.get_name_offset(func_name)
header_filename = "%s.h" % filename
for name, cb in to_invoke.iteritems():
log_info("Learning %s over %s with %s" % (func_name,
filename, name))
cmdline = cb(filename, func_name, header_filename, cont)
print " ".join(cmdline)
sibyl = subprocess.Popen(cmdline, env=os.environ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = sibyl.communicate()
if sibyl.returncode != 0:
log_error("Failed to learn with error:")
print stderr
fail = True
continue
log_info("Testing generated class")
mod = imp.new_module("testclass")
exec stdout in mod.__dict__
classTest = getattr(mod, "TESTS")[0]
tl = TestLauncher(filename, machine, ABI_AMD64_SYSTEMV, [classTest],
config.jit_engine)
possible_funcs = tl.run(func_addr)
if tl.possible_funcs and possible_funcs == [filename]:
log_success("Generated class recognize the function " \
"'%s'" % func_name)
else:
log_error("Generated class failed to recognize the function " \
"'%s'" % func_name)
fail = True
# Clean
log_info( "Remove old files" )
os.system("make clean")
return fail
def test_find(args):
if args.func_heuristic:
get_funcs = get_funcs_heuristics
else:
get_funcs = get_funcs_exe_source
# Compil tests
log_info("Remove old files")
os.system("make clean")
log_info("Compile C files")
status = os.system("make")
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x for x in files if x.endswith(".c")]
log_info("Found:\n\t- " + "\n\t- ".join(c_files))
for c_file in c_files:
filename = c_file[:-2]
log_info(" %s:" % filename)
# to_check: (addr, expected found)
# extra: possible extra match
to_check, extra = get_funcs(c_file, filename)
print "\n".join("0x%08x: %s" % (addr, funcname)
for (addr, funcname) in to_check)
# Launch Sibyl
log_info("Launch Sibyl")
options = ["-j", "gcc", "-i", "5", "-b", "ABIStdCall_x86_32"]
if not args.arch_heuristic:
options += ["-a", "x86_32"]
cmd = ["sibyl", "find"] + options + [filename]
cmd += [hex(addr) for addr, _ in to_check]
print " ".join(cmd)
sibyl = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Parse result
found = []
stdout, stderr = sibyl.communicate()
for line in stdout.split("\n"):
if not line or not " : " in line:
continue
addr, func = line.split(" : ")
found.append((int(addr, 0), func))
if sibyl.returncode:
log_error("Process exits with a %d code" % sibyl.returncode)
print stderr
exit(sibyl.returncode)
log_info("Evaluate results")
i = 0
for element in found:
if element not in to_check:
offset, name = element
if offset in extra.get(name, []):
# Present in symtab but not in C source file
print "[+] Additionnal found: %s (@0x%08x)" % (name,
offset)
else:
alt_names = [
aname for aname, offsets in extra.iteritems()
if offset in offsets
]
log_error("Bad found: %s (@0x%08x -> '%s')" %
(name, offset, ",".join(alt_names)))
else:
i += 1
for element in to_check:
if element not in found:
log_error("Unable to find: %s (@0x%08x)" %
(element[1], element[0]))
log_success("Found %d/%d correct elements" % (i, len(to_check)))
log_info("Remove old files")
os.system("make clean")
return False
def test_learn(args):
machine = Machine("x86_64")
# Compil tests
log_info("Remove old files")
os.system("make clean")
log_info("Compile C files")
status = os.system("make")
assert status == 0
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x[:-2] for x in files if x.endswith(".c")]
# Ways to invoke
to_invoke = {
"Miasm": invoke_miasm,
}
if args.pin_tracer:
to_invoke["PIN"] = invoke_pin
# Learn + test
fail = False
for filename in c_files:
if filename in unsupported:
log_error("Skip %s (unsupported)" % filename)
continue
with open(filename) as fdesc:
cont = Container.from_stream(fdesc)
func_name = filename
func_addr = cont.loc_db.get_name_offset(func_name)
header_filename = "%s.h" % filename
for name, cb in to_invoke.iteritems():
log_info("Learning %s over %s with %s" %
(func_name, filename, name))
cmdline = cb(filename, func_name, header_filename, cont)
print " ".join(cmdline)
sibyl = subprocess.Popen(cmdline,
env=os.environ,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = sibyl.communicate()
if sibyl.returncode != 0:
log_error("Failed to learn with error:")
print stderr
fail = True
continue
log_info("Testing generated class")
mod = imp.new_module("testclass")
exec stdout in mod.__dict__
classTest = getattr(mod, "TESTS")[0]
tl = TestLauncher(filename, machine, ABI_AMD64_SYSTEMV,
[classTest], config.jit_engine)
possible_funcs = tl.run(func_addr)
if tl.possible_funcs and possible_funcs == [filename]:
log_success("Generated class recognize the function " \
"'%s'" % func_name)
else:
log_error("Generated class failed to recognize the function " \
"'%s'" % func_name)
fail = True
# Clean
log_info("Remove old files")
os.system("make clean")
return fail
def test_find(args):
if args.func_heuristic:
get_funcs = get_funcs_heuristics
else:
get_funcs = get_funcs_exe_source
# Compil tests
log_info( "Remove old files" )
os.system("make clean")
log_info( "Compile C files" )
status = os.system("make")
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x for x in files if x.endswith(".c")]
log_info( "Found:\n\t- " + "\n\t- ".join(c_files) )
for c_file in c_files:
filename = c_file[:-2]
log_info( " %s:" % filename )
# to_check: (addr, expected found)
# extra: possible extra match
to_check, extra = get_funcs(c_file, filename)
print "\n".join("0x%08x: %s" % (addr, funcname)
for (addr, funcname) in to_check)
# Launch Sibyl
log_info( "Launch Sibyl" )
options = ["-j", "gcc", "-i", "5", "-b", "ABIStdCall_x86_32"]
if not args.arch_heuristic:
options += ["-a", "x86_32"]
cmd = ["sibyl", "find"] + options + [filename]
cmd += [hex(addr) for addr, _ in to_check]
print " ".join(cmd)
sibyl = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Parse result
found = []
stdout, stderr = sibyl.communicate()
for line in stdout.split("\n"):
if not line or not " : " in line:
continue
addr, func = line.split(" : ")
found.append((int(addr, 0), func))
if sibyl.returncode:
log_error("Process exits with a %d code" % sibyl.returncode)
print stderr
exit(sibyl.returncode)
log_info( "Evaluate results" )
i = 0
for element in found:
if element not in to_check:
offset, name = element
if offset in extra.get(name, []):
# Present in symtab but not in C source file
print "[+] Additionnal found: %s (@0x%08x)" % (name, offset)
else:
alt_names = [aname
for aname, offsets in extra.iteritems()
if offset in offsets]
log_error("Bad found: %s (@0x%08x -> '%s')" % (name,
offset,
",".join(alt_names)))
else:
i += 1
for element in to_check:
if element not in found:
log_error("Unable to find: %s (@0x%08x)" % (element[1], element[0]))
log_success("Found %d/%d correct elements" % (i, len(to_check)))
log_info( "Remove old files" )
os.system("make clean")
return False
def test_find(args):
custom_tag = "my_"
whitelist_funcs = ["main"]
# Compil tests
log_info("Remove old files")
os.system("make clean")
log_info("Compile C files")
status = os.system("make")
# Find test names
c_files = []
for cur_dir, sub_dir, files in os.walk("."):
c_files += [x for x in files if x.endswith(".c")]
log_info("Found:\n\t- " + "\n\t- ".join(c_files))
m = re.compile("\w+[ \*]+(\w+)\(.*\)")
for c_file in c_files:
# Get function defined in the source
with open(c_file) as fdesc:
data = fdesc.read()
filename = c_file[:-2]
log_info(" %s:" % filename)
funcs = []
for p in m.finditer(data):
funcs.append(p.groups()[0])
funcs = list(x for x in set(funcs) if x not in whitelist_funcs)
# Find corresponding binary offset
to_check = []
with open(filename) as fdesc:
elf = ELF(fdesc.read())
symbols = {}
for name, symb in elf.getsectionbyname(".symtab").symbols.iteritems():
offset = symb.value
if name.startswith("__"):
name = name[2:]
symbols.setdefault(name, set()).add(offset)
if name in funcs:
if name.startswith(custom_tag):
## Custom tags can be used to write equivalent functions like
## 'my_strlen' for a custom strlen
name = name[len(custom_tag):]
to_check.append((offset, name))
print "\n".join("0x%08x: %s" % (addr, funcname)
for (addr, funcname) in to_check)
# Launch Sibyl
log_info("Launch Sibyl")
options = ["-j", "gcc", "-i", "5", "-b", "ABIStdCall_x86_32"]
if not args.arch_heuristic:
options += ["-a", "x86_32"]
cmd = ["sibyl", "find"] + options + [filename]
if not args.func_heuristic:
cmd += [hex(addr) for addr, f in to_check]
print " ".join(cmd)
sibyl = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Parse result
found = []
stdout, stderr = sibyl.communicate()
for line in stdout.split("\n"):
if not line:
continue
addr, func = line.split(" : ")
found.append((int(addr, 16), func))
if sibyl.returncode:
log_error("Process exits with a %d code" % sibyl.returncode)
print stderr
exit(sibyl.returncode)
log_info("Evaluate results")
i = 0
for element in found:
if element not in to_check:
offset, name = element
if offset in symbols.get(name, []):
# Present in symtab but not in C source file
print "[+] Additionnal found: %s (@0x%08x)" % (name,
offset)
else:
alt_names = [
aname for aname, offsets in symbols.iteritems()
if offset in offsets
]
log_error("Bad found: %s (@0x%08x -> '%s')" %
(name, offset, ",".join(alt_names)))
else:
i += 1
for element in to_check:
if element not in found:
log_error("Unable to find: %s (@0x%08x)" %
(element[1], element[0]))
log_success("Found %d/%d correct elements" % (i, len(to_check)))
log_info("Remove old files")
os.system("make clean")