def export(picklefile, zipfile, syms, debug):
global leaks
set_debuglevel(debug)
assert (syms is not None)
SymbolInfo.open(syms)
assert (loadleaksglob(picklefile))
export_leaks(leaks, zipfile, syms)
def getLocalIp(ip):
"""Find local ip using SymbolInfo."""
if SymbolInfo.isopen():
sym = SymbolInfo.lookup(ip) # Type: SymbolInfo
if sym is not None:
if sym.img.dynamic:
return ip - sym.img.lower
return ip
def specific(randompickle, callback, pickle, syms, xml, debug):
global printer
global leaks
set_debuglevel(debug)
SymbolInfo.open(syms)
leaks = loadpickle(randompickle)
specific_leakage_test(leaks, callback)
if pickle is not None:
storepickle(pickle, leaks)
if xml is not None:
print_leaks(leaks, XmlLeakPrinter(xml), True)
def diff(file1, file2, syms, pickle, xml, fast, debug):
global printer
set_debuglevel(debug)
SymbolInfo.open(syms)
if pickle is not None:
loadleaksglob(pickle)
iterate_queue([file1, file2], fast)
if pickle is not None:
storepickle(pickle, leaks)
if xml is not None:
print_leaks(leaks, XmlLeakPrinter(xml), True)
def show(picklefile, syms, xml, leakout, debug):
global printer
global leaks
set_debuglevel(debug)
assert (syms is not None)
SymbolInfo.open(syms)
assert (loadleaksglob(picklefile))
leaks = collapse_cfleaks(leaks, True, 1, '')
if xml is not None:
print_leaks(leaks, XmlLeakPrinter(xml), True)
if leakout is not None:
print_leaks(leaks, BinLeakPrinter(leakout))
def merge(pickle_a, pickle_b, syms, xml, pickle, debug):
global printer
global leaks
set_debuglevel(debug)
SymbolInfo.open(syms)
leaks = loadpickle(pickle_a)
leakB = loadpickle(pickle_b)
merge_leaks(leakB)
if pickle is not None:
storepickle(pickle, leaks)
if xml is not None:
print_leaks(leaks, XmlLeakPrinter(xml), True)
def generic(fixedpickle, randompickle, pickle, syms, xml, debug):
global printer
global leaks
set_debuglevel(debug)
SymbolInfo.open(syms)
fixed = loadpickle(fixedpickle)
random = loadpickle(randompickle)
generic_leakage_test(fixed, random)
leaks = random
if pickle is not None:
storepickle(pickle, leaks)
if xml is not None:
print_leaks(leaks, XmlLeakPrinter(xml), True)
def doprint(self, text, ip, leak):
self.outstream.write(" " * self.depth)
if len(text) > 0:
self.outstream.write(text + " ")
if SymbolInfo.isopen():
sym = SymbolInfo.lookup(ip)
if sym is not None:
self.outstream.write(sym.strat(ip))
else:
self.outstream.write(hex(ip))
else:
self.outstream.write(hex(ip))
self.outstream.write("\n")
if leak is not None:
leak.doprint(self)
def __init__(self, leak):
self.dataleaks = MergeMap(DataLeak)
self.cfleaks = MergeMap(CFLeak)
self.sym = SymbolInfo.lookup(leak.ip)
if self.sym is None:
img = Image("Unknown", 0, 0, False)
self.sym = Symbol(0, 0, "UnknownSym", img, "")
self.fentry = self.sym.addr
self.append(leak)
def export_ip(ip, datafs, imgmap, info_map):
if ip is None or ip == 0:
return
if not ip in info_map:
sym = SymbolInfo.lookup(ip)
assert (sym is not None)
if sym.img.dynamic:
addr = ip - sym.img.lower
else:
addr = ip
bin_file_path = sym.img.name
asm_file_path = bin_file_path + ".asm"
# Add binary (ELF) + ASM objdump to datafs
if not bin_file_path in imgmap:
try:
datafs.add_file(bin_file_path)
except:
debug(0, "Error: Binary file missing: %s", (bin_file_path))
return
asm_dump = ""
try:
debug(1, "[ASM] objdump %s", (str(bin_file_path)))
# asm_dump = subprocess.check_output(["objdump", "-Dj", ".text", bin_file_path], universal_newlines=True)
with datafs.create_file(asm_file_path) as f:
subprocess.call(["objdump", "-dS", bin_file_path],
universal_newlines=True,
stdout=f)
f.seek(0)
asm_dump = f.read().decode('utf-8')
except subprocess.CalledProcessError as err:
debug(0, "[ASM] objdump %s failed with error_code: %s",
(str(bin_file_path), str(err.returncode)))
asm_dump = None
imgmap[bin_file_path] = asm_dump
if not ip in info_map:
# Search for leak in asm dump
asm_dump = imgmap[bin_file_path]
asm_line_nr = getAsmFileInfo(addr, asm_dump)
if asm_line_nr < 0:
debug(1, "[ASM] unavailable for %s in %s",
(hex(addr), bin_file_path))
# Search for leak in source code
src_file_path, src_line_nr = getSourceFileInfo(
hex(addr), bin_file_path)
if src_file_path is not None and os.path.exists(src_file_path):
datafs.add_file(src_file_path)
else:
if src_file_path is None:
debug(1, "[SRC] unavailable for %s in %s",
(hex(addr), bin_file_path))
else:
debug(1, "[SRC] source file %s missing", (src_file_path))
ip_info = IpInfoShort(asm_file_path, asm_line_nr, src_file_path,
src_line_nr)
info_map[ip] = ip_info
def getCtxName(ip):
"""Find context name of ip.
Returns:
A string containing the context name, or the hex presentation
of the ip of no SymbolInfo is available.
"""
if SymbolInfo.isopen():
sym = SymbolInfo.lookup(ip) # Type: SymbolInfo
if sym is not None:
name = ""
if sym.img is not None and sym.img.dynamic:
name += "(+%x)" % (ip - sym.img.lower)
name += " %s(%s)" % (sym.getname(), sym.type)
return name
else:
return hex(ip)
else:
return hex(ip)
def resetSymbolInfo():
SymbolInfo.close()
def setupSymbolInfo(file_path):
global datafs
datafs = DataFS(file_path, write=False)
with datafs.get_file("allsyms.txt") as f:
SymbolInfo.open(f)
def add_elf_syms(symfile, newsymfile):
with open(symfile, 'r') as f:
SymbolInfo.open(f)
SymbolInfo.reload_syms_from_elf()
with open(newsymfile, 'w') as f:
SymbolInfo.write(f)
def specific_leakage_test(random, callback):
# load callback function
try:
with open(callback) as fp:
code = compile(fp.read(), callback, "exec")
splcb = types.ModuleType("<config>")
exec(code, splcb.__dict__)
except Exception as e:
debug(0, "Unable to load specific leakage test callback function!")
debug(0, str(e))
assert (False)
assert (splcb.specific_leakage_callback)
xtarget = str(os.path.splitext(os.path.basename(callback))[0])
# print test types
debug(1, "Test Types:")
debug(1, " 2 .... number of accesses per address")
debug(1, " 3 .... position of address during access")
debug(1, "")
# process leaks
randomleaks = extract_leakdiff_to_array(random, LeaksOnly=True)
debug(0, "Got %d leaks.", (len(randomleaks)))
sys.stdout.flush()
for i in range(0, len(randomleaks)):
rl = randomleaks[i]
noleakdetected = True
leaktype = "dataleak" if isinstance(rl, DataLeak) else "cfleak"
debug(1, "Testing %s@%x...", (leaktype, rl.ip))
cursym = SymbolInfo.lookup(rl.ip)
if (cursym is not None) and (len(cursym.name) > 0):
cursym = cursym.name[0]
else:
cursym = None
rl.status.spperformed.add(xtarget)
# sanity check
if len(rl.evidence) == 0:
debug(0, "Testing %s@%x...", (leaktype, rl.ip))
debug(0, " warning: no evidences")
continue
# gather information -- leaks
rdic = {}
cnt_t2 = 0
rdic_pos = {}
cnt_t3 = 0
keys = []
for e in rl.evidence:
if len(e.entries) == 0:
continue
if e.source != EvidenceSource.Specific.value:
continue
# all entries
selentries = e.entries
# gather information -- keys
keys.append(e.key)
# gather information -- type2
chist = Counter(selentries)
rset = set(rdic.keys())
cset = set(chist.keys())
for s in list(cset - rset):
rdic[s] = [0] * cnt_t2
for c in rdic.keys():
if c in chist.keys():
rdic[c] += [chist[c]]
else:
rdic[c] += [0]
cnt_t2 += 1
# gather information -- type3
cs = set(selentries)
for c in cs:
cp = [pos for pos, j in enumerate(selentries) if j == c]
if c not in rdic_pos.keys():
rdic_pos[c] = [-1] * cnt_t3
rdic_pos[c] += [int(numpy.round(numpy.median(cp)))]
for s in list(set(rdic_pos.keys()) - cs):
rdic_pos[s] += [-1]
cnt_t3 += 1
# postprocessing -- type2
for c in rdic.keys():
rdic[c] = numpy.asarray(rdic[c], dtype=numpy.uint)
# postprocessing -- type3
for c in rdic_pos.keys():
rdic_pos[c] = numpy.array(rdic_pos[c], dtype=numpy.int)
# convert keys with callback
# the callback always returns the matrix X:
#
# x_{0,0} x_{0,1} ... x_{0,N}
# x_{1,0} x_{1,1} ... x_{1,N}
# ...
# x_{M,0} x_{M,1} ... x_{M,N}
#
# M ... number of keys (one row per key)
# N ... number of properties in X (one column per property)
X = splcb.specific_leakage_callback(keys)
if X.shape[0] != len(keys):
debug(0, "Testing %s@%x...", (leaktype, rl.ip))
debug(0, " warning: callback returned wrong matrix!")
continue
######
# Test2: number of accesses per address
######
for j in range(0, X.shape[1]):
lfound = False
for c in rdic.keys():
(R, L, I) = rdctest.RDC.test(X[:, j], rdic[c], 0.9999)
if I == False:
lfound = True
noleakdetected = False
cleak = SPLeak(NSPType.Type2, j, c, None, R, L, I, xtarget,
0.9999)
rl.status.spleak.add(cleak)
debug(1, " [Test2] -- P%d -- %x -- %s",
(j, c, str(cleak)))
if not lfound:
debug(1, " [Test2] -- P%d -- nothing found", (j))
######
# Test3: position of address during access
######
for j in range(0, X.shape[1]):
lfound = False
for c in rdic_pos.keys():
(R, L, I) = rdctest.RDC.test(X[:, j], rdic_pos[c], 0.9999)
if I == False:
lfound = True
noleakdetected = False
cleak = SPLeak(NSPType.Type3, j, c, None, R, L, I, xtarget,
0.9999)
rl.status.spleak.add(cleak)
debug(1, " [Test3] -- P%d -- %x -- %s",
(j, c, str(cleak)))
if not lfound:
debug(1, " [Test3] -- P%d -- nothing found", (j))
# add noleak element to document that no leakage was found
if noleakdetected:
cnl = SPLeak(NSPType.Noleak, target=xtarget)
rl.status.spleak.add(cnl)
debug(1, " %s", str(cnl))
# progress
if len(randomleaks) > 100:
if (i % int(len(randomleaks) / 10)) == 0:
debug(0, "[Progress] %6.2f%%",
((i * 100.0) / len(randomleaks)))
else:
debug(0, "[Progress] Finished %d", (i + 1))
sys.stdout.flush()
debug(0, "[Progress] 100.00%%")
sys.stdout.flush()
def generic_leakage_test(fixed, random):
fixedleaks = extract_leakdiff_to_array(fixed)
randomleaks = extract_leakdiff_to_array(random)
assert (len(fixedleaks) == len(randomleaks))
# print test types
debug(1, "Test Types:")
debug(1, " 1a ... number of addresses")
debug(1, " 1b ... number of unique addresses")
debug(1, " 2 .... number of accesses per address")
debug(1, "")
# iterate over leaks
debug(0, "Got %d trace differences.", (len(fixedleaks)))
sys.stdout.flush()
for i in range(0, len(fixedleaks)):
fl = fixedleaks[i]
rl = randomleaks[i]
assert (fl.ip == rl.ip)
noleakdetected = True
msgwarning = ""
msgleak = ""
cursym = SymbolInfo.lookup(fl.ip)
if (cursym is not None) and (len(cursym.name) > 0):
cursym = cursym.name[0]
else:
cursym = None
# always test
leaktype = "dataleak" if isinstance(fl, DataLeak) else "cfleak"
msgwarning += "Testing %s@%x...\n" % (leaktype, fl.ip)
msgleak += "Testing %s@%x...\n" % (leaktype, fl.ip)
fl.status.nsperformed = True
rl.status.nsperformed = True
# sanity check
cont = False
if len(fl.evidence) == 0:
msgwarning += " warning: no evidences for fixed\n"
cont = True
if len(rl.evidence) == 0:
msgwarning += " warning: no evidences for random\n"
cont = True
if cont:
debug(0, msgwarning.rstrip())
continue
# init
fnum = {}
fnum_uniq = {}
rnum = {}
rnum_uniq = {}
fdic = {}
rdic = {}
# gather information (fixed)
for e in fl.evidence:
if len(e.entries) == 0:
continue
if e.source != EvidenceSource.Generic.value:
continue
# all entries
selentries = e.entries
# Type1a/b
cn = len(selentries)
cnu = len(set(selentries))
if cn in fnum.keys():
fnum[cn] += 1
else:
fnum[cn] = 1
if cnu in fnum_uniq.keys():
fnum_uniq[cnu] += 1
else:
fnum_uniq[cnu] = 1
# Type2
counts = Counter(selentries)
for c in counts.keys():
if c in fdic:
fdic[c] += counts[c]
else:
fdic[c] = counts[c]
# gather information (random)
for e in rl.evidence:
if len(e.entries) == 0:
continue
if e.source != EvidenceSource.Generic.value:
continue
# all entries
selentries = e.entries
# Type1a/b
cn = len(selentries)
cnu = len(set(selentries))
if cn in rnum.keys():
rnum[cn] += 1
else:
rnum[cn] = 1
if cnu in rnum_uniq.keys():
rnum_uniq[cnu] += 1
else:
rnum_uniq[cnu] = 1
# Type2
counts = Counter(selentries)
for c in counts.keys():
if c in rdic:
rdic[c] += counts[c]
else:
rdic[c] = counts[c]
# sanity check
if len(fnum) == 0 or len(rnum) == 0 or len(fnum_uniq) == 0 or len(
rnum_uniq) == 0:
continue
######
# Test1a: number of addresses
######
# sanity check
cont = False
if len(fnum) == 0:
cont = True
if len(rnum) == 0:
cont = True
# test
if not cont:
# solve entry mismatches
fset = set(fnum.keys())
rset = set(rnum.keys())
if fset != rset:
for s in list(fset - rset):
rnum[s] = 0
for s in list(rset - fset):
fnum[s] = 0
# compile histograms
fhist = numpy.array([fnum[j] for j in sorted(fnum.keys())],
dtype=numpy.float32)
rhist = numpy.array([rnum[j] for j in sorted(rnum.keys())],
dtype=numpy.float32)
fhist_len = numpy.int32(numpy.sum(fhist))
rhist_len = numpy.int32(numpy.sum(rhist))
# sanity check
cont = False
if fhist_len < 30:
cont = True
if rhist_len < 30:
cont = True
# stat test
if not cont:
(D, L) = kuipertest.kp_histogram(fhist, rhist, fhist_len,
rhist_len, 0.9999)
R = (D > L)
if R:
noleakdetected = False
cfl = NSLeak(NSPType.Type1a, None, D, L, 0.9999, R)
crl = NSLeak(NSPType.Type1a, None, D, L, 0.9999, R)
fl.status.nsleak.add(cfl)
rl.status.nsleak.add(crl)
msgleak += " [Test1a] -- %s\n" % str(cfl)
######
# Test1b: number of unique addresses
######
# sanity check
cont = False
if len(fnum_uniq) == 0:
cont = True
if len(rnum_uniq) == 0:
cont = True
# test
if not cont:
# solve entry mismatches
fset = set(fnum_uniq.keys())
rset = set(rnum_uniq.keys())
if fset != rset:
for s in list(fset - rset):
rnum_uniq[s] = 0
for s in list(rset - fset):
fnum_uniq[s] = 0
# compile histograms
fhist = numpy.array(
[fnum_uniq[j] for j in sorted(fnum_uniq.keys())],
dtype=numpy.float32)
rhist = numpy.array(
[rnum_uniq[j] for j in sorted(rnum_uniq.keys())],
dtype=numpy.float32)
fhist_len = numpy.int32(numpy.sum(fhist))
rhist_len = numpy.int32(numpy.sum(rhist))
# sanity check
cont = False
if fhist_len < 30:
cont = True
if rhist_len < 30:
cont = True
# stat test
if not cont:
(D, L) = kuipertest.kp_histogram(fhist, rhist, fhist_len,
rhist_len, 0.9999)
R = (D > L)
if R:
noleakdetected = False
cfl = NSLeak(NSPType.Type1b, None, D, L, 0.9999, R)
crl = NSLeak(NSPType.Type1b, None, D, L, 0.9999, R)
fl.status.nsleak.add(cfl)
rl.status.nsleak.add(crl)
msgleak += " [Test1b] -- %s\n" % str(cfl)
######
# Test2: number of accesses per address
######
# sanity check
cont = False
if len(fdic.keys()) == 0:
cont = True
if len(rdic.keys()) == 0:
cont = True
# test
if not cont:
# solve entry mismatches
fset = set(fdic.keys())
rset = set(rdic.keys())
if fset != rset:
for s in list(fset - rset):
rdic[s] = 0
for s in list(rset - fset):
fdic[s] = 0
# compile histograms
fhist = numpy.array([fdic[j] for j in sorted(fdic.keys())],
dtype=numpy.float32)
rhist = numpy.array([rdic[j] for j in sorted(rdic.keys())],
dtype=numpy.float32)
fhist_len = numpy.int32(numpy.sum(fhist))
rhist_len = numpy.int32(numpy.sum(rhist))
# sanity check
cont = False
if fhist_len < 30:
cont = True
if rhist_len < 30:
cont = True
# stat test
if not cont:
(D, L) = kuipertest.kp_histogram(fhist, rhist, fhist_len,
rhist_len, 0.9999)
R = (D > L)
if R:
noleakdetected = False
cfl = NSLeak(NSPType.Type2, None, D, L, 0.9999, R)
crl = NSLeak(NSPType.Type2, None, D, L, 0.9999, R)
fl.status.nsleak.add(cfl)
rl.status.nsleak.add(crl)
msgleak += " [Test2] -- %s\n" % str(cfl)
# add noleak element to document that no leakage was found
if noleakdetected:
cnl = NSLeak(NSPType.Noleak)
fl.status.nsleak.add(cnl)
rl.status.nsleak.add(cnl)
debug(1, "Testing %s@%x...", (leaktype, fl.ip))
debug(1, " %s", (str(cnl)))
else:
debug(1, msgleak.rstrip())
# progress
if len(fixedleaks) > 100:
if (i % int(len(fixedleaks) / 10)) == 0:
debug(0, "[Progress] %6.2f%%", ((i * 100.0) / len(fixedleaks)))
else:
debug(0, "[Progress] Finished %d", (i + 1))
sys.stdout.flush()
debug(0, "[Progress] 100.00%%")
sys.stdout.flush()
