def wireshark_diff():
data = load_workbook("/home/bert/Documents/data/wireshark.xlsx", read_only=True)[u'Sheet3']
suffix_obj = suffixtree()
wb = Workbook()
ws = wb.active
db1 = Graph("http://127.0.0.1:7473/db/data/")
db2 = Graph()
for row in data.rows:
vuln_seg = row[0].value
patched_name = vuln_seg[:14] + "PATCHED_" + row[2].value
vuln_name = vuln_seg[:14] + "VULN_" + row[2].value
var_map = get_type_mapping_table(db2, vuln_name)
try:
ret = search_vuln_seg_in_func(db1, vuln_seg, row[2].value,var_map, db2, patched_name, suffix_obj)
ws.append(ret)
except Exception as e:
print e
ws.append((vuln_seg, patched_name, "failed"))
wb.save("/home/bert/Documents/data/wireshark_diff.xlsx")
suffix_obj.close()
print "wireshark all works done"
def wireshark_diff():
data = load_workbook("/home/bert/Documents/data/wireshark.xlsx",
read_only=True)[u'Sheet3']
suffix_obj = suffixtree()
wb = Workbook()
ws = wb.active
db1 = Graph("http://127.0.0.1:7473/db/data/")
db2 = Graph()
for row in data.rows:
vuln_seg = row[0].value
patched_name = vuln_seg[:14] + "PATCHED_" + row[2].value
vuln_name = vuln_seg[:14] + "VULN_" + row[2].value
var_map = get_type_mapping_table(db2, vuln_name)
try:
ret = search_vuln_seg_in_func(db1, vuln_seg, row[2].value, var_map,
db2, patched_name, suffix_obj)
ws.append(ret)
except Exception as e:
print e
ws.append((vuln_seg, patched_name, "failed"))
wb.save("/home/bert/Documents/data/wireshark_diff.xlsx")
suffix_obj.close()
print "wireshark all works done"
def firefox_code_reuse(table_name):
result_db = sqlite3.connect(
"/home/bert/Documents/data/firefox_code_reuse.db")
result_db.execute('''create table if not exists %s(
vuln_segement CHAR(50) NOT NULL,
reuse_func CHAR(50) NOT NULL,
status CHAR(10) NOT NULL,
distinct_type_and_const BOOLEAN,
distinct_const_no_type BOOLEAN,
distinct_type_no_const BOOLEAN,
no_type_no_const BOOLEAN,
no_mapping BOOLEAN)
''' % table_name)
db = Graph("http://127.0.0.1:7474/db/data/")
suffix_obj = suffixtree()
worksheet = load_workbook(
"/home/bert/Documents/data/firefox_reuse.xlsx").active
for row in worksheet.rows:
#check
ret = result_db.execute(
"select * from %s where vuln_segement=? and reuse_func=?" %
table_name, (row[0].value, row[2].value))
if ret.fetchone():
continue
vuln_seg = row[0].value
vuln_name = vuln_seg[:14] + "VULN_" + row[1].value
try:
var_map = get_type_mapping_table(db, vuln_name)
ret = search_vuln_seg_in_func(db, row[0].value, row[1].value,
var_map, db, row[2].value,
suffix_obj)
if ret[2] == "success":
result_db.execute(
"insert into %s values(?,?,?,?,?,?,?,?)" % table_name, ret)
else:
result_db.execute(
"insert into %s(vuln_segement, reuse_func, status) values(?,?,?)"
% table_name, ret)
result_db.commit()
except Exception as e:
result_db.execute(
"insert into %s(vuln_segement, reuse_func, status) values(?,?,?)"
% table_name, (row[0].value, row[2].value, "failed"))
print e
print "firefox reuse works done"
def astlevel_comp_proc():
db_conn = get_connection()
if db_conn is None:
print u"数据库连接失败"
return
#选择所有ffmpeg的漏洞函数
cur = db_conn.cursor()
cur.execute("select * from vulnerability_info")
rets = cur.fetchall()
func_names = []
for ret in rets:
vuln_info = vulnerability_info(ret)
cve_info = vuln_info.get_cve_info(db_conn)
soft = cve_info.get_soft(db_conn)
if soft.software_name == "ffmpeg":
func_names.append(cve_info.cveid.upper().replace("-", "_") + "_VULN_" + vuln_info.vuln_func )
#特征数据库，默认开启在7474端口
db2 = Graph() #默认连接7474端口
db1 = Graph("http://localhost:7475/db/data") #假设7475端口是某ffmpeg的图形数据库
suffix_tree_obj = suffixtree()
wb = Workbook()
ws = wb.active
ws.title = u"AST函数级漏洞查找测试结果"
header = [u'漏洞函数名', u"漏洞文件", u"漏洞函数", "distinct_type_and_const" , "distinct_const_no_type",
"distinct_type_no_const", "no_type_no_const", "耗时"]
ws.append(header)
wb.save("ast_func.xlsx")
all_funcs = get_all_functions(db2)
for name in func_names:
try:
func_similarity_astLevel(db1, all_funcs, db2, name, suffix_tree_obj, ws)
wb.save("ast_func.xlsx")
except:
print "error occured"
suffix_tree_obj.close()
print "all works done!"
def firefox_code_reuse(table_name):
result_db = sqlite3.connect("/home/bert/Documents/data/firefox_code_reuse.db")
result_db.execute('''create table if not exists %s(
vuln_segement CHAR(50) NOT NULL,
reuse_func CHAR(50) NOT NULL,
status CHAR(10) NOT NULL,
distinct_type_and_const BOOLEAN,
distinct_const_no_type BOOLEAN,
distinct_type_no_const BOOLEAN,
no_type_no_const BOOLEAN,
no_mapping BOOLEAN)
''' % table_name)
db = Graph("http://127.0.0.1:7474/db/data/")
suffix_obj = suffixtree()
worksheet = load_workbook("/home/bert/Documents/data/firefox_reuse.xlsx").active
for row in worksheet.rows:
#check
ret = result_db.execute("select * from %s where vuln_segement=? and reuse_func=?" % table_name, (row[0].value, row[2].value))
if ret.fetchone():
continue
vuln_seg = row[0].value
vuln_name = vuln_seg[:14] + "VULN_" + row[1].value
try:
var_map = get_type_mapping_table(db, vuln_name)
ret = search_vuln_seg_in_func(db, row[0].value, row[1].value, var_map, db, row[2].value, suffix_obj)
if ret[2] == "success":
result_db.execute("insert into %s values(?,?,?,?,?,?,?,?)" % table_name, ret)
else:
result_db.execute("insert into %s(vuln_segement, reuse_func, status) values(?,?,?)" % table_name, ret)
result_db.commit()
except Exception as e:
result_db.execute("insert into %s(vuln_segement, reuse_func, status) values(?,?,?)" % table_name,
(row[0].value, row[2].value, "failed") )
print e
print "firefox reuse works done"
def ffmpeg_test(softdb):
segements = get_ffmpeg_segements()
soft_funcs = get_ffmpeg_funcs(softdb)
suffixtree_obj = suffixtree()
wb = Workbook()
ws = wb.active
func_db = sqlite3.connect("/home/bert/Documents/data/all_funcs.db")
for func in soft_funcs:
for seg in segements:
ret = func_db.execute("select * from all_funcs where func_name='%s'" % seg)
seg_ast = ret.fetchone()
seg_ast1 = re.sub(r"^FunctionDef\([0-9]+\);CompoundStatement\([0-9]+\);", "", seg_ast[4])
seg_ast2 = re.sub(r"^FunctionDef\([0-9]+\);CompoundStatement\([0-9]+\);", "", seg_ast[5])
seg_ast3 = re.sub(r"^FunctionDef\([0-9]+\);CompoundStatement\([0-9]+\);", "", seg_ast[6])
seg_ast4 = re.sub(r"^FunctionDef\([0-9]+\);CompoundStatement\([0-9]+\);", "", seg_ast[7])
seg_ast5 = re.sub(r"^FunctionDef\([0-9]+\);CompoundStatement\([0-9]+\);", "", seg_ast[8])
print "[%s] processing %s VS %s" % (
datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S"),
seg, func[1])
try:
ast1_ret = suffixtree_obj.search(func[4], seg_ast1)
ast2_ret = suffixtree_obj.search(func[5], seg_ast2)
ast3_ret = suffixtree_obj.search(func[6], seg_ast3)
ast4_ret = suffixtree_obj.search(func[7], seg_ast4)
ast_nomap = suffixtree_obj.search(func[8], seg_ast5) #nomap
if ast1_ret or ast2_ret or ast3_ret or ast4_ret or ast_nomap:
line = (seg, func[1], func[0], func[2], ast1_ret, ast2_ret, ast3_ret, ast4_ret, ast_nomap)
ws.append(line)
wb.save(os.path.basename(softdb)[:-3] + ".xlsx")
except Exception,e:
print e
def vuln_patch_comp_proc():
db_conn = get_connection()
if db_conn is None:
print u"数据库连接失败"
return
neo4jdb = Graph()
suffix_tree_obj = suffixtree()
cur = db_conn.cursor()
cur.execute("select * from vulnerability_info")
rets = cur.fetchall()
infos = []
for ret in rets:
soft = vulnerability_info(ret).get_cve_info(db_conn).get_soft(db_conn)
if soft.software_name == "ffmpeg":
infos.append(ret)
wb = Workbook()
ws = wb.active
ws.title = u"测试结果"
header = [u'CVE编号', u"软件版本", u"漏洞函数", u"漏洞文件",u"状态", "distinct_type_and_const" , "distinct_const_no_type",
"distinct_type_no_const", "no_type_no_const", "cost"]
ws.append(header)
for info in infos:
try:
vuln_patch_compare(db_conn, neo4jdb, vulnerability_info(info), ws, suffix_tree_obj)
wb.save("ast_result.xlsx")
except Exception as e:
print e
suffix_tree_obj.close()
print "all works done!"
def func_similarity_segement_level(db1, funcs, db2, func_name, db_table):
# @db1 待比对数据库
# @db2 代码段数据库
# @func_name 代码段构成的函数名
neo4j_db1 = Graph(db1)
neo4j_db2 = Graph(db2)
suffix_tree_obj = suffixtree()
#sqlite
db_conn = sqlite3.connect("/home/bert/Documents/data/soft_test.db")
db_conn.execute("""create table if not exists %s(
func_id INT PRIMARY KEY,
func_name CHAR(100) NOT NULL,
file CHAR(200) NOT NULL,
vuln_segement CHAR(100) NOT NULL,
distinct_type_and_const BOOLEAN,
distinct_const_no_type BOOLEAN,
distinct_type_no_const BOOLEAN,
no_type_no_const BOOLEAN)""" % db_table)
db_conn.commit()
target_func = get_function_ast_root(neo4j_db2, func_name)
if target_func is None:
print "%s is not found" % func_name
return
ret = serializedAST(neo4j_db2).genSerilizedAST(target_func)
pattern1 = ";".join(ret[0][2:])
pattern2 = ";".join(ret[1][2:])
pattern3 = ";".join(ret[2][2:])
pattern4 = ";".join(ret[3][2:])
for func in funcs:
print "[%s] processing %s VS %s" % (
datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S"),
func[1], func_name)
ast_root = get_function_ast_root(neo4j_db1, func[0])
if ast_root is None:
print "function not found:", func[0], func[1]
tmp = serializedAST(neo4j_db1).genSerilizedAST(ast_root)
s1 = ";".join(tmp[0])
s2 = ";".join(tmp[1])
s3 = ";".join(tmp[2])
s4 = ";".join(tmp[3])
report = {}
try:
if suffix_tree_obj.search(s1, pattern1):
report['distinct_type_and_const'] = True
else:
report['distinct_type_and_const'] = False
if suffix_tree_obj.search(s2, pattern2):
report['distinct_const_no_type'] = True
else:
report['distinct_const_no_type'] = False
if suffix_tree_obj.search(s3, pattern3):
report['distinct_type_no_const'] = True
else:
report['distinct_type_no_const'] = False
if suffix_tree_obj.search(s4, pattern4):
report['no_type_no_const'] = True
else:
report['no_type_no_const'] = False
query = "insert into %s values(?,?,?,?,?,?,?,?)" % db_table
db_conn.execute(query, (func[0], func[1], func[2], func_name, report['distinct_type_and_const'],
report['distinct_const_no_type'],
report['distinct_type_no_const'],
report['no_type_no_const'])
)
db_conn.commit()
except Exception,e:
log_file = open("suffix_tree_error.log","a")
log_file.writelines(
[datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S") + " " + e.__str__(),
s1, pattern1])
log_file.flush()
log_file.close()
except Exception:
print u"数据库连接失败:7474"
wb = load_workbook("test3.xlsx", read_only=True)
ws = wb[u'Sheet3']
workbook = Workbook()
worksheet = workbook.active
worksheet.title = u"AST代码段测试结果"
header = [
u'漏洞段', u"无漏洞段", u"计算状态", u"区分类型和常量", u"区分常量不区分类型", u"区分类型不区分常量",
u"不区分常量和类型", u"耗时", u"原漏洞函数", u"类型映射"
]
worksheet.append(header)
suffix_tree_obj = suffixtree()
for row in ws.rows:
type_mapping = {'other': 'v'}
if row[2].value != 0:
func_name = row[0].value[:19] + row[2].value
type_mapping = get_type_mapping_table(org_db, func_name)
try:
segement_ast_similarity_process(row[0].value, row[1].value,
neo4jdb, row[2].value,
type_mapping, worksheet,
suffix_tree_obj)
workbook.save("ast_segement_result.xlsx")
except Exception as e:
print "process " + row[0].value + "error"
print e
try:
org_db = py2neo.Graph()
except Exception:
print u"数据库连接失败:7474"
wb = load_workbook("test3.xlsx", read_only=True)
ws = wb[u"Sheet3"]
workbook = Workbook()
worksheet = workbook.active
worksheet.title = u"AST代码段测试结果"
header = [u"漏洞段", u"无漏洞段", u"计算状态", u"区分类型和常量", u"区分常量不区分类型", u"区分类型不区分常量", u"不区分常量和类型", u"耗时", u"原漏洞函数", u"类型映射"]
worksheet.append(header)
suffix_tree_obj = suffixtree()
for row in ws.rows:
type_mapping = {"other": "v"}
if row[2].value != 0:
func_name = row[0].value[:19] + row[2].value
type_mapping = get_type_mapping_table(org_db, func_name)
try:
segement_ast_similarity_process(
row[0].value, row[1].value, neo4jdb, row[2].value, type_mapping, worksheet, suffix_tree_obj
)
workbook.save("ast_segement_result.xlsx")
except Exception as e:
print "process " + row[0].value + "error"
print e
suffix_tree_obj.close()