def summarize_results(cve_list, bugged_dll_file, bugged_function_file,
dll_matrix, function_matrix, xref_matrix,
results_file_name):
header = [
"cve_number", "bugged_dll", "bugged_function", "dll_precision",
"dll_recall", "dll_comps_number", "dll_failed_number",
"function_precision", "function_recall", "xref_precision",
"xref_recall"
]
csv_lines = [header]
for cve in cve_list:
fuzzing_dir = os.path.join(r"C:\vulnerabilities\ImageMagick_exploited",
cve, "fuzzing")
bugged_dll = open(os.path.join(fuzzing_dir,
bugged_dll_file)).read().replace(
"\n", "")
bugged_function = open(os.path.join(
fuzzing_dir, bugged_function_file)).read().replace("\n", "")
dll_instance = readPlanningFile(os.path.join(fuzzing_dir, dll_matrix))
dll_precision, dll_recall = dll_instance.calc_precision_recall()
function_precision, function_recall = readPlanningFile(
os.path.join(fuzzing_dir,
function_matrix)).calc_precision_recall()
xref_precision, xref_recall = xref_diagnose(
readPlanningFile(os.path.join(fuzzing_dir, xref_matrix)))
csv_lines.append([
cve, bugged_dll, bugged_function, dll_precision, dll_recall,
function_precision, function_recall, xref_precision, xref_recall
])
with open(results_file_name, "wb") as f:
writer = csv.writer(f)
writer.writerows(csv_lines)
def get_results_objects_for_instance(instance_to_diagnose, sep):
optimized_matrix = tempfile.mktemp(prefix="optimized_")
reduced_matrix = tempfile.mktemp(prefix="reduced_")
merged_matrix = tempfile.mktemp(prefix="merged_")
merged_reduced_matrix = tempfile.mktemp(prefix="merged_reduced_")
save_ds_to_matrix_file(instance_to_diagnose.initials_to_DS().optimize(),
optimized_matrix)
optimized_instance = readPlanningFile(optimized_matrix)
optimized_instance.diagnose()
optimized_results = get_results_by_sep(optimized_instance, sep)
save_ds_to_matrix_file(
optimized_instance.initials_to_DS().remove_duplicate_tests(),
reduced_matrix)
write_merged_matrix(optimized_instance, merged_matrix)
reduced_instance = readPlanningFile(reduced_matrix)
reduced_instance.diagnose()
reduced_results = get_results_by_sep(reduced_instance, sep)
merged_instance = readPlanningFile(merged_matrix)
merged_instance.diagnose()
merged_results = get_results_by_sep(merged_instance, sep)
save_ds_to_matrix_file(
merged_instance.initials_to_DS().remove_duplicate_tests(),
merged_reduced_matrix)
merged_reduced_instance = readPlanningFile(merged_reduced_matrix)
merged_reduced_instance.diagnose()
merged_reduced_results = get_results_by_sep(merged_reduced_instance, sep)
return optimized_results, reduced_results, merged_results, merged_reduced_results
def check_influence():
base = readPlanningFile(r"c:\temp\base_matrix.txt")
base.diagnose()
base_results = Diagnosis_Results(base.diagnoses, base.initial_tests,
base.error)
added = readPlanningFile(r"c:\temp\added_matrix.txt")
added.diagnose()
added_results = Diagnosis_Results(added.diagnoses, added.initial_tests,
added.error)
pass
def abstraction():
write_planning_file(
r"c:\temp\yemp_matrix.txt", ["a"],
[["T1", ["a", "b", "d"], 1], ["T2", ["b"], 0],
["T3", ["a", "b", "c"], 1], ["T4", ["a", "b", "c"], 0]])
instance = readPlanningFile(r"c:\temp\yemp_matrix.txt")
write_planning_file(r"c:\temp\yemp_matrix.txt", ["a"],
[["T1", ["a", "b", "d"], 1], ["T2", ["b"], 0],
["T3", ["a", "b"], 1], ["T4", ["a", "b"], 0]])
instance = readPlanningFile(r"c:\temp\yemp_matrix.txt")
write_planning_file(r"c:\temp\yemp_matrix.txt", ["a"],
[["T1", ["a", "b"], 1], ["T2", ["b"], 0],
["T3", ["a", "b"], 1], ["T4", ["a", "b"], 0]])
instance = readPlanningFile(r"c:\temp\yemp_matrix.txt")
print "a"
def executeTests():
web_prediction_results = utilsConf.get_configuration().web_prediction_results
matrix_path = os.path.join(web_prediction_results, "matrix_{0}_{1}.matrix")
outcomes_path = os.path.join(web_prediction_results, "outcomes.json")
diagnoses_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.matrix")
diagnoses_json_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.json")
tested_repo = utilsConf.to_short_path(os.path.join(utilsConf.get_configuration().workingDir, "testedVer", "repo"))
test_runner = TestRunner(tested_repo, AmirTracer(tested_repo, utilsConf.get_configuration().amir_tracer, utilsConf.get_configuration().DebuggerTests))
test_runner.run()
tests = test_runner.get_tests()
json_observations = map(lambda test: test_runner.observations[test].as_dict(), tests)
with open(outcomes_path, "wb") as f:
f.write(json.dumps(json_observations))
for granularity in utilsConf.get_configuration().prediction_files:
for bugged_type in utilsConf.get_configuration().prediction_files[granularity]:
components_priors = get_components_probabilities(bugged_type, granularity, test_runner, tests)
tests_details = map(
lambda test_name: (test_name, list(set(test_runner.tracer.traces[test_name].get_trace(granularity)) & set(components_priors.keys())),
test_runner.observations[test_name].get_observation()),
tests)
matrix = matrix_path.format(granularity, bugged_type)
write_planning_file(matrix, [], filter(lambda test: len(test[1]) > 0, tests_details),
priors=components_priors)
inst = readPlanningFile(matrix)
inst.diagnose()
named_diagnoses = sorted(inst.get_named_diagnoses(), key=lambda d: d.probability, reverse=True)
with open(diagnoses_path.format(granularity, bugged_type), "wb") as diagnosis_file:
diagnosis_file.writelines("\n".join(map(lambda d: repr(d), named_diagnoses)))
with open(diagnoses_json_path.format(granularity, bugged_type), "wb") as diagnosis_json:
diagnosis_json.writelines(json.dumps(map(lambda d: dict([('_name', d[0])] + d[1].as_dict().items()), enumerate(named_diagnoses))))
return test_runner
def executeTests():
web_prediction_results = utilsConf.get_configuration().web_prediction_results
matrix_path = os.path.join(web_prediction_results, "matrix_{0}_{1}.matrix")
outcomes_path = os.path.join(web_prediction_results, "outcomes.json")
diagnoses_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.matrix")
diagnoses_json_path = os.path.join(web_prediction_results, "diagnosis_{0}_{1}.json")
tested_repo = utilsConf.to_short_path(os.path.join(utilsConf.get_configuration().workingDir, "version_to_test_trace", "repo"))
utilsConf.open_subprocess(["git", "-C", tested_repo, 'checkout', '--', '.'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
test_runner = TestRunner(tested_repo, utilsConf.get_configuration().traces_dir)
test_runner.run()
tests = test_runner.get_tests()
json_observations = map(lambda test: test_runner.observations[test].as_dict(), tests)
with open(outcomes_path, "wb") as f:
f.write(json.dumps(json_observations))
for bugged_type in utilsConf.get_configuration().prediction_files:
for granularity in utilsConf.get_configuration().prediction_files[bugged_type]:
components_priors = get_components_probabilities(bugged_type, granularity, test_runner, tests)
tests_details = map(
lambda test_name: (test_name, list(set(test_runner.traces[test_name].get_trace(granularity)) & set(components_priors.keys())),
test_runner.observations[test_name].get_observation()),
tests)
matrix = matrix_path.format(granularity, bugged_type)
write_planning_file(matrix, [], filter(lambda test: len(test[1]) > 0, tests_details),
priors=components_priors)
inst = readPlanningFile(matrix)
inst.diagnose()
named_diagnoses = sorted(inst.get_named_diagnoses(), key=lambda d: round(d.probability, 2), reverse=True)
with open(diagnoses_path.format(granularity, bugged_type), "wb") as diagnosis_file:
diagnosis_file.writelines("\n".join(map(lambda d: repr(d), named_diagnoses)))
with open(diagnoses_json_path.format(granularity, bugged_type), "wb") as diagnosis_json:
diagnosis_json.writelines(json.dumps(map(lambda d: dict([('_name', d[0])] + d[1].as_dict().items()), enumerate(named_diagnoses))))
return test_runner
def full_results(base_dir,
cve_list,
dll_matrix_file_name=consts.DLL_MATRIX,
function_matrix_file_name=consts.FUNCTION_MATRIX,
dominator_matrix_file_name=consts.DOMINATOR_MATRIX,
xref_matrix_file_name=consts.XREF_MATRIX,
results_file_name=None):
header = ["cve_number", "granularity", "matrix_type"]
csv_lines = []
added_results_header = False
for cve in cve_list:
fuzzing_dir = os.path.join(base_dir, cve, "fuzzing")
dll_matrix = os.path.join(fuzzing_dir, dll_matrix_file_name)
function_matrix = os.path.join(fuzzing_dir, function_matrix_file_name)
dominator_matrix = os.path.join(fuzzing_dir,
dominator_matrix_file_name)
xref_matrix = os.path.join(fuzzing_dir, xref_matrix_file_name)
for granularity, instance_file, sep in zip(
["dll", "function", "dominator", "code blocks"],
[dll_matrix, function_matrix, dominator_matrix, xref_matrix],
[None, "&", "&", "&"]):
# for granularity, instance_file, sep in zip(["dll", "entry_points", "function", "xref"], [dll_matrix, entry_matrix, function_matrix, xref_matrix], [None, None, None, "$"]):
print cve, granularity
if not os.path.exists(instance_file):
continue
csv_lines.extend(
instance_diagnosis(added_results_header, cve,
granularity, header,
readPlanningFile(instance_file), sep))
added_results_header = True
with open(results_file_name, "wb") as f:
writer = csv.writer(f)
writer.writerows(csv_lines)
def calc_result_fo_planning_file(planning_file, out_file):
global instance
instance = readPlanningFile(planning_file)
precision, recall = instance.calc_precision_recall()
csv_output = [["precision", "recall"], [precision, recall]]
print planning_file, precision, recall
with open(out_file, "wb") as f:
writer = csv.writer(f)
writer.writerows(csv_output)
def get_instance_results(planning_file):
"""
return the following results for the planning file:
precision
recall
wasted
#comps
#failed comps
"""
instance = readPlanningFile(planning_file)
precision, recall = instance.calc_precision_recall()
wasted = instance.calc_wasted_components()
ds = instance.initials_to_DS()
comps_num = len(ds.get_components())
failed_comps = len(ds.get_components_in_failed_tests())
def create_instances(self):
MATRIX_PATH = os.path.join(utilsConf.get_configuration().experiments,
"{ITERATION}_{BUG_ID}_{GRANULARITY}_{BUGGED_TYPE}.matrix")
DESCRIPTION = 'sample bug id {BUG_ID} with bug_passing_probability = {PROB} with garnularity of {GRANULARITY} and bugged type {BUGGED_TYPE}'
i = 0.0
results = AvgResults()
while i < self.num_instances:
bug = random.choice(self.bugs)
buggy_components = set()
for component in set(self.components_priors.keys()):
for buggy in bug.get_buggy_components(self.granularity, self.bugged_type):
if component in buggy:
buggy_components.add(component)
if len(buggy_components) == 0:
continue
tests = reduce(set.__or__,
map(lambda x: self.test_runner.get_packages_tests().get('.'.join(x[:random.randint(0, len(x))]), set()),
map(lambda file_name: file_name.replace('.java', '').split('.'), buggy_components)),
set())
if len(tests) < self.tests_per_instance:
continue
relevant_tests = random.sample(tests, self.tests_per_instance)
tests_details = []
for test_name in relevant_tests:
trace = list(set(self.test_runner.tracer.traces[test_name].get_trace(self.granularity)) & set(
self.components_priors.keys()))
tests_details.append((test_name, trace, self.sample_observation(trace, buggy_components)))
if sum(map(lambda x: x[2], tests_details)) == 0:
continue
matrix = MATRIX_PATH.format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
write_planning_file(matrix, list(buggy_components),
filter(lambda test: len(test[1]) > 0, tests_details),
priors=self.components_priors,
description=DESCRIPTION.format(BUG_ID=bug.bug_id, PROB=self.bug_passing_probability,
GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type))
inst = readPlanningFile(matrix)
inst.diagnose()
res = Diagnosis_Results(inst.diagnoses, inst.initial_tests, inst.error)
results.update(res)
print "created instance num {ITERATION} with bugid {BUG_ID} for granularity {GRANULARITY} and type {BUGGED_TYPE}".\
format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
i += 1
return results.results
def check_fuzzing_influence(base_dir,
cves,
dll_matrix_file_name=consts.DLL_MATRIX,
function_matrix_file_name=consts.FUNCTION_MATRIX,
dominator_matrix_file_name=consts.DOMINATOR_MATRIX,
xref_matrix_file_name=consts.XREF_MATRIX,
results_file_name=None):
number_files_to_read = range(1, 20) + [20, 50, 100, 250, 400]
tmp_matrix = tempfile.mktemp(dir=r"C:\temp", prefix="tmp_matrix")
print tmp_matrix
header = ["cve_number", "granularity", "matrix_type", "fuzzing_files"]
csv_lines = []
added_results_header = False
for cve in cves:
fuzzing_dir = os.path.join(base_dir, cve, "fuzzing")
if not os.path.exists(fuzzing_dir):
continue
dll_matrix = os.path.join(fuzzing_dir, dll_matrix_file_name)
function_matrix = os.path.join(fuzzing_dir, function_matrix_file_name)
dominator_matrix = os.path.join(fuzzing_dir,
dominator_matrix_file_name)
xref_matrix = os.path.join(fuzzing_dir, xref_matrix_file_name)
for granularity, instance_file, sep in zip(
["dll", "function", "dominator", "code blocks"],
[dll_matrix, function_matrix, dominator_matrix, xref_matrix],
[None, None, "&", "&"]):
for number in number_files_to_read:
instance = readPlanningFile(instance_file)
instance.initial_tests = instance.initial_tests[:number]
print cve, granularity, number
csv_lines.extend(
instance_diagnosis(added_results_header, cve, granularity,
header, instance, sep, [number]))
added_results_header = True
with open(results_file_name, "wb") as f:
writer = csv.writer(f)
writer.writerows(csv_lines)
from sfl_diagnoser.Diagnoser.diagnoserUtils import readPlanningFile
from sfl_diagnoser.Diagnoser.Diagnosis_Results import Diagnosis_Results
import sfl_diagnoser.Diagnoser.ExperimentInstance
inst = readPlanningFile(
"C:\\Users\\eyalhad\\Desktop\\SFL-diagnoser\\MatrixFile2.txt")
inst.diagnose()
results = Diagnosis_Results(inst.diagnoses, inst.initial_tests, inst.error)
results.get_metrics_names()
results.get_metrics_values()
ei = sfl_diagnoser.Diagnoser.ExperimentInstance.addTests(
inst, inst.hp_next_by_prob())
i = 5
# fileName = 'CVE-2017-9204-transformed'
# fileName = 'CVE-2017-9205-transformed'
# fileName = 'CVE-2017-9206-transformed'
# fileName = 'CVE-2017-9207-transformed'
################################## file path ###################################
# file123 = open('sfl_diagnoser/output-single/xrefs/' + fileName + '-out.txt' ,'w')
# file123 = open('sfl_diagnoser/output/xrefs/' + fileName + '-out.txt' ,'w')
file123 = open(
'sfl_diagnoser/output-single/dominators/' + fileName + '-out.txt', 'w')
# file123 = open('sfl_diagnoser/output/dominators/' + fileName + '-out.txt' ,'w')
# inst = diagnoserUtils.readPlanningFile('sfl_diagnoser/input-single/xrefs/' + fileName + '.txt')
# inst = diagnoserUtils.readPlanningFile('sfl_diagnoser/input/xrefs/' + fileName + '.txt')
inst = diagnoserUtils.readPlanningFile(
'sfl_diagnoser/input-single/dominators/' + fileName + '.txt')
# inst = diagnoserUtils.readPlanningFile('sfl_diagnoser/input/dominators/' + fileName + '.txt')
################################## calculate diagnoses ###################################
inst.diagnose()
results = Diagnosis_Results.Diagnosis_Results(inst.diagnoses,
inst.initial_tests, inst.error)
################################## check diagnoses size from file ###################################
#get components numbers and prob
probabilities = ([x.probability for x in inst.diagnoses])
probability = probabilities[0]
print("\n")
print("prob: " + str(probability))
from sfl_diagnoser.Diagnoser.diagnoserUtils import readPlanningFile, write_planning_file, write_merged_matrix, writePlanningFileProbabilities, write_merged_matrix_with_new_probabilities
# preprocessing - put the probabilities into the new matrix
write_merged_matrix_with_new_probabilities(
"./all_methods_prediction.csv", "./matrices/Math_21.txt",
"./matrices/Math_21-with-probs.txt")
base = readPlanningFile(r"./matrices/Math_21-with-probs.txt")
base.diagnose()
from sfl_diagnoser.Diagnoser.Diagnosis_Results import Diagnosis_Results
res = Diagnosis_Results(base.diagnoses, base.initial_tests, base.error)
print res.get_metrics_names()
print res.get_metrics_values()
print res.diagnoses
print res.get_new_probabilities()
new_probabilities = res.get_new_probabilities()
writePlanningFileProbabilities("./matrices/Math_21-with-probs.txt",
"./matrices/Math_21-with-probs-new.txt",
new_probabilities)
# round 2
print "------------"
base = readPlanningFile(r"./matrices/Math_21-with-probs-new.txt")
base.diagnose()
from sfl_diagnoser.Diagnoser.Diagnosis_Results import Diagnosis_Results
res = Diagnosis_Results(base.diagnoses, base.initial_tests, base.error)
print res.get_metrics_names()
print res.get_metrics_values()
print res.diagnoses
# ei = sfl_diagnoser.Diagnoser.ExperimentInstance.addTests(inst, inst.hp_next())
#
# # check_influence()
# # exit()
# base = readPlanningFile(r"C:\Users\User\Downloads\MatrixFile4.txt")
# from sfl_diagnoser.Planner.HP_Random import main_HP
# main_HP(base)
abstraction()
write_planning_file(r"c:\temp\yemp_matrix.txt", ["a"],
[["T1", ["a", "c"], 1], ["T2", ["b"], 0],
["T3", ["a", "b"], 1], ["T4", ["a", "b", "c"], 0]])
write_planning_file(r"c:\temp\yemp_matrix2.txt", ["a"],
[["T1", ["a", "b"], 1], ["T2", ["b"], 0],
["T3", ["a", "b"], 1], ["T4", ["a", "b"], 0]])
instance = readPlanningFile(r"c:\temp\yemp_matrix.txt")
instance = readPlanningFile(r"c:\temp\yemp_matrix2.txt")
instance = readPlanningFile(
r"C:\Users\User\Dropbox\softwareMbd (1)\Amir_AAAI18\vulnerability example\code_blocks_matrix.txt"
)
instance.get_named_diagnoses()
named = get_xref_diagnoses(instance.get_named_diagnoses(), "#")
# instance = readPlanningFile(r"c:\temp\merged_matrix.txt")
function_instance = readPlanningFile(
r"C:\vulnerabilities\ImageMagick_exploited\CVE-2016-8866\fuzzing\function_matrix.txt"
)
xref_instance = readPlanningFile(
r"C:\vulnerabilities\ImageMagick_exploited\CVE-2016-8866\fuzzing\xref_matrix.txt"
)
# write_merged_matrix(function_instance, r"c:\temp\function_merged_matrix.txt")
write_merged_matrix(xref_instance, r"c:\temp\xref_merged_matrix.txt")
def create_instances(self):
MATRIX_PATH = os.path.join(
utilsConf.get_configuration().experiments,
"{ITERATION}_{BUG_ID}_{GRANULARITY}_{BUGGED_TYPE}.matrix")
DESCRIPTION = 'sample bug id {BUG_ID} with bug_passing_probability = {PROB} with garnularity of {GRANULARITY} and bugged type {BUGGED_TYPE}'
i = 0.0
results = AvgResults()
while i < self.num_instances:
bug = random.choice(self.bugs)
buggy_components = set()
for component in set(self.components_priors.keys()):
for buggy in bug.get_buggy_components(self.granularity,
self.bugged_type):
if component in buggy:
buggy_components.add(component)
if len(buggy_components) == 0:
continue
tests = reduce(
set.__or__,
map(
lambda x: self.test_runner.get_packages_tests().get(
'.'.join(x[:random.randint(0, len(x))]), set()),
map(
lambda file_name: file_name.replace('.java', '').split(
'.'), buggy_components)), set())
if len(tests) < self.tests_per_instance:
continue
relevant_tests = random.sample(tests, self.tests_per_instance)
tests_details = []
for test_name in relevant_tests:
trace = list(
set(self.test_runner.tracer.traces[test_name].get_trace(
self.granularity))
& set(self.components_priors.keys()))
tests_details.append(
(test_name, trace,
self.sample_observation(trace, buggy_components)))
if sum(map(lambda x: x[2], tests_details)) == 0:
continue
matrix = MATRIX_PATH.format(ITERATION=i,
BUG_ID=bug.bug_id,
GRANULARITY=self.granularity,
BUGGED_TYPE=self.bugged_type)
write_planning_file(matrix,
list(buggy_components),
filter(lambda test: len(test[1]) > 0,
tests_details),
priors=self.components_priors,
description=DESCRIPTION.format(
BUG_ID=bug.bug_id,
PROB=self.bug_passing_probability,
GRANULARITY=self.granularity,
BUGGED_TYPE=self.bugged_type))
inst = readPlanningFile(matrix)
inst.diagnose()
res = Diagnosis_Results(inst.diagnoses, inst.initial_tests,
inst.error)
results.update(res)
print "created instance num {ITERATION} with bugid {BUG_ID} for granularity {GRANULARITY} and type {BUGGED_TYPE}".\
format(ITERATION=i, BUG_ID=bug.bug_id, GRANULARITY=self.granularity, BUGGED_TYPE=self.bugged_type)
i += 1
return results.results