def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Helgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
os.chdir(os.path.expanduser(benchmark_path))
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Helgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
os.chdir(os.path.expanduser(benchmark_path))
class UBSan:
def get_name(self):
return self.name
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "UBSan"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=undefined -fsanitize=integer", LD="clang")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
if len(stderr) > 0:
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if "runtime error" in (stdout + stderr).lower():
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class ASan:
def get_name(self):
return self.name
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "ASan"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -O2 -fsanitize=address", LD="clang")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
if len(stderr) > 0:
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class Valgrind:
def get_name(self):
return self.name
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Valgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="gcc", CFLAGS="", LD="gcc")
self.pipeline.run_bechmark(self, ["valgrind", "--error-exitcode=10"], 6)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class Helgrind:
def get_name(self):
return self.name
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Helgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="gcc", CFLAGS="-g", LD="gcc")
self.pipeline.run_bechmark(
self, ["valgrind", "--tool=helgrind", "--error-exitcode=10"], 6)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
return self.tp_set
def get_fp_set(self):
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class Valgrind:
def get_name(self):
return self.name
def build(self):
if "Makefile" in os.listdir(os.getcwd()):
subprocess.check_call(["make", "clean"])
subprocess.check_call(["autoreconf", "--install"])
subprocess.check_call(["automake"])
subprocess.check_call(["./configure", "CFLAGS=-g"])
subprocess.check_call(["make"], stderr=subprocess.STDOUT)
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Valgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="gcc", CFLAGS="", LD="gcc")
self.pipeline.run_bechmark(self, ["valgrind", "--error-exitcode=10"],
6)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
def cleanup(self):
self.logger.close_log()
class TSan:
def get_name(self):
return self.name
def build(self):
if "Makefile" in os.listdir(os.getcwd()):
subprocess.check_call(["make", "clean"])
subprocess.check_call(["autoreconf", "--install"])
subprocess.check_call(["automake"])
subprocess.check_call(["./configure", "CFLAGS=-g"])
subprocess.check_call(["make"], stderr=subprocess.STDOUT)
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "TSan"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=thread", LD="clang")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
if len(stderr) > 0:
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if "ThreadSanitizer: reported" in stderr:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set))
Tool.cleanup(self)
self.logger.close_log()
class RVMatch:
def get_name(self):
return self.name
def check_output(self, output):
print "Checking output " + output
error_regex = re.compile('(UB|CV|IMPL|L|USP)\-([A-Z]+[0-9]*)')
if re.search(error_regex, output):
return True
return False
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "RVMatch"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="kcc", CFLAGS="-flint", LD="kcc")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
output = stdout + stderr
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if self.check_output(output):
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(output, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(output, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(output, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class Helgrind:
def get_name(self):
return self.name
def build(self):
if "Makefile" in os.listdir(os.getcwd()):
subprocess.check_call(["make", "clean"])
subprocess.check_call(["autoreconf", "--install"])
subprocess.check_call(["automake"])
subprocess.check_call(["./configure", "CFLAGS=-g"])
subprocess.check_call(["make"], stderr=subprocess.STDOUT)
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "Helgrind"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="gcc", CFLAGS="-g", LD="gcc")
self.pipeline.run_bechmark(self, ["valgrind", "--tool=helgrind", "--error-exitcode=10"], 6)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
return self.tp_set
def get_fp_set(self):
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
def cleanup(self):
Tool.cleanup(self)
self.logger.close_log()
class RVMatch:
def get_name(self):
return self.name
def check_output(self, output):
print "Checking output " + output
error_regex = re.compile('(UB|CV|IMPL|L|USP)\-([A-Z]+[0-9]*)')
if re.search(error_regex, output):
return True
return False
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "RVMatch"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="kcc", CFLAGS="-flint", LD="kcc")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
output = stdout + stderr
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if self.check_output(output):
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(output, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(output, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(output, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
self.pipeline.clean_benchmark()
self.logger.close_log()
class ASan:
def get_name(self):
return self.name
def build(self):
if "Makefile" in os.listdir(os.getcwd()):
subprocess.check_call(["make", "clean"])
subprocess.check_call(["autoreconf", "--install"])
subprocess.check_call(["automake"])
subprocess.check_call(["./configure", "CFLAGS=-g"])
subprocess.check_call(["make"], stderr=subprocess.STDOUT)
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "ASan"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -O2 -fsanitize=address", LD="clang")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
if len(stderr) > 0:
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
print "Numbers for " + self.name
print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set))
print "TP Count = " + str(len(self.tp_set))
print "FP Count = " + str(len(self.fp_set))
print "Negatives Count = " + str(self.neg_count)
Tool.cleanup(self)
self.logger.close_log()
Tool.cleanup(self)
self.logger.close_log()
class MSan:
def get_name(self):
return self.name
def __init__(self, benchmark_path, log_file_path):
self.pipeline = MakePipeline(benchmark_path)
self.name = "MSan"
self.logger = Logger(log_file_path, self.name)
self.output_dict = {}
self.tp_set = set([])
self.fp_set = set([])
self.neg_count = 0
os.chdir(os.path.expanduser(benchmark_path))
def run(self):
self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=memory", LD="clang")
self.pipeline.run_bechmark(self, [], 2)
return self.output_dict
def get_output_dict(self):
return self.output_dict
def get_tp_set(self):
print len(self.tp_set)
return self.tp_set
def get_fp_set(self):
print len(self.fp_set)
return self.fp_set
def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j):
if len(stderr) > 0:
print(stderr)
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
if exit_code != 0:
if "w_Defects" in cur_dir:
self.output_dict[i]["TP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "TP")
self.tp_set.add((i, j))
else:
self.output_dict[i]["FP"] += 1
self.logger.log_output(stderr, i, cur_dir, j, "FP")
self.fp_set.add((i, j))
else:
self.logger.log_output(stdout, i, cur_dir, j, "NEG")
self.neg_count += 1
def analyze_timeout(self, cur_dir, i, j):
if i not in self.output_dict:
self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0}
self.output_dict[i]["count"] += 1
self.logger.log_output("", i, cur_dir, j, "NEG")
self.neg_count += 1
def cleanup(self):
print "Numbers for " + self.name
print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set))
print "TP Count = " + str(len(self.tp_set))
print "FP Count = " + str(len(self.fp_set))
print "Negatives Count = " + str(self.neg_count)
Tool.cleanup(self)
self.logger.close_log()
