def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Helgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) os.chdir(os.path.expanduser(benchmark_path))
class UBSan: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "UBSan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=undefined -fsanitize=integer", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if "runtime error" in (stdout + stderr).lower(): if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class ASan: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "ASan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -O2 -fsanitize=address", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class Valgrind: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Valgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="gcc", CFLAGS="", LD="gcc") self.pipeline.run_bechmark(self, ["valgrind", "--error-exitcode=10"], 6) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class Helgrind: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Helgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="gcc", CFLAGS="-g", LD="gcc") self.pipeline.run_bechmark( self, ["valgrind", "--tool=helgrind", "--error-exitcode=10"], 6) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): return self.tp_set def get_fp_set(self): return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class Valgrind: def get_name(self): return self.name def build(self): if "Makefile" in os.listdir(os.getcwd()): subprocess.check_call(["make", "clean"]) subprocess.check_call(["autoreconf", "--install"]) subprocess.check_call(["automake"]) subprocess.check_call(["./configure", "CFLAGS=-g"]) subprocess.check_call(["make"], stderr=subprocess.STDOUT) def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Valgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="gcc", CFLAGS="", LD="gcc") self.pipeline.run_bechmark(self, ["valgrind", "--error-exitcode=10"], 6) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") def cleanup(self): self.logger.close_log()
class TSan: def get_name(self): return self.name def build(self): if "Makefile" in os.listdir(os.getcwd()): subprocess.check_call(["make", "clean"]) subprocess.check_call(["autoreconf", "--install"]) subprocess.check_call(["automake"]) subprocess.check_call(["./configure", "CFLAGS=-g"]) subprocess.check_call(["make"], stderr=subprocess.STDOUT) def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "TSan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=thread", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if "ThreadSanitizer: reported" in stderr: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set)) Tool.cleanup(self) self.logger.close_log()
class RVMatch: def get_name(self): return self.name def check_output(self, output): print "Checking output " + output error_regex = re.compile('(UB|CV|IMPL|L|USP)\-([A-Z]+[0-9]*)') if re.search(error_regex, output): return True return False def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "RVMatch" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="kcc", CFLAGS="-flint", LD="kcc") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): output = stdout + stderr if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if self.check_output(output): if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(output, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(output, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(output, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): self.pipeline.clean_benchmark() self.logger.close_log()
class Helgrind: def get_name(self): return self.name def build(self): if "Makefile" in os.listdir(os.getcwd()): subprocess.check_call(["make", "clean"]) subprocess.check_call(["autoreconf", "--install"]) subprocess.check_call(["automake"]) subprocess.check_call(["./configure", "CFLAGS=-g"]) subprocess.check_call(["make"], stderr=subprocess.STDOUT) def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "Helgrind" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="gcc", CFLAGS="-g", LD="gcc") self.pipeline.run_bechmark(self, ["valgrind", "--tool=helgrind", "--error-exitcode=10"], 6) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): return self.tp_set def get_fp_set(self): return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") def cleanup(self): Tool.cleanup(self) self.logger.close_log()
class ASan: def get_name(self): return self.name def build(self): if "Makefile" in os.listdir(os.getcwd()): subprocess.check_call(["make", "clean"]) subprocess.check_call(["autoreconf", "--install"]) subprocess.check_call(["automake"]) subprocess.check_call(["./configure", "CFLAGS=-g"]) subprocess.check_call(["make"], stderr=subprocess.STDOUT) def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "ASan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -O2 -fsanitize=address", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): print "Numbers for " + self.name print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set)) print "TP Count = " + str(len(self.tp_set)) print "FP Count = " + str(len(self.fp_set)) print "Negatives Count = " + str(self.neg_count) Tool.cleanup(self) self.logger.close_log() Tool.cleanup(self) self.logger.close_log()
class MSan: def get_name(self): return self.name def __init__(self, benchmark_path, log_file_path): self.pipeline = MakePipeline(benchmark_path) self.name = "MSan" self.logger = Logger(log_file_path, self.name) self.output_dict = {} self.tp_set = set([]) self.fp_set = set([]) self.neg_count = 0 os.chdir(os.path.expanduser(benchmark_path)) def run(self): self.pipeline.build_benchmark(CC="clang", CFLAGS="-g -fsanitize=memory", LD="clang") self.pipeline.run_bechmark(self, [], 2) return self.output_dict def get_output_dict(self): return self.output_dict def get_tp_set(self): print len(self.tp_set) return self.tp_set def get_fp_set(self): print len(self.fp_set) return self.fp_set def analyze_output(self, exit_code, stdout, stderr, cur_dir, i, j): if len(stderr) > 0: print(stderr) if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 if exit_code != 0: if "w_Defects" in cur_dir: self.output_dict[i]["TP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "TP") self.tp_set.add((i, j)) else: self.output_dict[i]["FP"] += 1 self.logger.log_output(stderr, i, cur_dir, j, "FP") self.fp_set.add((i, j)) else: self.logger.log_output(stdout, i, cur_dir, j, "NEG") self.neg_count += 1 def analyze_timeout(self, cur_dir, i, j): if i not in self.output_dict: self.output_dict[i] = {"count": 0, "TP": 0, "FP": 0} self.output_dict[i]["count"] += 1 self.logger.log_output("", i, cur_dir, j, "NEG") self.neg_count += 1 def cleanup(self): print "Numbers for " + self.name print "Total Count = " + str(self.neg_count + len(self.tp_set) + len(self.fp_set)) print "TP Count = " + str(len(self.tp_set)) print "FP Count = " + str(len(self.fp_set)) print "Negatives Count = " + str(self.neg_count) Tool.cleanup(self) self.logger.close_log()