def get_num_passing_failing_variants(mutated_project_dir, list_of_stms, spectrum_coverage_prefix): failing_passing_variants_of_stms = init_num_of_failing_passing_variants( list_of_stms) variants_list = get_all_variant_dirs(mutated_project_dir) total_fails = 0 total_passes = 0 for variant_dir in variants_list: test_coverage_dir = get_test_coverage_dir(variant_dir) spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version( spectrum_coverage_prefix) spectrum_failed_coverage_file_dir = join_path(test_coverage_dir, spectrum_failed_file) spectrum_passed_file = get_spectrum_passed_coverage_file_name_with_version( spectrum_coverage_prefix) spectrum_passed_coverage_file_dir = join_path(test_coverage_dir, spectrum_passed_file) if os.path.isfile(spectrum_failed_coverage_file_dir): failing_passing_variants_of_stms = read_data_from_coverage_file( failing_passing_variants_of_stms, spectrum_failed_coverage_file_dir, VARIANT_NUM_OF_FAILS) total_fails += 1 if not os.path.isfile( spectrum_failed_coverage_file_dir) and os.path.isfile( spectrum_passed_coverage_file_dir): failing_passing_variants_of_stms = read_data_from_coverage_file( failing_passing_variants_of_stms, spectrum_passed_coverage_file_dir, VARIANT_NUM_OF_PASSES) total_passes += 1 return failing_passing_variants_of_stms, total_fails, total_passes
def statement_coverage(variant_dir, spectrum_coverage_prefix): global NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME = spectrum_coverage_prefix + SPECTRUM_PASSED_COVERAGE_FILE_NAME test_coverage_dir = get_test_coverage_dir(variant_dir) spectrum_passed_coverage_file = join_path(test_coverage_dir, NEW_SPECTRUM_PASSED_COVERAGE_FILE_NAME) if not os.path.isfile(spectrum_passed_coverage_file): spectrum_passed_coverage_file = join_path(test_coverage_dir, SPECTRUM_PASSED_COVERAGE_FILE_NAME) num_of_stm = 0 untested_stm = 0 if os.path.isfile(spectrum_passed_coverage_file): try: tree = ET.parse(spectrum_passed_coverage_file) root = tree.getroot() project = root.find("project") for package in project: for file in package: for line in file: num_of_stm += 1 if (int(line.get("count")) == 0): untested_stm += 1 except: logging.info("Exception when parsing %s", spectrum_passed_coverage_file) else: logging.info("spectrum passed coveraged file does not exist in %s", variant_dir) coverage_rate = (num_of_stm - untested_stm) / num_of_stm return coverage_rate
def check_variant_final_test_output(variant_dir, junit_mode=JunitMode.FAST): coverage_dir = get_test_coverage_dir(variant_dir) if is_path_exist(join_path(variant_dir, FAILED_TEST_FLAG_FILE_NAME)): if junit_mode == JunitMode.FULL_COVERAGE and not is_path_exist( join_path(coverage_dir, FAILED_TEST_COVERAGE_FOLDER_NAME)): return None return False elif is_path_exist(join_path(variant_dir, PASSED_TEST_FLAG_FILE_NAME)): if junit_mode == JunitMode.FULL_COVERAGE and not is_path_exist( join_path(coverage_dir, PASSED_TEST_COVERAGE_FOLDER_NAME)): return None return True else: return None
def read_coverage_file(mutated_project_dir): variants_dir = get_variants_dir(mutated_project_dir) variants_list = os.listdir(variants_dir) data = {} for variant in variants_list: variant_dir = get_variant_dir(mutated_project_dir, variant) test_coverage_dir = get_test_coverage_dir(variant_dir) coverage_file = join_path(test_coverage_dir, SPECTRUM_FAILED_COVERAGE_FILE_NAME) if os.path.isfile(coverage_file): data[variant] = [] try: tree = ET.parse(coverage_file) root = tree.getroot() project = root.find("project") for package in project: for file in package: for line in file: id = line.get('featureClass') + "." + line.get('featureLineNum') if id not in data[variant] and int(line.get('count')) != 0: data[variant].append(id) except: logging.info("Exception when parsing %s", coverage_file) return data
def get_buggy_statement(mutated_project_name, mutated_project_dir): mutated_log_file_path = join_path(mutated_project_dir, mutated_project_name + ".mutant.log") mutated_log_file = open(mutated_log_file_path, "r") mutated_log_file_content = mutated_log_file.readline().split(":") buggy_line_number_position_in_log_file = 1 return (".").join(mutated_project_name.split(".")[0:-1]) + "." + mutated_log_file_content[ buggy_line_number_position_in_log_file]
def statement_coverage_of_variants(project_dir, spectrum_coverage_prefix=""): stm_coverage_variants = {} variants_dir = get_variants_dir(project_dir) for variant in list_dir(variants_dir): variant_dir = join_path(variants_dir, variant) testing_coverage = statement_coverage(variant_dir, spectrum_coverage_prefix) stm_coverage_variants[variant] = testing_coverage return stm_coverage_variants
def build_incremental_spectrum_coverages_with_each_test_suite( coverage_file_paths, test_coverage_dir, version): failed_coverage_file_names = [] passed_coverage_file_names = [] failed_coverage_dir = join_path(test_coverage_dir, FAILED_TEST_COVERAGE_FOLDER_NAME) passed_coverage_dir = join_path(test_coverage_dir, PASSED_TEST_COVERAGE_FOLDER_NAME) for file_path in coverage_file_paths: if file_path.startswith(failed_coverage_dir): coverage_file_name = file_path.replace(failed_coverage_dir, "").strip("/") failed_coverage_file_names.append(coverage_file_name) continue if file_path.startswith(passed_coverage_dir): coverage_file_name = file_path.replace(passed_coverage_dir, "").strip("/") passed_coverage_file_names.append(coverage_file_name) continue if failed_coverage_file_names: spectrum_coverage_file_path = get_failed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) rebuild_spectrum_coverage( input_coverage_dir=failed_coverage_dir, spectrum_output_path=spectrum_coverage_file_path, specific_test_cases=failed_coverage_file_names, random=False, max_test_cases=-1) if passed_coverage_file_names: spectrum_coverage_file_path = get_passed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) rebuild_spectrum_coverage( input_coverage_dir=passed_coverage_dir, spectrum_output_path=spectrum_coverage_file_path, specific_test_cases=passed_coverage_file_names, random=False, max_test_cases=-1)
def get_multiple_buggy_statements(mutated_project_name, mutated_project_dir): mutated_log_file_path = join_path(mutated_project_dir, mutated_project_name + ".mutant.log") mutated_log_file = open(mutated_log_file_path, "r") bugs_content = mutated_log_file.readlines() buggy_line_number_position_in_log_file = 1 buggy_statements = [] for item in bugs_content: contents = item.split(":") buggy_statements.append( (".").join(contents[0].split(".")[0:-1]) + "." + contents[buggy_line_number_position_in_log_file]) return buggy_statements
def save_purified_test_suite(failed_variant_dir, new_test_class_name, old_test_file_path, composed_test_suite_source_code): # write new test file to temp_src/ for providing simple slicing (without needing to import JUnit) new_test_file_path = join_path(get_outer_dir(old_test_file_path), new_test_class_name + SOURCE_CODE_EXTENSION) new_test_file_path = new_test_file_path[:len(failed_variant_dir)] + re.sub( f"^/{TEST_FOLDER_NAME}/(main/)?", f"/{TEMP_SRC_FOLDER_NAME}/", new_test_file_path[len(failed_variant_dir):]) with open(new_test_file_path, "w+") as output_file: output_file.write(composed_test_suite_source_code) return new_test_file_path
def aggreate_results(experimental_dirs, num_of_examed_stms): experimental_folder_dir = EXPERIMENT_RESULT_FOLDER aggregation_folder_dir = join_path(experimental_folder_dir, "result_aggreation") print(aggregation_folder_dir) if not os.path.exists(aggregation_folder_dir): os.makedirs(aggregation_folder_dir) all_bugs_file_dir = join_path(aggregation_folder_dir, "all_bugs_temp.xlsx") write_all_bugs_to_a_file(all_bugs_file_dir, experimental_dirs) aggregation_file = join_path(aggregation_folder_dir, "aggreation_result.xlsx") comparison_data = summary_result(all_bugs_file_dir, aggregation_file) hitx_file_dir = join_path(aggregation_folder_dir, "hitx.xlsx") summary_hitx(hitx_file_dir, all_bugs_file_dir, num_of_examed_stms) pbl_file = join_path(aggregation_folder_dir, "pbl.xlsx") summary_pbl(all_bugs_file_dir, pbl_file, num_of_examed_stms) percentage_case_found_bug_file = join_path( aggregation_folder_dir, "percentage_case_found_bug_file.xlsx") summary_percentage_of_cases_found_bugs(all_bugs_file_dir, percentage_case_found_bug_file, num_of_examed_stms) os.remove(all_bugs_file_dir)
def rebuild_incremental_spectrum_coverages_for_variant(variant_dir, version=""): raw_version = version test_coverage_dir = get_test_coverage_dir(variant_dir) failed_coverage_dir = join_path(test_coverage_dir, FAILED_TEST_COVERAGE_FOLDER_NAME) failed_coverage_file_paths = TestingCoverageManager.get_all_coverage_file_paths( failed_coverage_dir) passed_coverage_dir = join_path(test_coverage_dir, PASSED_TEST_COVERAGE_FOLDER_NAME) passed_coverage_file_paths = TestingCoverageManager.get_all_coverage_file_paths( passed_coverage_dir) test_suites = get_incremental_test_suites(failed_coverage_file_paths, passed_coverage_file_paths) for i, test_suite in enumerate(test_suites): coverage_file_paths = test_suite version = f"INoT_{i + 1}_{raw_version}" build_incremental_spectrum_coverages_with_each_test_suite( coverage_file_paths, test_coverage_dir, version)
def get_coverage_infor_of_variants(variant, variant_dir, failing_variants, features_coverage_info, search_spaces, spectrum_coverage_prefix): test_coverage_dir = get_test_coverage_dir(variant_dir) spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version( spectrum_coverage_prefix) spectrum_failed_coverage_file_dir = join_path(test_coverage_dir, spectrum_failed_file) spectrum_passed_file = get_spectrum_passed_coverage_file_name_with_version( spectrum_coverage_prefix) spectrum_passed_coverage_file_dir = join_path(test_coverage_dir, spectrum_passed_file) if variant in failing_variants: if os.path.isfile(spectrum_failed_coverage_file_dir): features_coverage_info = read_coverage_info( variant, features_coverage_info, search_spaces, spectrum_failed_coverage_file_dir, VARIANTS_FAILED) if os.path.isfile(spectrum_passed_coverage_file_dir): features_coverage_info = read_coverage_info( variant, features_coverage_info, search_spaces, spectrum_passed_coverage_file_dir, VARIANTS_FAILED) else: if os.path.isfile(spectrum_failed_coverage_file_dir): features_coverage_info = read_coverage_info( variant, features_coverage_info, search_spaces, spectrum_failed_coverage_file_dir, VARIANTS_PASSED) if os.path.isfile(spectrum_passed_coverage_file_dir): features_coverage_info = read_coverage_info( variant, features_coverage_info, search_spaces, spectrum_passed_coverage_file_dir, VARIANTS_PASSED) return features_coverage_info
def rebuild_spectrum_coverage_for_variant(variant_dir, version=None, random=True): test_coverage_dir = get_test_coverage_dir(variant_dir) failed_coverage_dir = join_path(test_coverage_dir, FAILED_TEST_COVERAGE_FOLDER_NAME) if is_path_exist(failed_coverage_dir): spectrum_failed_coverage_file_path = get_failed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) rebuild_spectrum_coverage( input_coverage_dir=failed_coverage_dir, spectrum_output_path=spectrum_failed_coverage_file_path, random=random) passed_coverage_dir = join_path(test_coverage_dir, PASSED_TEST_COVERAGE_FOLDER_NAME) if is_path_exist(passed_coverage_dir): spectrum_passed_coverage_file_path = get_passed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) rebuild_spectrum_coverage( input_coverage_dir=passed_coverage_dir, spectrum_output_path=spectrum_passed_coverage_file_path, random=random)
def write_runtime_to_file(system_result_dir, run_time, file_name): experiment_file_name = join_path(system_result_dir, file_name) if os.path.exists(experiment_file_name): return wb = Workbook(experiment_file_name) sheet = wb.add_worksheet("run_time") row = 0 for item in run_time.keys(): sheet.write(row, 0, item) col = 1 for time in run_time[item]: sheet.write(row, col, time) col += 1 row += 1 wb.close()
def summary_result(all_bugs_file, summary_file): summary_file_dir = join_path(EXPERIMENT_RESULT_FOLDER, summary_file) wb = Workbook(summary_file_dir) sheet = wb.add_worksheet("sheet1") excel_data_df = pandas.read_excel(all_bugs_file, sheet_name=None) row = 0 header_column = [] for item in default_data_column: if item in excel_data_df[TARANTULA]: header_column.append(item) write_header_in_sumary_file(row, sheet, header_column) row += 1 comparison_data = calculate_average_in_a_file(excel_data_df, row, sheet, header_column) wb.close() return comparison_data
def rebuild_spectrum_coverage_from_specific_test_cases( test_coverage_dir, by_test_result_folder_name, test_cases, version=""): by_result_coverage_dir = join_path(test_coverage_dir, by_test_result_folder_name) if is_path_exist(by_result_coverage_dir): if by_test_result_folder_name == FAILED_TEST_COVERAGE_FOLDER_NAME: spectrum_coverage_file_path = get_failed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) else: spectrum_coverage_file_path = get_passed_spectrum_coverage_file_path_with_version( test_coverage_dir, version) rebuild_spectrum_coverage( input_coverage_dir=by_result_coverage_dir, spectrum_output_path=spectrum_coverage_file_path, specific_test_cases=test_cases, random=False, max_test_cases=-1)
def generate_junit_test_cases(lib_paths, variant_dir): logger.info( f"Generating JUnit Test for variant [{get_file_name_without_ext(variant_dir)}]" ) compiled_classes_dir = get_compiled_source_classes_dir(variant_dir) evosuite_temp_dir = join_path(".evosuite_" + hash_md5(variant_dir)) delete_dir(evosuite_temp_dir) test_cases_dir = get_test_dir(variant_dir, force_mkdir=False) delete_dir(test_cases_dir) output_log = execute_shell_command( f'java -jar {EVOSUITE_PLUGIN_PATH}', extra_args=[ { "-projectCP": ":".join([compiled_classes_dir] + lib_paths) }, { "-seed": 1583738192420 }, { "-target": compiled_classes_dir }, { "-continuous": "execute" }, { "-Dctg_memory": "4000" }, { "-Dctg_cores": "4" }, { "-Dctg_dir": evosuite_temp_dir }, { "-Dctg_export_folder": test_cases_dir }, ], log_to_file=True)
def is_var_bug(mutated_project_dir, filter_coverage, spectrum_coverage_prefix=""): num_of_failing_variants = 0 num_of_passing_variants = 0 variants_dir = get_variants_dir(mutated_project_dir) variants_list = list_dir(variants_dir) for variant in variants_list: variant_dir = get_variant_dir(mutated_project_dir, variant) test_coverage_dir = get_test_coverage_dir(variant_dir) stm_coverage = statement_coverage(variant_dir, spectrum_coverage_prefix) spectrum_failed_file = get_spectrum_failed_coverage_file_name_with_version( spectrum_coverage_prefix) failed_file = join_path(test_coverage_dir, spectrum_failed_file) if os.path.isfile(failed_file): num_of_failing_variants += 1 elif stm_coverage >= filter_coverage: num_of_passing_variants += 1 if num_of_failing_variants >= 1 and num_of_passing_variants >= 1: return 1 return 0
def summary_percentage_of_cases_found_bugs(all_bugs_file, summary_file, num_of_examed_stms): summary_file_dir = join_path(EXPERIMENT_RESULT_FOLDER, summary_file) wb = Workbook(summary_file_dir) sheets = [] num_sheet = 0 excel_data_df = pandas.read_excel(all_bugs_file, sheet_name=None) rank_column = [] for item in default_rank_column: if item in excel_data_df[TARANTULA]: rank_column.append(item) for spectrum_expression_type in [TARANTULA, OP2, OCHIAI, BARINEL, DSTAR]: sheets.append(wb.add_worksheet(spectrum_expression_type)) sheet = sheets[num_sheet] num_sheet += 1 row = 0 col = 0 sheet.write(row, col, "NUM OF EXAMED STMS") col += 1 for item in rank_column: sheet.write(row, col, item) col += 1 row += 1 for num_stm in range(1, num_of_examed_stms + 1): col = 0 sheet.write(row, col, num_stm) col = +1 average_value_list = percentage_of_cases_found_bugs(excel_data_df, rank_column, spectrum_expression_type, num_stm) for metric in rank_column: sheet.write(row, col, average_value_list[metric]) col += 1 row += 1 wb.close()
def run_batch_junit_test_cases_on_project(project_dir, custom_ant=None, custom_variant_dirs=None): if not custom_variant_dirs: variants_dir = get_variants_dir(project_dir) custom_variant_dirs = list_dir(variants_dir, full_path=True) lib_paths = get_dependency_lib_dirs(project_dir) for variant_dir in custom_variant_dirs: are_all_tests_passed = run_batch_junit_test_cases( variant_dir=variant_dir, lib_paths=lib_paths, custom_ant=custom_ant) if are_all_tests_passed is True: file_name = PASSED_TEST_FLAG_FILE_NAME elif are_all_tests_passed is False: file_name = FAILED_TEST_FLAG_FILE_NAME else: logger.warning( f"Invalid are_all_tests_passed value {are_all_tests_passed} on variant {variant_dir}" ) file_name = FAILED_TEST_FLAG_FILE_NAME test_flag_file = join_path(variant_dir, file_name) touch_file(test_flag_file)
def compose_by_config(project_dir, config_file_path): logger.info( f"Composing [{get_file_name(project_dir)}] project's source code with config file [{get_file_name_without_ext(config_file_path)}]" ) config_name = get_file_name_without_ext(config_file_path) output_dir = get_variant_dir(project_dir, config_name) execute_shell_command(f'java -jar {PLUGIN_PATH}', extra_args=[{ "--expression": config_file_path }, { "--base-directory": get_feature_source_code_dir(project_dir) }, { "--output-directory": output_dir }, { "--export_roles_json": "" }, { "--featureAnnotationJava": "" }]) output_src_dir = join_path(output_dir, config_name) if is_path_exist(output_src_dir): renamed_folder_dir = get_src_dir(output_dir) move_file(output_src_dir, renamed_folder_dir) return output_dir
def get_failed_test_info_from_junit_report(failed_variant_dir): junit_report_path = get_junit_report_path(failed_variant_dir) failed_test_info_list = [] test_dir = get_test_dir(failed_variant_dir, force_mkdir=False) with open(junit_report_path) as input_file: soup = BeautifulSoup(input_file, "html.parser") for elm in soup.find_all('tr', {'class': 'Error'}): if not str(elm.parent.previous_sibling.previous_sibling.text ).startswith("TestCase"): continue test_file_name = elm.parent.find_previous_sibling( "a")["name"].strip() + SOURCE_CODE_EXTENSION tr = elm td_children = list(tr.findChildren("td", recursive=False)) test_case_name = td_children[0].text.strip() test_case_stack_trace_elm = td_children[2].find("code") failed_test_info = None for code_elm in list(test_case_stack_trace_elm.children)[5:]: if not isinstance(code_elm, NavigableString): continue trace_info = str(code_elm) if test_file_name in trace_info: qualified_test_case_class_name = re.search( f"([a-zA-Z._]+).{test_case_name}", trace_info).group(1) test_case_file_path = join_path( test_dir, *qualified_test_case_class_name.split(".")[:-1], test_file_name) failed_assertion_line_number = int( re.search(f"{SOURCE_CODE_EXTENSION}:(\d+)\)", trace_info).group(1)) failed_test_info = (test_case_file_path, test_case_name, failed_assertion_line_number) break if not failed_test_info: # "java.lang.StackOverflowError" does not show detail source test file, so all the test is taken test_case_file_path = find_file_by_wildcard(test_dir, "**/" + test_file_name, recursive=True) test_case_method_signature = TEST_CASE_METHOD_SIGNATURE_TEMPLATE.format( test_case_name=test_case_name) source_code_lines = open(test_case_file_path).readlines() indentation_count = -1 for index, line in enumerate(source_code_lines): if test_case_method_signature in line: indentation_count = len(line) - len(line.lstrip()) elif indentation_count >= 0 and line.startswith( " " * indentation_count + "}"): if 'verifyException("' in source_code_lines[index - 2]: failed_assertion_line_number = index - 8 else: failed_assertion_line_number = index break failed_test_info = (test_case_file_path, test_case_name, failed_assertion_line_number) failed_test_info_list.append(failed_test_info) return failed_test_info_list
def get_mutation_operators(mutated_project_name, mutated_project_dir): mutated_log_file_path = join_path(mutated_project_dir, mutated_project_name + ".mutant.log") mutated_log_file = open(mutated_log_file_path, "r") mutation_operators = [l.split(":")[0] for l in mutated_log_file.readlines()] return mutation_operators
def get_single_mutation_operator(mutated_project_name, mutated_project_dir): mutated_log_file_path = join_path(mutated_project_dir, mutated_project_name + ".mutant.log") mutated_log_file = open(mutated_log_file_path, "r") bug_content = mutated_log_file.readline().split(":") return bug_content[0]
def multiple_bugs_ranking(system_name, buggy_systems_folder, sbfl_metrics, alpha=0.5, normalization=NORMALIZATION_ENABLE, aggregation=AGGREGATION_ARITHMETIC_MEAN, filtering_coverage_rate=0.1, coverage_version=""): if os.path.exists(buggy_systems_folder): mutated_projects = list_dir(buggy_systems_folder) result_folder_dir = join_path(EXPERIMENT_RESULT_FOLDER, "w=" + str(alpha)) if not os.path.exists(result_folder_dir): os.makedirs(result_folder_dir) system_result_dir = join_path(result_folder_dir, system_name) if not os.path.exists(system_result_dir): os.makedirs(system_result_dir) normalization_result_dir = join_path(system_result_dir, normalization) if not os.path.exists(normalization_result_dir): os.makedirs(normalization_result_dir) aggregation_result_dir = join_path(normalization_result_dir, aggregation) if not os.path.exists(aggregation_result_dir): os.makedirs(aggregation_result_dir) sheet = [] row = 0 experiment_file_name = join_path(aggregation_result_dir, system_name + "_ranking_result.xlsx") wb = Workbook(experiment_file_name) for i in range(0, len(sbfl_metrics)): sheet.append(wb.add_worksheet(sbfl_metrics[i])) write_header_in_result_file(row, sheet[i]) row += 1 num_of_bugs = 0 for mutated_project_name in mutated_projects: num_of_bugs += 1 mutated_project_dir = join_path(buggy_systems_folder, mutated_project_name) suspicious_isolation(mutated_project_dir, filtering_coverage_rate, coverage_version) search_spaces = get_suspicious_space(mutated_project_dir, filtering_coverage_rate, coverage_version) buggy_statements = get_multiple_buggy_statements( mutated_project_name, mutated_project_dir) row_temp = row if system_name == "ZipMe": is_a_var_bug = is_var_bug_by_config(mutated_project_dir, ["Base", "Compress"]) else: is_a_var_bug = is_var_bug_by_config(mutated_project_dir, ["Base"]) ranking_results, varcop_ranking_time = ranking_multiple_bugs( buggy_statements, mutated_project_dir, search_spaces, sbfl_metrics, aggregation, normalization, coverage_version, filtering_coverage_rate, alpha) fb_ranking_results = features_ranking_multiple_bugs( buggy_statements, mutated_project_dir, search_spaces, filtering_coverage_rate, sbfl_metrics) for metric in range(0, len(sbfl_metrics)): sheet[metric].write(row_temp, BUG_ID_COL, mutated_project_name) row = write_result_to_file( row_temp, sheet[metric], ranking_results[sbfl_metrics[metric]], fb_ranking_results[sbfl_metrics[metric]], search_spaces, is_a_var_bug) wb.close()