def merge_lmatrix_into_right(lmatrix_file, rmatrix_file): if not os.path.isfile(rmatrix_file): shutil.copy2(lmatrix_file, rmatrix_file) else: lmatrix = common_matrices.ExecutionMatrix(filename=lmatrix_file) rmatrix = common_matrices.ExecutionMatrix(filename=rmatrix_file) rmatrix.update_with_other_matrix(lmatrix) rmatrix.serialize()
def merge_lmatrix_into_right(lmatrix_file, rmatrix_file): if not os.path.isfile(rmatrix_file): shutil.copy2(lmatrix_file, rmatrix_file) else: lmatrix = common_matrices.ExecutionMatrix(filename=lmatrix_file) rmatrix = common_matrices.ExecutionMatrix(filename=rmatrix_file) rmatrix.update_with_other_matrix(lmatrix, override_existing=True, \ ask_confirmation_with_exist_missing=True) rmatrix.serialize()
def load(resdir, fault_revealing=True): # load fault revealing tests fail_tests = None if fault_revealing: f_file = os.path.join(resdir, "fail_test_checking", "fault_reveling_tests.txt") fail_tests = [] with open(f_file) as f: for line in f: fail_tests.append(line.strip()) # load all tests post_pf_file = os.path.join(resdir, "post", "RESULTS_DATA", "matrices", "PASSFAIL.csv") post_pf_mat = common_matrices.ExecutionMatrix(post_pf_file) all_tests_post = post_pf_mat.get_nonkey_colname_list() pre_pf_file = os.path.join(resdir, "pre", "RESULTS_DATA", "matrices", "PASSFAIL.csv") pre_pf_mat = common_matrices.ExecutionMatrix(pre_pf_file) all_tests_pre = pre_pf_mat.get_nonkey_colname_list() # Conider test thta are not flaky in both pre and post all_tests = list(set(all_tests_pre) & set(all_tests_post)) # load execution outputs pre_orig_outlog_file = os.path.join(resdir, "pre", "RESULTS_DATA", "testexecution_outputs", "program_output.json") pre_muts_outlog_file = os.path.join(resdir, "pre", "RESULTS_DATA", "testexecution_outputs", "STRONG_MUTATION_output.json") post_orig_outlog_file = os.path.join(resdir, "post", "RESULTS_DATA", "testexecution_outputs", "program_output.json") post_muts_outlog_file = os.path.join(resdir, "post", "RESULTS_DATA", "testexecution_outputs", "STRONG_MUTATION_output.json") pre_orig_outlog = common_matrices.OutputLogData(pre_orig_outlog_file) pre_muts_outlog = common_matrices.OutputLogData(pre_muts_outlog_file) post_orig_outlog = common_matrices.OutputLogData(post_orig_outlog_file) post_muts_outlog = common_matrices.OutputLogData(post_muts_outlog_file) # Compute relevant mutants relevant_mutants_to_relevant_tests = get_relevant_mutants_to_relevant_tests( pre_orig_outlog, pre_muts_outlog, post_orig_outlog, post_muts_outlog) # load matrices and compute mutant killtest mapping post_sm_file = os.path.join(resdir, "post", "RESULTS_DATA", "matrices", "STRONG_MUTATION.csv") post_sm_mat = common_matrices.ExecutionMatrix(post_sm_file) mutants_to_killingtests = post_sm_mat.query_active_columns_of_rows() tests_to_killed_mutants = post_sm_mat.query_active_rows_of_columns() # return data return all_tests, fail_tests, relevant_mutants_to_relevant_tests, mutants_to_killingtests, tests_to_killed_mutants
def getHardToPropagateMutants (strong_mutant_kill_matrix_file, \ weak_mutant_kill_matrix_file, \ threshold=0.10, \ selected_tests=None): """ Return the pair of propagation ratio and list of hard to propagate mutant (strongly killed by less than threshold proportion of test that weakly kill them, 0 < threshold < 1 ). """ # check threshold ERROR_HANDLER.assert_true(threshold > 0 and threshold < 1, \ "Invalid threshold, must be in interval (0,1)", \ __file__) # load matrix sm_matrix = \ common_matrices.ExecutionMatrix(strong_mutant_kill_matrix_file) wm_matrix = \ common_matrices.ExecutionMatrix(weak_mutant_kill_matrix_file) # get mutants_to_killing_tests sm_mutants_to_killing_tests = sm_matrix.query_active_columns_of_rows() wm_mutants_to_killing_tests = wm_matrix.query_active_columns_of_rows() all_tests = sm_matrix.get_nonkey_colname_list() ERROR_HANDLER.assert_true(set(sm_mutants_to_killing_tests) == \ set(wm_mutants_to_killing_tests), \ "strong and weak mutant killing matrices " "have different mutants", __file__) ERROR_HANDLER.assert_true(set(all_tests) == \ set(wm_matrix.get_nonkey_colname_list()), \ "strong and weak mutant killing matrices " "have different tests", __file__) if selected_tests is not None: _filter_out_tests(all_tests, selected_tests, \ sm_mutants_to_killing_tests) _filter_out_tests(all_tests, selected_tests, \ wm_mutants_to_killing_tests) all_tests = selected_tests dualkillratio = {} for mut, sm_killtests in sm_mutants_to_killing_tests.items(): wm_killtests = wm_mutants_to_killing_tests[mut] if len(wm_killtests) == 0: dualkillratio[mut] = 1 else: dualkillratio[mut] = len(sm_killtests) * 1.0 / len(wm_killtests) return dualkillratio, [mut for mut, h in dualkillratio.items() if h <= threshold]
def getHardToKillMutants (mutant_kill_matrix_file, threshold=0.025, \ selected_tests=None): """ Return the pair of kill ratio and list of hard to kill mutant (kill by less than threshold proportion of test, 0 < threshold < 1 ). """ # check threshold ERROR_HANDLER.assert_true(threshold > 0 and threshold < 1, \ "Invalid threshold, must be in interval (0,1)", \ __file__) # load matrix matrix = common_matrices.ExecutionMatrix(mutant_kill_matrix_file) # get mutants_to_killing_tests mutants_to_killing_tests = matrix.query_active_columns_of_rows() all_tests = matrix.get_nonkey_colname_list() if selected_tests is not None: _filter_out_tests(all_tests, selected_tests, mutants_to_killing_tests) all_tests = selected_tests killratio = {} for mut, killtests in mutants_to_killing_tests.items(): if len(killtests) == 0: killratio[mut] = 1 else: killratio[mut] = len(killtests) * 1.0 / len(all_tests) return killratio, [mut for mut, h in killratio.items() if h <= threshold]
def getFaultRevealingMutants (strong_mutant_kill_matrix_file, \ expected_program_output_file, \ program_output_file, \ threshold=1.0, \ selected_tests=None): """ This function compute the set of fault revealing mutants. The inputs are: - mutant kill matrix file, - expected program output file, Used to see which test fails - obtained program output file, Used to see which test fails - threshold, in case a relaxed fault revealing is looked for - selected tests, in case part of the tests should be used :return: A pair is returned, with first element the set of fault revealing Mutants, and second element, a dict with key the mutants and values the fault revelation ratio ('# test kill and find fault' divided (/) '# test that kill') For equivalent mutants, the division isinvalid, we set the value to -1 """ prog_out = common_matrices.OutputLogData(filename=program_output_file) exp_prog_out = common_matrices.OutputLogData(\ filename=expected_program_output_file) _, prog_out_uniq = list(prog_out.get_zip_objective_and_data())[0] _, exp_prog_out_uniq = list(exp_prog_out.get_zip_objective_and_data())[0] if set(prog_out_uniq) != set(exp_prog_out_uniq): logging.warning("Test mismatch between program output and expected!") intersect = set(prog_out_uniq) & set(exp_prog_out_uniq) fault_tests = set() for elem in intersect: ol_equiv = common_matrices.OutputLogData.outlogdata_equiv(\ prog_out_uniq[elem], exp_prog_out_uniq[elem]) if not ol_equiv: ERROR_HANDLER.assert_true (elem not in fault_tests, \ "duplicate test", __file__) fault_tests.add(elem) # get mutant to killing test dict kill_matrix = common_matrices.ExecutionMatrix(\ filename=strong_mutant_kill_matrix_file) mut_to_killtests = kill_matrix.query_active_columns_of_rows() # remove unselected tests if selected_tests is not None: selected_tests = set(selected_tests) for mut, tests in mut_to_kiltests.items(): mut_to_killtests[mut] = set(tests) & selected_tests mutant_to_fr = {} mut, tests in mut_to_killtests.items(): kill_fr = len(tests & fault_tests) kill_all = len(tests) if kill_all > 0: # Killable mutant_to_fr[mut] = -1.0 else: mutant_to_fr[mut] = kill_fr * 1.0 / kill_all
def fault_analysis(cm_corebench_scripts_dir, c_id, conf_py, in_muteria_outdir, out_top_dir): if not os.path.isdir(out_top_dir): os.mkdir(out_top_dir) in_res_data_dir = os.path.join(in_muteria_outdir, 'latest', 'RESULTS_DATA') testtools_workdir = os.path.join(in_muteria_outdir, 'latest', 'testscases_workdir') pass_fail_matrix = os.path.join(in_res_data_dir, "matrices", "PASSFAIL.csv") pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix) test_list = list(pf_mat.get_nonkey_colname_list()) #semu_tool_dirs = [d for d in os.listdir(testtools_workdir) if d.startswith('semu_cmp-')] # get fault tests get_commit_fault_tests(cm_corebench_scripts_dir, c_id, conf_py, in_res_data_dir, out_top_dir) # get test to timestamp test_timestamp_file = os.path.join(out_top_dir, "test_to_timestamp.json") test2timestamp = {} ## get tests by tools tools2tests = {} for test in test_list: alias, _ = DriversUtils.reverse_meta_element(test) # XXX: only SEMU if not alias.startswith('semu_cmp-'): continue if alias not in tools2tests: tools2tests[alias] = set() tools2tests[alias].add(test) ## untar tests dir for alias, tests in tools2tests.items(): d = os.path.join(testtools_workdir, alias) assert os.path.isdir( d), "test tool dir " + d + " missing for alias " + alias test_tar = os.path.join(d, 'tests_files.tar.gz') es = common_fs.TarGz.decompressDir(test_tar) assert es is None, "decompress error: " + es tests_files = os.path.join(d, 'tests_files') assert os.path.isdir(tests_files), "dir missing after decompress" for test in tests: _, simple_test = DriversUtils.reverse_meta_element(test) gt = TestcasesToolSemu._get_generation_time_of_test( simple_test, tests_files) test2timestamp[test] = gt shutil.rmtree(tests_files) common_fs.dumpJSON(test2timestamp, test_timestamp_file, pretty=True)
def compute_stats(config, explorer): # get the matrix of each test criterion coverages = {} total_to = {} for c in config.ENABLED_CRITERIA.get_val(): if explorer.file_exists(fd_structure.CRITERIA_MATRIX[c]): mat_file = explorer.get_existing_file_pathname(\ fd_structure.CRITERIA_MATRIX[c]) mat = common_matrices.ExecutionMatrix(filename=mat_file) row2collist = mat.query_active_columns_of_rows() cov = len([k for k, v in row2collist.items() if len(v) > 0]) tot = len(row2collist) coverages[c.get_str()] = '{:.2f}'.format(cov * 100.0 / tot) total_to[c.get_str()] = tot # JSON out_json = {} for c in coverages: out_json[c] = { 'coverage': coverages[c], '# test objectives': total_to[c] } common_fs.dumpJSON(out_json, explorer.get_file_pathname(\ fd_structure.STATS_MAIN_FILE_JSON)) # HTML template_file = os.path.join(os.path.dirname(\ os.path.abspath(__file__)), 'summary_report.html') report_file = explorer.get_file_pathname(\ fd_structure.STATS_MAIN_FILE_HTML) rendered = Template(open(template_file).read()).render( \ {'coverages':coverages, 'total_to':total_to}) with open(report_file, 'w') as f: f.write(rendered) try: webbrowser.get() webbrowser.open('file://' + report_file, new=2) except Exception as e: logging.warning("webbrowser error: " + str(e)) #~ def compute_stats() #~ class DataHandling
def load(matrice_file): # load matrices and compute mutant killtest mapping sm_mat = common_matrices.ExecutionMatrix(matrice_file) print("[{}] Getting mutants_to_killingtests ...".format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) mutants_to_killingtests = sm_mat.query_active_columns_of_rows() print("[{}] Getting tests_to_killed_mutants ...".format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) tests_to_killed_mutants = sm_mat.query_active_rows_of_columns() all_tests = sm_mat.get_nonkey_colname_list() print("[{}] Loaded".format( datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) # return data return all_tests, mutants_to_killingtests, tests_to_killed_mutants
def getSubsumingMutants (mutant_kill_matrix_file, clustered=True, \ selected_tests=None): """ Get the subsuming mutants from the matrix file :returns: a tuple of (1) list of equivalent mutants (2) the list of tuple of subsuming mutant (Each tuple contain the mutants that are subsuming each others) or list of all subsuming mutants. """ # load matrix matrix = common_matrices.ExecutionMatrix(mutant_kill_matrix_file) # get mutants_to_killing_tests mutants_to_killing_tests = matrix.query_active_columns_of_rows() if selected_tests is not None: _filter_out_tests(matrix.get_nonkey_colname_list(), selected_tests, \ mutants_to_killing_tests) return algorithms.getSubsumingMutants(mutants_to_killing_tests, \ clustered=clustered)
def compute_stats(config, explorer, checkpointer): # get the matrix of each test criterion coverages = {} total_to = {} number_of_testcases = None for c in config.ENABLED_CRITERIA.get_val(): if explorer.file_exists(fd_structure.CRITERIA_MATRIX[c]): mat_file = explorer.get_existing_file_pathname(\ fd_structure.CRITERIA_MATRIX[c]) mat = common_matrices.ExecutionMatrix(filename=mat_file) row2collist = mat.query_active_columns_of_rows() cov = len([k for k,v in row2collist.items() if len(v) > 0]) tot = len(row2collist) coverages[c.get_str()] = 'n.a.' if tot == 0 else \ '{:.2f}'.format(cov * 100.0 / tot) total_to[c.get_str()] = tot if number_of_testcases is None: number_of_testcases = len(mat.get_nonkey_colname_list()) # JSON out_json = {} out_json['TOTAL EXECUTION TIME (s)'] = \ checkpointer.get_execution_time() out_json['NUMBER OF TESTCASES'] = number_of_testcases out_json['CRITERIA'] = {} for c in coverages: out_json['CRITERIA'][c] = {'coverage': coverages[c], '# test objectives': total_to[c]} common_fs.dumpJSON(out_json, explorer.get_file_pathname(\ fd_structure.STATS_MAIN_FILE_JSON), \ pretty=True) # HTML template_file = os.path.join(os.path.dirname(\ os.path.abspath(__file__)), 'summary_report.html') report_file = explorer.get_file_pathname(\ fd_structure.STATS_MAIN_FILE_HTML) def format_execution_time(exec_time): n_day = int(exec_time // (24 * 3600)) exec_time = exec_time % (24 * 3600) n_hour = int(exec_time // 3600) exec_time %= 3600 n_minutes = int(exec_time // 60) exec_time %= 60 n_seconds = int(round(exec_time)) res = "" for val, unit in [(n_day, 'day'), (n_hour, 'hour'), \ (n_minutes, 'minutes'), (n_seconds, 'second')]: if val > 0: s = ' ' if val == 1 else 's ' res += str(val) + ' ' + unit + s return res #~ def format_execution_time() total_exec_time = format_execution_time(\ checkpointer.get_execution_time()) rendered = Template(open(template_file).read()).render( \ { 'total_execution_time': total_exec_time, 'number_of_testcases': number_of_testcases, 'coverages':coverages, 'total_to':total_to, }) with open(report_file, 'w') as f: f.write(rendered) try: webbrowser.get() webbrowser.open('file://' + report_file,new=2) except Exception as e: logging.warning("webbrowser error: "+str(e))
def get_subsuming_elements(matrix_file): mat = common_matrices.ExecutionMatrix(filename=matrix_file) elem_to_tests = mat.query_active_columns_of_rows() equiv, subs_clusters = stats_algo.getSubsumingMutants(\ elem_to_tests, clustered=True) return equiv, subs_clusters
def update_matrix_to_cover_when_difference(cls, \ target_matrix_file, target_outdata_file, \ comparing_vector_file, comparing_outdata_file): ERROR_HANDLER.assert_true(target_matrix_file is not None \ and comparing_vector_file is not None, \ "target or comparing matrix is None", __file__) target_matrix = common_matrices.ExecutionMatrix(\ filename=target_matrix_file) comparing_vector = common_matrices.ExecutionMatrix(\ filename=comparing_vector_file) ERROR_HANDLER.assert_true(len(\ set(target_matrix.get_nonkey_colname_list()) - \ set(comparing_vector.get_nonkey_colname_list()) \ ) == 0, "Mismatch of columns", __file__) # Get uncertain target_uncertain_cols_dict = \ target_matrix.query_uncertain_columns_of_rows() vector_uncertain_cols = \ comparing_vector.query_uncertain_columns_of_rows() vector_uncertain_cols = \ set(vector_uncertain_cols[list(vector_uncertain_cols)[0]]) # Check if outdata and proceed accordingly if target_outdata_file is not None and \ comparing_outdata_file is not None: # outdata are set use outdata values to decide of differences target_outdata = common_matrices.OutputLogData(\ filename=target_outdata_file) vector_outdata = common_matrices.OutputLogData(\ filename=comparing_outdata_file) _, vector_outdata_uniq = list(\ vector_outdata.get_zip_objective_and_data())[0] ## Compare using output timedout_key = common_matrices.OutputLogData.TIMEDOUT key_to_diffs = {} for key, key_data in target_outdata.get_zip_objective_and_data(): intersect = set(vector_outdata_uniq) & set(key_data) ERROR_HANDLER.assert_true(set(key_data) == intersect, \ "The elements in target must all be in vector",\ __file__) key_to_diffs[key] = set() for elem in intersect: ol_equiv = common_matrices.OutputLogData.outlogdata_equiv(\ key_data[elem], vector_outdata_uniq[elem]) if not ol_equiv: key_to_diffs[key].add(elem) else: # outdata is not set use difference of matrices ## obtain the active cols for each rows target_active_cols_dict = \ target_matrix.query_active_columns_of_rows() vector_active_cols = \ comparing_vector.query_active_columns_of_rows() vector_active_cols = \ set(vector_active_cols[list(vector_active_cols)[0]]) ## for each row of target matrix, get diff target_matrix_allcol = set(target_matrix.get_nonkey_colname_list()) key_to_diffs = {} for row_key, row_active in target_active_cols_dict.items(): key_to_diffs[row_key] = (set(row_active) - vector_active_cols)\ | (vector_active_cols - set(row_active)) key_to_diffs[row_key] &= target_matrix_allcol # Clear matrix to inactive target_matrix.clear_cells_to_value(target_matrix.getInactiveCellVal()) # Update matrix based on diff for key, diffs in key_to_diffs.items(): # gather the uncertain and set to uncertain uncertain = vector_uncertain_cols | \ set(target_uncertain_cols_dict[key]) set_uncertain = uncertain & diffs diffs -= uncertain values = {} for col in set_uncertain: values[col] = target_matrix.getUncertainCellDefaultVal() for col in diffs: values[col] = target_matrix.getActiveCellDefaultVal() if len(values) > 0: target_matrix.update_cells(key, values) target_matrix.serialize()
def _get_input_bitcode_file(self, code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=None): meta_mu_src = self.driver_config.get_meta_mutant_source() # XXX Case of manual annotation if meta_mu_src == MetaMuSource.ANNOTATION: with open(self.cand_muts_file, 'w') as f: # Single mutant (id 1, corresponding to old version) f.write(str(1) + '\n') return super(TestcasesToolSemu, self)._get_input_bitcode_file(\ code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=meta_criteria_tool_obj) if type(meta_mu_src) == str: # XXX: The actual path to the meta is specified return meta_mu_src # XXX: Case of other mutation tools like Mart # get the meta criterion file from MART or any compatible tool. mutant_gen_tool_name = meta_mu_src.get_field_value() mut_tool_alias_to_obj = \ meta_criteria_tool_obj.get_criteria_tools_by_name(\ mutant_gen_tool_name) if len(mut_tool_alias_to_obj) == 0: logging.warning(\ 'SEMu requires {} to generate mutants but none used'.format(\ mutant_gen_tool_name)) ERROR_HANDLER.assert_true(len(mut_tool_alias_to_obj) == 1, \ "SEMu supports tests generation from" "a single .bc file for now (todo).", __file__) t_alias2metamu_bc = {} t_alias2mutantInfos = {} for alias, obj in mut_tool_alias_to_obj.items(): dest_bc = rel_path_map[list(rel_path_map)[0]] + '.bc' shutil.copy2(obj.get_test_gen_metamutant_bc(), dest_bc) t_alias2metamu_bc[alias] = dest_bc t_alias2mutantInfos[alias] = obj.get_criterion_info_object(None) # XXX: get mutants ids by functions self.mutants_by_funcs = {} single_alias = list(t_alias2mutantInfos)[0] single_tool_obj = t_alias2mutantInfos[single_alias] cand_muts = list(single_tool_obj.get_elements_list()) for mut in single_tool_obj.get_elements_list(): func = single_tool_obj.get_element_data(mut)[\ 'mutant_function_name'] #meta_mut = DriversUtils.make_meta_element(mut, single_alias) if func not in self.mutants_by_funcs: self.mutants_by_funcs[func] = set() self.mutants_by_funcs[func].add(mut) #meta_mut) # XXX: get candidate mutants list if self.driver_config.get_target_only_live_mutants() \ and os.path.isfile(self.sm_mat_file): sm_mat = common_matrices.ExecutionMatrix(\ filename=self.sm_mat_file) mut2killing_tests = sm_mat.query_active_columns_of_rows() alive_muts = [m for m, k_t in mut2killing_tests.items() \ if len(k_t) == 0] cand_muts = [] for meta_m in alive_muts: t_alias, m = DriversUtils.reverse_meta_element(meta_m) if t_alias in t_alias2metamu_bc: # There is a single one cand_muts.append(m) with open(self.cand_muts_file, 'w') as f: for m in cand_muts: f.write(str(m) + '\n') return t_alias2metamu_bc[list(t_alias2metamu_bc)[0]]
def _get_fault_tests(cm_corebench_scripts_dir, c_id, conf_py, in_res_data_dir, outdir, get_difference=True): if not os.path.isdir(outdir): os.mkdir(outdir) exe_dir = os.path.join(cm_corebench_scripts_dir, "bug_fixing_exes", c_id) exe_file = os.listdir(os.path.join(exe_dir, "old")) if len(exe_file) != 1: error_exit("not one file for old") exe_file = exe_file[0] fail_test_execution = os.path.join(outdir, "fail_test_checking") if os.path.isdir(fail_test_execution): shutil.rmtree(fail_test_execution) if not os.path.isdir(fail_test_execution): os.makedirs(fail_test_execution) bug_finding_tests_list = os.path.join(fail_test_execution, "fault_reveling_tests.txt") # temporary test_list_file = os.path.join(fail_test_execution, "test_list.tmp") pass_fail_matrix = os.path.join(in_res_data_dir, "matrices", "PASSFAIL.csv") pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix) with open(test_list_file, "w") as f: for test in pf_mat.get_nonkey_colname_list(): f.write(test + "\n") nohash = ['--nohashoutlog'] #nohash = [] print("# info: running bug-fix old ...") version = "old" custom_exe = os.path.join(exe_dir, version, exe_file) stdin = "{}\n{}\n{}\n{}\n".format( "tests", os.path.join(fail_test_execution, version), '{"src/' + exe_file + '": "' + custom_exe + '"}', test_list_file) if os.system(" ".join([ "printf", "'" + stdin + "'", "|", "muteria", "--config", conf_py, "--lang", "c", "customexec" ] + nohash)) != 0: assert False, "bug-fix old failed" print("# info: running bug-fix new ...") version = "new" custom_exe = os.path.join(exe_dir, version, exe_file) stdin = "{}\n{}\n{}\n{}\n".format( "tests", os.path.join(fail_test_execution, version), '{"src/' + exe_file + '": "' + custom_exe + '"}', test_list_file) if os.system(" ".join([ "printf", "'" + stdin + "'", "|", "muteria", "--config", conf_py, "--lang", "c", "customexec" ] + nohash)) != 0: assert False, "bug-fix new failed" _extract_list(fail_test_execution, bug_finding_tests_list) if get_difference: # get differences in bug introducing ## get diff exe file diff_exe_dir = os.path.join(os.path.dirname(in_res_data_dir), 'code_build_workdir') map_file_ = os.path.join(diff_exe_dir, 'files_map') map_ = common_fs.loadJSON(map_file_) diff_exe_file = None for f, dest in map_.items(): if os.path.basename(f) == exe_file: assert diff_exe_file is None, "multiple exefile found in defference" diff_exe_file = os.path.join(diff_exe_dir, dest) assert os.path.isfile( diff_exe_file), "missing diff_exe_file: " + diff_exe_file assert diff_exe_file is not None, "diff exe file not found" ## Execution diff_test_execution = os.path.join(outdir, "diff_test_checking") diff_finding_tests_list = os.path.join(diff_test_execution, "diff_reveling_tests.txt") if os.path.isdir(diff_test_execution): shutil.rmtree(diff_test_execution) if not os.path.isdir(diff_test_execution): os.makedirs(diff_test_execution) print("# info: running bug-intro old ...") version = "old" stdin = "{}\n{}\n{}\n{}\n".format( "tests", os.path.join(diff_test_execution, version), '{"src/' + exe_file + '": "' + diff_exe_file + '"}', test_list_file) if os.system(" ".join([ "printf", "'" + stdin + "'", "|", "KLEE_CHANGE_RUNTIME_SET_OLD_VERSION=on", "muteria", "--config", conf_py, "--lang", "c", "customexec" ] + nohash)) != 0: assert False, "bug-intro old failed" print("# info: running bug-intro new ...") version = "new" stdin = "{}\n{}\n{}\n{}\n".format( "tests", os.path.join(diff_test_execution, version), '{"src/' + exe_file + '": "' + diff_exe_file + '"}', test_list_file) if os.system(" ".join([ "printf", "'" + stdin + "'", "|", "muteria", "--config", conf_py, "--lang", "c", "customexec" ] + nohash)) != 0: assert False, "bug-intro new failed" _extract_list(diff_test_execution, diff_finding_tests_list) os.remove(test_list_file)
def _call_generation_run(self, runtool, args): # Delete any klee-out-* for d in os.listdir(self.tests_working_dir): if d.startswith('klee-out-'): shutil.rmtree(os.path.join(self.tests_working_dir, d)) call_shadow_wrapper_file = os.path.join(self.tests_working_dir, \ "shadow_wrap") devtest_toolalias = self.parent_meta_tool.get_devtest_toolalias() ERROR_HANDLER.assert_true(devtest_toolalias is not None, \ "devtest must be used when using shadow_se", __file__) #test_list = list(self.code_builds_factory.repository_manager\ # .get_dev_tests_list()) test_list = [] for meta_test in self.parent_meta_tool.get_testcase_info_object(\ candidate_tool_aliases=[devtest_toolalias])\ .get_tests_list(): toolalias, test = DriversUtils.reverse_meta_element(meta_test) ERROR_HANDLER.assert_true(toolalias == devtest_toolalias, \ "BUG in above get_testcase_info_object", __file__) test_list.append(test) # Get list of klee_change, klee_get_true/false locations. klee_change_stmts = [] get_lines_callback_obj = self.GetLinesCallbackObject() get_lines_callback_obj.set_pre_callback_args(self.code_builds_factory\ .repository_manager.revert_src_list_files) get_lines_callback_obj.set_post_callback_args(klee_change_stmts) pre_ret, post_ret = self.code_builds_factory.repository_manager\ .custom_read_access(get_lines_callback_obj) ERROR_HANDLER.assert_true(pre_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "pre failed", __file__) ERROR_HANDLER.assert_true(post_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "post failed", __file__) ERROR_HANDLER.assert_true(len(klee_change_stmts) > 0, \ "No klee_change statement in the sources", __file__) # Filter only tests that cover those locations, # if there is stmt coverage matrix stmt_cov_mat_file = self.head_explorer.get_file_pathname(\ fd_structure.CRITERIA_MATRIX[criteria.TestCriteria\ .STATEMENT_COVERAGE]) cov_tests = None if os.path.isfile(stmt_cov_mat_file): stmt_cov_mat = common_matrices.ExecutionMatrix(\ filename=stmt_cov_mat_file) # due to possible wrapper test splitting we update test_list here tmp_test_list = [] for mt in stmt_cov_mat.get_nonkey_colname_list(): alias, t = DriversUtils.reverse_meta_element(mt) if alias == devtest_toolalias: tmp_test_list.append(t) test_list = tmp_test_list meta_stmts = list(stmt_cov_mat.get_keys()) tool_aliases = set() for meta_stmt in meta_stmts: alias, stmt = DriversUtils.reverse_meta_element(meta_stmt) tool_aliases.add(alias) klee_change_meta_stmts = [] for alias in tool_aliases: klee_change_meta_stmts += [\ DriversUtils.make_meta_element(e, alias) \ for e in klee_change_stmts] klee_change_meta_stmts = list(set(meta_stmts) & \ set(klee_change_meta_stmts)) cov_tests = set() if len(klee_change_meta_stmts) > 0: for _, t in stmt_cov_mat.query_active_columns_of_rows(\ row_key_list=klee_change_meta_stmts).items(): cov_tests |= set(t) else: logging.warning('No test covers the patch (SHADOW)!') # ERROR_HANDLER.assert_true(len(klee_change_meta_stmts) > 0, \ # "No test covers the patch", __file__) # tests will be generated in the same dir that has the input .bc file os.mkdir(self.tests_storage_dir) # obtain candidate tests cand_testpair_list = [] for test in test_list: meta_test = DriversUtils.make_meta_element(test, devtest_toolalias) if cov_tests is not None and meta_test not in cov_tests: continue cand_testpair_list.append((test, meta_test)) # Adjust the max-time in args ## locate max-time per_test_hard_timeout = None per_test_timeout = None if len(cand_testpair_list) > 0: cur_max_time = float(self.get_value_in_arglist(args, 'max-time')) if self.driver_config.get_gen_timeout_is_per_test(): per_test_timeout = cur_max_time else: per_test_timeout = max(60, \ cur_max_time / len(cand_testpair_list)) self.set_value_in_arglist(args, 'max-time', str(per_test_timeout)) # give time to dump remaning states per_test_hard_timeout = per_test_timeout + \ self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE #per_test_hard_timeout = 300 #DBG # Set the wrapper with open(call_shadow_wrapper_file, 'w') as wf: wf.write('#! /bin/bash\n\n') wf.write('set -u\n') wf.write('set -o pipefail\n\n') wf.write('ulimit -s unlimited\n') # timeout the shadow execution (some test create daemon which) # are not killed by test timeout. ALSO MAKE SURE TO DESACTIVATE # IN TEST SCRIPT TIMEOUT kill_after = 30 wf.write('time_out_cmd="/usr/bin/timeout --kill-after={}s {}"\n'.\ format(kill_after, per_test_hard_timeout)) # kill after and time for timeout to act per_test_hard_timeout += kill_after + 60 #wf.write(' '.join(['exec', runtool] + args + ['"${@:1}"']) + '\n') wf.write('\nstdindata="{}/klee-last/{}"\n'.format(\ self.tests_working_dir, \ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('tmpstdindata="{}/{}"\n\n'.format(self.tests_working_dir,\ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('if [ -t 0 ] # check if stdin do not exist\n') wf.write('then\n') wf.write(' '.join(['\t(', '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('\t/usr/bin/touch $tmpstdindata\n') wf.write('else\n') wf.write('\t(/bin/cat - > $tmpstdindata ) || EXIT_CODE=1\n') wf.write(' '.join(['\t(', '/bin/cat $tmpstdindata | ', \ '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('fi\n\n') wf.write('/bin/mv $tmpstdindata $stdindata || EXIT_CODE=2\n') wf.write('\n# Provoke "unbound variable" if KLEE fails\n') wf.write('# Preserve the KLEE exit code\n') wf.write('exit $EXIT_CODE\n') os.chmod(call_shadow_wrapper_file, 0o775) # run test exes, _ = self.code_builds_factory.repository_manager\ .get_relative_exe_path_map() ERROR_HANDLER.assert_true(len(exes) == 1, \ "Must have a single exe", __file__) exe_path_map = {e: call_shadow_wrapper_file for e in exes} env_vars = {} self._dir_chmod777(self.tests_storage_dir) for test, meta_test in cand_testpair_list: self.parent_meta_tool.execute_testcase(meta_test, exe_path_map, \ env_vars, timeout=per_test_hard_timeout,\ with_output_summary=False) #logging.debug("DBG: Just executed test '{}'".format(meta_test)) #input(">>>> ") #DBG # copy the klee out test_out = os.path.join(self.tests_storage_dir, \ self.get_sorage_name_of_test(test)) os.mkdir(test_out) for d in glob.glob(self.tests_working_dir + "/klee-out-*"): # make sure we can do anything with it self._dir_chmod777(d) if not self.keep_first_test: first_test = os.path.join(d, 'test000001.ktest') if os.path.isfile(first_test): shutil.move(first_test, first_test + '.disable') shutil.move(d, test_out) ERROR_HANDLER.assert_true(len(list(os.listdir(test_out))) > 0, \ "Shadow generated no test for tescase: "+test,\ __file__) if os.path.islink(os.path.join(self.tests_working_dir, \ 'klee-last')): os.unlink(os.path.join(self.tests_working_dir, 'klee-last')) # store klee_change locs common_fs.dumpJSON(klee_change_stmts, self.klee_change_locs_list_file)
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \ stop_on_failure=False, \ per_test_timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ fault_test_execution_matrix_file=None, \ fault_test_execution_execoutput_file=None, \ with_outlog_hash=True, \ test_prioritization_module=None, \ parallel_test_count=1, \ parallel_test_scheduler=None, \ restart_checkpointer=False, finish_destroy_checkpointer=True): ''' Execute the list of test cases with the given executable and say, for each test case, whether it failed :param meta_testcases: list of test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the tests :param env_vars: dict of environment variables to set before executing each test ({<variable>: <value>}) :param stop_on_failure: decide whether to stop the test execution once a test fails :param fault_test_execution_matrix_file: Optional matrix file to store the tests' pass fail execution data :param fault_test_execution_execoutput_file: Optional output log file to store the tests' execution actual output (hashed) :param with_outlog_hash: decide whether to return outlog hash :param test_prioritization_module: Specify the test prioritization module. (TODO: Implement support) :param parallel_test_count: Specify the number of parallel test Execution. must be an integer >= 1 :param parallel_test_scheduler: Specify the function that will handle parallel test scheduling by tool, using the test execution optimizer. (TODO: Implement support) :type restart_checkointer: bool :param restart_checkointer: Decide whether to discard checkpoint and restart anew. :type finish_destroy_checkpointer: bool :param finish_destroy_checkpointer: Decide whether to automatically destroy the checkpointer when done or not Useful is caller has a checkpointer to update. :returns: dict of testcase and their failed verdict. {<test case name>: <True if failed, False if passed, UNCERTAIN_TEST_VERDICT if uncertain>} If stop_on_failure is True, only return the tests that have been executed until the failure ''' # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(test_prioritization_module is None, \ "Must implement test prioritization support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_count <= 1, \ "Must implement parallel tests execution support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \ "Must implement parallel tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() ERROR_HANDLER.assert_true(parallel_test_count > 0, \ "invalid parallel test execution count: {}. {}".format( \ parallel_test_count, "must be >= 1")) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests" cp_task_id = 1 checkpoint_handler = \ CheckPointHandler(self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): logging.warning("%s %s" %("The function 'runtests' is finished", \ "according to checkpoint, but called again. None returned")) if common_mix.confirm_execution("%s %s" % ( \ "Function 'runtests' is already", \ "finished, do you want to restart?")): checkpoint_handler.restart() logging.info("Restarting the finished 'runtests'") else: ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\ "Execution halted. Cannot continue because no value", \ " can be returned. Check the results of the", \ "finished execution"), call_location=__file__) # @Checkpoint: Get the saved payload (data kapt for each tool) # pair list of testfailed verdict and execution output meta_test_failedverdicts_outlog = \ checkpoint_handler.get_optional_payload() if meta_test_failedverdicts_outlog is None: meta_test_failedverdicts_outlog = [{}, {}] # Make sure the tests are unique ERROR_HANDLER.assert_true(len(meta_testcases) == \ len(set(meta_testcases)), \ "not all tests are unique", __file__) testcases_by_tool = {} for meta_testcase in meta_testcases: ttoolalias, testcase = \ DriversUtils.reverse_meta_element(meta_testcase) if ttoolalias not in testcases_by_tool: testcases_by_tool[ttoolalias] = [] testcases_by_tool[ttoolalias].append(testcase) found_a_failure = False for tpos, ttoolalias in enumerate(testcases_by_tool.keys()): # @Checkpoint: Check whether already executed if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias): continue # Actual execution ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_outlog_hash=with_outlog_hash) for testcase in test_failed_verdicts: meta_testcase = \ DriversUtils.make_meta_element(testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.COMMAND_UNCERTAIN: found_a_failure = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) if stop_on_failure and found_a_failure: # @Checkpoint: Chekpointing for remaining tools for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]: checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=rem_tool, \ opt_payload=meta_test_failedverdicts_outlog) break if stop_on_failure: # Make sure the non executed test has the uncertain value (None) if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases): for meta_testcase in set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0]): meta_test_failedverdicts_outlog[0][meta_testcase] = \ common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT meta_test_failedverdicts_outlog[1][meta_testcase] = \ common_matrices.OutputLogData.\ UNCERTAIN_TEST_OUTLOGDATA ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \ len(meta_testcases), \ "Not all tests have a verdict reported", __file__) if fault_test_execution_matrix_file is not None: # Load or Create the matrix fault_test_execution_matrix = common_matrices.ExecutionMatrix( \ filename=fault_test_execution_matrix_file, \ non_key_col_list=meta_testcases) ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \ "matrix must be empty", __file__) failverdict2val = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ fault_test_execution_matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ fault_test_execution_matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ fault_test_execution_matrix.getUncertainCellDefaultVal(), } cells_dict = {} for meta_testcase in meta_test_failedverdicts_outlog[0]: cells_dict[meta_testcase] = failverdict2val[\ meta_test_failedverdicts_outlog[0][meta_testcase]] fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \ cells_dict, serialize=True) if fault_test_execution_execoutput_file is None: meta_test_failedverdicts_outlog[1] = None else: # Load or Create the data object fault_test_execution_execoutput = common_matrices.OutputLogData( \ filename=fault_test_execution_execoutput_file) ERROR_HANDLER.assert_true(\ fault_test_execution_execoutput.is_empty(), \ "outlog data must be empty", __file__) fault_test_execution_execoutput.add_data(\ {self.PROGRAM_EXECOUTPUT_KEY: \ meta_test_failedverdicts_outlog[1]}, \ serialize=True) # @Checkpoint: Finished detailed_exectime = {} for ttoolalias in testcases_by_tool.keys(): tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ttoolalias] = (\ tt.get_checkpointer().get_execution_time(),\ tt.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished( \ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy() return meta_test_failedverdicts_outlog
def runtests_criteria_coverage (self, testcases, criterion_to_matrix, \ criterion_to_executionoutput=None, criteria_element_list_by_criteria=None, \ re_instrument_code=False, \ cover_criteria_elements_once=False, prioritization_module_by_criteria=None, parallel_count=1, \ parallel_criteria_test_scheduler=None,\ restart_checkpointer=False, \ finish_destroy_checkpointer=True): ''' Executes the instrumented executable code with testscases and returns the different code coverage matrices. :param testcases: list of testcases to execute :param criterion_to_matrix: dict of <criterion, Matrix file where to store coverage>. :param criterion_to_executionoutput: dict of <criterion, execoutput file where to store coverage>. :param criteria_element_list_by_criteria: dictionary representing the list of criteria elements (stmts, branches, mutants) to consider in the test execution matices. Key is the criterion and the value the list of elements :param re_instrument_code: Decide whether to instrument code before running the tests. (Example when instrumentation was not specifically called. This is True by default) :param cover_criteria_elements_once: Specify whether to cover criteria elements once is enough, meaning that we stop analysing a criterion element once a test covers it. The remaining test covering verdict will be UNKNOWN. :param prioritization_module_by_criteria: dict of prioritization module by criteria. None means no prioritization used. :type \parallel_count: :param \parallel_count: :type \parallel_criteria_test_scheduler: :param \parallel_criteria_test_scheduler: scheduler that organize parallelism across criteria tools. (TODO: Implement support) :type \restart_checkpointer: :param \restart_checkpointer: :type finish_destroy_checkpointer: :param finish_destroy_checkpointer: ''' # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(parallel_count <= 1, \ "Must implement parallel execution support here", \ __file__) ERROR_HANDLER.assert_true(parallel_criteria_test_scheduler is None, \ "Must implement parallel codes tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity ERROR_HANDLER.assert_true(parallel_count > 0, \ "invalid parallel execution count: {}. {}".format( \ parallel_count, "must be >= 1")) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests_criteria_coverage" cp_task_id = 1 checkpoint_handler = CheckPointHandler( \ self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): return ERROR_HANDLER.assert_true(len(criterion_to_matrix) > 0, \ "no criterion is enabled", __file__) ERROR_HANDLER.assert_true(len(set(criterion_to_matrix) - \ set(self.tools_config_by_criterion_dict)) == 0, \ "Passed matrices output are more than tool specified", \ __file__) if criterion_to_executionoutput is not None: ERROR_HANDLER.assert_true(set(criterion_to_matrix) == \ set(criterion_to_executionoutput), \ "criteria mismatch between matrix and output", \ __file__) tool2criteria = self._get_tool2criteria(criterion_to_matrix.keys()) matrices_dir_tmp = os.path.join(self.criteria_working_dir, \ "codecov_dir.tmp") if os.path.isdir(matrices_dir_tmp): if restart_checkpointer: shutil.rmtree(matrices_dir_tmp) os.mkdir(matrices_dir_tmp) else: os.mkdir(matrices_dir_tmp) if criteria_element_list_by_criteria is None: criteria_element_list_by_criteria = \ {c: None for c in criterion_to_matrix} # get criteria elements by tools criteria_elem_list_by_tool = {} for criterion in criteria_element_list_by_criteria: if criteria_element_list_by_criteria[criterion] is None: for t_conf in self.tools_config_by_criterion_dict[criterion]: toolalias = t_conf.get_tool_config_alias() if toolalias not in criteria_elem_list_by_tool: criteria_elem_list_by_tool[toolalias] = {} criteria_elem_list_by_tool[toolalias][criterion] = None continue ERROR_HANDLER.assert_true(\ len(criteria_element_list_by_criteria[criterion]) != 0, \ "Empty criteria element list for criterion "\ +criterion.get_str(), __file__) for crit_elem in criteria_element_list_by_criteria[criterion]: toolalias, elem = DriversUtils.reverse_meta_element(crit_elem) if toolalias not in criteria_elem_list_by_tool: criteria_elem_list_by_tool[toolalias] = {} if criterion not in criteria_elem_list_by_tool[toolalias]: criteria_elem_list_by_tool[toolalias][criterion] = [] criteria_elem_list_by_tool[toolalias][criterion].append(elem) ERROR_HANDLER.assert_true(len(set(criteria_elem_list_by_tool) - \ set(self.criteria_configured_tools)) == 0, \ "some tools in data not registered", __file__) crit2tool2matrixfile = {cv: {} for cv in criterion_to_matrix} crit2tool2outhashfile = {cv: {} for cv in criterion_to_executionoutput} for ctoolalias in tool2criteria: _criteria2matrix = {} _criteria2outhash = {} for criterion in tool2criteria[ctoolalias]: _criteria2matrix[criterion] = os.path.join(matrices_dir_tmp, \ criterion.get_field_value() + '-' + ctoolalias + '.csv') if criterion_to_executionoutput is None or \ criterion_to_executionoutput[criterion] is None: _criteria2outhash[criterion] = None else: _criteria2outhash[criterion] = \ os.path.join(matrices_dir_tmp, \ criterion.get_field_value() + '-' + ctoolalias + '.outloghash.json') crit2tool2matrixfile[criterion][ctoolalias] = \ _criteria2matrix[criterion] crit2tool2outhashfile[criterion][ctoolalias] = \ _criteria2outhash[criterion] # @Checkpoint: Check whether already executed if checkpoint_handler.is_to_execute( \ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ctoolalias): for criterion in _criteria2matrix: _criteria2matrix[criterion] = \ common_matrices.ExecutionMatrix( \ filename=_criteria2matrix[criterion], \ non_key_col_list=testcases) if _criteria2outhash[criterion] is not None: _criteria2outhash[criterion] = \ common_matrices.OutputLogData( \ filename=_criteria2outhash[criterion]) # Actual execution ctool = self.criteria_configured_tools[ctoolalias][\ self.TOOL_OBJ_KEY] ctool.runtests_criteria_coverage(testcases, \ criteria_element_list_by_criteria=\ criteria_elem_list_by_tool[toolalias],\ criterion_to_matrix=_criteria2matrix, \ criterion_to_executionoutput=\ _criteria2outhash,\ re_instrument_code=re_instrument_code, \ cover_criteria_elements_once=\ cover_criteria_elements_once, \ prioritization_module_by_criteria=\ prioritization_module_by_criteria) # Checkpointing checkpoint_handler.do_checkpoint( \ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ctoolalias) # Aggregate the matrices and out hashes ## Create reult matrices and out hashes result_matrices = {} result_outloghashes = {} for criterion in criterion_to_matrix: result_matrices[criterion] = common_matrices.ExecutionMatrix( \ filename=criterion_to_matrix[criterion], \ non_key_col_list=testcases) if criterion_to_executionoutput[criterion] is None: result_outloghashes[criterion] = None else: result_outloghashes[criterion] = \ common_matrices.OutputLogData(filename=\ criterion_to_executionoutput[criterion]) ERROR_HANDLER.assert_true(\ crit2tool2outhashfile[criterion] is not None, "Bug: log enabled but hidden from tool", __file__) ## Actual aggregate logging.debug("saving results ...") for criterion in result_matrices: result_matrix = result_matrices[criterion] result_outloghash = result_outloghashes[criterion] for mtoolalias in crit2tool2matrixfile[criterion]: tool_matrix = common_matrices.ExecutionMatrix(\ filename=crit2tool2matrixfile[criterion][mtoolalias]) # Check columns ERROR_HANDLER.assert_true(tool_matrix.get_key_colname() == \ result_matrix.get_key_colname(), \ "mismatch on key column name", __file__) ERROR_HANDLER.assert_true( \ set(tool_matrix.get_nonkey_colname_list()) == \ set(result_matrix.get_nonkey_colname_list()), \ "mismatch on non key column names", __file__) # bring in the data key2nonkeydict = tool_matrix.to_pandas_df().\ set_index(tool_matrix.get_key_colname(), drop=True).\ to_dict(orient="index") for c_key in key2nonkeydict: meta_c_key = DriversUtils.make_meta_element(\ str(c_key), mtoolalias) result_matrix.add_row_by_key(meta_c_key, key2nonkeydict[c_key], serialize=False) # out log hash if crit2tool2outhashfile[criterion] is not None: tool_outloghash = common_matrices.OutputLogData(\ filename=\ crit2tool2outhashfile[criterion][mtoolalias]) for objective, objective_data in \ tool_outloghash.get_zip_objective_and_data(): meta_objective = DriversUtils.make_meta_element(\ str(objective), mtoolalias) result_outloghash.add_data( {meta_objective: objective_data}, \ serialize=False) # @Checkpoint: Check whether already executed if checkpoint_handler.is_to_execute( \ func_name=cp_func_name, \ taskid=cp_task_id + 1, tool=criterion.get_str()): # Serialized the computed matrix result_matrix.serialize() if result_outloghash is not None: result_outloghash.serialize() # @Checkpoint: Checkpointing checkpoint_handler.do_checkpoint( \ func_name=cp_func_name, \ taskid=cp_task_id + 1, tool=criterion.get_str()) # Delete the temporary tool matrix's directory if os.path.isdir(matrices_dir_tmp): shutil.rmtree(matrices_dir_tmp) # @Checkpoint: Finished detailed_exectime = {} for ctoolalias in tool2criteria: ct = self.criteria_configured_tools[ctoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ctoolalias] = (\ ct.get_checkpointer().get_execution_time(),\ ct.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished(\ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy()
def update_matrix_to_cover_when_diference(cls, \ target_matrix_file, target_outdata_file, \ comparing_vector_file, comparing_outdata_file): ERROR_HANDLER.assert_true(target_matrix_file is not None \ and comparing_vector_file is not None, \ "target or comparing matrix is None", __file__) target_matrix = common_matrices.ExecutionMatrix(\ filename=target_matrix_file) comparing_vector = common_matrices.ExecutionMatrix(\ filename=comparing_vector_file) ERROR_HANDLER.assert_true(\ set(target_matrix.get_nonkey_colname_list()) == \ set(comparing_vector.get_nonkey_colname_list()), \ "Mismatch of columns", __file__) # Get uncertain target_uncertain_cols_dict = \ target_matrix.query_uncertain_columns_of_rows() vector_uncertain_cols = \ comparing_vector.query_uncertain_columns_of_rows() vector_uncertain_cols = \ set(vector_uncertain_cols[list(vector_uncertain_cols)[0]]) # Check if outdata and proceed accordingly if target_outdata_file is not None and \ comparing_outdata_file is not None: # outdata are set use outdata values to decide of differences target_outdata = common_matrices.OutputLogData(\ filename=target_outdata_file) vector_outdata = common_matrices.OutputLogData(\ filename=comparing_outdata_file) vector_outdata_uniq, _ = list(\ vector_outdata.get_zip_objective_and_data())[0] ## Compare using output key_to_diffs = {} for key, key_data in target_outdata.get_zip_objective_and_data(): intersect = set(vector_outdata_uniq) & set(key_data) key_to_diffs[key] = (set(vector_outdata_uniq) | \ set(key_data)) - intersect for elem in intersect: if key_data[elem] != vector_outdata_uniq[elem]: key_to_diffs[key].add(elem) else: # outdata is not set use difference of matrices ## obtain the active cols for each rows target_active_cols_dict = \ target_matrix.query_active_columns_of_rows() vector_active_cols = \ comparing_vector.query_active_columns_of_rows() vector_active_cols = \ set(vector_active_cols[list(vector_active_cols)[0]]) ## for each row of target matrix, get diff key_to_diffs = {} for row_key, row_active in target_active_cols_dict.items(): key_to_diffs[row_key] = (set(row_active) - vector_active_cols)\ | (vector_active_cols - set(row_active)) # Update matrix based on diff for key, diffs in key_to_diffs.items(): # gather the uncertain and set to uncertain uncertain = vector_uncertain_cols | \ set(target_uncertain_cols_dict[key]) set_uncertain = uncertain & diffs diffs -= uncertain values = {} for col in set_uncertain: values[col] = target_matrix.getUncertainCellDefaultVal() for col in diffs: values[col] = target_matrix.getActiveCellDefaultVal() if len(values) > 0: target_matrix.update_cells(key, values) target_matrix.serialize() #~ def update_matrix_to_cover_when_diference() #~class DriversUtils()
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \ stop_on_failure=False, \ per_test_timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ fault_test_execution_matrix_file=None, \ fault_test_execution_execoutput_file=None, \ with_output_summary=True, \ hash_outlog=None, \ test_prioritization_module=None, \ parallel_test_count=1, \ parallel_test_scheduler=None, \ restart_checkpointer=False, finish_destroy_checkpointer=True): ''' Execute the list of test cases with the given executable and say, for each test case, whether it failed :param meta_testcases: list of test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the tests :param env_vars: dict of environment variables to set before executing each test ({<variable>: <value>}) :param stop_on_failure: decide whether to stop the test execution once a test fails :param fault_test_execution_matrix_file: Optional matrix file to store the tests' pass fail execution data :param fault_test_execution_execoutput_file: Optional output log file to store the tests' execution actual output (hashed) :param with_output_summary: decide whether to return outlog hash :type hash_outlog: bool :hash_outlog: decide whether to hash the outlog or not :param test_prioritization_module: Specify the test prioritization module. (TODO: Implement support) :param parallel_test_count: Specify the number of parallel test Execution. must be an integer >= 1 or None. When None, the max possible value is used. :param parallel_test_scheduler: Specify the function that will handle parallel test scheduling by tool, using the test execution optimizer. (TODO: Implement support) :type restart_checkointer: bool :param restart_checkointer: Decide whether to discard checkpoint and restart anew. :type finish_destroy_checkpointer: bool :param finish_destroy_checkpointer: Decide whether to automatically destroy the checkpointer when done or not Useful is caller has a checkpointer to update. :returns: dict of testcase and their failed verdict. {<test case name>: <True if failed, False if passed, UNCERTAIN_TEST_VERDICT if uncertain>} If stop_on_failure is True, only return the tests that have been executed until the failure ''' ERROR_HANDLER.assert_true(meta_testcases is not None, \ "Must specify testcases", __file__) # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(test_prioritization_module is None, \ "Must implement test prioritization support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \ "Must implement parallel tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() if hash_outlog is None: hash_outlog = self.hash_outlog ERROR_HANDLER.assert_true(parallel_test_count is None \ or parallel_test_count >= 1, \ "invalid parallel tests count ({})".format(\ parallel_test_count), __file__) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests" cp_task_id = 1 checkpoint_handler = \ CheckPointHandler(self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): logging.warning("%s %s" %("The function 'runtests' is finished", \ "according to checkpoint, but called again. None returned")) if common_mix.confirm_execution("%s %s" % ( \ "Function 'runtests' is already", \ "finished, do you want to restart?")): checkpoint_handler.restart() logging.info("Restarting the finished 'runtests'") else: ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\ "Execution halted. Cannot continue because no value", \ " can be returned. Check the results of the", \ "finished execution"), call_location=__file__) # @Checkpoint: Get the saved payload (data kapt for each tool) # pair list of testfailed verdict and execution output meta_test_failedverdicts_outlog = \ checkpoint_handler.get_optional_payload() if meta_test_failedverdicts_outlog is None: meta_test_failedverdicts_outlog = [{}, {}] # Make sure the tests are unique ERROR_HANDLER.assert_true(len(meta_testcases) == \ len(set(meta_testcases)), \ "not all tests are unique", __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases_backup = meta_testcases meta_testcases = set(meta_testcases) dups_remove_meta_testcases = meta_testcases & \ set(self.tests_duplicates_map) dup_toadd_test = {self.tests_duplicates_map[v] for v in \ dups_remove_meta_testcases} - meta_testcases meta_testcases = (meta_testcases - dups_remove_meta_testcases) \ | dup_toadd_test testcases_by_tool = {} for meta_testcase in meta_testcases: ttoolalias, testcase = \ DriversUtils.reverse_meta_element(meta_testcase) if ttoolalias not in testcases_by_tool: testcases_by_tool[ttoolalias] = [] testcases_by_tool[ttoolalias].append(testcase) candidate_aliases = [] for tpos, ttoolalias in enumerate(testcases_by_tool.keys()): # @Checkpoint: Check whether already executed if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias): continue candidate_aliases.append(ttoolalias) # parallelism strategies PARA_FULL_DOUBLE = 0 PARA_ALT_TOOLS_AND_TESTS = 1 PARA_TOOLS_ONLY = 2 PARA_TOOLS_TESTS_AS_TOOLS = 3 parallel_strategy = PARA_TOOLS_ONLY # minimum number of tests (accross) for parallelism ptest_tresh = 5 # minimum number of tests (of the given tool) for tool parallelism sub_ptest_thresh = 3 shared_loc = multiprocessing.RLock() parallel_test_count_by_tool = {ta: 1 for ta in candidate_aliases} # tool with parallel test exec # TODO: find way to pass parallel count here if parallel_test_count is None: #parallel_test_count = min(10, multiprocessing.cpu_count()) parallel_test_count = min(20, 2 * multiprocessing.cpu_count()) cand_alias_joblib = [] cand_alias_for = [] para_tools = [] para_tools = [tt for tt in candidate_aliases if \ (len(testcases_by_tool[tt]) >= sub_ptest_thresh \ and self.testcases_configured_tools[tt]\ [self.TOOL_OBJ_KEY].can_run_tests_in_parallel()) ] actual_parallel_cond = len(candidate_aliases) > 1 \ and len(meta_testcases) >= ptest_tresh \ and parallel_test_count is not None \ and parallel_test_count > 1 if parallel_strategy == PARA_ALT_TOOLS_AND_TESTS: # the para_tools will run without parallelism, give them all threads for tt in para_tools: parallel_test_count_by_tool[tt] = parallel_test_count seq_tools = list(set(candidate_aliases) - set(para_tools)) if len(seq_tools) > 1 and actual_parallel_cond: cand_alias_joblib = seq_tools cand_alias_for = para_tools else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_ONLY: if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_FULL_DOUBLE: # use parallel sub_parallel_count = 0 if parallel_test_count is None else \ parallel_test_count - len(parallel_test_count_by_tool) if sub_parallel_count > 0: para_tools.sort(reverse=True, \ key=lambda x: len(testcases_by_tool[x])) para_tools_n_tests = sum(\ [len(testcases_by_tool[tt]) for tt in para_tools]) used = 0 for tt in para_tools: quota = int(len(testcases_by_tool[tt]) * \ sub_parallel_count / para_tools_n_tests) parallel_test_count_by_tool[tt] += quota used += quota for tt in para_tools: if used == sub_parallel_count: break parallel_test_count_by_tool[tt] += 1 if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_TESTS_AS_TOOLS: # split the tests of one tool and # make the same tool run multiple times ERROR_HANDLER.error_exit("To Be implemented: same tool many times") else: ERROR_HANDLER.error_exit("Invalid parallel startegy") def tool_parallel_test_exec(ttoolalias): # Actual execution found_a_failure = False # Whether the execution was unsuccessful test_error = False ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=\ with_output_summary, \ hash_outlog=hash_outlog, \ parallel_count=\ parallel_test_count_by_tool[ttoolalias]) with shared_loc: for testcase in test_failed_verdicts: meta_testcase = DriversUtils.make_meta_element(\ testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if not found_a_failure \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.FAIL_TEST_VERDICT: found_a_failure = True if not test_error \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.TEST_EXECUTION_ERROR: test_error = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) return found_a_failure, test_error #~ def tool_parallel_test_exec() if len(cand_alias_joblib) > 0: parallel_count_ = min(len(cand_alias_joblib), parallel_test_count) joblib.Parallel(n_jobs=parallel_count_, require='sharedmem')\ (joblib.delayed(tool_parallel_test_exec)(ttoolalias) \ for ttoolalias in cand_alias_joblib) if len(cand_alias_for) > 0: for tpos, ttoolalias in enumerate(cand_alias_for): found_a_failure, test_error = \ tool_parallel_test_exec(ttoolalias) if stop_on_failure and found_a_failure: # @Checkpoint: Chekpointing for remaining tools for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]: checkpoint_handler.do_checkpoint(\ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=rem_tool, \ opt_payload=meta_test_failedverdicts_outlog) break if stop_on_failure: # Make sure the non executed test has the uncertain value (None) if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases): for meta_testcase in set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0]): meta_test_failedverdicts_outlog[0][meta_testcase] = \ common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT meta_test_failedverdicts_outlog[1][meta_testcase] = \ common_matrices.OutputLogData.\ UNCERTAIN_TEST_OUTLOGDATA ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \ len(meta_testcases), \ "mismatch between number of tests and reported verdicts:" + " Tests without verdict are {};".format(\ set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0])) \ + " Test not in testlist are {}.".format(\ set(meta_test_failedverdicts_outlog[0]) - \ set(meta_testcases)), \ __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases = meta_testcases_backup for i in (0, 1): for mtest in dups_remove_meta_testcases: # add to results meta_test_failedverdicts_outlog[i][mtest] = copy.deepcopy(\ meta_test_failedverdicts_outlog[i]\ [self.tests_duplicates_map[mtest]]) for mtest in dup_toadd_test: # remove from results del meta_test_failedverdicts_outlog[i][mtest] if fault_test_execution_matrix_file is not None: # Load or Create the matrix fault_test_execution_matrix = common_matrices.ExecutionMatrix( \ filename=fault_test_execution_matrix_file, \ non_key_col_list=meta_testcases) ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \ "matrix must be empty. Filename is:" " "+fault_test_execution_matrix_file, __file__) failverdict2val = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ fault_test_execution_matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ fault_test_execution_matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ fault_test_execution_matrix.getUncertainCellDefaultVal(), } cells_dict = {} for meta_testcase in meta_test_failedverdicts_outlog[0]: cells_dict[meta_testcase] = failverdict2val[\ meta_test_failedverdicts_outlog[0][meta_testcase]] fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \ cells_dict, serialize=True) if fault_test_execution_execoutput_file is not None: # Load or Create the data object fault_test_execution_execoutput = common_matrices.OutputLogData( \ filename=fault_test_execution_execoutput_file) ERROR_HANDLER.assert_true(\ fault_test_execution_execoutput.is_empty(), \ "outlog data must be empty", __file__) fault_test_execution_execoutput.add_data(\ {self.PROGRAM_EXECOUTPUT_KEY: \ meta_test_failedverdicts_outlog[1]}, \ serialize=True) # @Checkpoint: Finished detailed_exectime = {} for ttoolalias in testcases_by_tool.keys(): tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ttoolalias] = (\ tt.get_checkpointer().get_execution_time(),\ tt.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished( \ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy() return meta_test_failedverdicts_outlog
def _get_test_objective_tests_from_matrix(self, matrix_file): matrix = common_matrices.ExecutionMatrix(filename=matrix_file) res = matrix.query_active_columns_of_rows() return res