def post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map): for kept, duplist in kepttest2duptest_map.items(): alias, _ = DriversUtils.reverse_meta_element(kept) if alias == origin: stored_map[kept] = [] for dup in duplist: dalias, dtest = DriversUtils.reverse_meta_element(dup) if dalias == origin: continue stored_map[kept].append(dup) tests_to_avoid.append(dup)
def fault_analysis(cm_corebench_scripts_dir, c_id, conf_py, in_muteria_outdir, out_top_dir): if not os.path.isdir(out_top_dir): os.mkdir(out_top_dir) in_res_data_dir = os.path.join(in_muteria_outdir, 'latest', 'RESULTS_DATA') testtools_workdir = os.path.join(in_muteria_outdir, 'latest', 'testscases_workdir') pass_fail_matrix = os.path.join(in_res_data_dir, "matrices", "PASSFAIL.csv") pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix) test_list = list(pf_mat.get_nonkey_colname_list()) #semu_tool_dirs = [d for d in os.listdir(testtools_workdir) if d.startswith('semu_cmp-')] # get fault tests get_commit_fault_tests(cm_corebench_scripts_dir, c_id, conf_py, in_res_data_dir, out_top_dir) # get test to timestamp test_timestamp_file = os.path.join(out_top_dir, "test_to_timestamp.json") test2timestamp = {} ## get tests by tools tools2tests = {} for test in test_list: alias, _ = DriversUtils.reverse_meta_element(test) # XXX: only SEMU if not alias.startswith('semu_cmp-'): continue if alias not in tools2tests: tools2tests[alias] = set() tools2tests[alias].add(test) ## untar tests dir for alias, tests in tools2tests.items(): d = os.path.join(testtools_workdir, alias) assert os.path.isdir( d), "test tool dir " + d + " missing for alias " + alias test_tar = os.path.join(d, 'tests_files.tar.gz') es = common_fs.TarGz.decompressDir(test_tar) assert es is None, "decompress error: " + es tests_files = os.path.join(d, 'tests_files') assert os.path.isdir(tests_files), "dir missing after decompress" for test in tests: _, simple_test = DriversUtils.reverse_meta_element(test) gt = TestcasesToolSemu._get_generation_time_of_test( simple_test, tests_files) test2timestamp[test] = gt shutil.rmtree(tests_files) common_fs.dumpJSON(test2timestamp, test_timestamp_file, pretty=True)
def get_too2relmuts(relmuts_to_reltests, toollist=None): res = {tool: set() for tool in toollist} if toollist is not None else {} for relmut, t_list in relmuts_to_reltests.items(): for meta_t in t_list: toolalias, test = DriversUtils.reverse_meta_element(meta_t) if toollist is None: if toolalias not in res: res[toolalias] = set() else: assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format( toolalias, toollist) res[toolalias].add(relmut) for toolalias in res: res[toolalias] = list(res[toolalias]) return res
def execute_testcase (self, meta_testcase, exe_path_map, env_vars, \ timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ with_output_summary=True, \ hash_outlog=None): ''' Execute a test case with the given executable and say whether it failed :param meta_testcase: string name of the test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the test :param env_vars: dict of environment variables to set before executing the test ({<variable>: <value>}) :type hash_outlog: bool :hash_outlog: decide whether to hash the outlog or not :returns: pair of: - boolean failed verdict of the test (True if failed, False otherwise) - test execution output log hash data object or None ''' if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() if hash_outlog is None: hash_outlog = self.hash_outlog # Find which test tool's the testcase is, then execute ttoolalias, testcase = DriversUtils.reverse_meta_element(meta_testcase) ERROR_HANDLER.assert_true( \ ttoolalias in self.testcases_configured_tools, \ "Test tool {} not registered".format(ttoolalias), \ __file__) ttool = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] return ttool.execute_testcase(testcase, exe_path_map, env_vars, \ timeout=timeout, \ use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=with_output_summary, \ hash_outlog=hash_outlog)
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \ stop_on_failure=False, \ per_test_timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ fault_test_execution_matrix_file=None, \ fault_test_execution_execoutput_file=None, \ with_output_summary=True, \ hash_outlog=None, \ test_prioritization_module=None, \ parallel_test_count=1, \ parallel_test_scheduler=None, \ restart_checkpointer=False, finish_destroy_checkpointer=True): ''' Execute the list of test cases with the given executable and say, for each test case, whether it failed :param meta_testcases: list of test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the tests :param env_vars: dict of environment variables to set before executing each test ({<variable>: <value>}) :param stop_on_failure: decide whether to stop the test execution once a test fails :param fault_test_execution_matrix_file: Optional matrix file to store the tests' pass fail execution data :param fault_test_execution_execoutput_file: Optional output log file to store the tests' execution actual output (hashed) :param with_output_summary: decide whether to return outlog hash :type hash_outlog: bool :hash_outlog: decide whether to hash the outlog or not :param test_prioritization_module: Specify the test prioritization module. (TODO: Implement support) :param parallel_test_count: Specify the number of parallel test Execution. must be an integer >= 1 or None. When None, the max possible value is used. :param parallel_test_scheduler: Specify the function that will handle parallel test scheduling by tool, using the test execution optimizer. (TODO: Implement support) :type restart_checkointer: bool :param restart_checkointer: Decide whether to discard checkpoint and restart anew. :type finish_destroy_checkpointer: bool :param finish_destroy_checkpointer: Decide whether to automatically destroy the checkpointer when done or not Useful is caller has a checkpointer to update. :returns: dict of testcase and their failed verdict. {<test case name>: <True if failed, False if passed, UNCERTAIN_TEST_VERDICT if uncertain>} If stop_on_failure is True, only return the tests that have been executed until the failure ''' ERROR_HANDLER.assert_true(meta_testcases is not None, \ "Must specify testcases", __file__) # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(test_prioritization_module is None, \ "Must implement test prioritization support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \ "Must implement parallel tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() if hash_outlog is None: hash_outlog = self.hash_outlog ERROR_HANDLER.assert_true(parallel_test_count is None \ or parallel_test_count >= 1, \ "invalid parallel tests count ({})".format(\ parallel_test_count), __file__) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests" cp_task_id = 1 checkpoint_handler = \ CheckPointHandler(self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): logging.warning("%s %s" %("The function 'runtests' is finished", \ "according to checkpoint, but called again. None returned")) if common_mix.confirm_execution("%s %s" % ( \ "Function 'runtests' is already", \ "finished, do you want to restart?")): checkpoint_handler.restart() logging.info("Restarting the finished 'runtests'") else: ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\ "Execution halted. Cannot continue because no value", \ " can be returned. Check the results of the", \ "finished execution"), call_location=__file__) # @Checkpoint: Get the saved payload (data kapt for each tool) # pair list of testfailed verdict and execution output meta_test_failedverdicts_outlog = \ checkpoint_handler.get_optional_payload() if meta_test_failedverdicts_outlog is None: meta_test_failedverdicts_outlog = [{}, {}] # Make sure the tests are unique ERROR_HANDLER.assert_true(len(meta_testcases) == \ len(set(meta_testcases)), \ "not all tests are unique", __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases_backup = meta_testcases meta_testcases = set(meta_testcases) dups_remove_meta_testcases = meta_testcases & \ set(self.tests_duplicates_map) dup_toadd_test = {self.tests_duplicates_map[v] for v in \ dups_remove_meta_testcases} - meta_testcases meta_testcases = (meta_testcases - dups_remove_meta_testcases) \ | dup_toadd_test testcases_by_tool = {} for meta_testcase in meta_testcases: ttoolalias, testcase = \ DriversUtils.reverse_meta_element(meta_testcase) if ttoolalias not in testcases_by_tool: testcases_by_tool[ttoolalias] = [] testcases_by_tool[ttoolalias].append(testcase) candidate_aliases = [] for tpos, ttoolalias in enumerate(testcases_by_tool.keys()): # @Checkpoint: Check whether already executed if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias): continue candidate_aliases.append(ttoolalias) # parallelism strategies PARA_FULL_DOUBLE = 0 PARA_ALT_TOOLS_AND_TESTS = 1 PARA_TOOLS_ONLY = 2 PARA_TOOLS_TESTS_AS_TOOLS = 3 parallel_strategy = PARA_TOOLS_ONLY # minimum number of tests (accross) for parallelism ptest_tresh = 5 # minimum number of tests (of the given tool) for tool parallelism sub_ptest_thresh = 3 shared_loc = multiprocessing.RLock() parallel_test_count_by_tool = {ta: 1 for ta in candidate_aliases} # tool with parallel test exec # TODO: find way to pass parallel count here if parallel_test_count is None: #parallel_test_count = min(10, multiprocessing.cpu_count()) parallel_test_count = min(20, 2 * multiprocessing.cpu_count()) cand_alias_joblib = [] cand_alias_for = [] para_tools = [] para_tools = [tt for tt in candidate_aliases if \ (len(testcases_by_tool[tt]) >= sub_ptest_thresh \ and self.testcases_configured_tools[tt]\ [self.TOOL_OBJ_KEY].can_run_tests_in_parallel()) ] actual_parallel_cond = len(candidate_aliases) > 1 \ and len(meta_testcases) >= ptest_tresh \ and parallel_test_count is not None \ and parallel_test_count > 1 if parallel_strategy == PARA_ALT_TOOLS_AND_TESTS: # the para_tools will run without parallelism, give them all threads for tt in para_tools: parallel_test_count_by_tool[tt] = parallel_test_count seq_tools = list(set(candidate_aliases) - set(para_tools)) if len(seq_tools) > 1 and actual_parallel_cond: cand_alias_joblib = seq_tools cand_alias_for = para_tools else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_ONLY: if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_FULL_DOUBLE: # use parallel sub_parallel_count = 0 if parallel_test_count is None else \ parallel_test_count - len(parallel_test_count_by_tool) if sub_parallel_count > 0: para_tools.sort(reverse=True, \ key=lambda x: len(testcases_by_tool[x])) para_tools_n_tests = sum(\ [len(testcases_by_tool[tt]) for tt in para_tools]) used = 0 for tt in para_tools: quota = int(len(testcases_by_tool[tt]) * \ sub_parallel_count / para_tools_n_tests) parallel_test_count_by_tool[tt] += quota used += quota for tt in para_tools: if used == sub_parallel_count: break parallel_test_count_by_tool[tt] += 1 if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_TESTS_AS_TOOLS: # split the tests of one tool and # make the same tool run multiple times ERROR_HANDLER.error_exit("To Be implemented: same tool many times") else: ERROR_HANDLER.error_exit("Invalid parallel startegy") def tool_parallel_test_exec(ttoolalias): # Actual execution found_a_failure = False # Whether the execution was unsuccessful test_error = False ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=\ with_output_summary, \ hash_outlog=hash_outlog, \ parallel_count=\ parallel_test_count_by_tool[ttoolalias]) with shared_loc: for testcase in test_failed_verdicts: meta_testcase = DriversUtils.make_meta_element(\ testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if not found_a_failure \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.FAIL_TEST_VERDICT: found_a_failure = True if not test_error \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.TEST_EXECUTION_ERROR: test_error = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) return found_a_failure, test_error #~ def tool_parallel_test_exec() if len(cand_alias_joblib) > 0: parallel_count_ = min(len(cand_alias_joblib), parallel_test_count) joblib.Parallel(n_jobs=parallel_count_, require='sharedmem')\ (joblib.delayed(tool_parallel_test_exec)(ttoolalias) \ for ttoolalias in cand_alias_joblib) if len(cand_alias_for) > 0: for tpos, ttoolalias in enumerate(cand_alias_for): found_a_failure, test_error = \ tool_parallel_test_exec(ttoolalias) if stop_on_failure and found_a_failure: # @Checkpoint: Chekpointing for remaining tools for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]: checkpoint_handler.do_checkpoint(\ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=rem_tool, \ opt_payload=meta_test_failedverdicts_outlog) break if stop_on_failure: # Make sure the non executed test has the uncertain value (None) if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases): for meta_testcase in set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0]): meta_test_failedverdicts_outlog[0][meta_testcase] = \ common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT meta_test_failedverdicts_outlog[1][meta_testcase] = \ common_matrices.OutputLogData.\ UNCERTAIN_TEST_OUTLOGDATA ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \ len(meta_testcases), \ "mismatch between number of tests and reported verdicts:" + " Tests without verdict are {};".format(\ set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0])) \ + " Test not in testlist are {}.".format(\ set(meta_test_failedverdicts_outlog[0]) - \ set(meta_testcases)), \ __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases = meta_testcases_backup for i in (0, 1): for mtest in dups_remove_meta_testcases: # add to results meta_test_failedverdicts_outlog[i][mtest] = copy.deepcopy(\ meta_test_failedverdicts_outlog[i]\ [self.tests_duplicates_map[mtest]]) for mtest in dup_toadd_test: # remove from results del meta_test_failedverdicts_outlog[i][mtest] if fault_test_execution_matrix_file is not None: # Load or Create the matrix fault_test_execution_matrix = common_matrices.ExecutionMatrix( \ filename=fault_test_execution_matrix_file, \ non_key_col_list=meta_testcases) ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \ "matrix must be empty. Filename is:" " "+fault_test_execution_matrix_file, __file__) failverdict2val = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ fault_test_execution_matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ fault_test_execution_matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ fault_test_execution_matrix.getUncertainCellDefaultVal(), } cells_dict = {} for meta_testcase in meta_test_failedverdicts_outlog[0]: cells_dict[meta_testcase] = failverdict2val[\ meta_test_failedverdicts_outlog[0][meta_testcase]] fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \ cells_dict, serialize=True) if fault_test_execution_execoutput_file is not None: # Load or Create the data object fault_test_execution_execoutput = common_matrices.OutputLogData( \ filename=fault_test_execution_execoutput_file) ERROR_HANDLER.assert_true(\ fault_test_execution_execoutput.is_empty(), \ "outlog data must be empty", __file__) fault_test_execution_execoutput.add_data(\ {self.PROGRAM_EXECOUTPUT_KEY: \ meta_test_failedverdicts_outlog[1]}, \ serialize=True) # @Checkpoint: Finished detailed_exectime = {} for ttoolalias in testcases_by_tool.keys(): tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ttoolalias] = (\ tt.get_checkpointer().get_execution_time(),\ tt.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished( \ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy() return meta_test_failedverdicts_outlog
def _call_generation_run(self, runtool, args): # Delete any klee-out-* for d in os.listdir(self.tests_working_dir): if d.startswith('klee-out-'): shutil.rmtree(os.path.join(self.tests_working_dir, d)) call_shadow_wrapper_file = os.path.join(self.tests_working_dir, \ "shadow_wrap") devtest_toolalias = self.parent_meta_tool.get_devtest_toolalias() ERROR_HANDLER.assert_true(devtest_toolalias is not None, \ "devtest must be used when using shadow_se", __file__) #test_list = list(self.code_builds_factory.repository_manager\ # .get_dev_tests_list()) test_list = [] for meta_test in self.parent_meta_tool.get_testcase_info_object(\ candidate_tool_aliases=[devtest_toolalias])\ .get_tests_list(): toolalias, test = DriversUtils.reverse_meta_element(meta_test) ERROR_HANDLER.assert_true(toolalias == devtest_toolalias, \ "BUG in above get_testcase_info_object", __file__) test_list.append(test) # Get list of klee_change, klee_get_true/false locations. klee_change_stmts = [] get_lines_callback_obj = self.GetLinesCallbackObject() get_lines_callback_obj.set_pre_callback_args(self.code_builds_factory\ .repository_manager.revert_src_list_files) get_lines_callback_obj.set_post_callback_args(klee_change_stmts) pre_ret, post_ret = self.code_builds_factory.repository_manager\ .custom_read_access(get_lines_callback_obj) ERROR_HANDLER.assert_true(pre_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "pre failed", __file__) ERROR_HANDLER.assert_true(post_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "post failed", __file__) ERROR_HANDLER.assert_true(len(klee_change_stmts) > 0, \ "No klee_change statement in the sources", __file__) # Filter only tests that cover those locations, # if there is stmt coverage matrix stmt_cov_mat_file = self.head_explorer.get_file_pathname(\ fd_structure.CRITERIA_MATRIX[criteria.TestCriteria\ .STATEMENT_COVERAGE]) cov_tests = None if os.path.isfile(stmt_cov_mat_file): stmt_cov_mat = common_matrices.ExecutionMatrix(\ filename=stmt_cov_mat_file) # due to possible wrapper test splitting we update test_list here tmp_test_list = [] for mt in stmt_cov_mat.get_nonkey_colname_list(): alias, t = DriversUtils.reverse_meta_element(mt) if alias == devtest_toolalias: tmp_test_list.append(t) test_list = tmp_test_list meta_stmts = list(stmt_cov_mat.get_keys()) tool_aliases = set() for meta_stmt in meta_stmts: alias, stmt = DriversUtils.reverse_meta_element(meta_stmt) tool_aliases.add(alias) klee_change_meta_stmts = [] for alias in tool_aliases: klee_change_meta_stmts += [\ DriversUtils.make_meta_element(e, alias) \ for e in klee_change_stmts] klee_change_meta_stmts = list(set(meta_stmts) & \ set(klee_change_meta_stmts)) cov_tests = set() if len(klee_change_meta_stmts) > 0: for _, t in stmt_cov_mat.query_active_columns_of_rows(\ row_key_list=klee_change_meta_stmts).items(): cov_tests |= set(t) else: logging.warning('No test covers the patch (SHADOW)!') # ERROR_HANDLER.assert_true(len(klee_change_meta_stmts) > 0, \ # "No test covers the patch", __file__) # tests will be generated in the same dir that has the input .bc file os.mkdir(self.tests_storage_dir) # obtain candidate tests cand_testpair_list = [] for test in test_list: meta_test = DriversUtils.make_meta_element(test, devtest_toolalias) if cov_tests is not None and meta_test not in cov_tests: continue cand_testpair_list.append((test, meta_test)) # Adjust the max-time in args ## locate max-time per_test_hard_timeout = None per_test_timeout = None if len(cand_testpair_list) > 0: cur_max_time = float(self.get_value_in_arglist(args, 'max-time')) if self.driver_config.get_gen_timeout_is_per_test(): per_test_timeout = cur_max_time else: per_test_timeout = max(60, \ cur_max_time / len(cand_testpair_list)) self.set_value_in_arglist(args, 'max-time', str(per_test_timeout)) # give time to dump remaning states per_test_hard_timeout = per_test_timeout + \ self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE #per_test_hard_timeout = 300 #DBG # Set the wrapper with open(call_shadow_wrapper_file, 'w') as wf: wf.write('#! /bin/bash\n\n') wf.write('set -u\n') wf.write('set -o pipefail\n\n') wf.write('ulimit -s unlimited\n') # timeout the shadow execution (some test create daemon which) # are not killed by test timeout. ALSO MAKE SURE TO DESACTIVATE # IN TEST SCRIPT TIMEOUT kill_after = 30 wf.write('time_out_cmd="/usr/bin/timeout --kill-after={}s {}"\n'.\ format(kill_after, per_test_hard_timeout)) # kill after and time for timeout to act per_test_hard_timeout += kill_after + 60 #wf.write(' '.join(['exec', runtool] + args + ['"${@:1}"']) + '\n') wf.write('\nstdindata="{}/klee-last/{}"\n'.format(\ self.tests_working_dir, \ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('tmpstdindata="{}/{}"\n\n'.format(self.tests_working_dir,\ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('if [ -t 0 ] # check if stdin do not exist\n') wf.write('then\n') wf.write(' '.join(['\t(', '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('\t/usr/bin/touch $tmpstdindata\n') wf.write('else\n') wf.write('\t(/bin/cat - > $tmpstdindata ) || EXIT_CODE=1\n') wf.write(' '.join(['\t(', '/bin/cat $tmpstdindata | ', \ '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('fi\n\n') wf.write('/bin/mv $tmpstdindata $stdindata || EXIT_CODE=2\n') wf.write('\n# Provoke "unbound variable" if KLEE fails\n') wf.write('# Preserve the KLEE exit code\n') wf.write('exit $EXIT_CODE\n') os.chmod(call_shadow_wrapper_file, 0o775) # run test exes, _ = self.code_builds_factory.repository_manager\ .get_relative_exe_path_map() ERROR_HANDLER.assert_true(len(exes) == 1, \ "Must have a single exe", __file__) exe_path_map = {e: call_shadow_wrapper_file for e in exes} env_vars = {} self._dir_chmod777(self.tests_storage_dir) for test, meta_test in cand_testpair_list: self.parent_meta_tool.execute_testcase(meta_test, exe_path_map, \ env_vars, timeout=per_test_hard_timeout,\ with_output_summary=False) #logging.debug("DBG: Just executed test '{}'".format(meta_test)) #input(">>>> ") #DBG # copy the klee out test_out = os.path.join(self.tests_storage_dir, \ self.get_sorage_name_of_test(test)) os.mkdir(test_out) for d in glob.glob(self.tests_working_dir + "/klee-out-*"): # make sure we can do anything with it self._dir_chmod777(d) if not self.keep_first_test: first_test = os.path.join(d, 'test000001.ktest') if os.path.isfile(first_test): shutil.move(first_test, first_test + '.disable') shutil.move(d, test_out) ERROR_HANDLER.assert_true(len(list(os.listdir(test_out))) > 0, \ "Shadow generated no test for tescase: "+test,\ __file__) if os.path.islink(os.path.join(self.tests_working_dir, \ 'klee-last')): os.unlink(os.path.join(self.tests_working_dir, 'klee-last')) # store klee_change locs common_fs.dumpJSON(klee_change_stmts, self.klee_change_locs_list_file)
def _get_input_bitcode_file(self, code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=None): meta_mu_src = self.driver_config.get_meta_mutant_source() # XXX Case of manual annotation if meta_mu_src == MetaMuSource.ANNOTATION: with open(self.cand_muts_file, 'w') as f: # Single mutant (id 1, corresponding to old version) f.write(str(1) + '\n') return super(TestcasesToolSemu, self)._get_input_bitcode_file(\ code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=meta_criteria_tool_obj) if type(meta_mu_src) == str: # XXX: The actual path to the meta is specified return meta_mu_src # XXX: Case of other mutation tools like Mart # get the meta criterion file from MART or any compatible tool. mutant_gen_tool_name = meta_mu_src.get_field_value() mut_tool_alias_to_obj = \ meta_criteria_tool_obj.get_criteria_tools_by_name(\ mutant_gen_tool_name) if len(mut_tool_alias_to_obj) == 0: logging.warning(\ 'SEMu requires {} to generate mutants but none used'.format(\ mutant_gen_tool_name)) ERROR_HANDLER.assert_true(len(mut_tool_alias_to_obj) == 1, \ "SEMu supports tests generation from" "a single .bc file for now (todo).", __file__) t_alias2metamu_bc = {} t_alias2mutantInfos = {} for alias, obj in mut_tool_alias_to_obj.items(): dest_bc = rel_path_map[list(rel_path_map)[0]] + '.bc' shutil.copy2(obj.get_test_gen_metamutant_bc(), dest_bc) t_alias2metamu_bc[alias] = dest_bc t_alias2mutantInfos[alias] = obj.get_criterion_info_object(None) # XXX: get mutants ids by functions self.mutants_by_funcs = {} single_alias = list(t_alias2mutantInfos)[0] single_tool_obj = t_alias2mutantInfos[single_alias] cand_muts = list(single_tool_obj.get_elements_list()) for mut in single_tool_obj.get_elements_list(): func = single_tool_obj.get_element_data(mut)[\ 'mutant_function_name'] #meta_mut = DriversUtils.make_meta_element(mut, single_alias) if func not in self.mutants_by_funcs: self.mutants_by_funcs[func] = set() self.mutants_by_funcs[func].add(mut) #meta_mut) # XXX: get candidate mutants list if self.driver_config.get_target_only_live_mutants() \ and os.path.isfile(self.sm_mat_file): sm_mat = common_matrices.ExecutionMatrix(\ filename=self.sm_mat_file) mut2killing_tests = sm_mat.query_active_columns_of_rows() alive_muts = [m for m, k_t in mut2killing_tests.items() \ if len(k_t) == 0] cand_muts = [] for meta_m in alive_muts: t_alias, m = DriversUtils.reverse_meta_element(meta_m) if t_alias in t_alias2metamu_bc: # There is a single one cand_muts.append(m) with open(self.cand_muts_file, 'w') as f: for m in cand_muts: f.write(str(m) + '\n') return t_alias2metamu_bc[list(t_alias2metamu_bc)[0]]
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \ stop_on_failure=False, \ per_test_timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ fault_test_execution_matrix_file=None, \ fault_test_execution_execoutput_file=None, \ with_outlog_hash=True, \ test_prioritization_module=None, \ parallel_test_count=1, \ parallel_test_scheduler=None, \ restart_checkpointer=False, finish_destroy_checkpointer=True): ''' Execute the list of test cases with the given executable and say, for each test case, whether it failed :param meta_testcases: list of test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the tests :param env_vars: dict of environment variables to set before executing each test ({<variable>: <value>}) :param stop_on_failure: decide whether to stop the test execution once a test fails :param fault_test_execution_matrix_file: Optional matrix file to store the tests' pass fail execution data :param fault_test_execution_execoutput_file: Optional output log file to store the tests' execution actual output (hashed) :param with_outlog_hash: decide whether to return outlog hash :param test_prioritization_module: Specify the test prioritization module. (TODO: Implement support) :param parallel_test_count: Specify the number of parallel test Execution. must be an integer >= 1 :param parallel_test_scheduler: Specify the function that will handle parallel test scheduling by tool, using the test execution optimizer. (TODO: Implement support) :type restart_checkointer: bool :param restart_checkointer: Decide whether to discard checkpoint and restart anew. :type finish_destroy_checkpointer: bool :param finish_destroy_checkpointer: Decide whether to automatically destroy the checkpointer when done or not Useful is caller has a checkpointer to update. :returns: dict of testcase and their failed verdict. {<test case name>: <True if failed, False if passed, UNCERTAIN_TEST_VERDICT if uncertain>} If stop_on_failure is True, only return the tests that have been executed until the failure ''' # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(test_prioritization_module is None, \ "Must implement test prioritization support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_count <= 1, \ "Must implement parallel tests execution support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \ "Must implement parallel tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() ERROR_HANDLER.assert_true(parallel_test_count > 0, \ "invalid parallel test execution count: {}. {}".format( \ parallel_test_count, "must be >= 1")) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests" cp_task_id = 1 checkpoint_handler = \ CheckPointHandler(self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): logging.warning("%s %s" %("The function 'runtests' is finished", \ "according to checkpoint, but called again. None returned")) if common_mix.confirm_execution("%s %s" % ( \ "Function 'runtests' is already", \ "finished, do you want to restart?")): checkpoint_handler.restart() logging.info("Restarting the finished 'runtests'") else: ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\ "Execution halted. Cannot continue because no value", \ " can be returned. Check the results of the", \ "finished execution"), call_location=__file__) # @Checkpoint: Get the saved payload (data kapt for each tool) # pair list of testfailed verdict and execution output meta_test_failedverdicts_outlog = \ checkpoint_handler.get_optional_payload() if meta_test_failedverdicts_outlog is None: meta_test_failedverdicts_outlog = [{}, {}] # Make sure the tests are unique ERROR_HANDLER.assert_true(len(meta_testcases) == \ len(set(meta_testcases)), \ "not all tests are unique", __file__) testcases_by_tool = {} for meta_testcase in meta_testcases: ttoolalias, testcase = \ DriversUtils.reverse_meta_element(meta_testcase) if ttoolalias not in testcases_by_tool: testcases_by_tool[ttoolalias] = [] testcases_by_tool[ttoolalias].append(testcase) found_a_failure = False for tpos, ttoolalias in enumerate(testcases_by_tool.keys()): # @Checkpoint: Check whether already executed if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias): continue # Actual execution ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_outlog_hash=with_outlog_hash) for testcase in test_failed_verdicts: meta_testcase = \ DriversUtils.make_meta_element(testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.COMMAND_UNCERTAIN: found_a_failure = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) if stop_on_failure and found_a_failure: # @Checkpoint: Chekpointing for remaining tools for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]: checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=rem_tool, \ opt_payload=meta_test_failedverdicts_outlog) break if stop_on_failure: # Make sure the non executed test has the uncertain value (None) if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases): for meta_testcase in set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0]): meta_test_failedverdicts_outlog[0][meta_testcase] = \ common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT meta_test_failedverdicts_outlog[1][meta_testcase] = \ common_matrices.OutputLogData.\ UNCERTAIN_TEST_OUTLOGDATA ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \ len(meta_testcases), \ "Not all tests have a verdict reported", __file__) if fault_test_execution_matrix_file is not None: # Load or Create the matrix fault_test_execution_matrix = common_matrices.ExecutionMatrix( \ filename=fault_test_execution_matrix_file, \ non_key_col_list=meta_testcases) ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \ "matrix must be empty", __file__) failverdict2val = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ fault_test_execution_matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ fault_test_execution_matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ fault_test_execution_matrix.getUncertainCellDefaultVal(), } cells_dict = {} for meta_testcase in meta_test_failedverdicts_outlog[0]: cells_dict[meta_testcase] = failverdict2val[\ meta_test_failedverdicts_outlog[0][meta_testcase]] fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \ cells_dict, serialize=True) if fault_test_execution_execoutput_file is None: meta_test_failedverdicts_outlog[1] = None else: # Load or Create the data object fault_test_execution_execoutput = common_matrices.OutputLogData( \ filename=fault_test_execution_execoutput_file) ERROR_HANDLER.assert_true(\ fault_test_execution_execoutput.is_empty(), \ "outlog data must be empty", __file__) fault_test_execution_execoutput.add_data(\ {self.PROGRAM_EXECOUTPUT_KEY: \ meta_test_failedverdicts_outlog[1]}, \ serialize=True) # @Checkpoint: Finished detailed_exectime = {} for ttoolalias in testcases_by_tool.keys(): tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ttoolalias] = (\ tt.get_checkpointer().get_execution_time(),\ tt.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished( \ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy() return meta_test_failedverdicts_outlog
def runtests_criteria_coverage (self, testcases, criterion_to_matrix, \ criterion_to_executionoutput=None, criteria_element_list_by_criteria=None, \ re_instrument_code=False, \ cover_criteria_elements_once=False, prioritization_module_by_criteria=None, parallel_count=1, \ parallel_criteria_test_scheduler=None,\ restart_checkpointer=False, \ finish_destroy_checkpointer=True): ''' Executes the instrumented executable code with testscases and returns the different code coverage matrices. :param testcases: list of testcases to execute :param criterion_to_matrix: dict of <criterion, Matrix file where to store coverage>. :param criterion_to_executionoutput: dict of <criterion, execoutput file where to store coverage>. :param criteria_element_list_by_criteria: dictionary representing the list of criteria elements (stmts, branches, mutants) to consider in the test execution matices. Key is the criterion and the value the list of elements :param re_instrument_code: Decide whether to instrument code before running the tests. (Example when instrumentation was not specifically called. This is True by default) :param cover_criteria_elements_once: Specify whether to cover criteria elements once is enough, meaning that we stop analysing a criterion element once a test covers it. The remaining test covering verdict will be UNKNOWN. :param prioritization_module_by_criteria: dict of prioritization module by criteria. None means no prioritization used. :type \parallel_count: :param \parallel_count: :type \parallel_criteria_test_scheduler: :param \parallel_criteria_test_scheduler: scheduler that organize parallelism across criteria tools. (TODO: Implement support) :type \restart_checkpointer: :param \restart_checkpointer: :type finish_destroy_checkpointer: :param finish_destroy_checkpointer: ''' # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(parallel_count <= 1, \ "Must implement parallel execution support here", \ __file__) ERROR_HANDLER.assert_true(parallel_criteria_test_scheduler is None, \ "Must implement parallel codes tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity ERROR_HANDLER.assert_true(parallel_count > 0, \ "invalid parallel execution count: {}. {}".format( \ parallel_count, "must be >= 1")) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests_criteria_coverage" cp_task_id = 1 checkpoint_handler = CheckPointHandler( \ self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): return ERROR_HANDLER.assert_true(len(criterion_to_matrix) > 0, \ "no criterion is enabled", __file__) ERROR_HANDLER.assert_true(len(set(criterion_to_matrix) - \ set(self.tools_config_by_criterion_dict)) == 0, \ "Passed matrices output are more than tool specified", \ __file__) if criterion_to_executionoutput is not None: ERROR_HANDLER.assert_true(set(criterion_to_matrix) == \ set(criterion_to_executionoutput), \ "criteria mismatch between matrix and output", \ __file__) tool2criteria = self._get_tool2criteria(criterion_to_matrix.keys()) matrices_dir_tmp = os.path.join(self.criteria_working_dir, \ "codecov_dir.tmp") if os.path.isdir(matrices_dir_tmp): if restart_checkpointer: shutil.rmtree(matrices_dir_tmp) os.mkdir(matrices_dir_tmp) else: os.mkdir(matrices_dir_tmp) if criteria_element_list_by_criteria is None: criteria_element_list_by_criteria = \ {c: None for c in criterion_to_matrix} # get criteria elements by tools criteria_elem_list_by_tool = {} for criterion in criteria_element_list_by_criteria: if criteria_element_list_by_criteria[criterion] is None: for t_conf in self.tools_config_by_criterion_dict[criterion]: toolalias = t_conf.get_tool_config_alias() if toolalias not in criteria_elem_list_by_tool: criteria_elem_list_by_tool[toolalias] = {} criteria_elem_list_by_tool[toolalias][criterion] = None continue ERROR_HANDLER.assert_true(\ len(criteria_element_list_by_criteria[criterion]) != 0, \ "Empty criteria element list for criterion "\ +criterion.get_str(), __file__) for crit_elem in criteria_element_list_by_criteria[criterion]: toolalias, elem = DriversUtils.reverse_meta_element(crit_elem) if toolalias not in criteria_elem_list_by_tool: criteria_elem_list_by_tool[toolalias] = {} if criterion not in criteria_elem_list_by_tool[toolalias]: criteria_elem_list_by_tool[toolalias][criterion] = [] criteria_elem_list_by_tool[toolalias][criterion].append(elem) ERROR_HANDLER.assert_true(len(set(criteria_elem_list_by_tool) - \ set(self.criteria_configured_tools)) == 0, \ "some tools in data not registered", __file__) crit2tool2matrixfile = {cv: {} for cv in criterion_to_matrix} crit2tool2outhashfile = {cv: {} for cv in criterion_to_executionoutput} for ctoolalias in tool2criteria: _criteria2matrix = {} _criteria2outhash = {} for criterion in tool2criteria[ctoolalias]: _criteria2matrix[criterion] = os.path.join(matrices_dir_tmp, \ criterion.get_field_value() + '-' + ctoolalias + '.csv') if criterion_to_executionoutput is None or \ criterion_to_executionoutput[criterion] is None: _criteria2outhash[criterion] = None else: _criteria2outhash[criterion] = \ os.path.join(matrices_dir_tmp, \ criterion.get_field_value() + '-' + ctoolalias + '.outloghash.json') crit2tool2matrixfile[criterion][ctoolalias] = \ _criteria2matrix[criterion] crit2tool2outhashfile[criterion][ctoolalias] = \ _criteria2outhash[criterion] # @Checkpoint: Check whether already executed if checkpoint_handler.is_to_execute( \ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ctoolalias): for criterion in _criteria2matrix: _criteria2matrix[criterion] = \ common_matrices.ExecutionMatrix( \ filename=_criteria2matrix[criterion], \ non_key_col_list=testcases) if _criteria2outhash[criterion] is not None: _criteria2outhash[criterion] = \ common_matrices.OutputLogData( \ filename=_criteria2outhash[criterion]) # Actual execution ctool = self.criteria_configured_tools[ctoolalias][\ self.TOOL_OBJ_KEY] ctool.runtests_criteria_coverage(testcases, \ criteria_element_list_by_criteria=\ criteria_elem_list_by_tool[toolalias],\ criterion_to_matrix=_criteria2matrix, \ criterion_to_executionoutput=\ _criteria2outhash,\ re_instrument_code=re_instrument_code, \ cover_criteria_elements_once=\ cover_criteria_elements_once, \ prioritization_module_by_criteria=\ prioritization_module_by_criteria) # Checkpointing checkpoint_handler.do_checkpoint( \ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ctoolalias) # Aggregate the matrices and out hashes ## Create reult matrices and out hashes result_matrices = {} result_outloghashes = {} for criterion in criterion_to_matrix: result_matrices[criterion] = common_matrices.ExecutionMatrix( \ filename=criterion_to_matrix[criterion], \ non_key_col_list=testcases) if criterion_to_executionoutput[criterion] is None: result_outloghashes[criterion] = None else: result_outloghashes[criterion] = \ common_matrices.OutputLogData(filename=\ criterion_to_executionoutput[criterion]) ERROR_HANDLER.assert_true(\ crit2tool2outhashfile[criterion] is not None, "Bug: log enabled but hidden from tool", __file__) ## Actual aggregate logging.debug("saving results ...") for criterion in result_matrices: result_matrix = result_matrices[criterion] result_outloghash = result_outloghashes[criterion] for mtoolalias in crit2tool2matrixfile[criterion]: tool_matrix = common_matrices.ExecutionMatrix(\ filename=crit2tool2matrixfile[criterion][mtoolalias]) # Check columns ERROR_HANDLER.assert_true(tool_matrix.get_key_colname() == \ result_matrix.get_key_colname(), \ "mismatch on key column name", __file__) ERROR_HANDLER.assert_true( \ set(tool_matrix.get_nonkey_colname_list()) == \ set(result_matrix.get_nonkey_colname_list()), \ "mismatch on non key column names", __file__) # bring in the data key2nonkeydict = tool_matrix.to_pandas_df().\ set_index(tool_matrix.get_key_colname(), drop=True).\ to_dict(orient="index") for c_key in key2nonkeydict: meta_c_key = DriversUtils.make_meta_element(\ str(c_key), mtoolalias) result_matrix.add_row_by_key(meta_c_key, key2nonkeydict[c_key], serialize=False) # out log hash if crit2tool2outhashfile[criterion] is not None: tool_outloghash = common_matrices.OutputLogData(\ filename=\ crit2tool2outhashfile[criterion][mtoolalias]) for objective, objective_data in \ tool_outloghash.get_zip_objective_and_data(): meta_objective = DriversUtils.make_meta_element(\ str(objective), mtoolalias) result_outloghash.add_data( {meta_objective: objective_data}, \ serialize=False) # @Checkpoint: Check whether already executed if checkpoint_handler.is_to_execute( \ func_name=cp_func_name, \ taskid=cp_task_id + 1, tool=criterion.get_str()): # Serialized the computed matrix result_matrix.serialize() if result_outloghash is not None: result_outloghash.serialize() # @Checkpoint: Checkpointing checkpoint_handler.do_checkpoint( \ func_name=cp_func_name, \ taskid=cp_task_id + 1, tool=criterion.get_str()) # Delete the temporary tool matrix's directory if os.path.isdir(matrices_dir_tmp): shutil.rmtree(matrices_dir_tmp) # @Checkpoint: Finished detailed_exectime = {} for ctoolalias in tool2criteria: ct = self.criteria_configured_tools[ctoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ctoolalias] = (\ ct.get_checkpointer().get_execution_time(),\ ct.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished(\ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy()
def summarize_data(outdir, summarized_data_dir, mutant_exec_res, \ mut_ex_prepa_data_dir): other_rel_to_reltests_file = os.path.join( mut_ex_prepa_data_dir, 'relevantmuts_to_relevanttests.json') other_rel_to_reltests = common_fs.loadJSON(other_rel_to_reltests_file) stored_map_file = os.path.join(mut_ex_prepa_data_dir, 'stored_test_map.json') stored_map = common_fs.loadJSON(stored_map_file) def get_too2relmuts(relmuts_to_reltests, toollist=None): res = {tool: set() for tool in toollist} if toollist is not None else {} for relmut, t_list in relmuts_to_reltests.items(): for meta_t in t_list: toolalias, test = DriversUtils.reverse_meta_element(meta_t) if toollist is None: if toolalias not in res: res[toolalias] = set() else: assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format( toolalias, toollist) res[toolalias].add(relmut) for toolalias in res: res[toolalias] = list(res[toolalias]) return res #~ def get_too2relmuts() other_tool_to_relmuts = get_too2relmuts(other_rel_to_reltests) all_tests, fail_tests, relevant_mutants_to_relevant_tests, \ mutants_to_killingtests, tests_to_killed_mutants = load.load (mutant_exec_res, fault_revealing=False) if not os.path.isdir(summarized_data_dir): os.mkdir(summarized_data_dir) cmp_result_file = os.path.join(summarized_data_dir, "compare_results.json") initial_relmuts_file = os.path.join(summarized_data_dir, "initial_relevant.json") rMS_file = os.path.join(summarized_data_dir, "rMS.json") # get relevant muts by tools # get tool list tinf = common_fs.loadJSON( os.path.join( mutant_exec_res, "post/RESULTS_DATA/other_copied_results/testcasesInfos.json")) tool2relmuts = get_too2relmuts( relevant_mutants_to_relevant_tests, toollist=list(set(tinf["CUSTOM"]) - {"custom_devtests"})) # update with stored map for relmut, t_list in other_rel_to_reltests.items(): for other_meta_t in t_list: if other_meta_t in stored_map: for meta_t in stored_map[other_meta_t]: toolalias, test = DriversUtils.reverse_meta_element(meta_t) assert toolalias in tool2relmuts, "PB2: " tool2relmuts[toolalias].append(relmut) for toolalias in tool2relmuts: tool2relmuts[toolalias] = list(set(tool2relmuts[toolalias])) common_fs.dumpJSON(tool2relmuts, cmp_result_file, pretty=True) common_fs.dumpJSON(other_tool_to_relmuts, initial_relmuts_file, pretty=True) # Compute and print the relevant mutation score per tool all_rel_muts = set() devtest_rel = set() for talias, rellist in tool2relmuts.items(): all_rel_muts |= set(rellist) for talias, rellist in other_tool_to_relmuts.items(): all_rel_muts |= set(rellist) if talias.startswith("custom_devtests"): devtest_rel |= set(rellist) rMS = {"ALL-GENONLY": {}, "ALL-ALL": {}, "ADDITIONAL": {}} for talias, rellist in tool2relmuts.items(): rMS["ALL-ALL"][talias] = len(set(rellist) | devtest_rel) * 100.0 / len(all_rel_muts) rMS["ALL-GENONLY"][talias] = len(rellist) * 100.0 / len(all_rel_muts) rMS["ADDITIONAL"][talias] = len(set( rellist) - devtest_rel) * 100.0 / len(all_rel_muts - devtest_rel) common_fs.dumpJSON(rMS, rMS_file, pretty=True)
def prepare_mutant_execution(outdir, muteria_output, original_conf, \ prepare_data_dir, \ avoid_meta_tests_list_file, avoid_meta_mutants_list_file): if not os.path.isdir(prepare_data_dir): os.mkdir(prepare_data_dir) # Store the test list per technique test_list_file = os.path.join(muteria_output, 'latest', 'RESULTS_DATA', \ 'other_copied_results', 'testcasesInfos.json') shutil.copy2(test_list_file, os.path.join(prepare_data_dir, \ 'gentests_'+os.path.basename(test_list_file))) relevant_exec_outfolder = os.path.join(os.path.dirname(original_conf), \ 'output', 'latest') # Do fdupes with relevant output tests of semu and shadow and store map r_shadow_tests_src = os.path.join(relevant_exec_outfolder, \ 'testscases_workdir', 'shadow_se', 'tests_files') r_shadow_tests_src_tar = r_shadow_tests_src + '.tar.gz' r_semu_tests_src = os.path.join(relevant_exec_outfolder, \ 'testscases_workdir', 'semu', 'tests_files') r_semu_tests_src_tar = r_semu_tests_src + '.tar.gz' cur_exec_outfolder = os.path.join(muteria_output, 'latest') cur_exec_tg_folder = os.path.join(cur_exec_outfolder, 'testscases_workdir') cur_shadow_tests_src = None cur_semu_tests_srcs = [] cur_semu_tests_src_tars = [] for f in os.listdir(cur_exec_tg_folder): if f.startswith('semu'): cur_semu_tests_srcs.append( os.path.join(cur_exec_tg_folder, f, 'tests_files')) cur_semu_tests_src_tars.append(cur_semu_tests_srcs[-1] + '.tar.gz') elif f.startswith('shadow_se'): if cur_shadow_tests_src is not None: error_exit("Multiple shadow_se folders exist") cur_shadow_tests_src = os.path.join(cur_exec_tg_folder, f, 'tests_files') cur_shadow_tests_src_tar = cur_shadow_tests_src + '.tar.gz' # use folder fdupes from KtestFormat in muteria ## decompress the folders common_fs.TarGz.decompressDir(r_shadow_tests_src_tar) common_fs.TarGz.decompressDir(r_semu_tests_src_tar) common_fs.TarGz.decompressDir(cur_shadow_tests_src_tar) for i in range(len(cur_semu_tests_src_tars)): common_fs.TarGz.decompressDir(cur_semu_tests_src_tars[i]) ## apply fdupes def post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map): for kept, duplist in kepttest2duptest_map.items(): alias, _ = DriversUtils.reverse_meta_element(kept) if alias == origin: stored_map[kept] = [] for dup in duplist: dalias, dtest = DriversUtils.reverse_meta_element(dup) if dalias == origin: continue stored_map[kept].append(dup) tests_to_avoid.append(dup) #~ def post_test_dup() stored_map = {} stored_map_file = os.path.join(prepare_data_dir, 'stored_test_map.json') tests_to_avoid = [] ### Shadow custom_bin = '/home/shadowvm/shadow/klee-change/Release+Asserts/bin/' folders = [r_shadow_tests_src, cur_shadow_tests_src] folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders} origin = folder2alias[r_shadow_tests_src] kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\ custom_bin, folders, folder2alias) post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map) ## remove the untar dirs for d in folders: shutil.rmtree(d) ### Semu custom_bin = None folders = [r_semu_tests_src] + cur_semu_tests_srcs folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders} origin = folder2alias[r_semu_tests_src] kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\ custom_bin, folders, folder2alias) post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map) for d in folders: shutil.rmtree(d) ## store dup map and tests to avoid common_fs.dumpJSON(stored_map, stored_map_file, pretty=True) common_fs.dumpJSON(tests_to_avoid, avoid_meta_tests_list_file, pretty=True) # XXX TODO: modify tg_conf to use tests to avoid # Get the criteria dir and add to the muteria out r_criteria_dir = os.path.join(relevant_exec_outfolder, 'criteria_workdir') dest_crit_work = os.path.join(muteria_output, 'latest', 'criteria_workdir') if os.path.isdir(os.path.join(dest_crit_work, 'mart_0')): shutil.rmtree(os.path.join(dest_crit_work, 'mart_0')) shutil.copytree(os.path.join(r_criteria_dir, 'mart_0'), os.path.join(dest_crit_work, 'mart_0')) # Get the selected mutants (mutants that are Not relevant w.r.t dev tests) r_res_top = os.path.dirname(os.path.dirname(original_conf)) r_res = os.path.join(r_res_top, 'res') r_res_tar = glob.glob(r_res_top + '/*res.tar.gz')[0] ## untar res if os.system('cd {} && tar -xzf {} --exclude {} --exclude {} && test -d res'.format(\ r_res_top, os.path.basename(r_res_tar), \ 'post/RESULTS_DATA/other_copied_results/Flakiness', \ 'pre/RESULTS_DATA/other_copied_results/Flakiness')) != 0: error_exit("untar re failed") ## get relevant mutants to relevant tests all_tests, fail_tests, relevant_mutants_to_relevant_tests, \ mutants_to_killingtests, tests_to_killed_mutants = load.load (r_res, fault_revealing=False) ## remove untar shutil.rmtree(r_res) ## filter mutants (remove those relevant w.r.t. dev tests) rel_to_reltests_file = os.path.join(prepare_data_dir, 'relevantmuts_to_relevanttests.json') mutants_to_avoid = [] ### mutants relevant w.r.t. devtests added to avoid for mut, r_tests in relevant_mutants_to_relevant_tests.items(): for rt in r_tests: alias, _ = DriversUtils.reverse_meta_element(rt) if alias.startswith("custom_devtests"): mutants_to_avoid.append(mut) break ## save filter mutants as to avoid common_fs.dumpJSON(relevant_mutants_to_relevant_tests, rel_to_reltests_file, pretty=True) common_fs.dumpJSON(mutants_to_avoid, avoid_meta_mutants_list_file, pretty=True)
def _execute_task(self, task): """ TODO: 1. implement the todos here """ # @Checkpointing: see whether the task is untouch to clear tmp task_untouched = self.cp_data.tasks_obj.task_is_untouched(task) # Execute base on the task: --------------------------------------- if task == checkpoint_tasks.Tasks.STARTING: pass elif task == checkpoint_tasks.Tasks.FINISHED: pass elif task == checkpoint_tasks.Tasks.TESTS_GENERATION_GUIDANCE: pass #TODO: Generate focus criteria and tests and store in json file elif task == checkpoint_tasks.Tasks.TESTS_GENERATION: # @Checkpointing if task_untouched: self.meta_testcase_tool.clear_working_dir() self.cp_data.tasks_obj.set_task_executing(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Generate the tests self.meta_testcase_tool.generate_tests(\ test_tool_type_list=self.cp_data.test_types) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Destroy meta test checkpointer self.meta_testcase_tool.get_checkpoint_state_object()\ .destroy_checkpoint() elif task == checkpoint_tasks.Tasks.\ TESTS_EXECUTION_SELECTION_PRIORITIZATION: out_file_key = outdir_struct.TMP_SELECTED_TESTS_LIST out_file = self.head_explorer.get_file_pathname(out_file_key) # @Checkpointing if task_untouched: self.head_explorer.remove_file_and_get(out_file_key) self.cp_data.tasks_obj.set_task_executing(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) all_tests = self.meta_testcase_tool.\ get_testcase_info_object().get_tests_list() candidate_aliases = \ self.meta_testcase_tool.get_candidate_tools_aliases(\ test_tool_type_list=self.cp_data.test_types) selected_tests = [] for meta_test in all_tests: toolalias, _ = DriversUtils.reverse_meta_element(meta_test) if toolalias in candidate_aliases: selected_tests.append(meta_test) # TODO: use the actual selector. For now just use all tests common_fs.dumpJSON(list(selected_tests), out_file) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Destroy meta test checkpointer #self.meta_testexec_optimization_tool.get_checkpoint_state_object()\ # .destroy_checkpoint() elif task == checkpoint_tasks.Tasks.PASS_FAIL_TESTS_EXECUTION: # Make sure that the Matrices dir exists self.head_explorer.get_or_create_and_get_dir(\ outdir_struct.RESULTS_MATRICES_DIR) matrix_file_key = outdir_struct.TMP_TEST_PASS_FAIL_MATRIX matrix_file = self.head_explorer.get_file_pathname(matrix_file_key) execoutput_file_key = \ outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT if self.config.GET_PASSFAIL_OUTPUT_SUMMARY: execoutput_file = self.head_explorer.get_file_pathname(\ execoutput_file_key) else: execoutput_file = None # @Checkpointing if task_untouched: self.head_explorer.remove_file_and_get(matrix_file_key) self.head_explorer.remove_file_and_get(execoutput_file_key) self.meta_testcase_tool.get_checkpoint_state_object()\ .restart_task() self.cp_data.tasks_obj.set_task_executing(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Execute tests test_list_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_SELECTED_TESTS_LIST) meta_testcases = common_fs.loadJSON(test_list_file) # Set test oracle self.test_oracle_manager.set_oracle(passfail=True) self.meta_testcase_tool.runtests(meta_testcases=meta_testcases, \ stop_on_failure=\ self.config.STOP_TESTS_EXECUTION_ON_FAILURE, \ recalculate_execution_times=True, \ fault_test_execution_matrix_file=matrix_file, \ fault_test_execution_execoutput_file=execoutput_file, \ test_prioritization_module=\ self.meta_testexec_optimization_tool, \ finish_destroy_checkpointer=False) # Unset test oracle self.test_oracle_manager.set_oracle(passfail=False) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Destroy meta test checkpointer self.meta_testcase_tool.get_checkpoint_state_object()\ .destroy_checkpoint() elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION_GUIDANCE: pass #TODO (Maybe could be used someday. for now, just skip it) elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION: # @Checkpointing if task_untouched: self.meta_criteria_tool.get_checkpoint_state_object()\ .restart_task() self.cp_data.tasks_obj.set_task_executing(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) self.meta_criteria_tool.instrument_code(criteria_enabled_list=\ self.config.ENABLED_CRITERIA.get_val(), \ finish_destroy_checkpointer=False) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Destroy meta test checkpointer self.meta_criteria_tool.get_checkpoint_state_object()\ .destroy_checkpoint() elif task == checkpoint_tasks.Tasks.\ CRITERIA_EXECUTION_SELECTION_PRIORITIZATION: pass #TODO (Maybe could be used someday. for now, just skip it) elif task == checkpoint_tasks.Tasks.CRITERIA_TESTS_EXECUTION: # Make sure that the Matrices dir exists self.head_explorer.get_or_create_and_get_dir(\ outdir_struct.RESULTS_MATRICES_DIR) matrix_files_keys = {} matrix_files = {} execoutput_files_keys = {} execoutput_files = {} for criterion in self.config.ENABLED_CRITERIA.get_val(): matrix_files_keys[criterion] = \ outdir_struct.TMP_CRITERIA_MATRIX[criterion] matrix_files[criterion] = \ self.head_explorer.get_file_pathname(\ matrix_files_keys[criterion]) execoutput_files_keys[criterion] = \ outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion] if criterion in self.config.CRITERIA_WITH_OUTPUT_SUMMARY: execoutput_files[criterion] = \ self.head_explorer.get_file_pathname(\ execoutput_files_keys[criterion]) else: execoutput_files[criterion] = None # @Checkpointing if task_untouched: for _, matrix_file_key in list(matrix_files_keys.items()): self.head_explorer.remove_file_and_get(matrix_file_key) for _, execoutput_file_key in list(\ execoutput_files_keys.items()): self.head_explorer.remove_file_and_get(execoutput_file_key) self.meta_criteria_tool.get_checkpoint_state_object()\ .restart_task() self.cp_data.tasks_obj.set_task_executing(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) criteria_set_sequence = self.config.CRITERIA_SEQUENCE.get_val() #if criteria_set_sequence is None: # criteria_set_sequence = criteria_pkg.CRITERIA_SEQUENCE for cs_pos, criteria_set in enumerate(criteria_set_sequence): criteria_set &= set(matrix_files) if len(criteria_set) == 0: continue # Was it already checkpointed w.r.t criteria set seq if self.cp_data.criteria_set_is_executed(cs_pos, criteria_set): continue # If we have a new criteria set id self.cp_data.switchto_new_criteria_set(cs_pos, criteria_set) # get matrices by criteria test_list_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_SELECTED_TESTS_LIST) meta_testcases = common_fs.loadJSON(test_list_file) criterion_to_matrix = {\ c: matrix_files[c] for c in criteria_set} # TODO: chack set based on the criteria for which execout is enabled criterion_to_execoutput = {\ c: execoutput_files[c] for c in criteria_set} # Set test oracle self.test_oracle_manager.set_oracle(criteria_on=criteria_set) # execute self.meta_criteria_tool.runtests_criteria_coverage( \ testcases=meta_testcases, \ criterion_to_matrix=criterion_to_matrix, \ criterion_to_executionoutput=\ criterion_to_execoutput, \ cover_criteria_elements_once=self.config.\ COVER_CRITERIA_ELEMENTS_ONCE.get_val(),\ prioritization_module_by_criteria=\ self.meta_criteriaexec_optimization_tools,\ finish_destroy_checkpointer=True) # Update matrix if needed to have output diff or such # TODO: add CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM to config for crit in criteria_set & set(self.config.\ CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM.\ get_val()): pf_matrix_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_TEST_PASS_FAIL_MATRIX) if self.config.GET_PASSFAIL_OUTPUT_SUMMARY.get_val(): pf_execoutput_file = \ self.head_explorer.get_file_pathname(\ outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT) else: pf_execoutput_file = None DriversUtils.update_matrix_to_cover_when_diference(\ criterion_to_matrix[crit], \ criterion_to_execoutput[crit], \ pf_matrix_file, pf_execoutput_file) # Unset test oracle self.test_oracle_manager.set_oracle(criteria_on=None) # @Checkpointing self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) elif task == checkpoint_tasks.Tasks.PASS_FAIL_STATS: # Make sure that the Matrices dir exists self.head_explorer.get_or_create_and_get_dir(\ outdir_struct.RESULTS_STATS_DIR) tmp_matrix_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_TEST_PASS_FAIL_MATRIX) matrix_file = self.head_explorer.get_file_pathname(\ outdir_struct.TEST_PASS_FAIL_MATRIX) tmp_execoutput_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT) execoutput_file = self.head_explorer.get_file_pathname(\ outdir_struct.PROGRAM_TESTEXECUTION_OUTPUT) # Merge TMP pass fail and potentially existing pass fail StatsComputer.merge_lmatrix_into_right(\ tmp_matrix_file, matrix_file) StatsComputer.merge_lexecoutput_into_right(\ tmp_execoutput_file, execoutput_file) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Cleanup self.head_explorer.remove_file_and_get(\ outdir_struct.TMP_TEST_PASS_FAIL_MATRIX) elif task == checkpoint_tasks.Tasks.CRITERIA_STATS: # Make sure that the Matrices dir exists self.head_explorer.get_or_create_and_get_dir(\ outdir_struct.RESULTS_STATS_DIR) # Merge TMP criteria and potentially existing criteria for criterion in self.config.ENABLED_CRITERIA.get_val(): tmp_matrix_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_CRITERIA_MATRIX[criterion]) matrix_file = self.head_explorer.get_file_pathname(\ outdir_struct.CRITERIA_MATRIX[criterion]) tmp_execoutput_file = self.head_explorer.get_file_pathname(\ outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion]) execoutput_file = self.head_explorer.get_file_pathname(\ outdir_struct.CRITERIA_EXECUTION_OUTPUT[criterion]) StatsComputer.merge_lmatrix_into_right(tmp_matrix_file, \ matrix_file) StatsComputer.merge_lexecoutput_into_right(\ tmp_execoutput_file, execoutput_file) # @Checkpointing self.cp_data.tasks_obj.set_task_completed(task) self.checkpointer.write_checkpoint(self.cp_data.get_json_obj()) # Cleanup for criterion in self.config.ENABLED_CRITERIA.get_val(): self.head_explorer.remove_file_and_get(\ outdir_struct.TMP_CRITERIA_MATRIX[criterion]) elif task == checkpoint_tasks.Tasks.AGGREGATED_STATS: # Compute the final stats (MS, ...) StatsComputer.compute_stats(self.config, self.head_explorer) #------------------------------------------------------------------- if not self.cp_data.tasks_obj.task_is_complete(task): # @Checkpoint: set task as done self.cp_data.tasks_obj.set_task_completed(task) # @Checkpoint: write checkpoint self.checkpointer.write_checkpoint(\ json_obj=self.cp_data.get_json_obj())