def cross_folder_fdupes(custom_bin, folders, folder2alias): dup_list, invalid = KTestTestFormat.ktest_fdupes(*folders, \ custom_replay_tool_binary_dir=custom_bin) ERROR_HANDLER.assert_true(len(invalid) == 0, \ "there should be no invalid here", __file__) kepttest2duptest_map = {} test2keptdup = {} # get fdupes with metatests # TODO: check that each result of relpath is really a test for dup_tuple in dup_list: k_dir = KTestTestFormat.get_dir(dup_tuple[0], folders) key = os.path.relpath(dup_tuple[0], k_dir) key = DriversUtils.make_meta_element(key, \ folder2alias[k_dir]) kepttest2duptest_map[key] = [] for dp in dup_tuple[1:]: v_dir = KTestTestFormat.get_dir(dp, folders) val = os.path.relpath(dp, v_dir) val = DriversUtils.make_meta_element(val, \ folder2alias[v_dir]) kepttest2duptest_map[key].append(val) test2keptdup[val] = key return kepttest2duptest_map, test2keptdup
def system_test_runner(prog, args_list, test_filename, repo_root_dir, \ exe_path_map=None, env_vars=None, timeout=None, \ collected_output=None, using_wrapper=False, \ dbg_log_execution_out=False): try: tmp_env = os.environ.copy() if env_vars is not None: #for e, v in env_vars.items(): # tmp_env[e] = v tmp_env.update(env_vars) if test_filename is not None: tmp_env[TEST_FILE_NAME_ENV_VAR] = test_filename if using_wrapper and timeout is not None: tmp_env[TEST_EXECUTION_TIMEOUT_ENV_VAR] = str(timeout) timeout = None if collected_output is None: retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog=prog, args_list=args_list, env=tmp_env,\ timeout=timeout, out_on=dbg_log_execution_out,\ err_on=dbg_log_execution_out, \ merge_err_to_out=True) else: retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog=prog, args_list=args_list, env=tmp_env,\ timeout=timeout, merge_err_to_out=True) collected_output.append(retcode) collected_output.append(out) collected_output.append( retcode in DriversUtils.EXEC_TIMED_OUT_RET_CODE) if dbg_log_execution_out: logging.debug("(DBG - Test Output):\n" + out) #except (ValueError, OSError) as os_e: except OSError as os_e: # ERROR logging.warning( "\ntest execution error in system_test_runner (bellow)") logging.warning(str(os_e)) logging.warning("COMMAND: " + " ".join([prog] + args_list)) if collected_output is not None: collected_output.append(None) collected_output.append(str(os_e)) collected_output.append(False) return GlobalConstants.TEST_EXECUTION_ERROR except ValueError as v_e: logging.warning( "\ntest execution valueerror in system_test_runner (bellow)") logging.warning(str(v_e)) logging.warning("COMMAND: " + " ".join([prog] + args_list)) return GlobalConstants.TEST_EXECUTION_ERROR # Parse the result hasfail = False hasfail |= (retcode != 0) return GlobalConstants.FAIL_TEST_VERDICT if hasfail else \ GlobalConstants.PASS_TEST_VERDICT
def _execute_a_test (self, testcase, exe_path_map, env_vars, \ callback_object=None, timeout=None, collect_output=False): """ Execute a test given that the executables have been set properly """ prog = 'klee-replay' if timeout is None: timeout = self.config.ONE_TEST_EXECUTION_TIMEOUT ERROR_HANDLER.assert_true(len(exe_path_map) == 1, \ "support a single exe for now", __file__) ERROR_HANDLER.assert_true(callback_object is None, \ 'TODO: handle callback_obj', __file__) repo_exe = list(exe_path_map.keys())[0] local_exe = os.path.join(self.klee_used_tmp_build_dir, repo_exe) if repo_exe not in self.repo_exe_to_local_to_remote: if not os.path.isdir(os.path.dirname(local_exe)): os.makedirs(os.path.dirname(local_exe)) self.repo_exe_to_local_to_remote[repo_exe] = {local_exe: None} remote_exe = exe_path_map[repo_exe] if remote_exe is None: remote_exe = repo_exe if remote_exe != self.repo_exe_to_local_to_remote[repo_exe][local_exe]: if remote_exe == repo_exe: self.code_builds_factory.repository_manager.\ set_repo_to_build_default(\ also_copy_to_map={repo_exe: local_exe}) else: shutil.copy2(remote_exe, local_exe) args = [local_exe, os.path.join(self.tests_storage_dir, testcase)] tmp_env = os.environ.copy() #tmp_env.update(env_vars) tmp_env['KLEE_REPLAY_TIMEOUT'] = str(timeout) if collect_output: retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog=prog, args_list=args, env=tmp_env, \ merge_err_to_out=True) output_err = (retcode, out) else: retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog=prog, args_list=args, env=tmp_env, \ out_on=False, err_on=False) output_err = None if retcode in (DriversUtils.EXEC_TIMED_OUT_RET_CODE, \ DriversUtils.EXEC_SEGFAULT_OUT_RET_CODE): verdict = common_mix.GlobalConstants.FAIL_TEST_VERDICT else: verdict = common_mix.GlobalConstants.PASS_TEST_VERDICT return verdict, output_err
def make_build_func(repo_root_dir, exe_rel_paths, compiler, flags_list, clean,\ reconfigure): """ Helper for GNU make """ def print_err(out, msg): out = out.splitlines() print(out, msg) #~ def print_err() cwd = os.getcwd() os.chdir(repo_root_dir) #try: tmp_env = os.environ.copy() if compiler is not None: tmp_env["CC"] = compiler if flags_list is not None: tmp_env["CFLAGS"] = " ".join(flags_list) if reconfigure: args_list = ['clean'] retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\ prog='make', args_list=args_list, \ env=tmp_env, merge_err_to_out=True) if retcode != 0: print_err(out, "reconfigure failed") os.chdir(cwd) return GlobalConstants.COMMAND_FAILURE if clean: args_list = ['clean'] retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\ prog='make', args_list=args_list, \ env=tmp_env, merge_err_to_out=True) if retcode != 0: print_err(out, "clean failed") os.chdir(cwd) return GlobalConstants.COMMAND_FAILURE retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\ prog='make', env=tmp_env, merge_err_to_out=True) if retcode != 0: print_err(out, "make") os.chdir(cwd) return GlobalConstants.COMMAND_FAILURE #except: # os.chdir(cwd) # assert False, "Build Unexpected Error in "+__file__ #return GlobalConstants.COMMAND_FAILURE os.chdir(cwd) return GlobalConstants.COMMAND_SUCCESS #~ def make_build_func()
def post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map): for kept, duplist in kepttest2duptest_map.items(): alias, _ = DriversUtils.reverse_meta_element(kept) if alias == origin: stored_map[kept] = [] for dup in duplist: dalias, dtest = DriversUtils.reverse_meta_element(dup) if dalias == origin: continue stored_map[kept].append(dup) tests_to_avoid.append(dup)
def fix_matrices(results_data_dir): import os from muteria.drivers import DriversUtils sm_mat = os.path.join(results_data_dir, 'matrices', 'STRONG_MUTATION.csv') sm_out = os.path.join(results_data_dir, 'testexecution_outputs', 'STRONG_MUTATION_output.json') p_mat = os.path.join(results_data_dir, 'matrices', 'PASSFAIL.csv') p_out = os.path.join(results_data_dir, 'testexecution_outputs', 'program_output.json') DriversUtils.update_matrix_to_cover_when_difference( sm_mat, sm_out, p_mat, p_out) print("Fix success for {}!".format(results_data_dir))
def fault_analysis(cm_corebench_scripts_dir, c_id, conf_py, in_muteria_outdir, out_top_dir): if not os.path.isdir(out_top_dir): os.mkdir(out_top_dir) in_res_data_dir = os.path.join(in_muteria_outdir, 'latest', 'RESULTS_DATA') testtools_workdir = os.path.join(in_muteria_outdir, 'latest', 'testscases_workdir') pass_fail_matrix = os.path.join(in_res_data_dir, "matrices", "PASSFAIL.csv") pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix) test_list = list(pf_mat.get_nonkey_colname_list()) #semu_tool_dirs = [d for d in os.listdir(testtools_workdir) if d.startswith('semu_cmp-')] # get fault tests get_commit_fault_tests(cm_corebench_scripts_dir, c_id, conf_py, in_res_data_dir, out_top_dir) # get test to timestamp test_timestamp_file = os.path.join(out_top_dir, "test_to_timestamp.json") test2timestamp = {} ## get tests by tools tools2tests = {} for test in test_list: alias, _ = DriversUtils.reverse_meta_element(test) # XXX: only SEMU if not alias.startswith('semu_cmp-'): continue if alias not in tools2tests: tools2tests[alias] = set() tools2tests[alias].add(test) ## untar tests dir for alias, tests in tools2tests.items(): d = os.path.join(testtools_workdir, alias) assert os.path.isdir( d), "test tool dir " + d + " missing for alias " + alias test_tar = os.path.join(d, 'tests_files.tar.gz') es = common_fs.TarGz.decompressDir(test_tar) assert es is None, "decompress error: " + es tests_files = os.path.join(d, 'tests_files') assert os.path.isdir(tests_files), "dir missing after decompress" for test in tests: _, simple_test = DriversUtils.reverse_meta_element(test) gt = TestcasesToolSemu._get_generation_time_of_test( simple_test, tests_files) test2timestamp[test] = gt shutil.rmtree(tests_files) common_fs.dumpJSON(test2timestamp, test_timestamp_file, pretty=True)
def reset(self, toolalias, test_objective_list, test_list, **kwargs): """ Reset the optimizer """ self.test_objective_ordered_list = copy.deepcopy(test_objective_list) self.pointer = 0 self.test_objective_to_test_execution_optimizer = { to: TestExecutionOptimizer(self.config, self.explorer) \ for to in self.test_objective_ordered_list } # get the test list per test objectives, based on the matrix opt_by_criterion = self._get_optimizing_criterion() matrix_file = self.explorer.get_existing_file_pathname(\ explorer.TMP_CRITERIA_MATRIX[opt_by_criterion]) test_list_per_test_objective = \ self._get_test_objective_tests_from_matrix(\ matrix_file=matrix_file) # Update test list for to, teo in self.test_objective_to_test_execution_optimizer.items(): alias_to = DriversUtils.make_meta_element(to, toolalias) ERROR_HANDLER.assert_true(\ alias_to in test_list_per_test_objective, \ "Bug: test objective missing("+str(to)+')') teo.reset(None, test_list_per_test_objective[alias_to], \ disable_reset=True)
def reset(self, toolalias, test_objective_list, test_list, **kwargs): """ Reset the optimizer """ # Make sure that all objectives in test_objective_list are in dictobj # and get test_of_testobjective test_of_to = {} for to in test_objective_list: meta_to = DriversUtils.make_meta_element(to, toolalias) ERROR_HANDLER.assert_true(meta_to in self.dictobj, \ "meta test objective {} not in self.dictobj"\ .format(meta_to), __file__) test_of_to[to] = self.dictobj[meta_to] # If None as testlist, use all tests if test_of_to[to] is None: test_of_to[to] = test_list else: test_of_to[to] = list(set(test_list) & set(test_of_to[to])) self.test_objective_ordered_list = copy.deepcopy(test_objective_list) self.pointer = 0 self.test_objective_to_test_execution_optimizer = { to: TestExecutionOptimizer(self.config, self.explorer) \ for to in self.test_objective_ordered_list } for to, teo in self.test_objective_to_test_execution_optimizer.items(): teo.reset(None, test_of_to[to], disable_reset=True)
def _extract_coverage_data_of_a_test(self, enabled_criteria, \ test_execution_verdict, result_dir_tmp): ''' read json files and extract data return: the dict of criteria with covering count ''' in_file = os.path.join(result_dir_tmp, self.cov_data_filename) cov_dat_obj = common_fs.loadJSON(in_file) cov_dat_obj = {TestCriteria[v]: cov_dat_obj[v] for v in cov_dat_obj} ERROR_HANDLER.assert_true(set(cov_dat_obj) == set(enabled_criteria), \ "mismatching criteria enabled", __file__) res = {c: {} for c in enabled_criteria} for c in cov_dat_obj: for filename, filedat in list(cov_dat_obj[c].items()): if filedat is not None: for id_, cov in list(filedat.items()): ident = DriversUtils.make_meta_element(\ str(id_), filename) res[c][ident] = cov # delete cov file os.remove(in_file) return res
def installed(cls, custom_binary_dir=None): """ Check that the tool is installed :return: bool reprenting whether the tool is installed or not (executable accessible on the path) - True: the tool is installed and works - False: the tool is not installed or do not work """ for prog in (cls.tool, ): if custom_binary_dir is not None: prog = os.path.join(custom_binary_dir, prog) if not DriversUtils.check_tool(prog=prog, args_list=['--version'],\ expected_exit_codes=[1]): return False if not DriversUtils.check_tool(prog=cls.stdbuf, args_list=['--version'],\ expected_exit_codes=[0]): return False return True
def _do_instrument_code (self, outputdir, exe_path_map, \ code_builds_factory, \ enabled_criteria, parallel_count=1): # Setup if os.path.isdir(self.instrumented_code_storage_dir): shutil.rmtree(self.instrumented_code_storage_dir) os.mkdir(self.instrumented_code_storage_dir) if os.path.isdir(self.gc_files_dir): shutil.rmtree(self.gc_files_dir) os.mkdir(self.gc_files_dir) prog = 'gcc' flags = ['--coverage', '-fprofile-dir='+self.gc_files_dir, '-O0'] additionals = ["-fkeep-inline-functions"] # get gcc version ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog, ['-dumpversion']) ERROR_HANDLER.assert_true(ret == 0, "'gcc -dumpversion' failed'") # if version > 6.5 if int(out.split('.')[0]) >= 6: if int(out.split('.')[0]) > 6 or int(out.split('.')[1]) > 5: additionals += ["-fkeep-static-functions"] flags += additionals rel_path_map = {} exes, _ = code_builds_factory.repository_manager.\ get_relative_exe_path_map() for exe in exes: filename = os.path.basename(exe) rel_path_map[exe] = os.path.join(\ self.instrumented_code_storage_dir, filename) self.instrument_callback_obj = self.InstrumentCallbackObject() self.instrument_callback_obj.set_post_callback_args(\ (self.gc_files_dir, rel_path_map)) pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\ src_fmt=CodeFormats.C_SOURCE,\ dest_fmt=CodeFormats.NATIVE_CODE,\ src_dest_files_paths_map=None,\ compiler=prog, flags_list=flags, clean_tmp=True, \ reconfigure=True, \ callback_object=self.instrument_callback_obj) # Check if ret == common_mix.GlobalConstants.COMMAND_FAILURE: ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\ 'built problematic'), __file__) # write down the rel_path_map ERROR_HANDLER.assert_true(not os.path.isfile(\ self.instrumentation_details), "must not exist here", __file__) common_fs.dumpJSON(rel_path_map, self.instrumentation_details)
def installed(cls, custom_binary_dir=None): """ Check that the tool is installed :return: bool reprenting whether the tool is installed or not (executable accessible on the path) - True: the tool is installed and works - False: the tool is not installed or do not work """ for prog in ('klee', 'klee-replay'): if not DriversUtils.check_tool(prog=prog, args_list=['--version'],\ expected_exit_codes=[1]): return False return True
def _do_instrument_code (self, exe_path_map, \ code_builds_factory, \ enabled_criteria, parallel_count=1): # Setup if os.path.isdir(self.instrumented_code_storage_dir): shutil.rmtree(self.instrumented_code_storage_dir) os.mkdir(self.instrumented_code_storage_dir) if os.path.isdir(self.mutant_data): shutil.rmtree(self.mutant_data) os.mkdir(self.mutant_data) prog = 'gpmutation' if self.custom_binary_dir is not None: prog = os.path.join(self.custom_binary_dir, prog) ERROR_HANDLER.assert_true(os.path.isfile(prog), \ "The tool {} is missing from the specified dir {}"\ .format(os.path.basename(prog), \ self.custom_binary_dir), __file__) exes, _ = code_builds_factory.repository_manager.\ get_relative_exe_path_map() ERROR_HANDLER.assert_true(len(exes) == 1, \ "Support only a singe exe", __file__) gpmutation_subj = os.path.basename(exes[0]) args=[gpmutation_subj, self.separate_muts_dir] # Execute GPMutation ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog, args_list=args) if (ret != 0): logging.error(out) logging.error(err) logging.error("\n>> CMD: " + " ".join([prog]+args) + '\n') ERROR_HANDLER.error_exit("gpmutation failed!", __file__) # write down the rel_path_map ERROR_HANDLER.assert_true(not os.path.isfile(\ self.instrumentation_details), "must not exist here", __file__) with open (self.instrumentation_details, 'w') as f: for exe in exes: f.write(exe+'\n') # Archive separated if on if self.archive_separated: err_msg = common_fs.TarGz.compressDir(self.separate_muts_dir, \ remove_in_directory=True) ERROR_HANDLER.assert_true(err_msg is None,\ "Compression failed: "+str(err_msg), __file__)
def _dir_chmod777(self, dirpath): try: for root_, dirs_, files_ in os.walk(dirpath): for sub_d in dirs_: if os.path.isdir(os.path.join(root_, sub_d)): os.chmod(os.path.join(root_, sub_d), 0o777) for f_ in files_: if os.path.isfile(os.path.join(root_, f_)): os.chmod(os.path.join(root_, f_), 0o777) except PermissionError: ret,_,_ = DriversUtils.execute_and_get_retcode_out_err('sudo', \ args_list=['chmod 777 -R {}'.format(dirpath)]) ERROR_HANDLER.assert_true(ret == 0, \ "'sudo chmod 777 -R "+dirpath+"' failed (returned "+\ str(ret)+")", __file__)
def _compute_testcases_info(self, candidate_tool_aliases=None): meta_testcase_info_obj = TestcasesInfoObject() if candidate_tool_aliases is None: candidate_tool_aliases = self.testcases_configured_tools.keys() for ttoolalias in candidate_tool_aliases: ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] tool_testcase_info = ttool.get_testcase_info_object() old2new_tests = {} for t_test in tool_testcase_info.get_tests_list(): meta_t_key = DriversUtils.make_meta_element(t_test, ttoolalias) old2new_tests[t_test] = meta_t_key meta_testcase_info_obj.update_using(ttoolalias, old2new_tests, \ tool_testcase_info) return meta_testcase_info_obj
def _collect_temporary_coverage_data(self, criteria_name_list, \ test_execution_verdict, \ used_environment_vars, \ result_dir_tmp): ''' get gcov files from gcda files into result_dir_tmp ''' prog = 'gcov' cov2flags = { TestCriteria.STATEMENT_COVERAGE: [], TestCriteria.BRANCH_COVERAGE: ['-b', '-c'], TestCriteria.FUNCTION_COVERAGE: ['-f'], } args_list = [] for criterion in criteria_name_list: args_list += cov2flags[criterion] gcda_files = self._get_gcda_list() raw_filename_list = [os.path.splitext(f)[0] for f in gcda_files] args_list += raw_filename_list if len(gcda_files) > 0: # TODO: When gcov generate coverage for different files with # same name filename bu located at diferent dir. Avoid override. # Go where the gcov will be looked for cwd = os.getcwd() os.chdir(self.gc_files_dir) # collect gcda (gcno) r, _, _ = DriversUtils.execute_and_get_retcode_out_err(prog=prog, \ args_list=args_list, out_on=False, \ err_on=False) os.chdir(cwd) if r != 0: ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\ 'error collecting coverage is problematic'), __file__) # delete gcda for gcda_f in gcda_files: os.remove(gcda_f) common_fs.dumpJSON(self._get_gcov_list(), \ os.path.join(result_dir_tmp,\ self.gcov_files_list_filename))
def get_too2relmuts(relmuts_to_reltests, toollist=None): res = {tool: set() for tool in toollist} if toollist is not None else {} for relmut, t_list in relmuts_to_reltests.items(): for meta_t in t_list: toolalias, test = DriversUtils.reverse_meta_element(meta_t) if toollist is None: if toolalias not in res: res[toolalias] = set() else: assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format( toolalias, toollist) res[toolalias].add(relmut) for toolalias in res: res[toolalias] = list(res[toolalias]) return res
def after_command(self): if self.op_retval == common_mix.GlobalConstants.COMMAND_FAILURE: return common_mix.GlobalConstants.COMMAND_FAILURE m_regex = re.compile('({}|{}|{})'.format('klee_change', \ 'klee_get_true', 'klee_get_false')) matched_lines = set() for src in self.source_files_to_objects: with open(os.path.join(self.repository_rootdir, src), \ encoding='UTF-8') as f: for lnum, line in enumerate(f.readlines()): if m_regex.search(line) is not None: matched_lines.add(DriversUtils.make_meta_element(\ str(lnum+1), src)) ret_lines = self.post_callback_args ret_lines.extend(matched_lines) return common_mix.GlobalConstants.COMMAND_SUCCESS
def tool_parallel_test_exec(ttoolalias): # Actual execution found_a_failure = False # Whether the execution was unsuccessful test_error = False ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=\ with_output_summary, \ hash_outlog=hash_outlog, \ parallel_count=\ parallel_test_count_by_tool[ttoolalias]) with shared_loc: for testcase in test_failed_verdicts: meta_testcase = DriversUtils.make_meta_element(\ testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if not found_a_failure \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.FAIL_TEST_VERDICT: found_a_failure = True if not test_error \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.TEST_EXECUTION_ERROR: test_error = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) return found_a_failure, test_error
def execute_testcase (self, meta_testcase, exe_path_map, env_vars, \ timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ with_output_summary=True, \ hash_outlog=None): ''' Execute a test case with the given executable and say whether it failed :param meta_testcase: string name of the test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the test :param env_vars: dict of environment variables to set before executing the test ({<variable>: <value>}) :type hash_outlog: bool :hash_outlog: decide whether to hash the outlog or not :returns: pair of: - boolean failed verdict of the test (True if failed, False otherwise) - test execution output log hash data object or None ''' if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() if hash_outlog is None: hash_outlog = self.hash_outlog # Find which test tool's the testcase is, then execute ttoolalias, testcase = DriversUtils.reverse_meta_element(meta_testcase) ERROR_HANDLER.assert_true( \ ttoolalias in self.testcases_configured_tools, \ "Test tool {} not registered".format(ttoolalias), \ __file__) ttool = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] return ttool.execute_testcase(testcase, exe_path_map, env_vars, \ timeout=timeout, \ use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=with_output_summary, \ hash_outlog=hash_outlog)
def python_unittest_runner(test_name, repo_root_dir, exe_path_map, env_vars, \ timeout, collected_output=None): # TODO: use exe_path_map def parse_test(s): return s.split('...')[0].replace(':','/').replace(' ','') cwd = os.getcwd() os.chdir(repo_root_dir) try: args_list = ['-m', 'unittest', test_name, '-v'] if collected_output is None: retcode, stdout, _ = DriversUtils.execute_and_get_retcode_out_err(\ prog=sys.executable, args_list=args_list, \ timeout=timeout, merge_err_to_out=True) stdout = stdout.splitlines() else: # collected_output is a list ([retcode, out_err_log]) # TODO: use wrapper? or parse out for particular codes... assert False, "TO BE Implemented" except: # ERROR os.chdir(cwd) return GlobalConstants.TEST_EXECUTION_ERROR # Parse the result subtests_verdicts = {} hasfail = False hasfail |= (retcode != 0) for s in stdout: if s.endswith('... FAIL'): hasfail = True subtests_verdicts[parse_test(s)] = True elif s.endswith('... ok'): subtests_verdicts[parse_test(s)] = False #print(subtests_verdicts) os.chdir(cwd) return GlobalConstants.FAIL_TEST_VERDICT if hasfail else \ GlobalConstants.PASS_TEST_VERDICT #~ def python_unittest_runner()
def _compute_criterion_info(self, criterion, candidate_tool_aliases=None): if criterion not in CriteriaToInfoObject: return None meta_criterion_info_obj = CriteriaToInfoObject[criterion]() if candidate_tool_aliases is None: candidate_tool_aliases = [] for _, config in list(self.tools_config_by_criterion_dict.items()): candidate_tool_aliases.append(config.get_tool_config_alias()) for ctoolalias in candidate_tool_aliases: ctool = self.criteria_configured_tools[ctoolalias]\ [self.TOOL_OBJ_KEY] tool_element_info = ctool.get_criterion_info_object(criterion) old2new_tests = {} for c_elem in tool_element_info.get_elements_list(): meta_c_key = DriversUtils.make_meta_element(c_elem, ctoolalias) old2new_tests[c_elem] = meta_c_key meta_criterion_info_obj.update_using(ctoolalias, old2new_tests, \ tool_element_info) return meta_criterion_info_obj
def make_build(repo_root_dir, exe_rel_paths, compiler, flags_list, clean,\ reconfigure): cwd = os.getcwd() os.chdir(repo_root_dir) if flags_list is None: flags_list = ["-DUSING_MUTERIA"] else: flags_list.append("-DUSING_MUTERIA") # try: tmp_env = os.environ.copy() if compiler is not None: tmp_env["CC"] = compiler if flags_list is not None: tmp_env["CFLAGS"] = " ".join(flags_list) args_list = ['build_project.sh'] if reconfigure: args_list.append('1') else: args_list.append('0') if clean: args_list.append('1') else: args_list.append('0') retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err( prog='bash', args_list=args_list, env=tmp_env, merge_err_to_out=True) def print_err(out, msg): #out = out.splitlines() #print(out, msg) logging.error(str(out) + msg) if retcode != 0: print_err(out, "make") os.chdir(cwd) return GlobalConstants.COMMAND_FAILURE os.chdir(cwd) return GlobalConstants.COMMAND_SUCCESS
def _get_input_bitcode_file(self, code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=None): meta_mu_src = self.driver_config.get_meta_mutant_source() # XXX Case of manual annotation if meta_mu_src == MetaMuSource.ANNOTATION: with open(self.cand_muts_file, 'w') as f: # Single mutant (id 1, corresponding to old version) f.write(str(1) + '\n') return super(TestcasesToolSemu, self)._get_input_bitcode_file(\ code_builds_factory, rel_path_map, \ meta_criteria_tool_obj=meta_criteria_tool_obj) if type(meta_mu_src) == str: # XXX: The actual path to the meta is specified return meta_mu_src # XXX: Case of other mutation tools like Mart # get the meta criterion file from MART or any compatible tool. mutant_gen_tool_name = meta_mu_src.get_field_value() mut_tool_alias_to_obj = \ meta_criteria_tool_obj.get_criteria_tools_by_name(\ mutant_gen_tool_name) if len(mut_tool_alias_to_obj) == 0: logging.warning(\ 'SEMu requires {} to generate mutants but none used'.format(\ mutant_gen_tool_name)) ERROR_HANDLER.assert_true(len(mut_tool_alias_to_obj) == 1, \ "SEMu supports tests generation from" "a single .bc file for now (todo).", __file__) t_alias2metamu_bc = {} t_alias2mutantInfos = {} for alias, obj in mut_tool_alias_to_obj.items(): dest_bc = rel_path_map[list(rel_path_map)[0]] + '.bc' shutil.copy2(obj.get_test_gen_metamutant_bc(), dest_bc) t_alias2metamu_bc[alias] = dest_bc t_alias2mutantInfos[alias] = obj.get_criterion_info_object(None) # XXX: get mutants ids by functions self.mutants_by_funcs = {} single_alias = list(t_alias2mutantInfos)[0] single_tool_obj = t_alias2mutantInfos[single_alias] cand_muts = list(single_tool_obj.get_elements_list()) for mut in single_tool_obj.get_elements_list(): func = single_tool_obj.get_element_data(mut)[\ 'mutant_function_name'] #meta_mut = DriversUtils.make_meta_element(mut, single_alias) if func not in self.mutants_by_funcs: self.mutants_by_funcs[func] = set() self.mutants_by_funcs[func].add(mut) #meta_mut) # XXX: get candidate mutants list if self.driver_config.get_target_only_live_mutants() \ and os.path.isfile(self.sm_mat_file): sm_mat = common_matrices.ExecutionMatrix(\ filename=self.sm_mat_file) mut2killing_tests = sm_mat.query_active_columns_of_rows() alive_muts = [m for m, k_t in mut2killing_tests.items() \ if len(k_t) == 0] cand_muts = [] for meta_m in alive_muts: t_alias, m = DriversUtils.reverse_meta_element(meta_m) if t_alias in t_alias2metamu_bc: # There is a single one cand_muts.append(m) with open(self.cand_muts_file, 'w') as f: for m in cand_muts: f.write(str(m) + '\n') return t_alias2metamu_bc[list(t_alias2metamu_bc)[0]]
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \ stop_on_failure=False, \ per_test_timeout=None, \ use_recorded_timeout_times=None, \ recalculate_execution_times=False, \ fault_test_execution_matrix_file=None, \ fault_test_execution_execoutput_file=None, \ with_output_summary=True, \ hash_outlog=None, \ test_prioritization_module=None, \ parallel_test_count=1, \ parallel_test_scheduler=None, \ restart_checkpointer=False, finish_destroy_checkpointer=True): ''' Execute the list of test cases with the given executable and say, for each test case, whether it failed :param meta_testcases: list of test cases to execute :param exe_path_map: string representing the file system path to the executable to execute with the tests :param env_vars: dict of environment variables to set before executing each test ({<variable>: <value>}) :param stop_on_failure: decide whether to stop the test execution once a test fails :param fault_test_execution_matrix_file: Optional matrix file to store the tests' pass fail execution data :param fault_test_execution_execoutput_file: Optional output log file to store the tests' execution actual output (hashed) :param with_output_summary: decide whether to return outlog hash :type hash_outlog: bool :hash_outlog: decide whether to hash the outlog or not :param test_prioritization_module: Specify the test prioritization module. (TODO: Implement support) :param parallel_test_count: Specify the number of parallel test Execution. must be an integer >= 1 or None. When None, the max possible value is used. :param parallel_test_scheduler: Specify the function that will handle parallel test scheduling by tool, using the test execution optimizer. (TODO: Implement support) :type restart_checkointer: bool :param restart_checkointer: Decide whether to discard checkpoint and restart anew. :type finish_destroy_checkpointer: bool :param finish_destroy_checkpointer: Decide whether to automatically destroy the checkpointer when done or not Useful is caller has a checkpointer to update. :returns: dict of testcase and their failed verdict. {<test case name>: <True if failed, False if passed, UNCERTAIN_TEST_VERDICT if uncertain>} If stop_on_failure is True, only return the tests that have been executed until the failure ''' ERROR_HANDLER.assert_true(meta_testcases is not None, \ "Must specify testcases", __file__) # FIXME: Make sure that the support are implemented for # parallelism and test prioritization. Remove the code bellow # once supported: ERROR_HANDLER.assert_true(test_prioritization_module is None, \ "Must implement test prioritization support here", \ __file__) ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \ "Must implement parallel tests execution support here", \ __file__) #~FIXMEnd # Check arguments Validity if exe_path_map is None: exe_path_map = self._get_default_exe_path_map() if hash_outlog is None: hash_outlog = self.hash_outlog ERROR_HANDLER.assert_true(parallel_test_count is None \ or parallel_test_count >= 1, \ "invalid parallel tests count ({})".format(\ parallel_test_count), __file__) # @Checkpoint: create a checkpoint handler cp_func_name = "runtests" cp_task_id = 1 checkpoint_handler = \ CheckPointHandler(self.get_checkpoint_state_object()) if restart_checkpointer: checkpoint_handler.restart() if checkpoint_handler.is_finished(): logging.warning("%s %s" %("The function 'runtests' is finished", \ "according to checkpoint, but called again. None returned")) if common_mix.confirm_execution("%s %s" % ( \ "Function 'runtests' is already", \ "finished, do you want to restart?")): checkpoint_handler.restart() logging.info("Restarting the finished 'runtests'") else: ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\ "Execution halted. Cannot continue because no value", \ " can be returned. Check the results of the", \ "finished execution"), call_location=__file__) # @Checkpoint: Get the saved payload (data kapt for each tool) # pair list of testfailed verdict and execution output meta_test_failedverdicts_outlog = \ checkpoint_handler.get_optional_payload() if meta_test_failedverdicts_outlog is None: meta_test_failedverdicts_outlog = [{}, {}] # Make sure the tests are unique ERROR_HANDLER.assert_true(len(meta_testcases) == \ len(set(meta_testcases)), \ "not all tests are unique", __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases_backup = meta_testcases meta_testcases = set(meta_testcases) dups_remove_meta_testcases = meta_testcases & \ set(self.tests_duplicates_map) dup_toadd_test = {self.tests_duplicates_map[v] for v in \ dups_remove_meta_testcases} - meta_testcases meta_testcases = (meta_testcases - dups_remove_meta_testcases) \ | dup_toadd_test testcases_by_tool = {} for meta_testcase in meta_testcases: ttoolalias, testcase = \ DriversUtils.reverse_meta_element(meta_testcase) if ttoolalias not in testcases_by_tool: testcases_by_tool[ttoolalias] = [] testcases_by_tool[ttoolalias].append(testcase) candidate_aliases = [] for tpos, ttoolalias in enumerate(testcases_by_tool.keys()): # @Checkpoint: Check whether already executed if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias): continue candidate_aliases.append(ttoolalias) # parallelism strategies PARA_FULL_DOUBLE = 0 PARA_ALT_TOOLS_AND_TESTS = 1 PARA_TOOLS_ONLY = 2 PARA_TOOLS_TESTS_AS_TOOLS = 3 parallel_strategy = PARA_TOOLS_ONLY # minimum number of tests (accross) for parallelism ptest_tresh = 5 # minimum number of tests (of the given tool) for tool parallelism sub_ptest_thresh = 3 shared_loc = multiprocessing.RLock() parallel_test_count_by_tool = {ta: 1 for ta in candidate_aliases} # tool with parallel test exec # TODO: find way to pass parallel count here if parallel_test_count is None: #parallel_test_count = min(10, multiprocessing.cpu_count()) parallel_test_count = min(20, 2 * multiprocessing.cpu_count()) cand_alias_joblib = [] cand_alias_for = [] para_tools = [] para_tools = [tt for tt in candidate_aliases if \ (len(testcases_by_tool[tt]) >= sub_ptest_thresh \ and self.testcases_configured_tools[tt]\ [self.TOOL_OBJ_KEY].can_run_tests_in_parallel()) ] actual_parallel_cond = len(candidate_aliases) > 1 \ and len(meta_testcases) >= ptest_tresh \ and parallel_test_count is not None \ and parallel_test_count > 1 if parallel_strategy == PARA_ALT_TOOLS_AND_TESTS: # the para_tools will run without parallelism, give them all threads for tt in para_tools: parallel_test_count_by_tool[tt] = parallel_test_count seq_tools = list(set(candidate_aliases) - set(para_tools)) if len(seq_tools) > 1 and actual_parallel_cond: cand_alias_joblib = seq_tools cand_alias_for = para_tools else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_ONLY: if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_FULL_DOUBLE: # use parallel sub_parallel_count = 0 if parallel_test_count is None else \ parallel_test_count - len(parallel_test_count_by_tool) if sub_parallel_count > 0: para_tools.sort(reverse=True, \ key=lambda x: len(testcases_by_tool[x])) para_tools_n_tests = sum(\ [len(testcases_by_tool[tt]) for tt in para_tools]) used = 0 for tt in para_tools: quota = int(len(testcases_by_tool[tt]) * \ sub_parallel_count / para_tools_n_tests) parallel_test_count_by_tool[tt] += quota used += quota for tt in para_tools: if used == sub_parallel_count: break parallel_test_count_by_tool[tt] += 1 if actual_parallel_cond: cand_alias_joblib = candidate_aliases else: cand_alias_for = candidate_aliases elif parallel_strategy == PARA_TOOLS_TESTS_AS_TOOLS: # split the tests of one tool and # make the same tool run multiple times ERROR_HANDLER.error_exit("To Be implemented: same tool many times") else: ERROR_HANDLER.error_exit("Invalid parallel startegy") def tool_parallel_test_exec(ttoolalias): # Actual execution found_a_failure = False # Whether the execution was unsuccessful test_error = False ttool = \ self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] test_failed_verdicts, test_execoutput = ttool.runtests( \ testcases_by_tool[ttoolalias], \ exe_path_map, env_vars, \ stop_on_failure, \ per_test_timeout=per_test_timeout, use_recorded_timeout_times=\ use_recorded_timeout_times, \ recalculate_execution_times=\ recalculate_execution_times, \ with_output_summary=\ with_output_summary, \ hash_outlog=hash_outlog, \ parallel_count=\ parallel_test_count_by_tool[ttoolalias]) with shared_loc: for testcase in test_failed_verdicts: meta_testcase = DriversUtils.make_meta_element(\ testcase, ttoolalias) meta_test_failedverdicts_outlog[0][meta_testcase] = \ test_failed_verdicts[testcase] meta_test_failedverdicts_outlog[1][meta_testcase] = \ test_execoutput[testcase] if not found_a_failure \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.FAIL_TEST_VERDICT: found_a_failure = True if not test_error \ and test_failed_verdicts[testcase] == \ common_mix.GlobalConstants.TEST_EXECUTION_ERROR: test_error = True # @Checkpoint: Chekpointing checkpoint_handler.do_checkpoint(func_name=cp_func_name, \ taskid=cp_task_id, \ tool=ttoolalias, \ opt_payload=meta_test_failedverdicts_outlog) return found_a_failure, test_error #~ def tool_parallel_test_exec() if len(cand_alias_joblib) > 0: parallel_count_ = min(len(cand_alias_joblib), parallel_test_count) joblib.Parallel(n_jobs=parallel_count_, require='sharedmem')\ (joblib.delayed(tool_parallel_test_exec)(ttoolalias) \ for ttoolalias in cand_alias_joblib) if len(cand_alias_for) > 0: for tpos, ttoolalias in enumerate(cand_alias_for): found_a_failure, test_error = \ tool_parallel_test_exec(ttoolalias) if stop_on_failure and found_a_failure: # @Checkpoint: Chekpointing for remaining tools for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]: checkpoint_handler.do_checkpoint(\ func_name=cp_func_name, \ taskid=cp_task_id, \ tool=rem_tool, \ opt_payload=meta_test_failedverdicts_outlog) break if stop_on_failure: # Make sure the non executed test has the uncertain value (None) if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases): for meta_testcase in set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0]): meta_test_failedverdicts_outlog[0][meta_testcase] = \ common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT meta_test_failedverdicts_outlog[1][meta_testcase] = \ common_matrices.OutputLogData.\ UNCERTAIN_TEST_OUTLOGDATA ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \ len(meta_testcases), \ "mismatch between number of tests and reported verdicts:" + " Tests without verdict are {};".format(\ set(meta_testcases) - \ set(meta_test_failedverdicts_outlog[0])) \ + " Test not in testlist are {}.".format(\ set(meta_test_failedverdicts_outlog[0]) - \ set(meta_testcases)), \ __file__) # For fdupes if len(self.tests_duplicates_map) > 0: meta_testcases = meta_testcases_backup for i in (0, 1): for mtest in dups_remove_meta_testcases: # add to results meta_test_failedverdicts_outlog[i][mtest] = copy.deepcopy(\ meta_test_failedverdicts_outlog[i]\ [self.tests_duplicates_map[mtest]]) for mtest in dup_toadd_test: # remove from results del meta_test_failedverdicts_outlog[i][mtest] if fault_test_execution_matrix_file is not None: # Load or Create the matrix fault_test_execution_matrix = common_matrices.ExecutionMatrix( \ filename=fault_test_execution_matrix_file, \ non_key_col_list=meta_testcases) ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \ "matrix must be empty. Filename is:" " "+fault_test_execution_matrix_file, __file__) failverdict2val = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ fault_test_execution_matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ fault_test_execution_matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ fault_test_execution_matrix.getUncertainCellDefaultVal(), } cells_dict = {} for meta_testcase in meta_test_failedverdicts_outlog[0]: cells_dict[meta_testcase] = failverdict2val[\ meta_test_failedverdicts_outlog[0][meta_testcase]] fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \ cells_dict, serialize=True) if fault_test_execution_execoutput_file is not None: # Load or Create the data object fault_test_execution_execoutput = common_matrices.OutputLogData( \ filename=fault_test_execution_execoutput_file) ERROR_HANDLER.assert_true(\ fault_test_execution_execoutput.is_empty(), \ "outlog data must be empty", __file__) fault_test_execution_execoutput.add_data(\ {self.PROGRAM_EXECOUTPUT_KEY: \ meta_test_failedverdicts_outlog[1]}, \ serialize=True) # @Checkpoint: Finished detailed_exectime = {} for ttoolalias in testcases_by_tool.keys(): tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY] detailed_exectime[ttoolalias] = (\ tt.get_checkpointer().get_execution_time(),\ tt.get_checkpointer().get_detailed_execution_time()) checkpoint_handler.set_finished( \ detailed_exectime_obj=detailed_exectime) if finish_destroy_checkpointer: checkpoint_handler.destroy() return meta_test_failedverdicts_outlog
def _do_instrument_code (self, outputdir, exe_path_map, \ code_builds_factory, \ enabled_criteria, parallel_count=1): # Setup if os.path.isdir(self.instrumented_code_storage_dir): shutil.rmtree(self.instrumented_code_storage_dir) os.mkdir(self.instrumented_code_storage_dir) if os.path.isdir(self.mutant_data): shutil.rmtree(self.mutant_data) os.mkdir(self.mutant_data) prog = 'mart' # get llvm compiler path ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog, ['--version']) llvm_compiler_path = None for line in out.splitlines(): line = line.strip() if line.startswith('LLVM tools dir:'): llvm_compiler_path = line.split()[3] #[1:-1] break ERROR_HANDLER.assert_true(llvm_compiler_path is not None, \ 'Problem getting llvm path for mart', __file__) # Build into LLVM back_llvm_compiler = 'clang' rel_path_map = {} exes, _ = code_builds_factory.repository_manager.\ get_relative_exe_path_map() for exe in exes: filename = os.path.basename(exe) rel_path_map[exe] = os.path.join(self.mutant_data, filename) pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\ src_fmt=CodeFormats.C_SOURCE,\ dest_fmt=CodeFormats.LLVM_BITCODE,\ src_dest_files_paths_map=rel_path_map,\ compiler=back_llvm_compiler, flags_list=['-g'], \ clean_tmp=True, reconfigure=True, \ llvm_compiler_path=llvm_compiler_path) if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR: ERROR_HANDLER.error_exit("Program {}.".format(\ 'LLVM (clang) built problematic'), __file__) # Update exe_map to reflect bitcode extension rel2bitcode = {} for r_file, b_file in list(rel_path_map.items()): bc = b_file + '.bc' ERROR_HANDLER.assert_true(os.path.isfile(bc), \ "Bitcode file not existing: "+bc, __file__) rel2bitcode[r_file] = bc ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \ "Support single bitcode module for now", __file__) bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]] # mart params bool_param, k_v_params = self._get_default_params() if TestCriteria.STRONG_MUTATION in enabled_criteria: bool_param['-write-mutants'] = True args = [bp for bp, en in list(bool_param.items()) if en] for k, v in list(k_v_params.items()): if v is not None: args += [k, v] args.append(bitcode_file) # Execute Mart cwd = os.getcwd() os.chdir(self.mutant_data) ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog, args) os.chdir(cwd) if (ret != 0): logging.error(out) logging.error(err) logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n') ERROR_HANDLER.error_exit("mart failed'", __file__) # write down the rel_path_map ERROR_HANDLER.assert_true(not os.path.isfile(\ self.instrumentation_details), "must not exist here", __file__) store_obj = {c.get_str(): {} for c in enabled_criteria} for k, v in list(rel_path_map.items()): exe_file = os.path.basename(v) if TestCriteria.WEAK_MUTATION in enabled_criteria: crit_str = TestCriteria.WEAK_MUTATION.get_str() store_obj[crit_str][k] = exe_file + '.WM' if TestCriteria.MUTANT_COVERAGE in enabled_criteria: crit_str = TestCriteria.MUTANT_COVERAGE.get_str() store_obj[crit_str][k] = exe_file + '.COV' if TestCriteria.STRONG_MUTATION in enabled_criteria: crit_str = TestCriteria.STRONG_MUTATION.get_str() store_obj[crit_str][k] = exe_file + '.MetaMu' common_fs.dumpJSON(store_obj, self.instrumentation_details)
def _runtest_separate_criterion_program (self, criterion, testcases, \ matrix, executionoutput, \ criteria_element_list, \ cover_criteria_elements_once=False, \ prioritization_module=None, \ test_parallel_count=1, serialize_period=5, checkpoint_handler=None, \ cp_calling_func_name=None, \ cp_calling_done_task_id=None, \ cp_calling_tool=None): ''' Note: Here the temporary matrix is used as checkpoint (with frequency the 'serialize_period' parameter). The checkpointer is mainly used for the execution time (TODO: support parallelism: per test outdata) ''' # FIXME: Support parallelism, then remove the code # bellow: ERROR_HANDLER.assert_true(test_parallel_count <= 1, \ "FIXME: Must first implement support for parallel mutatio") #~ FXIMEnd # @Checkpoint: validate if checkpoint_handler is not None: ERROR_HANDLER.assert_true(cp_calling_func_name is not None) ERROR_HANDLER.assert_true(cp_calling_done_task_id is not None) ERROR_HANDLER.assert_true(cp_calling_tool is not None) cp_data = checkpoint_handler.get_optional_payload() if cp_data is None: cp_data = [{}, {}] else: cp_data = [{}, {}] failverdict_to_val_map = { common_mix.GlobalConstants.FAIL_TEST_VERDICT: \ matrix.getActiveCellDefaultVal(), common_mix.GlobalConstants.PASS_TEST_VERDICT: \ matrix.getInactiveCellVal(), common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \ matrix.getUncertainCellDefaultVal() } assert serialize_period >= 1, \ "Serialize period must be an integer in [1,inf[" # matrix based checkpoint completed_elems = set(cp_data[0]) if criteria_element_list is None: criteria_element_list = self.get_criterion_info_object(criterion).\ get_elements_list() ERROR_HANDLER.assert_true(prioritization_module is not None, "prioritization module must be passed") timeout_times = \ self.config.SEPARATED_TEST_EXECUTION_EXTRA_TIMEOUT_TIMES # in case the test list is empty, do nothing if len(testcases) > 0: # main loop for elements execution num_elems = len(criteria_element_list) pos = -1 + len(completed_elems) ## prepare the optimizer prioritization_module.reset(self.config.get_tool_config_alias(), \ criteria_element_list, testcases) while prioritization_module.has_next_test_objective(): #pos += 1 # Done bellow in the case where it was not completed element = prioritization_module.get_next_test_objective() # @Checkpointing: check if already executed if element in completed_elems: continue else: pos += 1 logging.debug("# Executing {} element {} ({}/{}) ...".format( \ criterion.get_str(), \ DriversUtils.make_meta_element(element, \ self.config.get_tool_config_alias()), \ pos, num_elems)) # execute element with the given testcases element_executable_path = \ self._get_criterion_element_executable_path(\ criterion, element) execution_environment_vars = \ self._get_criterion_element_environment_vars(\ criterion, element) # run optimizer with all tests of targeting the test objective may_cov_tests = prioritization_module\ .get_test_execution_optimizer(element)\ .select_tests(100, is_proportion=True) cannot_cov_tests = set(testcases) - set(may_cov_tests) fail_verdicts, exec_outs_by_tests = \ self.meta_test_generation_obj.runtests(\ meta_testcases=may_cov_tests, \ exe_path_map=element_executable_path, \ env_vars=execution_environment_vars, \ stop_on_failure=\ cover_criteria_elements_once, \ use_recorded_timeout_times=\ timeout_times, \ with_output_summary=(executionoutput \ is not None), \ parallel_test_count=None, \ restart_checkpointer=True) prioritization_module.feedback(element, fail_verdicts) fail_verdicts.update({\ v: common_mix.GlobalConstants.PASS_TEST_VERDICT \ for v in cannot_cov_tests}) # put in row format for matrix matrix_row_key = element matrix_row_values = \ {tc:failverdict_to_val_map[fail_verdicts[tc]] \ for tc in fail_verdicts} serialize_on = (pos % serialize_period == 0) cp_data[0][matrix_row_key] = matrix_row_values if executionoutput is not None: cp_data[1][element] = exec_outs_by_tests # @Checkpointing: for time if serialize_on and checkpoint_handler is not None: checkpoint_handler.do_checkpoint( \ func_name=cp_calling_func_name, \ taskid=cp_calling_done_task_id, \ tool=cp_calling_tool, \ opt_payload=cp_data) # Write the execution data into the matrix for matrix_row_key, matrix_row_values in list(cp_data[0].items()): matrix.add_row_by_key(matrix_row_key, matrix_row_values, \ serialize=False) # TODO: Make the following two instructions atomic # final serialization (in case #Muts not multiple od serialize_period) matrix.serialize() # Write the execution output data if executionoutput is not None: if len(cp_data[1]) > 0: executionoutput.add_data(cp_data[1], serialize=True) else: executionoutput.serialize()
def _call_generation_run(self, runtool, args): # Delete any klee-out-* for d in os.listdir(self.tests_working_dir): if d.startswith('klee-out-'): shutil.rmtree(os.path.join(self.tests_working_dir, d)) call_shadow_wrapper_file = os.path.join(self.tests_working_dir, \ "shadow_wrap") devtest_toolalias = self.parent_meta_tool.get_devtest_toolalias() ERROR_HANDLER.assert_true(devtest_toolalias is not None, \ "devtest must be used when using shadow_se", __file__) #test_list = list(self.code_builds_factory.repository_manager\ # .get_dev_tests_list()) test_list = [] for meta_test in self.parent_meta_tool.get_testcase_info_object(\ candidate_tool_aliases=[devtest_toolalias])\ .get_tests_list(): toolalias, test = DriversUtils.reverse_meta_element(meta_test) ERROR_HANDLER.assert_true(toolalias == devtest_toolalias, \ "BUG in above get_testcase_info_object", __file__) test_list.append(test) # Get list of klee_change, klee_get_true/false locations. klee_change_stmts = [] get_lines_callback_obj = self.GetLinesCallbackObject() get_lines_callback_obj.set_pre_callback_args(self.code_builds_factory\ .repository_manager.revert_src_list_files) get_lines_callback_obj.set_post_callback_args(klee_change_stmts) pre_ret, post_ret = self.code_builds_factory.repository_manager\ .custom_read_access(get_lines_callback_obj) ERROR_HANDLER.assert_true(pre_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "pre failed", __file__) ERROR_HANDLER.assert_true(post_ret == \ common_mix.GlobalConstants.COMMAND_SUCCESS,\ "post failed", __file__) ERROR_HANDLER.assert_true(len(klee_change_stmts) > 0, \ "No klee_change statement in the sources", __file__) # Filter only tests that cover those locations, # if there is stmt coverage matrix stmt_cov_mat_file = self.head_explorer.get_file_pathname(\ fd_structure.CRITERIA_MATRIX[criteria.TestCriteria\ .STATEMENT_COVERAGE]) cov_tests = None if os.path.isfile(stmt_cov_mat_file): stmt_cov_mat = common_matrices.ExecutionMatrix(\ filename=stmt_cov_mat_file) # due to possible wrapper test splitting we update test_list here tmp_test_list = [] for mt in stmt_cov_mat.get_nonkey_colname_list(): alias, t = DriversUtils.reverse_meta_element(mt) if alias == devtest_toolalias: tmp_test_list.append(t) test_list = tmp_test_list meta_stmts = list(stmt_cov_mat.get_keys()) tool_aliases = set() for meta_stmt in meta_stmts: alias, stmt = DriversUtils.reverse_meta_element(meta_stmt) tool_aliases.add(alias) klee_change_meta_stmts = [] for alias in tool_aliases: klee_change_meta_stmts += [\ DriversUtils.make_meta_element(e, alias) \ for e in klee_change_stmts] klee_change_meta_stmts = list(set(meta_stmts) & \ set(klee_change_meta_stmts)) cov_tests = set() if len(klee_change_meta_stmts) > 0: for _, t in stmt_cov_mat.query_active_columns_of_rows(\ row_key_list=klee_change_meta_stmts).items(): cov_tests |= set(t) else: logging.warning('No test covers the patch (SHADOW)!') # ERROR_HANDLER.assert_true(len(klee_change_meta_stmts) > 0, \ # "No test covers the patch", __file__) # tests will be generated in the same dir that has the input .bc file os.mkdir(self.tests_storage_dir) # obtain candidate tests cand_testpair_list = [] for test in test_list: meta_test = DriversUtils.make_meta_element(test, devtest_toolalias) if cov_tests is not None and meta_test not in cov_tests: continue cand_testpair_list.append((test, meta_test)) # Adjust the max-time in args ## locate max-time per_test_hard_timeout = None per_test_timeout = None if len(cand_testpair_list) > 0: cur_max_time = float(self.get_value_in_arglist(args, 'max-time')) if self.driver_config.get_gen_timeout_is_per_test(): per_test_timeout = cur_max_time else: per_test_timeout = max(60, \ cur_max_time / len(cand_testpair_list)) self.set_value_in_arglist(args, 'max-time', str(per_test_timeout)) # give time to dump remaning states per_test_hard_timeout = per_test_timeout + \ self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE #per_test_hard_timeout = 300 #DBG # Set the wrapper with open(call_shadow_wrapper_file, 'w') as wf: wf.write('#! /bin/bash\n\n') wf.write('set -u\n') wf.write('set -o pipefail\n\n') wf.write('ulimit -s unlimited\n') # timeout the shadow execution (some test create daemon which) # are not killed by test timeout. ALSO MAKE SURE TO DESACTIVATE # IN TEST SCRIPT TIMEOUT kill_after = 30 wf.write('time_out_cmd="/usr/bin/timeout --kill-after={}s {}"\n'.\ format(kill_after, per_test_hard_timeout)) # kill after and time for timeout to act per_test_hard_timeout += kill_after + 60 #wf.write(' '.join(['exec', runtool] + args + ['"${@:1}"']) + '\n') wf.write('\nstdindata="{}/klee-last/{}"\n'.format(\ self.tests_working_dir, \ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('tmpstdindata="{}/{}"\n\n'.format(self.tests_working_dir,\ KTestTestFormat.STDIN_KTEST_DATA_FILE)) wf.write('if [ -t 0 ] # check if stdin do not exist\n') wf.write('then\n') wf.write(' '.join(['\t(', '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('\t/usr/bin/touch $tmpstdindata\n') wf.write('else\n') wf.write('\t(/bin/cat - > $tmpstdindata ) || EXIT_CODE=1\n') wf.write(' '.join(['\t(', '/bin/cat $tmpstdindata | ', \ '$time_out_cmd', runtool] + args + \ ['"${@:1}"', ') ; EXIT_CODE=$?', '\n'])) wf.write('fi\n\n') wf.write('/bin/mv $tmpstdindata $stdindata || EXIT_CODE=2\n') wf.write('\n# Provoke "unbound variable" if KLEE fails\n') wf.write('# Preserve the KLEE exit code\n') wf.write('exit $EXIT_CODE\n') os.chmod(call_shadow_wrapper_file, 0o775) # run test exes, _ = self.code_builds_factory.repository_manager\ .get_relative_exe_path_map() ERROR_HANDLER.assert_true(len(exes) == 1, \ "Must have a single exe", __file__) exe_path_map = {e: call_shadow_wrapper_file for e in exes} env_vars = {} self._dir_chmod777(self.tests_storage_dir) for test, meta_test in cand_testpair_list: self.parent_meta_tool.execute_testcase(meta_test, exe_path_map, \ env_vars, timeout=per_test_hard_timeout,\ with_output_summary=False) #logging.debug("DBG: Just executed test '{}'".format(meta_test)) #input(">>>> ") #DBG # copy the klee out test_out = os.path.join(self.tests_storage_dir, \ self.get_sorage_name_of_test(test)) os.mkdir(test_out) for d in glob.glob(self.tests_working_dir + "/klee-out-*"): # make sure we can do anything with it self._dir_chmod777(d) if not self.keep_first_test: first_test = os.path.join(d, 'test000001.ktest') if os.path.isfile(first_test): shutil.move(first_test, first_test + '.disable') shutil.move(d, test_out) ERROR_HANDLER.assert_true(len(list(os.listdir(test_out))) > 0, \ "Shadow generated no test for tescase: "+test,\ __file__) if os.path.islink(os.path.join(self.tests_working_dir, \ 'klee-last')): os.unlink(os.path.join(self.tests_working_dir, 'klee-last')) # store klee_change locs common_fs.dumpJSON(klee_change_stmts, self.klee_change_locs_list_file)
def _do_generate_tests (self, exe_path_map, outputdir, \ code_builds_factory, max_time=None): # Setup if os.path.isdir(self.tests_working_dir): shutil.rmtree(self.tests_working_dir) os.mkdir(self.tests_working_dir) if os.path.isdir(self.tests_storage_dir): shutil.rmtree(self.tests_storage_dir) prog = 'klee' default_sym_args = ['-sym-arg', '5'] back_llvm_compiler = None #'clang' rel_path_map = {} exes, _ = code_builds_factory.repository_manager.\ get_relative_exe_path_map() for exe in exes: filename = os.path.basename(exe) rel_path_map[exe] = os.path.join(self.tests_working_dir, filename) pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\ src_fmt=CodeFormats.C_SOURCE,\ dest_fmt=CodeFormats.LLVM_BITCODE,\ src_dest_files_paths_map=rel_path_map,\ compiler=back_llvm_compiler, \ clean_tmp=True, reconfigure=True) if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR: ERROR_HANDLER.error_exit("Program {}.".format(\ 'LLVM built problematic'), __file__) # Update exe_map to reflect bitcode extension rel2bitcode = {} for r_file, b_file in list(rel_path_map.items()): bc = b_file + '.bc' ERROR_HANDLER.assert_true(os.path.isfile(bc), \ "Bitcode file not existing: "+bc, __file__) rel2bitcode[r_file] = bc ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \ "Support single bitcode module for now", __file__) bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]] # klee params bool_param, k_v_params = self._get_default_params() if max_time is not None: k_v_params['-max-time'] = str(max_time) args = [bp for bp, en in list(bool_param.items()) if en] for k, v in list(k_v_params.items()): if v is not None: args += [k, str(v)] args.append(bitcode_file) # sym args klee_sym_args = default_sym_args uc = self.config.get_tool_user_custom() if uc is not None: post_bc_cmd = uc.POST_TARGET_CMD_ORDERED_FLAGS_LIST if post_bc_cmd is not None: klee_sym_args = [] for tup in post_bc_cmd: klee_sym_args += list(tup) args += klee_sym_args # Execute Klee ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\ prog, args) if (ret != 0): logging.error(out) logging.error(err) logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n') ERROR_HANDLER.error_exit("klee failed'", __file__) store_obj = {r: os.path.basename(b) for r, b in rel2bitcode.items()} common_fs.dumpJSON(store_obj, self.test_details_file) #~ def _do_generate_tests() #~ class CustomTestcases