def _update_tests_duplicates_map(self, recompute=False):
if os.path.isfile(self.duplicated_tests_info_file) and not recompute:
self.tests_duplicates_map = common_fs.loadJSON(
self.duplicated_tests_info_file)
return
groups = {}
for _, dat in self.testcases_configured_tools.items():
tobj = dat[self.TOOL_OBJ_KEY]
fmt = tobj.get_test_format_class()
if fmt is None:
continue
if fmt not in groups:
groups[fmt] = []
groups[fmt].append(tobj)
self.tests_duplicates_map = {}
# for each group do fdupes and merge all
for fmt, tt_list in groups.items():
if len(tt_list) > 1:
kepttest2duptest_map, test2keptdup = \
fmt.cross_tool_fdupes(*tt_list)
self.tests_duplicates_map.update(test2keptdup)
if os.path.isdir(os.path.dirname(self.duplicated_tests_info_file)):
common_fs.dumpJSON(self.tests_duplicates_map, \
self.duplicated_tests_info_file, pretty=True)
def _do_instrument_code (self, outputdir, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.gc_files_dir):
shutil.rmtree(self.gc_files_dir)
os.mkdir(self.gc_files_dir)
prog = 'gcc'
flags = ['--coverage', '-fprofile-dir='+self.gc_files_dir, '-O0']
additionals = ["-fkeep-inline-functions"]
# get gcc version
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, ['-dumpversion'])
ERROR_HANDLER.assert_true(ret == 0, "'gcc -dumpversion' failed'")
# if version > 6.5
if int(out.split('.')[0]) >= 6:
if int(out.split('.')[0]) > 6 or int(out.split('.')[1]) > 5:
additionals += ["-fkeep-static-functions"]
flags += additionals
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(\
self.instrumented_code_storage_dir, filename)
self.instrument_callback_obj = self.InstrumentCallbackObject()
self.instrument_callback_obj.set_post_callback_args(\
(self.gc_files_dir, rel_path_map))
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.NATIVE_CODE,\
src_dest_files_paths_map=None,\
compiler=prog, flags_list=flags, clean_tmp=True, \
reconfigure=True, \
callback_object=self.instrument_callback_obj)
# Check
if ret == common_mix.GlobalConstants.COMMAND_FAILURE:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'built problematic'), __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
common_fs.dumpJSON(rel_path_map, self.instrumentation_details)
def __init__(self, repository_manager, workdir=None):
self.repository_manager = repository_manager
self.workdir = workdir
self.src_dest_fmt_to_handling_obj = {}
self.code_conversion_tracker_file = None
self.stored_files_mapping = None
self.stored_files_mapping_file = None
if self.workdir is not None:
self.repo_path_saved_file = os.path.join(self.workdir, \
"saved_repo_path")
self.code_conversion_tracker_file = \
os.path.join(self.workdir, "is_converting")
self.stored_files_mapping_file = \
os.path.join(self.workdir, "files_map")
exes, src_obj_map = \
self.repository_manager.get_relative_exe_path_map()
if os.path.isdir(self.workdir):
tmp = common_fs.loadJSON(self.stored_files_mapping_file)
self.stored_files_mapping = {k: os.path.join(self.workdir, v) \
for k,v in tmp.items()}
else:
count = 0
self.stored_files_mapping = {}
tmp = {}
for f in exes:
tmp[f] = str(count)
self.stored_files_mapping[f] = \
os.path.join(self.workdir, str(count))
count += 1
for s, o in list(src_obj_map.items()):
tmp[o] = str(count)
self.stored_files_mapping[o] = \
os.path.join(self.workdir, str(count))
count += 1
os.mkdir(self.workdir)
common_fs.dumpJSON(tmp, \
self.stored_files_mapping_file, pretty=True)
# Initialize
for src_fmt, obj_cls in formatfrom_function_tuples:
if isinstance(obj_cls, ccs.IdentityCodeConverter):
self._fmt_from_to_registration(src_fmt, src_fmt, obj_cls)
else:
ERROR_HANDLER.assert_true(\
src_fmt in obj_cls.get_source_formats(), \
"{} {} {} {}".format( \
"Error in 'formatfrom_function_tuples'",
"src_fmt", src_fmt, "not in corresponding obj..."),
__file__)
for dest_fmt in obj_cls.get_destination_formats_for(src_fmt):
self._fmt_from_to_registration(src_fmt, dest_fmt, obj_cls)
def fault_analysis(cm_corebench_scripts_dir, c_id, conf_py, in_muteria_outdir,
out_top_dir):
if not os.path.isdir(out_top_dir):
os.mkdir(out_top_dir)
in_res_data_dir = os.path.join(in_muteria_outdir, 'latest', 'RESULTS_DATA')
testtools_workdir = os.path.join(in_muteria_outdir, 'latest',
'testscases_workdir')
pass_fail_matrix = os.path.join(in_res_data_dir, "matrices",
"PASSFAIL.csv")
pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix)
test_list = list(pf_mat.get_nonkey_colname_list())
#semu_tool_dirs = [d for d in os.listdir(testtools_workdir) if d.startswith('semu_cmp-')]
# get fault tests
get_commit_fault_tests(cm_corebench_scripts_dir, c_id, conf_py,
in_res_data_dir, out_top_dir)
# get test to timestamp
test_timestamp_file = os.path.join(out_top_dir, "test_to_timestamp.json")
test2timestamp = {}
## get tests by tools
tools2tests = {}
for test in test_list:
alias, _ = DriversUtils.reverse_meta_element(test)
# XXX: only SEMU
if not alias.startswith('semu_cmp-'):
continue
if alias not in tools2tests:
tools2tests[alias] = set()
tools2tests[alias].add(test)
## untar tests dir
for alias, tests in tools2tests.items():
d = os.path.join(testtools_workdir, alias)
assert os.path.isdir(
d), "test tool dir " + d + " missing for alias " + alias
test_tar = os.path.join(d, 'tests_files.tar.gz')
es = common_fs.TarGz.decompressDir(test_tar)
assert es is None, "decompress error: " + es
tests_files = os.path.join(d, 'tests_files')
assert os.path.isdir(tests_files), "dir missing after decompress"
for test in tests:
_, simple_test = DriversUtils.reverse_meta_element(test)
gt = TestcasesToolSemu._get_generation_time_of_test(
simple_test, tests_files)
test2timestamp[test] = gt
shutil.rmtree(tests_files)
common_fs.dumpJSON(test2timestamp, test_timestamp_file, pretty=True)
def _collect_temporary_coverage_data(self, criteria_name_list, \
test_execution_verdict, \
used_environment_vars, \
result_dir_tmp):
''' get gcov files from gcda files into result_dir_tmp
'''
prog = 'gcov'
cov2flags = {
TestCriteria.STATEMENT_COVERAGE: [],
TestCriteria.BRANCH_COVERAGE: ['-b', '-c'],
TestCriteria.FUNCTION_COVERAGE: ['-f'],
}
args_list = []
for criterion in criteria_name_list:
args_list += cov2flags[criterion]
gcda_files = self._get_gcda_list()
raw_filename_list = [os.path.splitext(f)[0] for f in gcda_files]
args_list += raw_filename_list
if len(gcda_files) > 0:
# TODO: When gcov generate coverage for different files with
# same name filename bu located at diferent dir. Avoid override.
# Go where the gcov will be looked for
cwd = os.getcwd()
os.chdir(self.gc_files_dir)
# collect gcda (gcno)
r, _, _ = DriversUtils.execute_and_get_retcode_out_err(prog=prog, \
args_list=args_list, out_on=False, \
err_on=False)
os.chdir(cwd)
if r != 0:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'error collecting coverage is problematic'), __file__)
# delete gcda
for gcda_f in gcda_files:
os.remove(gcda_f)
common_fs.dumpJSON(self._get_gcov_list(), \
os.path.join(result_dir_tmp,\
self.gcov_files_list_filename))
def compute_stats(config, explorer):
# get the matrix of each test criterion
coverages = {}
total_to = {}
for c in config.ENABLED_CRITERIA.get_val():
if explorer.file_exists(fd_structure.CRITERIA_MATRIX[c]):
mat_file = explorer.get_existing_file_pathname(\
fd_structure.CRITERIA_MATRIX[c])
mat = common_matrices.ExecutionMatrix(filename=mat_file)
row2collist = mat.query_active_columns_of_rows()
cov = len([k for k, v in row2collist.items() if len(v) > 0])
tot = len(row2collist)
coverages[c.get_str()] = '{:.2f}'.format(cov * 100.0 / tot)
total_to[c.get_str()] = tot
# JSON
out_json = {}
for c in coverages:
out_json[c] = {
'coverage': coverages[c],
'# test objectives': total_to[c]
}
common_fs.dumpJSON(out_json, explorer.get_file_pathname(\
fd_structure.STATS_MAIN_FILE_JSON))
# HTML
template_file = os.path.join(os.path.dirname(\
os.path.abspath(__file__)), 'summary_report.html')
report_file = explorer.get_file_pathname(\
fd_structure.STATS_MAIN_FILE_HTML)
rendered = Template(open(template_file).read()).render( \
{'coverages':coverages, 'total_to':total_to})
with open(report_file, 'w') as f:
f.write(rendered)
try:
webbrowser.get()
webbrowser.open('file://' + report_file, new=2)
except Exception as e:
logging.warning("webbrowser error: " + str(e))
#~ def compute_stats()
#~ class DataHandling
def test_dumpJSON(self):
jfilename = os.path.join(self._worktmpdir, "jsontmp.json")
exp = {"x": 1, "y": [2, 3, "zz"]}
res = common_fs.dumpJSON(exp, jfilename)
# dumpJSON succeded
self.assertEqual(res, None)
with open(jfilename) as fp:
res = json.load(fp)
# dumpJSON was correct
self.assertEqual(res, exp)
os.remove(jfilename)
def _do_generate_tests (self, exe_path_map, code_builds_factory, \
meta_criteria_tool_obj=None, \
max_time=None):
dtl = self.code_builds_factory.repository_manager.get_dev_tests_list()
ERROR_HANDLER.assert_true(dtl is not None, "invalid dev_test_list", \
__file__)
if os.path.isdir(self.tests_storage_dir):
shutil.rmtree(self.tests_storage_dir)
os.mkdir(self.tests_storage_dir)
if self.wrapper_test_splitting:
# Execute the tests with the counting wrapper
logging.debug("## Splitting tests with wrapper")
split_workdir = os.path.join(self.tests_storage_dir)
new_exe_path_map = self.wrapper_obj.get_test_splitting_wrapper()\
.set_wrapper(split_workdir, exe_path_map)
sub_test_list = []
for test in dtl:
logging.debug('splitting test '+test) #DBG
self.wrapper_obj.get_test_splitting_wrapper()\
.switch_to_new_test()
self.execute_testcase(test, new_exe_path_map, {})
n_subtest, args = \
self.wrapper_obj.get_test_splitting_wrapper()\
.collect_data()
for i in range(n_subtest):
sub_test_list.append([test+self.splittest_ext+str(i), \
args[i] if i < len(args) else None])
self.wrapper_obj.get_test_splitting_wrapper().cleanup()
# update dtl with counter tests
dtl = sub_test_list
else:
dtl = [[t, None] for t in dtl]
common_fs.dumpJSON(dtl, self.test_list_storage_file)
def serialize(self):
""" Serialize the matrix to its corresponding file if not None
"""
if self.filename is not None:
common_fs.dumpJSON(self.data, self.filename, pretty=True)
def _do_instrument_code (self, outputdir, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.mutant_data):
shutil.rmtree(self.mutant_data)
os.mkdir(self.mutant_data)
prog = 'mart'
# get llvm compiler path
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, ['--version'])
llvm_compiler_path = None
for line in out.splitlines():
line = line.strip()
if line.startswith('LLVM tools dir:'):
llvm_compiler_path = line.split()[3] #[1:-1]
break
ERROR_HANDLER.assert_true(llvm_compiler_path is not None, \
'Problem getting llvm path for mart', __file__)
# Build into LLVM
back_llvm_compiler = 'clang'
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(self.mutant_data, filename)
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.LLVM_BITCODE,\
src_dest_files_paths_map=rel_path_map,\
compiler=back_llvm_compiler, flags_list=['-g'], \
clean_tmp=True, reconfigure=True, \
llvm_compiler_path=llvm_compiler_path)
if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR:
ERROR_HANDLER.error_exit("Program {}.".format(\
'LLVM (clang) built problematic'), __file__)
# Update exe_map to reflect bitcode extension
rel2bitcode = {}
for r_file, b_file in list(rel_path_map.items()):
bc = b_file + '.bc'
ERROR_HANDLER.assert_true(os.path.isfile(bc), \
"Bitcode file not existing: "+bc, __file__)
rel2bitcode[r_file] = bc
ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \
"Support single bitcode module for now", __file__)
bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]]
# mart params
bool_param, k_v_params = self._get_default_params()
if TestCriteria.STRONG_MUTATION in enabled_criteria:
bool_param['-write-mutants'] = True
args = [bp for bp, en in list(bool_param.items()) if en]
for k, v in list(k_v_params.items()):
if v is not None:
args += [k, v]
args.append(bitcode_file)
# Execute Mart
cwd = os.getcwd()
os.chdir(self.mutant_data)
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args)
os.chdir(cwd)
if (ret != 0):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n')
ERROR_HANDLER.error_exit("mart failed'", __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
store_obj = {c.get_str(): {} for c in enabled_criteria}
for k, v in list(rel_path_map.items()):
exe_file = os.path.basename(v)
if TestCriteria.WEAK_MUTATION in enabled_criteria:
crit_str = TestCriteria.WEAK_MUTATION.get_str()
store_obj[crit_str][k] = exe_file + '.WM'
if TestCriteria.MUTANT_COVERAGE in enabled_criteria:
crit_str = TestCriteria.MUTANT_COVERAGE.get_str()
store_obj[crit_str][k] = exe_file + '.COV'
if TestCriteria.STRONG_MUTATION in enabled_criteria:
crit_str = TestCriteria.STRONG_MUTATION.get_str()
store_obj[crit_str][k] = exe_file + '.MetaMu'
common_fs.dumpJSON(store_obj, self.instrumentation_details)
def write_to_file(self, filepath):
common_fs.dumpJSON(self, filepath)
def write_to_file(self, file_path):
common_fs.dumpJSON(self.data, file_path, pretty=True)
def _do_generate_tests (self, exe_path_map, outputdir, \
code_builds_factory, max_time=None):
# Setup
if os.path.isdir(self.tests_working_dir):
shutil.rmtree(self.tests_working_dir)
os.mkdir(self.tests_working_dir)
if os.path.isdir(self.tests_storage_dir):
shutil.rmtree(self.tests_storage_dir)
prog = 'klee'
default_sym_args = ['-sym-arg', '5']
back_llvm_compiler = None #'clang'
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(self.tests_working_dir, filename)
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.LLVM_BITCODE,\
src_dest_files_paths_map=rel_path_map,\
compiler=back_llvm_compiler, \
clean_tmp=True, reconfigure=True)
if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR:
ERROR_HANDLER.error_exit("Program {}.".format(\
'LLVM built problematic'), __file__)
# Update exe_map to reflect bitcode extension
rel2bitcode = {}
for r_file, b_file in list(rel_path_map.items()):
bc = b_file + '.bc'
ERROR_HANDLER.assert_true(os.path.isfile(bc), \
"Bitcode file not existing: "+bc, __file__)
rel2bitcode[r_file] = bc
ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \
"Support single bitcode module for now", __file__)
bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]]
# klee params
bool_param, k_v_params = self._get_default_params()
if max_time is not None:
k_v_params['-max-time'] = str(max_time)
args = [bp for bp, en in list(bool_param.items()) if en]
for k, v in list(k_v_params.items()):
if v is not None:
args += [k, str(v)]
args.append(bitcode_file)
# sym args
klee_sym_args = default_sym_args
uc = self.config.get_tool_user_custom()
if uc is not None:
post_bc_cmd = uc.POST_TARGET_CMD_ORDERED_FLAGS_LIST
if post_bc_cmd is not None:
klee_sym_args = []
for tup in post_bc_cmd:
klee_sym_args += list(tup)
args += klee_sym_args
# Execute Klee
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args)
if (ret != 0):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n')
ERROR_HANDLER.error_exit("klee failed'", __file__)
store_obj = {r: os.path.basename(b) for r, b in rel2bitcode.items()}
common_fs.dumpJSON(store_obj, self.test_details_file)
#~ def _do_generate_tests()
#~ class CustomTestcases
def summarize_data(outdir, summarized_data_dir, mutant_exec_res, \
mut_ex_prepa_data_dir):
other_rel_to_reltests_file = os.path.join(
mut_ex_prepa_data_dir, 'relevantmuts_to_relevanttests.json')
other_rel_to_reltests = common_fs.loadJSON(other_rel_to_reltests_file)
stored_map_file = os.path.join(mut_ex_prepa_data_dir,
'stored_test_map.json')
stored_map = common_fs.loadJSON(stored_map_file)
def get_too2relmuts(relmuts_to_reltests, toollist=None):
res = {tool: set()
for tool in toollist} if toollist is not None else {}
for relmut, t_list in relmuts_to_reltests.items():
for meta_t in t_list:
toolalias, test = DriversUtils.reverse_meta_element(meta_t)
if toollist is None:
if toolalias not in res:
res[toolalias] = set()
else:
assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format(
toolalias, toollist)
res[toolalias].add(relmut)
for toolalias in res:
res[toolalias] = list(res[toolalias])
return res
#~ def get_too2relmuts()
other_tool_to_relmuts = get_too2relmuts(other_rel_to_reltests)
all_tests, fail_tests, relevant_mutants_to_relevant_tests, \
mutants_to_killingtests, tests_to_killed_mutants = load.load (mutant_exec_res, fault_revealing=False)
if not os.path.isdir(summarized_data_dir):
os.mkdir(summarized_data_dir)
cmp_result_file = os.path.join(summarized_data_dir, "compare_results.json")
initial_relmuts_file = os.path.join(summarized_data_dir,
"initial_relevant.json")
rMS_file = os.path.join(summarized_data_dir, "rMS.json")
# get relevant muts by tools
# get tool list
tinf = common_fs.loadJSON(
os.path.join(
mutant_exec_res,
"post/RESULTS_DATA/other_copied_results/testcasesInfos.json"))
tool2relmuts = get_too2relmuts(
relevant_mutants_to_relevant_tests,
toollist=list(set(tinf["CUSTOM"]) - {"custom_devtests"}))
# update with stored map
for relmut, t_list in other_rel_to_reltests.items():
for other_meta_t in t_list:
if other_meta_t in stored_map:
for meta_t in stored_map[other_meta_t]:
toolalias, test = DriversUtils.reverse_meta_element(meta_t)
assert toolalias in tool2relmuts, "PB2: "
tool2relmuts[toolalias].append(relmut)
for toolalias in tool2relmuts:
tool2relmuts[toolalias] = list(set(tool2relmuts[toolalias]))
common_fs.dumpJSON(tool2relmuts, cmp_result_file, pretty=True)
common_fs.dumpJSON(other_tool_to_relmuts,
initial_relmuts_file,
pretty=True)
# Compute and print the relevant mutation score per tool
all_rel_muts = set()
devtest_rel = set()
for talias, rellist in tool2relmuts.items():
all_rel_muts |= set(rellist)
for talias, rellist in other_tool_to_relmuts.items():
all_rel_muts |= set(rellist)
if talias.startswith("custom_devtests"):
devtest_rel |= set(rellist)
rMS = {"ALL-GENONLY": {}, "ALL-ALL": {}, "ADDITIONAL": {}}
for talias, rellist in tool2relmuts.items():
rMS["ALL-ALL"][talias] = len(set(rellist)
| devtest_rel) * 100.0 / len(all_rel_muts)
rMS["ALL-GENONLY"][talias] = len(rellist) * 100.0 / len(all_rel_muts)
rMS["ADDITIONAL"][talias] = len(set(
rellist) - devtest_rel) * 100.0 / len(all_rel_muts - devtest_rel)
common_fs.dumpJSON(rMS, rMS_file, pretty=True)
def prepare_mutant_execution(outdir, muteria_output, original_conf, \
prepare_data_dir, \
avoid_meta_tests_list_file, avoid_meta_mutants_list_file):
if not os.path.isdir(prepare_data_dir):
os.mkdir(prepare_data_dir)
# Store the test list per technique
test_list_file = os.path.join(muteria_output, 'latest', 'RESULTS_DATA', \
'other_copied_results', 'testcasesInfos.json')
shutil.copy2(test_list_file, os.path.join(prepare_data_dir, \
'gentests_'+os.path.basename(test_list_file)))
relevant_exec_outfolder = os.path.join(os.path.dirname(original_conf), \
'output', 'latest')
# Do fdupes with relevant output tests of semu and shadow and store map
r_shadow_tests_src = os.path.join(relevant_exec_outfolder, \
'testscases_workdir', 'shadow_se', 'tests_files')
r_shadow_tests_src_tar = r_shadow_tests_src + '.tar.gz'
r_semu_tests_src = os.path.join(relevant_exec_outfolder, \
'testscases_workdir', 'semu', 'tests_files')
r_semu_tests_src_tar = r_semu_tests_src + '.tar.gz'
cur_exec_outfolder = os.path.join(muteria_output, 'latest')
cur_exec_tg_folder = os.path.join(cur_exec_outfolder, 'testscases_workdir')
cur_shadow_tests_src = None
cur_semu_tests_srcs = []
cur_semu_tests_src_tars = []
for f in os.listdir(cur_exec_tg_folder):
if f.startswith('semu'):
cur_semu_tests_srcs.append(
os.path.join(cur_exec_tg_folder, f, 'tests_files'))
cur_semu_tests_src_tars.append(cur_semu_tests_srcs[-1] + '.tar.gz')
elif f.startswith('shadow_se'):
if cur_shadow_tests_src is not None:
error_exit("Multiple shadow_se folders exist")
cur_shadow_tests_src = os.path.join(cur_exec_tg_folder, f,
'tests_files')
cur_shadow_tests_src_tar = cur_shadow_tests_src + '.tar.gz'
# use folder fdupes from KtestFormat in muteria
## decompress the folders
common_fs.TarGz.decompressDir(r_shadow_tests_src_tar)
common_fs.TarGz.decompressDir(r_semu_tests_src_tar)
common_fs.TarGz.decompressDir(cur_shadow_tests_src_tar)
for i in range(len(cur_semu_tests_src_tars)):
common_fs.TarGz.decompressDir(cur_semu_tests_src_tars[i])
## apply fdupes
def post_test_dup(stored_map, tests_to_avoid, origin,
kepttest2duptest_map):
for kept, duplist in kepttest2duptest_map.items():
alias, _ = DriversUtils.reverse_meta_element(kept)
if alias == origin:
stored_map[kept] = []
for dup in duplist:
dalias, dtest = DriversUtils.reverse_meta_element(dup)
if dalias == origin:
continue
stored_map[kept].append(dup)
tests_to_avoid.append(dup)
#~ def post_test_dup()
stored_map = {}
stored_map_file = os.path.join(prepare_data_dir, 'stored_test_map.json')
tests_to_avoid = []
### Shadow
custom_bin = '/home/shadowvm/shadow/klee-change/Release+Asserts/bin/'
folders = [r_shadow_tests_src, cur_shadow_tests_src]
folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders}
origin = folder2alias[r_shadow_tests_src]
kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\
custom_bin, folders, folder2alias)
post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map)
## remove the untar dirs
for d in folders:
shutil.rmtree(d)
### Semu
custom_bin = None
folders = [r_semu_tests_src] + cur_semu_tests_srcs
folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders}
origin = folder2alias[r_semu_tests_src]
kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\
custom_bin, folders, folder2alias)
post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map)
for d in folders:
shutil.rmtree(d)
## store dup map and tests to avoid
common_fs.dumpJSON(stored_map, stored_map_file, pretty=True)
common_fs.dumpJSON(tests_to_avoid, avoid_meta_tests_list_file, pretty=True)
# XXX TODO: modify tg_conf to use tests to avoid
# Get the criteria dir and add to the muteria out
r_criteria_dir = os.path.join(relevant_exec_outfolder, 'criteria_workdir')
dest_crit_work = os.path.join(muteria_output, 'latest', 'criteria_workdir')
if os.path.isdir(os.path.join(dest_crit_work, 'mart_0')):
shutil.rmtree(os.path.join(dest_crit_work, 'mart_0'))
shutil.copytree(os.path.join(r_criteria_dir, 'mart_0'),
os.path.join(dest_crit_work, 'mart_0'))
# Get the selected mutants (mutants that are Not relevant w.r.t dev tests)
r_res_top = os.path.dirname(os.path.dirname(original_conf))
r_res = os.path.join(r_res_top, 'res')
r_res_tar = glob.glob(r_res_top + '/*res.tar.gz')[0]
## untar res
if os.system('cd {} && tar -xzf {} --exclude {} --exclude {} && test -d res'.format(\
r_res_top, os.path.basename(r_res_tar), \
'post/RESULTS_DATA/other_copied_results/Flakiness', \
'pre/RESULTS_DATA/other_copied_results/Flakiness')) != 0:
error_exit("untar re failed")
## get relevant mutants to relevant tests
all_tests, fail_tests, relevant_mutants_to_relevant_tests, \
mutants_to_killingtests, tests_to_killed_mutants = load.load (r_res, fault_revealing=False)
## remove untar
shutil.rmtree(r_res)
## filter mutants (remove those relevant w.r.t. dev tests)
rel_to_reltests_file = os.path.join(prepare_data_dir,
'relevantmuts_to_relevanttests.json')
mutants_to_avoid = []
### mutants relevant w.r.t. devtests added to avoid
for mut, r_tests in relevant_mutants_to_relevant_tests.items():
for rt in r_tests:
alias, _ = DriversUtils.reverse_meta_element(rt)
if alias.startswith("custom_devtests"):
mutants_to_avoid.append(mut)
break
## save filter mutants as to avoid
common_fs.dumpJSON(relevant_mutants_to_relevant_tests,
rel_to_reltests_file,
pretty=True)
common_fs.dumpJSON(mutants_to_avoid,
avoid_meta_mutants_list_file,
pretty=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('outdir', help="outdir of the execution")
parser.add_argument('original_conf', help="config of the project")
parser.add_argument('--cleanstart', action="store_true", \
help="delete out and restart")
parser.add_argument('--only_gentests', action="store_true", \
help="only generate tests, no mutant execution")
parser.add_argument('--only_getseeds', action="store_true", \
help="only collect the seeds, no tests generation or mutant execution")
parser.add_argument('--no_summary', action="store_true", \
help="Disable Summary")
args = parser.parse_args()
outdir = os.path.abspath(args.outdir)
original_conf = os.path.abspath(args.original_conf)
if args.cleanstart:
if os.path.isdir(outdir):
shutil.rmtree(outdir)
if not os.path.isdir(outdir):
os.mkdir(outdir)
# XXX
do_remove_added_devtests = args.only_gentests
seeds_dir = os.path.join(outdir, "ktests_seeds")
checkpoint_file = os.path.join(outdir, 'checkpoint.json')
tg_conf = os.path.join(outdir, "tg_conf.py")
muteria_output = os.path.join(outdir, "muteria_output")
mut_ex_prepa_data_dir = os.path.join(outdir, "mut_ex_prepa_data")
summarized_data_dir = os.path.join(outdir, "summarized_data_dir")
mutant_exec_res = os.path.join(outdir, "mutant_exec_res")
avoid_meta_mutants_list_file = os.path.join(outdir, mut_ex_prepa_data_dir, \
"avoid_meta_mutants_list_file.txt")
avoid_meta_tests_list_file = os.path.join(outdir, mut_ex_prepa_data_dir, \
"avoid_meta_tests_list_file.txt")
# load added dev tests
added_devtest_map_file = os.path.join(os.path.dirname(__file__),
'added_tests',
'added_tests_map.json')
added_devtest_map = common_fs.loadJSON(added_devtest_map_file)
cur_id = os.path.basename(outdir)
if do_remove_added_devtests:
if cur_id not in added_devtest_map:
error_exit("do_remove_added_devtests is true but id not in map")
toskip_added_devtest = added_devtest_map[cur_id]
else:
toskip_added_devtest = []
# check checkpoint and load if necessary
cp_data = {
SEED_COLLECTION: False,
TEST_GENERATION: False,
MUTANT_EXECUTION_PREPA: False,
MUTANT_EXECUTION: False,
DATA_SUMMARIZATION: False,
}
if args.only_getseeds:
cp_data[TEST_GENERATION] = True
cp_data[MUTANT_EXECUTION_PREPA] = True
cp_data[MUTANT_EXECUTION] = True
cp_data[DATA_SUMMARIZATION] = True
if args.only_gentests:
cp_data[MUTANT_EXECUTION_PREPA] = True
cp_data[MUTANT_EXECUTION] = True
if args.no_summary:
cp_data[DATA_SUMMARIZATION] = True
if os.path.isfile(checkpoint_file):
cp_data = common_fs.loadJSON(checkpoint_file)
if not cp_data[SEED_COLLECTION]:
print("#(DBG): Executing {} ...".format(SEED_COLLECTION))
collect_seeds(outdir, seeds_dir, muteria_output, original_conf,
toskip_added_devtest)
cp_data[SEED_COLLECTION] = True
common_fs.dumpJSON(cp_data, checkpoint_file, pretty=True)
if not cp_data[TEST_GENERATION]:
print("#(DBG): Executing {} ...".format(TEST_GENERATION))
generate_tests (outdir, seeds_dir, muteria_output, original_conf, \
tg_conf, args.only_gentests, toskip_added_devtest, args.only_gentests)
cp_data[TEST_GENERATION] = True
common_fs.dumpJSON(cp_data, checkpoint_file, pretty=True)
if not cp_data[MUTANT_EXECUTION_PREPA]:
print("#(DBG): Executing {} ...".format(MUTANT_EXECUTION_PREPA))
prepare_mutant_execution(outdir, muteria_output, original_conf, \
mut_ex_prepa_data_dir, avoid_meta_tests_list_file,\
avoid_meta_mutants_list_file)
cp_data[MUTANT_EXECUTION_PREPA] = True
common_fs.dumpJSON(cp_data, checkpoint_file, pretty=True)
if not cp_data[MUTANT_EXECUTION]:
print("#(DBG): Executing {} ...".format(MUTANT_EXECUTION))
mutant_execution(outdir, avoid_meta_tests_list_file, avoid_meta_mutants_list_file, \
muteria_output, tg_conf, mutant_exec_res)
cp_data[MUTANT_EXECUTION] = True
common_fs.dumpJSON(cp_data, checkpoint_file, pretty=True)
if not cp_data[DATA_SUMMARIZATION]:
print("#(DBG): Executing {} ...".format(DATA_SUMMARIZATION))
if args.only_gentests:
cm_corebench_scripts_dir = os.path.join(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(original_conf)))),
"cm_corebench_scripts")
c_id = os.path.basename(outdir)
get_fault_tests.fault_analysis (cm_corebench_scripts_dir, c_id, tg_conf, \
muteria_output, summarized_data_dir)
else:
summarize_data(outdir, summarized_data_dir, mutant_exec_res, \
mut_ex_prepa_data_dir)
cp_data[DATA_SUMMARIZATION] = True
common_fs.dumpJSON(cp_data, checkpoint_file, pretty=True)
print("\n# DONE!\n")
def _do_instrument_code (self, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.used_srcs_dir):
shutil.rmtree(self.used_srcs_dir)
os.mkdir(self.used_srcs_dir)
if os.path.isdir(self.preload_dir):
shutil.rmtree(self.preload_dir)
os.mkdir(self.preload_dir)
rel_path_map = {}
exes, src_inter_map = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for f in src_inter_map:
rel_path_map[f] = os.path.join(self.used_srcs_dir, f)
relloc = os.path.join(self.used_srcs_dir, os.path.dirname(f))
if not os.path.isdir(relloc):
os.makedirs(relloc)
not_none_dest = {k: v for k, v in list(rel_path_map.items())}
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.PYTHON_SOURCE,\
dest_fmt=CodeFormats.PYTHON_SOURCE,\
src_dest_files_paths_map=not_none_dest)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
common_fs.dumpJSON(rel_path_map, self.instrumentation_details)
if ret == common_mix.GlobalConstants.COMMAND_FAILURE:
ERROR_HANDLER.error_exit("Problem with copying python sources", \
__file__)
# Create config and preload
with open(self.preload_file, "w") as f:
f.write("import coverage\ncoverage.process_startup()\n")
concurrencies = ['thread', 'multiprocessing', 'gevent', 'greenlet', \
'eventlet']
concurrency_support = 'thread'
#config = configparser.ConfigParser()
config = {}
config['run'] = {
'include': \
[os.path.join('*',v) for v in rel_path_map.keys()],
'data_file': self.raw_data_file,
'branch': (\
TestCriteria.BRANCH_COVERAGE in enabled_criteria),
'parallel': True,
'concurrency': concurrency_support,
}
with open(self.config_file, 'w') as f:
#config.write(f)
for k, v in list(config.items()):
f.write('[' + k + ']' + '\n')
for kk, vv in list(v.items()):
if type(vv) == list:
f.write(kk + ' = \n')
for vvv in vv:
f.write('\t' + str(vvv) + '\n')
else:
f.write(kk + ' = ' + str(vv) + '\n')
#~ def _do_instrument_code()
#~ class CriteriaToolCoveragePy
def _collect_temporary_coverage_data(self, criteria_name_list, \
test_execution_verdict, \
used_environment_vars, \
result_dir_tmp, \
testcase):
''' get gcov files from gcda files into result_dir_tmp
'''
prog = 'gcov'
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
cov2flags = {
TestCriteria.STATEMENT_COVERAGE: [],
TestCriteria.BRANCH_COVERAGE: ['-b', '-c'],
TestCriteria.FUNCTION_COVERAGE: ['-f'],
}
args_list = []
for criterion in criteria_name_list:
args_list += cov2flags[criterion]
gcda_files = self._get_gcda_list()
raw_filename_list = [os.path.splitext(f)[0] for f in gcda_files]
args_list += raw_filename_list
if len(gcda_files) > 0:
# TODO: When gcov generate coverage for different files with
# same name filename but located at diferent dir. Avoid override.
# Go where the gcov will be looked for
cwd = os.getcwd()
os.chdir(self.gc_files_dir)
# collect gcda (gcno)
r, _, err_str = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, \
args_list=args_list, out_on=False, \
err_on=True, merge_err_to_out=False)
os.chdir(cwd)
if r != 0: # or err_str:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'collecting coverage is problematic. ')+
"The error msg is {}. \nThe command:\n{}".format(\
err_str, " ".join([prog]+args_list)), __file__)
dot_gcov_file_list = self._get_gcov_list()
# Sources of interest
_, src_map = self.code_builds_factory.repository_manager.\
get_relative_exe_path_map()
base_dot_gcov = [os.path.basename(f) for f in dot_gcov_file_list]
base_raw = [os.path.basename(f) for f in raw_filename_list]
interest = [os.path.basename(s) for s, o in src_map.items()]
interest = [s+'.gcov' for s in interest if os.path.splitext(s)[0] \
in base_raw]
missed = set(interest) - set(base_dot_gcov)
# FIXME: Check for partial failure (compare number of gcno and gcov)
ERROR_HANDLER.assert_true(len(missed) == 0,
"{} did not generate the '.gcov' files {}.".format(\
prog, missed)+ "\nIts stderr is {}.\n CMD: {}".format(\
err_str, " ".join([prog]+args_list)), __file__)
# delete gcda
for gcda_f in gcda_files:
os.remove(gcda_f)
else:
if not self.driver_config.get_allow_missing_coverage():
ERROR_HANDLER.error_exit(\
"Testcase '{}' did not generate gcda, {}".format(\
testcase, "when allow missing coverage is disabled"))
dot_gcov_file_list = []
common_fs.dumpJSON(dot_gcov_file_list, \
os.path.join(result_dir_tmp,\
self.gcov_files_list_filename))
def _setup_repository(self):
# Make sure the repo dir exists
ERROR_HANDLER.assert_true(os.path.isdir(self.repository_rootdir), \
"given repository dir is not existing: {}". format( \
self.repository_rootdir), __file__)
# make sure the repo dir is a git repo
# if no, ask the user whether to initialize and initialize or abort
# if yes or user accepted initialize, get the git object for the repo
try:
repo = git_repo(self.repository_rootdir)
gitobj = repo.git
except git_exc.InvalidGitRepositoryError:
make_it_git = common_mix.confirm_execution("{} {} {} {}".format(\
"The given repository directory is not", \
"a git repository, this must be a git", \
"repository to proceed.\n Do you want to",\
"set is as git repository?"))
if make_it_git:
repo = git_repo.init(self.repository_rootdir)
gitobj = repo.git
else:
ERROR_HANDLER.error_exit("{} {}".format(\
"Must make the repository as git repository,", \
"then re-run"), __file__)
# Check whether the repo is already managed by muteria
## if using branch
if self.delete_created_on_revert_as_initial:
if self.test_branch_name not in self._get_branches_list(\
self.repository_rootdir):
## Not managed, create branch
self._make_testing_branch(self.repository_rootdir, \
self.test_branch_name)
# checkout
gitobj.checkout(self.test_branch_name)
# Actual Check whether the repo is already managed by muteria
# There must be a directory DEFAULT_MUTERIA_REPO_META_FOLDER
src_in_prev = set()
if os.path.isdir(self.muteria_metadir):
## Managed
prev_src_list, prev_use_branch = \
common_fs.loadJSON(self.muteria_metadir_info_file)
ERROR_HANDLER.assert_true(prev_use_branch == \
self.delete_created_on_revert_as_initial, \
"{} {} {} {} {}".format(\
"unmatching repo backup type.", \
"previously, use branch was", \
prev_use_branch, ", now it is",\
self.delete_created_on_revert_as_initial), \
__file__)
src_in_prev = set(prev_src_list) & set(self.source_files_list)
prev_src_list = set(prev_src_list) - set(self.source_files_list)
remain_prev_src_set = {src for src in prev_src_list \
if os.path.isfile(self.repo_abs_path(src))}
# make sure that all prev_src are in initial state
untracked_diff_files = self._get_untracked_and_diffed(repo)
prev_untracked_diff = remain_prev_src_set & untracked_diff_files
if len(prev_untracked_diff) > 0:
bypass = common_mix.confirm_execution(\
"{} {} {} {} {}".format(
"the following files were previously used as",\
"src files by muteria and are now untracked:",\
prev_untracked_diff, \
"\nDo you want to handle it or bypass it?",\
"Choose yes to bypass: "******"{} {}".format(\
"Do you want to automatically restore the",\
"The untracked previous source and continue?"))
if revert_them:
for src in prev_untracked_diff:
self.revert_repository_file(src, gitobj=gitobj)
else:
ERROR_HANDLER.error_exit(\
"Handle it manually and restart the execution")
# update the info_file
if set(prev_src_list) != set(self.source_files_list):
common_fs.dumpJSON([self.source_files_list, \
self.delete_created_on_revert_as_initial],\
self.muteria_metadir_info_file)
else:
## Not managed
os.mkdir(self.muteria_metadir)
common_fs.dumpJSON([self.source_files_list, \
self.delete_created_on_revert_as_initial],\
self.muteria_metadir_info_file)
# Make sure all source files of interest are tracked
untracked_diff_files = self._get_untracked_and_diffed(repo)
untracked_diff_src_in_prev = untracked_diff_files & src_in_prev
if len(untracked_diff_src_in_prev) > 0:
if common_mix.confirm_execution(\
"{} {} {} {} {}".format("The following source",\
"files of interest are untracked", \
"and will be reverted (previous execution unfinished):", \
src_in_prev, \
"do you want to revert them?")):
for src in src_in_prev:
self.revert_repository_file(src, gitobj=gitobj)
else:
ERROR_HANDLER.error_exit("{} {}".format(\
"Handle untracked source files manually", \
"then restart the execution"))
untracked_diff_files = self._get_untracked_and_diffed(repo)
untracked_diff_src_files = set(self.source_files_list) & \
untracked_diff_files
if len(untracked_diff_src_files) > 0:
if common_mix.confirm_execution(\
"{} {} {} {}".format("The following source",\
"files of interest are untracked:", \
untracked_diff_src_files, \
"do you want to track them?")):
repo.index.add(list(untracked_diff_src_files))
else:
ERROR_HANDLER.error_exit("{} {}".format(\
"Handle untracked source files manually", \
"then restart the execution"))
def _runtests(self, testcases, exe_path_map, env_vars, \
stop_on_failure=False, per_test_timeout=None, \
use_recorded_timeout_times=None, \
recalculate_execution_times=False, \
with_output_summary=True, hash_outlog=True, \
parallel_count=1):
'''
Execute the list of test cases with the given executable and
say, for each test case, whether it failed.
Note: Re-implement this if there the tool implements ways to faster
execute multiple test cases.
:param testcases: list of test cases to execute
:param exe_path_map: string representing the file system path to
the executable to execute with the tests
:param env_vars: dict of environment variables to set before
executing each test ({<variable>: <value>})
:param stop_on_failure: decide whether to stop the test execution once
a test fails
:returns: plitair of:
- dict of testcase and their failed verdict.
{<test case name>: <True if failed, False if passed,
UNCERTAIN_TEST_VERDICT if uncertain>}
If stop_on_failure is True, only return the tests that have
been executed until the failure
- test execution output log hash data object or None
'''
# @Checkpoint: create a checkpoint handler (for time)
checkpoint_handler = CheckPointHandler(self.get_checkpointer())
if checkpoint_handler.is_finished():
logging.warning("{} {} {}".format( \
"The function 'runtests' is finished according", \
"to checkpoint, but called again. None returned", \
"\nPlease Confirm reexecution..."))
if common_mix.confirm_execution("{} {}".format( \
"Function 'runtests' is already", \
"finished, do yo want to restart?")):
checkpoint_handler.restart()
logging.info("Restarting the finished 'runtests'")
else:
ERROR_HANDLER.error_exit(err_string="{} {} {}".format( \
"Execution halted. Cannot continue because no value", \
" can be returned. Check the results of the", \
"finished execution"), call_location=__file__)
if per_test_timeout is None:
per_test_timeout = {tc: None for tc in testcases}
if use_recorded_timeout_times is not None:
ERROR_HANDLER.assert_true(use_recorded_timeout_times > 0, \
"use_recorded_timeout_times must be "
"positive if not None", __file__)
per_test_timeout.update({x: (y * use_recorded_timeout_times) \
for x, y in self.test_execution_time.items()})
else:
ERROR_HANDLER.assert_true(use_recorded_timeout_times is None, \
"use_recorded_timeout_times must not be set "
"when per_test_timeout is set", __file__)
# Prepare the exes
self._prepare_executable(exe_path_map, env_vars, \
collect_output=with_output_summary)
self._set_env_vars(env_vars)
test_failed_verdicts = {}
test_outlog_hash = {}
processbar = tqdm.tqdm(testcases, leave=False, dynamic_ncols=True)
# Parallel stuffs
def test_exec_iteration(testcase):
processbar.set_description("Running Test {} (x{})".format(\
testcase, parallel_count))
start_time = time.time()
test_failed, execoutlog_hash = \
self._oracle_execute_a_test(testcase, exe_path_map, \
env_vars, \
timeout=per_test_timeout[testcase], \
with_output_summary=with_output_summary, \
hash_outlog=hash_outlog)
#if testcase.endswith('.ktest'): # DBG - fix hang
# logging.debug("KTEST {} is done".format(testcase))
# Record exec time if not existing
with self.shared_loc:
if recalculate_execution_times:
self.test_execution_time[testcase] = \
max(1, int(time.time() - start_time)) \
* self.config.RECORDED_TEST_TIMEOUT_FACTOR
test_failed_verdicts[testcase] = test_failed
test_outlog_hash[testcase] = execoutlog_hash
return test_failed
#~ def test_exec_iteration()
if self.can_run_tests_in_parallel() and parallel_count is not None \
and parallel_count > 1:
parallel_count = min(len(testcases), parallel_count)
joblib.Parallel(n_jobs=parallel_count, require='sharedmem')\
(joblib.delayed(test_exec_iteration)(testcase) \
for testcase in processbar)
else:
parallel_count = 1 # to be printed in progress
for testcase in processbar:
test_failed = test_exec_iteration(testcase)
if stop_on_failure and test_failed != \
common_mix.GlobalConstants.PASS_TEST_VERDICT:
break
if recalculate_execution_times:
common_fs.dumpJSON(self.test_execution_time, \
self.test_execution_time_storage_file, pretty=True)
# Restore back the exes
self._restore_env_vars()
self._restore_default_executable(exe_path_map, env_vars, \
collect_output=with_output_summary)
if stop_on_failure:
# Make sure the non executed test has the uncertain value (None)
if len(test_failed_verdicts) < len(testcases):
for testcase in set(testcases) - set(test_failed_verdicts):
test_failed_verdicts[testcase] = \
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT
test_outlog_hash[testcase] = common_matrices.\
OutputLogData.UNCERTAIN_TEST_OUTLOGDATA
# @Checkpoint: Finished (for time)
checkpoint_handler.set_finished(None)
if not with_output_summary:
test_outlog_hash = None
ERROR_HANDLER.assert_true(len(testcases) == len(test_failed_verdicts),\
"Mismatch between testcases and test_failed_verdict (BUG)",\
__file__)
return test_failed_verdicts, test_outlog_hash
def compute_stats(config, explorer, checkpointer):
# get the matrix of each test criterion
coverages = {}
total_to = {}
number_of_testcases = None
for c in config.ENABLED_CRITERIA.get_val():
if explorer.file_exists(fd_structure.CRITERIA_MATRIX[c]):
mat_file = explorer.get_existing_file_pathname(\
fd_structure.CRITERIA_MATRIX[c])
mat = common_matrices.ExecutionMatrix(filename=mat_file)
row2collist = mat.query_active_columns_of_rows()
cov = len([k for k,v in row2collist.items() if len(v) > 0])
tot = len(row2collist)
coverages[c.get_str()] = 'n.a.' if tot == 0 else \
'{:.2f}'.format(cov * 100.0 / tot)
total_to[c.get_str()] = tot
if number_of_testcases is None:
number_of_testcases = len(mat.get_nonkey_colname_list())
# JSON
out_json = {}
out_json['TOTAL EXECUTION TIME (s)'] = \
checkpointer.get_execution_time()
out_json['NUMBER OF TESTCASES'] = number_of_testcases
out_json['CRITERIA'] = {}
for c in coverages:
out_json['CRITERIA'][c] = {'coverage': coverages[c],
'# test objectives': total_to[c]}
common_fs.dumpJSON(out_json, explorer.get_file_pathname(\
fd_structure.STATS_MAIN_FILE_JSON), \
pretty=True)
# HTML
template_file = os.path.join(os.path.dirname(\
os.path.abspath(__file__)), 'summary_report.html')
report_file = explorer.get_file_pathname(\
fd_structure.STATS_MAIN_FILE_HTML)
def format_execution_time(exec_time):
n_day = int(exec_time // (24 * 3600))
exec_time = exec_time % (24 * 3600)
n_hour = int(exec_time // 3600)
exec_time %= 3600
n_minutes = int(exec_time // 60)
exec_time %= 60
n_seconds = int(round(exec_time))
res = ""
for val, unit in [(n_day, 'day'), (n_hour, 'hour'), \
(n_minutes, 'minutes'), (n_seconds, 'second')]:
if val > 0:
s = ' ' if val == 1 else 's '
res += str(val) + ' ' + unit + s
return res
#~ def format_execution_time()
total_exec_time = format_execution_time(\
checkpointer.get_execution_time())
rendered = Template(open(template_file).read()).render( \
{
'total_execution_time': total_exec_time,
'number_of_testcases': number_of_testcases,
'coverages':coverages,
'total_to':total_to,
})
with open(report_file, 'w') as f:
f.write(rendered)
try:
webbrowser.get()
webbrowser.open('file://' + report_file,new=2)
except Exception as e:
logging.warning("webbrowser error: "+str(e))
def _runtests(self, testcases, exe_path_map, env_vars, \
stop_on_failure=False, per_test_timeout=None, \
use_recorded_timeout_times=None, \
recalculate_execution_times=False, \
with_outlog_hash=True, parallel_count=1):
'''
Execute the list of test cases with the given executable and
say, for each test case, whether it failed.
Note: Re-implement this if there the tool implements ways to faster
execute multiple test cases.
:param testcases: list of test cases to execute
:param exe_path_map: string representing the file system path to
the executable to execute with the tests
:param env_vars: dict of environment variables to set before
executing each test ({<variable>: <value>})
:param stop_on_failure: decide whether to stop the test execution once
a test fails
:returns: plitair of:
- dict of testcase and their failed verdict.
{<test case name>: <True if failed, False if passed,
UNCERTAIN_TEST_VERDICT if uncertain>}
If stop_on_failure is True, only return the tests that have
been executed until the failure
- test execution output log hash data object or None
'''
# @Checkpoint: create a checkpoint handler (for time)
checkpoint_handler = CheckPointHandler(self.get_checkpointer())
if checkpoint_handler.is_finished():
logging.warning("{} {} {}".format( \
"The function 'runtests' is finished according", \
"to checkpoint, but called again. None returned", \
"\nPlease Confirm reexecution..."))
if common_mix.confirm_execution("{} {}".format( \
"Function 'runtests' is already", \
"finished, do yo want to restart?")):
checkpoint_handler.restart()
logging.info("Restarting the finished 'runtests'")
else:
ERROR_HANDLER.error_exit(err_string="{} {} {}".format( \
"Execution halted. Cannot continue because no value", \
" can be returned. Check the results of the", \
"finished execution"), call_location=__file__)
if per_test_timeout is None:
per_test_timeout = {tc: None for tc in testcases}
if use_recorded_timeout_times is not None:
ERROR_HANDLER.assert_true(use_recorded_timeout_times > 0, \
"use_recorded_timeout_times must be "
"positive if not None", __file__)
per_test_timeout.update({x: (y * use_recorded_timeout_times) \
for x, y in self.test_execution_time.items()})
else:
ERROR_HANDLER.assert_true(use_recorded_timeout_times is None, \
"use_recorded_timeout_times must not be set "
"when per_test_timeout is set", __file__)
# Prepare the exes
self._prepare_executable(exe_path_map, env_vars, \
collect_output=with_outlog_hash)
self._set_env_vars(env_vars)
test_failed_verdicts = {}
test_outlog_hash = {}
for testcase in testcases:
start_time = time.time()
test_failed, execoutlog_hash = \
self._oracle_execute_a_test(testcase, exe_path_map, \
env_vars, \
timeout=per_test_timeout[testcase], \
with_outlog_hash=with_outlog_hash)
# Record exec time if not existing
if recalculate_execution_times:
self.test_execution_time[testcase] = \
1 + int(time.time() - start_time)
test_failed_verdicts[testcase] = test_failed
test_outlog_hash[testcase] = execoutlog_hash
if stop_on_failure and test_failed != \
common_mix.GlobalConstants.PASS_TEST_VERDICT:
break
if recalculate_execution_times:
common_fs.dumpJSON(self.test_execution_time, \
self.test_execution_time_storage_file, pretty=True)
# Restore back the exes
self._restore_env_vars()
self._restore_default_executable(exe_path_map, env_vars, \
collect_output=with_outlog_hash)
if stop_on_failure:
# Make sure the non executed test has the uncertain value (None)
if len(test_failed_verdicts) < len(testcases):
for testcase in set(testcases) - set(test_failed_verdicts):
test_failed_verdicts[testcase] = \
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT
test_outlog_hash[testcase] = common_matrices.\
OutputLogData.UNCERTAIN_TEST_OUTLOGDATA
# @Checkpoint: Finished (for time)
checkpoint_handler.set_finished(None)
if not with_outlog_hash:
test_outlog_hash = None
return test_failed_verdicts, test_outlog_hash
def _collect_temporary_coverage_data(self, criteria_name_list, \
test_execution_verdict, \
used_environment_vars, \
result_dir_tmp, \
testcase):
''' extract coverage data into json file in result_dir_tmp
'''
cov_obj = coverage.Coverage(config_file=self.config_file)
cov_obj.combine()
tmp_dat_obj = cov_obj.get_data()
in_dat_files = tmp_dat_obj.measured_files()
# Get file map
try:
self.exes_rel
except AttributeError:
obj = common_fs.loadJSON(self.instrumentation_details)
self.exes_abs = []
self.exes_rel = []
for rp, ap in list(obj.items()):
self.exes_rel.append(rp)
self.exes_abs.append(ap)
self.exes_rel, self.exes_abs = zip(*sorted(\
zip(self.exes_rel, self.exes_abs),\
key=lambda x: x.count(os.path.sep)\
))
# get executables stmt and branches
self.executable_lines = {}
self.executable_arcs = {}
for fn in self.exes_abs:
pser = coverage.parser.PythonParser(filename=fn)
pser.parse_source()
self.executable_lines[fn] = pser.statements
self.executable_arcs[fn] = pser.arcs()
dat_obj = coverage.CoverageData()
if len(in_dat_files) > 0:
file_map = self.PathAliases(in_dat_files, self.exes_rel, \
self.used_srcs_dir, \
self._get_latest_top_output_dir(), \
self.code_builds_factory.\
repository_manager.repo_abs_path(''))
dat_obj.update(tmp_dat_obj, aliases=file_map)
# Get the coverages
res = {c: {} for c in criteria_name_list}
if TestCriteria.STATEMENT_COVERAGE in criteria_name_list:
for rel_fn, abs_fn in zip(self.exes_rel, self.exes_abs):
res[TestCriteria.STATEMENT_COVERAGE][rel_fn] = \
{ln: 0 for ln in self.executable_lines[abs_fn]}
ln_list = dat_obj.lines(abs_fn)
if ln_list is not None:
res[TestCriteria.STATEMENT_COVERAGE][rel_fn].\
update({ln:1 for ln in ln_list})
if TestCriteria.BRANCH_COVERAGE in criteria_name_list:
for rel_fn, abs_fn in zip(self.exes_rel, self.exes_abs):
res[TestCriteria.BRANCH_COVERAGE][rel_fn] = \
{str(an): 0 for an in self.executable_arcs[abs_fn]}
an_list = dat_obj.arcs(abs_fn)
if an_list is not None:
res[TestCriteria.BRANCH_COVERAGE][rel_fn].\
update({str(an):1 for an in an_list})
# save the temporary coverage
res_str = {v.get_str(): res[v] for v in res}
common_fs.dumpJSON(res_str, os.path.join(result_dir_tmp,\
self.cov_data_filename))
cov_obj.erase()
# clean any possible raw data file
for file_ in glob.glob(self.raw_data_file + "*"):
os.remove(file_)
def _do_instrument_code (self, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.gc_files_dir):
try:
shutil.rmtree(self.gc_files_dir)
except PermissionError:
self._dir_chmod777(self.gc_files_dir)
shutil.rmtree(self.gc_files_dir)
os.mkdir(self.gc_files_dir, mode=0o777)
prog = 'gcc'
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
flags = [
'-g', '--coverage', '-fprofile-dir=' + self.gc_files_dir, '-O0'
]
additionals = ["-fkeep-inline-functions"]
#additionals.append('-fprofile-abs-path')
#additionals.append('-fprofile-prefix-path='+code_builds_factory\
# .repository_manager.get_repository_dir_path())
# get gcc version
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args_list=['-dumpversion'])
ERROR_HANDLER.assert_true(ret == 0, "'gcc -dumpversion' failed'")
# if version > 6.5
if int(out.split('.')[0]) >= 6:
if int(out.split('.')[0]) > 6 or int(out.split('.')[1]) > 5:
additionals += ["-fkeep-static-functions"]
flags += additionals
rel_rel_path_map = {}
rel_abs_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
ERROR_HANDLER.assert_true(len(exes) == 1, \
"Only single exe supported in GCOV", __file__)
for exe in exes:
filename = os.path.basename(exe)
rel_rel_path_map[exe] = filename
rel_abs_path_map[exe] = os.path.join(\
self.instrumented_code_storage_dir, filename)
self.instrument_callback_obj = self.InstrumentCallbackObject()
self.instrument_callback_obj.set_post_callback_args(\
(self.gc_files_dir, rel_abs_path_map))
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.NATIVE_CODE,\
src_dest_files_paths_map=None,\
compiler=prog, flags_list=flags, clean_tmp=True, \
reconfigure=True, \
callback_object=self.instrument_callback_obj)
# Check
if ret == common_mix.GlobalConstants.COMMAND_FAILURE:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'built problematic'), __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
common_fs.dumpJSON(rel_rel_path_map, self.instrumentation_details)
def _call_generation_run(self, runtool, args):
# Delete any klee-out-*
for d in os.listdir(self.tests_working_dir):
if d.startswith('klee-out-'):
shutil.rmtree(os.path.join(self.tests_working_dir, d))
call_shadow_wrapper_file = os.path.join(self.tests_working_dir, \
"shadow_wrap")
devtest_toolalias = self.parent_meta_tool.get_devtest_toolalias()
ERROR_HANDLER.assert_true(devtest_toolalias is not None, \
"devtest must be used when using shadow_se", __file__)
#test_list = list(self.code_builds_factory.repository_manager\
# .get_dev_tests_list())
test_list = []
for meta_test in self.parent_meta_tool.get_testcase_info_object(\
candidate_tool_aliases=[devtest_toolalias])\
.get_tests_list():
toolalias, test = DriversUtils.reverse_meta_element(meta_test)
ERROR_HANDLER.assert_true(toolalias == devtest_toolalias, \
"BUG in above get_testcase_info_object", __file__)
test_list.append(test)
# Get list of klee_change, klee_get_true/false locations.
klee_change_stmts = []
get_lines_callback_obj = self.GetLinesCallbackObject()
get_lines_callback_obj.set_pre_callback_args(self.code_builds_factory\
.repository_manager.revert_src_list_files)
get_lines_callback_obj.set_post_callback_args(klee_change_stmts)
pre_ret, post_ret = self.code_builds_factory.repository_manager\
.custom_read_access(get_lines_callback_obj)
ERROR_HANDLER.assert_true(pre_ret == \
common_mix.GlobalConstants.COMMAND_SUCCESS,\
"pre failed", __file__)
ERROR_HANDLER.assert_true(post_ret == \
common_mix.GlobalConstants.COMMAND_SUCCESS,\
"post failed", __file__)
ERROR_HANDLER.assert_true(len(klee_change_stmts) > 0, \
"No klee_change statement in the sources", __file__)
# Filter only tests that cover those locations,
# if there is stmt coverage matrix
stmt_cov_mat_file = self.head_explorer.get_file_pathname(\
fd_structure.CRITERIA_MATRIX[criteria.TestCriteria\
.STATEMENT_COVERAGE])
cov_tests = None
if os.path.isfile(stmt_cov_mat_file):
stmt_cov_mat = common_matrices.ExecutionMatrix(\
filename=stmt_cov_mat_file)
# due to possible wrapper test splitting we update test_list here
tmp_test_list = []
for mt in stmt_cov_mat.get_nonkey_colname_list():
alias, t = DriversUtils.reverse_meta_element(mt)
if alias == devtest_toolalias:
tmp_test_list.append(t)
test_list = tmp_test_list
meta_stmts = list(stmt_cov_mat.get_keys())
tool_aliases = set()
for meta_stmt in meta_stmts:
alias, stmt = DriversUtils.reverse_meta_element(meta_stmt)
tool_aliases.add(alias)
klee_change_meta_stmts = []
for alias in tool_aliases:
klee_change_meta_stmts += [\
DriversUtils.make_meta_element(e, alias) \
for e in klee_change_stmts]
klee_change_meta_stmts = list(set(meta_stmts) & \
set(klee_change_meta_stmts))
cov_tests = set()
if len(klee_change_meta_stmts) > 0:
for _, t in stmt_cov_mat.query_active_columns_of_rows(\
row_key_list=klee_change_meta_stmts).items():
cov_tests |= set(t)
else:
logging.warning('No test covers the patch (SHADOW)!')
# ERROR_HANDLER.assert_true(len(klee_change_meta_stmts) > 0, \
# "No test covers the patch", __file__)
# tests will be generated in the same dir that has the input .bc file
os.mkdir(self.tests_storage_dir)
# obtain candidate tests
cand_testpair_list = []
for test in test_list:
meta_test = DriversUtils.make_meta_element(test, devtest_toolalias)
if cov_tests is not None and meta_test not in cov_tests:
continue
cand_testpair_list.append((test, meta_test))
# Adjust the max-time in args
## locate max-time
per_test_hard_timeout = None
per_test_timeout = None
if len(cand_testpair_list) > 0:
cur_max_time = float(self.get_value_in_arglist(args, 'max-time'))
if self.driver_config.get_gen_timeout_is_per_test():
per_test_timeout = cur_max_time
else:
per_test_timeout = max(60, \
cur_max_time / len(cand_testpair_list))
self.set_value_in_arglist(args, 'max-time', str(per_test_timeout))
# give time to dump remaning states
per_test_hard_timeout = per_test_timeout + \
self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE
#per_test_hard_timeout = 300 #DBG
# Set the wrapper
with open(call_shadow_wrapper_file, 'w') as wf:
wf.write('#! /bin/bash\n\n')
wf.write('set -u\n')
wf.write('set -o pipefail\n\n')
wf.write('ulimit -s unlimited\n')
# timeout the shadow execution (some test create daemon which)
# are not killed by test timeout. ALSO MAKE SURE TO DESACTIVATE
# IN TEST SCRIPT TIMEOUT
kill_after = 30
wf.write('time_out_cmd="/usr/bin/timeout --kill-after={}s {}"\n'.\
format(kill_after, per_test_hard_timeout))
# kill after and time for timeout to act
per_test_hard_timeout += kill_after + 60
#wf.write(' '.join(['exec', runtool] + args + ['"${@:1}"']) + '\n')
wf.write('\nstdindata="{}/klee-last/{}"\n'.format(\
self.tests_working_dir, \
KTestTestFormat.STDIN_KTEST_DATA_FILE))
wf.write('tmpstdindata="{}/{}"\n\n'.format(self.tests_working_dir,\
KTestTestFormat.STDIN_KTEST_DATA_FILE))
wf.write('if [ -t 0 ] # check if stdin do not exist\n')
wf.write('then\n')
wf.write(' '.join(['\t(', '$time_out_cmd', runtool] + args + \
['"${@:1}"', ') ; EXIT_CODE=$?', '\n']))
wf.write('\t/usr/bin/touch $tmpstdindata\n')
wf.write('else\n')
wf.write('\t(/bin/cat - > $tmpstdindata ) || EXIT_CODE=1\n')
wf.write(' '.join(['\t(', '/bin/cat $tmpstdindata | ', \
'$time_out_cmd', runtool] + args + \
['"${@:1}"', ') ; EXIT_CODE=$?', '\n']))
wf.write('fi\n\n')
wf.write('/bin/mv $tmpstdindata $stdindata || EXIT_CODE=2\n')
wf.write('\n# Provoke "unbound variable" if KLEE fails\n')
wf.write('# Preserve the KLEE exit code\n')
wf.write('exit $EXIT_CODE\n')
os.chmod(call_shadow_wrapper_file, 0o775)
# run test
exes, _ = self.code_builds_factory.repository_manager\
.get_relative_exe_path_map()
ERROR_HANDLER.assert_true(len(exes) == 1, \
"Must have a single exe", __file__)
exe_path_map = {e: call_shadow_wrapper_file for e in exes}
env_vars = {}
self._dir_chmod777(self.tests_storage_dir)
for test, meta_test in cand_testpair_list:
self.parent_meta_tool.execute_testcase(meta_test, exe_path_map, \
env_vars, timeout=per_test_hard_timeout,\
with_output_summary=False)
#logging.debug("DBG: Just executed test '{}'".format(meta_test))
#input(">>>> ") #DBG
# copy the klee out
test_out = os.path.join(self.tests_storage_dir, \
self.get_sorage_name_of_test(test))
os.mkdir(test_out)
for d in glob.glob(self.tests_working_dir + "/klee-out-*"):
# make sure we can do anything with it
self._dir_chmod777(d)
if not self.keep_first_test:
first_test = os.path.join(d, 'test000001.ktest')
if os.path.isfile(first_test):
shutil.move(first_test, first_test + '.disable')
shutil.move(d, test_out)
ERROR_HANDLER.assert_true(len(list(os.listdir(test_out))) > 0, \
"Shadow generated no test for tescase: "+test,\
__file__)
if os.path.islink(os.path.join(self.tests_working_dir, \
'klee-last')):
os.unlink(os.path.join(self.tests_working_dir, 'klee-last'))
# store klee_change locs
common_fs.dumpJSON(klee_change_stmts, self.klee_change_locs_list_file)
def _do_generate_tests (self, exe_path_map, code_builds_factory, \
meta_criteria_tool_obj=None, \
max_time=None):
# Check the passed exe_path_map
if exe_path_map is not None:
for r_exe, v_exe in exe_path_map.items():
ERROR_HANDLER.assert_true(v_exe is None, \
"Klee driver does not use passed exe_path_map", \
__file__)
# Setup
if os.path.isdir(self.tests_working_dir):
try:
shutil.rmtree(self.tests_working_dir)
except PermissionError:
self._dir_chmod777(self.tests_working_dir)
shutil.rmtree(self.tests_working_dir)
os.mkdir(self.tests_working_dir)
if os.path.isdir(self.tests_storage_dir):
shutil.rmtree(self.tests_storage_dir)
prog = self._get_tool_name()
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(self.tests_working_dir, filename)
# Get bitcode file
bitcode_file = self._get_input_bitcode_file(code_builds_factory, \
rel_path_map, \
meta_criteria_tool_obj=meta_criteria_tool_obj)
repo_rel_exe_file = list(rel_path_map)[0]
# klee params
bool_param, k_v_params = self._get_default_params()
if max_time is not None:
k_v_params['-max-time'] = str(max_time)
# Consider pre user custom
uc = self.config.get_tool_user_custom()
pre_args = []
if uc is not None:
pre_bc_cmd = uc.PRE_TARGET_CMD_ORDERED_FLAGS_LIST
if pre_bc_cmd is not None:
for tup in pre_bc_cmd:
key = tup[0][1:] if tup[0].startswith('--') else tup[0]
bool_disabled = False
if key in k_v_params:
del k_v_params[key]
if key in bool_param:
del bool_param[key]
if len(tup) == 2:
ERROR_HANDLER.assert_true(\
type(tup[1]) == bool, \
"bool arg is either enabled or "
"disabled. val must be bool", __file__)
if not tup[1]:
bool_disabled = True
tup = tup[:1]
else:
ERROR_HANDLER.assert_true(len(tup) == 1, \
"Invalid bool param: "+key, __file__)
if not bool_disabled:
pre_args.extend(list(tup))
args = [bp for bp, en in list(bool_param.items()) if en]
for k, v in list(k_v_params.items()):
if v is not None:
args += [k, str(v)]
args.extend(pre_args)
args.append(bitcode_file)
args += self._get_sym_args(args)
self._call_generation_run(prog, list(args))
# XXX: remove duplicate tests
kepttest2duptest_map = {}
folders = [self.tests_storage_dir]
if self.gen_tests_no_dup_with_seeds:
seed_dir = self.get_value_in_arglist(args, self.SEED_DIR_ARG_NAME)
if seed_dir is not None:
seed_dir = os.path.normpath(os.path.abspath(seed_dir))
folders.append(seed_dir)
dup_list, invalid = KTestTestFormat.ktest_fdupes(*folders, \
custom_replay_tool_binary_dir=self.custom_binary_dir)
if len(invalid) > 0:
logging.warning(\
"{} generated ktests are invalid ({})".format(\
len(invalid), invalid))
for kt in invalid:
if KTestTestFormat.get_dir(kt, folders) == \
self.tests_storage_dir:
os.remove(kt)
for dup_tuple in dup_list:
key = os.path.relpath(dup_tuple[0], \
KTestTestFormat.get_dir(dup_tuple[0], folders))
kepttest2duptest_map[key] = [os.path.relpath(dp, \
KTestTestFormat.get_dir(dp, folders)) \
for dp in dup_tuple[1:]]
for df in dup_tuple[1:]:
if KTestTestFormat.get_dir(df, folders) == \
self.tests_storage_dir:
os.remove(df)
common_fs.dumpJSON(kepttest2duptest_map, self.keptktest2dupktests)
# Copy replay tool into test folder
klee_replay_pathname = KTestTestFormat.get_test_replay_tool(\
custom_replay_tool_binary_dir=self.custom_binary_dir)
shutil.copy2(klee_replay_pathname, self.tests_storage_dir)
store_obj = {repo_rel_exe_file: os.path.basename(bitcode_file)}
common_fs.dumpJSON(store_obj, self.test_details_file)
# Compute must exist dirs
test2file = {}
for kt in self.get_testcase_info_object().get_tests_list():
test2file[kt] = os.path.join(self.tests_storage_dir, kt)
ktest2reqdir = Misc.get_must_exist_dirs_of_ktests(test2file, \
self.custom_binary_dir)
ktest2reqdir = {kt: v for kt, v in ktest2reqdir.items() if len(v) > 0}
self.ktest_with_must_exist_dir = ktest2reqdir
if len(ktest2reqdir) > 0:
common_fs.dumpJSON(ktest2reqdir, \
self.ktest_with_must_exist_dir_file)
def check_get_flakiness(self, meta_testcases, repeat_count=2, \
get_flaky_tests_outputs=True):
"""
Check if tests have flakiness by running multiple times
:return: The list of flaky tests
"""
ERROR_HANDLER.assert_true(
repeat_count > 1, "Cannot check flakiness"
" with less than on repetition", __file__)
if os.path.isdir(self.flakiness_workdir):
shutil.rmtree(self.flakiness_workdir)
os.mkdir(self.flakiness_workdir)
outlog_files = [os.path.join(self.flakiness_workdir, \
str(of)+'-out.json') for of in range(repeat_count)]
matrix_files = [os.path.join(self.flakiness_workdir, \
str(of)+'-mat.csv') for of in range(repeat_count)]
def run(rep, test_list, hash_outlog):
self.runtests(test_list, \
fault_test_execution_matrix_file=matrix_files[rep], \
fault_test_execution_execoutput_file=\
outlog_files[rep],\
with_output_summary=True, \
hash_outlog=hash_outlog)
#~ def run()
# Execute with repetition and get output summaries
for rep in range(repeat_count):
run(rep, meta_testcases, True)
# get flaky tests list
flaky_tests = set()
fix_outdata = common_matrices.OutputLogData(filename=outlog_files[0])
fix_outdata = list(fix_outdata.get_zip_objective_and_data())[0][1]
for i in range(1, repeat_count):
other_outdata = common_matrices.OutputLogData(\
filename=outlog_files[i])
other_outdata = list(\
other_outdata.get_zip_objective_and_data())[0][1]
for test, t_dat_fix in fix_outdata.items():
if test in flaky_tests:
continue
t_dat_other = other_outdata[test]
if not common_matrices.OutputLogData.outlogdata_equiv(\
t_dat_fix, t_dat_other):
flaky_tests.add(test)
## cleanup
for f in outlog_files + matrix_files:
if os.path.isfile(f):
os.remove(f)
# get flaky tests outputs
if get_flaky_tests_outputs and len(flaky_tests) > 0:
for rep in range(repeat_count):
run(rep, list(flaky_tests), False)
flaky_test_list_file = os.path.join(self.flakiness_workdir, \
"flaky_test_list.json")
logging.warning("There were some flaky tests (see file {})"\
.format(flaky_test_list_file))
common_fs.dumpJSON(list(flaky_tests), flaky_test_list_file, \
pretty=True)
return list(flaky_tests)
def _execute_task(self, task):
""" TODO:
1. implement the todos here
"""
# @Checkpointing: see whether the task is untouch to clear tmp
task_untouched = self.cp_data.tasks_obj.task_is_untouched(task)
# Execute base on the task: ---------------------------------------
if task == checkpoint_tasks.Tasks.STARTING:
pass
elif task == checkpoint_tasks.Tasks.FINISHED:
pass
elif task == checkpoint_tasks.Tasks.TESTS_GENERATION_GUIDANCE:
pass #TODO: Generate focus criteria and tests and store in json file
elif task == checkpoint_tasks.Tasks.TESTS_GENERATION:
# @Checkpointing
if task_untouched:
self.meta_testcase_tool.clear_working_dir()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Generate the tests
self.meta_testcase_tool.generate_tests(\
test_tool_type_list=self.cp_data.test_types)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_testcase_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.\
TESTS_EXECUTION_SELECTION_PRIORITIZATION:
out_file_key = outdir_struct.TMP_SELECTED_TESTS_LIST
out_file = self.head_explorer.get_file_pathname(out_file_key)
# @Checkpointing
if task_untouched:
self.head_explorer.remove_file_and_get(out_file_key)
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
all_tests = self.meta_testcase_tool.\
get_testcase_info_object().get_tests_list()
candidate_aliases = \
self.meta_testcase_tool.get_candidate_tools_aliases(\
test_tool_type_list=self.cp_data.test_types)
selected_tests = []
for meta_test in all_tests:
toolalias, _ = DriversUtils.reverse_meta_element(meta_test)
if toolalias in candidate_aliases:
selected_tests.append(meta_test)
# TODO: use the actual selector. For now just use all tests
common_fs.dumpJSON(list(selected_tests), out_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
#self.meta_testexec_optimization_tool.get_checkpoint_state_object()\
# .destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.PASS_FAIL_TESTS_EXECUTION:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_MATRICES_DIR)
matrix_file_key = outdir_struct.TMP_TEST_PASS_FAIL_MATRIX
matrix_file = self.head_explorer.get_file_pathname(matrix_file_key)
execoutput_file_key = \
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT
if self.config.GET_PASSFAIL_OUTPUT_SUMMARY:
execoutput_file = self.head_explorer.get_file_pathname(\
execoutput_file_key)
else:
execoutput_file = None
# @Checkpointing
if task_untouched:
self.head_explorer.remove_file_and_get(matrix_file_key)
self.head_explorer.remove_file_and_get(execoutput_file_key)
self.meta_testcase_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Execute tests
test_list_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_SELECTED_TESTS_LIST)
meta_testcases = common_fs.loadJSON(test_list_file)
# Set test oracle
self.test_oracle_manager.set_oracle(passfail=True)
self.meta_testcase_tool.runtests(meta_testcases=meta_testcases, \
stop_on_failure=\
self.config.STOP_TESTS_EXECUTION_ON_FAILURE, \
recalculate_execution_times=True, \
fault_test_execution_matrix_file=matrix_file, \
fault_test_execution_execoutput_file=execoutput_file, \
test_prioritization_module=\
self.meta_testexec_optimization_tool, \
finish_destroy_checkpointer=False)
# Unset test oracle
self.test_oracle_manager.set_oracle(passfail=False)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_testcase_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION_GUIDANCE:
pass #TODO (Maybe could be used someday. for now, just skip it)
elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION:
# @Checkpointing
if task_untouched:
self.meta_criteria_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
self.meta_criteria_tool.instrument_code(criteria_enabled_list=\
self.config.ENABLED_CRITERIA.get_val(), \
finish_destroy_checkpointer=False)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_criteria_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.\
CRITERIA_EXECUTION_SELECTION_PRIORITIZATION:
pass #TODO (Maybe could be used someday. for now, just skip it)
elif task == checkpoint_tasks.Tasks.CRITERIA_TESTS_EXECUTION:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_MATRICES_DIR)
matrix_files_keys = {}
matrix_files = {}
execoutput_files_keys = {}
execoutput_files = {}
for criterion in self.config.ENABLED_CRITERIA.get_val():
matrix_files_keys[criterion] = \
outdir_struct.TMP_CRITERIA_MATRIX[criterion]
matrix_files[criterion] = \
self.head_explorer.get_file_pathname(\
matrix_files_keys[criterion])
execoutput_files_keys[criterion] = \
outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion]
if criterion in self.config.CRITERIA_WITH_OUTPUT_SUMMARY:
execoutput_files[criterion] = \
self.head_explorer.get_file_pathname(\
execoutput_files_keys[criterion])
else:
execoutput_files[criterion] = None
# @Checkpointing
if task_untouched:
for _, matrix_file_key in list(matrix_files_keys.items()):
self.head_explorer.remove_file_and_get(matrix_file_key)
for _, execoutput_file_key in list(\
execoutput_files_keys.items()):
self.head_explorer.remove_file_and_get(execoutput_file_key)
self.meta_criteria_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
criteria_set_sequence = self.config.CRITERIA_SEQUENCE.get_val()
#if criteria_set_sequence is None:
# criteria_set_sequence = criteria_pkg.CRITERIA_SEQUENCE
for cs_pos, criteria_set in enumerate(criteria_set_sequence):
criteria_set &= set(matrix_files)
if len(criteria_set) == 0:
continue
# Was it already checkpointed w.r.t criteria set seq
if self.cp_data.criteria_set_is_executed(cs_pos, criteria_set):
continue
# If we have a new criteria set id
self.cp_data.switchto_new_criteria_set(cs_pos, criteria_set)
# get matrices by criteria
test_list_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_SELECTED_TESTS_LIST)
meta_testcases = common_fs.loadJSON(test_list_file)
criterion_to_matrix = {\
c: matrix_files[c] for c in criteria_set}
# TODO: chack set based on the criteria for which execout is enabled
criterion_to_execoutput = {\
c: execoutput_files[c] for c in criteria_set}
# Set test oracle
self.test_oracle_manager.set_oracle(criteria_on=criteria_set)
# execute
self.meta_criteria_tool.runtests_criteria_coverage( \
testcases=meta_testcases, \
criterion_to_matrix=criterion_to_matrix, \
criterion_to_executionoutput=\
criterion_to_execoutput, \
cover_criteria_elements_once=self.config.\
COVER_CRITERIA_ELEMENTS_ONCE.get_val(),\
prioritization_module_by_criteria=\
self.meta_criteriaexec_optimization_tools,\
finish_destroy_checkpointer=True)
# Update matrix if needed to have output diff or such
# TODO: add CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM to config
for crit in criteria_set & set(self.config.\
CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM.\
get_val()):
pf_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
if self.config.GET_PASSFAIL_OUTPUT_SUMMARY.get_val():
pf_execoutput_file = \
self.head_explorer.get_file_pathname(\
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT)
else:
pf_execoutput_file = None
DriversUtils.update_matrix_to_cover_when_diference(\
criterion_to_matrix[crit], \
criterion_to_execoutput[crit], \
pf_matrix_file, pf_execoutput_file)
# Unset test oracle
self.test_oracle_manager.set_oracle(criteria_on=None)
# @Checkpointing
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
elif task == checkpoint_tasks.Tasks.PASS_FAIL_STATS:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_STATS_DIR)
tmp_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TEST_PASS_FAIL_MATRIX)
tmp_execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT)
execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.PROGRAM_TESTEXECUTION_OUTPUT)
# Merge TMP pass fail and potentially existing pass fail
StatsComputer.merge_lmatrix_into_right(\
tmp_matrix_file, matrix_file)
StatsComputer.merge_lexecoutput_into_right(\
tmp_execoutput_file, execoutput_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Cleanup
self.head_explorer.remove_file_and_get(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
elif task == checkpoint_tasks.Tasks.CRITERIA_STATS:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_STATS_DIR)
# Merge TMP criteria and potentially existing criteria
for criterion in self.config.ENABLED_CRITERIA.get_val():
tmp_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_CRITERIA_MATRIX[criterion])
matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.CRITERIA_MATRIX[criterion])
tmp_execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion])
execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.CRITERIA_EXECUTION_OUTPUT[criterion])
StatsComputer.merge_lmatrix_into_right(tmp_matrix_file, \
matrix_file)
StatsComputer.merge_lexecoutput_into_right(\
tmp_execoutput_file, execoutput_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Cleanup
for criterion in self.config.ENABLED_CRITERIA.get_val():
self.head_explorer.remove_file_and_get(\
outdir_struct.TMP_CRITERIA_MATRIX[criterion])
elif task == checkpoint_tasks.Tasks.AGGREGATED_STATS:
# Compute the final stats (MS, ...)
StatsComputer.compute_stats(self.config, self.head_explorer)
#-------------------------------------------------------------------
if not self.cp_data.tasks_obj.task_is_complete(task):
# @Checkpoint: set task as done
self.cp_data.tasks_obj.set_task_completed(task)
# @Checkpoint: write checkpoint
self.checkpointer.write_checkpoint(\
json_obj=self.cp_data.get_json_obj())
