def post_test_dup(stored_map, tests_to_avoid, origin,
kepttest2duptest_map):
for kept, duplist in kepttest2duptest_map.items():
alias, _ = DriversUtils.reverse_meta_element(kept)
if alias == origin:
stored_map[kept] = []
for dup in duplist:
dalias, dtest = DriversUtils.reverse_meta_element(dup)
if dalias == origin:
continue
stored_map[kept].append(dup)
tests_to_avoid.append(dup)
def fault_analysis(cm_corebench_scripts_dir, c_id, conf_py, in_muteria_outdir,
out_top_dir):
if not os.path.isdir(out_top_dir):
os.mkdir(out_top_dir)
in_res_data_dir = os.path.join(in_muteria_outdir, 'latest', 'RESULTS_DATA')
testtools_workdir = os.path.join(in_muteria_outdir, 'latest',
'testscases_workdir')
pass_fail_matrix = os.path.join(in_res_data_dir, "matrices",
"PASSFAIL.csv")
pf_mat = common_matrices.ExecutionMatrix(filename=pass_fail_matrix)
test_list = list(pf_mat.get_nonkey_colname_list())
#semu_tool_dirs = [d for d in os.listdir(testtools_workdir) if d.startswith('semu_cmp-')]
# get fault tests
get_commit_fault_tests(cm_corebench_scripts_dir, c_id, conf_py,
in_res_data_dir, out_top_dir)
# get test to timestamp
test_timestamp_file = os.path.join(out_top_dir, "test_to_timestamp.json")
test2timestamp = {}
## get tests by tools
tools2tests = {}
for test in test_list:
alias, _ = DriversUtils.reverse_meta_element(test)
# XXX: only SEMU
if not alias.startswith('semu_cmp-'):
continue
if alias not in tools2tests:
tools2tests[alias] = set()
tools2tests[alias].add(test)
## untar tests dir
for alias, tests in tools2tests.items():
d = os.path.join(testtools_workdir, alias)
assert os.path.isdir(
d), "test tool dir " + d + " missing for alias " + alias
test_tar = os.path.join(d, 'tests_files.tar.gz')
es = common_fs.TarGz.decompressDir(test_tar)
assert es is None, "decompress error: " + es
tests_files = os.path.join(d, 'tests_files')
assert os.path.isdir(tests_files), "dir missing after decompress"
for test in tests:
_, simple_test = DriversUtils.reverse_meta_element(test)
gt = TestcasesToolSemu._get_generation_time_of_test(
simple_test, tests_files)
test2timestamp[test] = gt
shutil.rmtree(tests_files)
common_fs.dumpJSON(test2timestamp, test_timestamp_file, pretty=True)
def get_too2relmuts(relmuts_to_reltests, toollist=None):
res = {tool: set()
for tool in toollist} if toollist is not None else {}
for relmut, t_list in relmuts_to_reltests.items():
for meta_t in t_list:
toolalias, test = DriversUtils.reverse_meta_element(meta_t)
if toollist is None:
if toolalias not in res:
res[toolalias] = set()
else:
assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format(
toolalias, toollist)
res[toolalias].add(relmut)
for toolalias in res:
res[toolalias] = list(res[toolalias])
return res
def execute_testcase (self, meta_testcase, exe_path_map, env_vars, \
timeout=None, \
use_recorded_timeout_times=None, \
recalculate_execution_times=False, \
with_output_summary=True, \
hash_outlog=None):
'''
Execute a test case with the given executable and
say whether it failed
:param meta_testcase: string name of the test cases to execute
:param exe_path_map: string representing the file system path to
the executable to execute with the test
:param env_vars: dict of environment variables to set before
executing the test ({<variable>: <value>})
:type hash_outlog: bool
:hash_outlog: decide whether to hash the outlog or not
:returns: pair of:
- boolean failed verdict of the test
(True if failed, False otherwise)
- test execution output log hash data object or None
'''
if exe_path_map is None:
exe_path_map = self._get_default_exe_path_map()
if hash_outlog is None:
hash_outlog = self.hash_outlog
# Find which test tool's the testcase is, then execute
ttoolalias, testcase = DriversUtils.reverse_meta_element(meta_testcase)
ERROR_HANDLER.assert_true( \
ttoolalias in self.testcases_configured_tools, \
"Test tool {} not registered".format(ttoolalias), \
__file__)
ttool = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY]
return ttool.execute_testcase(testcase, exe_path_map, env_vars, \
timeout=timeout, \
use_recorded_timeout_times=\
use_recorded_timeout_times, \
recalculate_execution_times=\
recalculate_execution_times, \
with_output_summary=with_output_summary, \
hash_outlog=hash_outlog)
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \
stop_on_failure=False, \
per_test_timeout=None, \
use_recorded_timeout_times=None, \
recalculate_execution_times=False, \
fault_test_execution_matrix_file=None, \
fault_test_execution_execoutput_file=None, \
with_output_summary=True, \
hash_outlog=None, \
test_prioritization_module=None, \
parallel_test_count=1, \
parallel_test_scheduler=None, \
restart_checkpointer=False,
finish_destroy_checkpointer=True):
'''
Execute the list of test cases with the given executable and
say, for each test case, whether it failed
:param meta_testcases: list of test cases to execute
:param exe_path_map: string representing the file system path to
the executable to execute with the tests
:param env_vars: dict of environment variables to set before
executing each test ({<variable>: <value>})
:param stop_on_failure: decide whether to stop the test
execution once a test fails
:param fault_test_execution_matrix_file: Optional matrix file
to store the tests' pass fail execution data
:param fault_test_execution_execoutput_file: Optional output log file
to store the tests' execution actual output (hashed)
:param with_output_summary: decide whether to return outlog hash
:type hash_outlog: bool
:hash_outlog: decide whether to hash the outlog or not
:param test_prioritization_module: Specify the test prioritization
module.
(TODO: Implement support)
:param parallel_test_count: Specify the number of parallel test
Execution. must be an integer >= 1 or None.
When None, the max possible value is used.
:param parallel_test_scheduler: Specify the function that will
handle parallel test scheduling by tool, using
the test execution optimizer.
(TODO: Implement support)
:type restart_checkointer: bool
:param restart_checkointer: Decide whether to discard checkpoint
and restart anew.
:type finish_destroy_checkpointer: bool
:param finish_destroy_checkpointer: Decide whether to automatically
destroy the checkpointer when done or not
Useful is caller has a checkpointer to update.
:returns: dict of testcase and their failed verdict.
{<test case name>: <True if failed, False if passed,
UNCERTAIN_TEST_VERDICT if uncertain>}
If stop_on_failure is True, only return the tests that
have been executed until the failure
'''
ERROR_HANDLER.assert_true(meta_testcases is not None, \
"Must specify testcases", __file__)
# FIXME: Make sure that the support are implemented for
# parallelism and test prioritization. Remove the code bellow
# once supported:
ERROR_HANDLER.assert_true(test_prioritization_module is None, \
"Must implement test prioritization support here", \
__file__)
ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \
"Must implement parallel tests execution support here", \
__file__)
#~FIXMEnd
# Check arguments Validity
if exe_path_map is None:
exe_path_map = self._get_default_exe_path_map()
if hash_outlog is None:
hash_outlog = self.hash_outlog
ERROR_HANDLER.assert_true(parallel_test_count is None \
or parallel_test_count >= 1, \
"invalid parallel tests count ({})".format(\
parallel_test_count), __file__)
# @Checkpoint: create a checkpoint handler
cp_func_name = "runtests"
cp_task_id = 1
checkpoint_handler = \
CheckPointHandler(self.get_checkpoint_state_object())
if restart_checkpointer:
checkpoint_handler.restart()
if checkpoint_handler.is_finished():
logging.warning("%s %s" %("The function 'runtests' is finished", \
"according to checkpoint, but called again. None returned"))
if common_mix.confirm_execution("%s %s" % ( \
"Function 'runtests' is already", \
"finished, do you want to restart?")):
checkpoint_handler.restart()
logging.info("Restarting the finished 'runtests'")
else:
ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\
"Execution halted. Cannot continue because no value", \
" can be returned. Check the results of the", \
"finished execution"), call_location=__file__)
# @Checkpoint: Get the saved payload (data kapt for each tool)
# pair list of testfailed verdict and execution output
meta_test_failedverdicts_outlog = \
checkpoint_handler.get_optional_payload()
if meta_test_failedverdicts_outlog is None:
meta_test_failedverdicts_outlog = [{}, {}]
# Make sure the tests are unique
ERROR_HANDLER.assert_true(len(meta_testcases) == \
len(set(meta_testcases)), \
"not all tests are unique", __file__)
# For fdupes
if len(self.tests_duplicates_map) > 0:
meta_testcases_backup = meta_testcases
meta_testcases = set(meta_testcases)
dups_remove_meta_testcases = meta_testcases & \
set(self.tests_duplicates_map)
dup_toadd_test = {self.tests_duplicates_map[v] for v in \
dups_remove_meta_testcases} - meta_testcases
meta_testcases = (meta_testcases - dups_remove_meta_testcases) \
| dup_toadd_test
testcases_by_tool = {}
for meta_testcase in meta_testcases:
ttoolalias, testcase = \
DriversUtils.reverse_meta_element(meta_testcase)
if ttoolalias not in testcases_by_tool:
testcases_by_tool[ttoolalias] = []
testcases_by_tool[ttoolalias].append(testcase)
candidate_aliases = []
for tpos, ttoolalias in enumerate(testcases_by_tool.keys()):
# @Checkpoint: Check whether already executed
if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ttoolalias):
continue
candidate_aliases.append(ttoolalias)
# parallelism strategies
PARA_FULL_DOUBLE = 0
PARA_ALT_TOOLS_AND_TESTS = 1
PARA_TOOLS_ONLY = 2
PARA_TOOLS_TESTS_AS_TOOLS = 3
parallel_strategy = PARA_TOOLS_ONLY
# minimum number of tests (accross) for parallelism
ptest_tresh = 5
# minimum number of tests (of the given tool) for tool parallelism
sub_ptest_thresh = 3
shared_loc = multiprocessing.RLock()
parallel_test_count_by_tool = {ta: 1 for ta in candidate_aliases}
# tool with parallel test exec
# TODO: find way to pass parallel count here
if parallel_test_count is None:
#parallel_test_count = min(10, multiprocessing.cpu_count())
parallel_test_count = min(20, 2 * multiprocessing.cpu_count())
cand_alias_joblib = []
cand_alias_for = []
para_tools = []
para_tools = [tt for tt in candidate_aliases if \
(len(testcases_by_tool[tt]) >= sub_ptest_thresh \
and self.testcases_configured_tools[tt]\
[self.TOOL_OBJ_KEY].can_run_tests_in_parallel())
]
actual_parallel_cond = len(candidate_aliases) > 1 \
and len(meta_testcases) >= ptest_tresh \
and parallel_test_count is not None \
and parallel_test_count > 1
if parallel_strategy == PARA_ALT_TOOLS_AND_TESTS:
# the para_tools will run without parallelism, give them all threads
for tt in para_tools:
parallel_test_count_by_tool[tt] = parallel_test_count
seq_tools = list(set(candidate_aliases) - set(para_tools))
if len(seq_tools) > 1 and actual_parallel_cond:
cand_alias_joblib = seq_tools
cand_alias_for = para_tools
else:
cand_alias_for = candidate_aliases
elif parallel_strategy == PARA_TOOLS_ONLY:
if actual_parallel_cond:
cand_alias_joblib = candidate_aliases
else:
cand_alias_for = candidate_aliases
elif parallel_strategy == PARA_FULL_DOUBLE:
# use parallel
sub_parallel_count = 0 if parallel_test_count is None else \
parallel_test_count - len(parallel_test_count_by_tool)
if sub_parallel_count > 0:
para_tools.sort(reverse=True, \
key=lambda x: len(testcases_by_tool[x]))
para_tools_n_tests = sum(\
[len(testcases_by_tool[tt]) for tt in para_tools])
used = 0
for tt in para_tools:
quota = int(len(testcases_by_tool[tt]) * \
sub_parallel_count / para_tools_n_tests)
parallel_test_count_by_tool[tt] += quota
used += quota
for tt in para_tools:
if used == sub_parallel_count:
break
parallel_test_count_by_tool[tt] += 1
if actual_parallel_cond:
cand_alias_joblib = candidate_aliases
else:
cand_alias_for = candidate_aliases
elif parallel_strategy == PARA_TOOLS_TESTS_AS_TOOLS:
# split the tests of one tool and
# make the same tool run multiple times
ERROR_HANDLER.error_exit("To Be implemented: same tool many times")
else:
ERROR_HANDLER.error_exit("Invalid parallel startegy")
def tool_parallel_test_exec(ttoolalias):
# Actual execution
found_a_failure = False
# Whether the execution was unsuccessful
test_error = False
ttool = \
self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY]
test_failed_verdicts, test_execoutput = ttool.runtests( \
testcases_by_tool[ttoolalias], \
exe_path_map, env_vars, \
stop_on_failure, \
per_test_timeout=per_test_timeout,
use_recorded_timeout_times=\
use_recorded_timeout_times, \
recalculate_execution_times=\
recalculate_execution_times, \
with_output_summary=\
with_output_summary, \
hash_outlog=hash_outlog, \
parallel_count=\
parallel_test_count_by_tool[ttoolalias])
with shared_loc:
for testcase in test_failed_verdicts:
meta_testcase = DriversUtils.make_meta_element(\
testcase, ttoolalias)
meta_test_failedverdicts_outlog[0][meta_testcase] = \
test_failed_verdicts[testcase]
meta_test_failedverdicts_outlog[1][meta_testcase] = \
test_execoutput[testcase]
if not found_a_failure \
and test_failed_verdicts[testcase] == \
common_mix.GlobalConstants.FAIL_TEST_VERDICT:
found_a_failure = True
if not test_error \
and test_failed_verdicts[testcase] == \
common_mix.GlobalConstants.TEST_EXECUTION_ERROR:
test_error = True
# @Checkpoint: Chekpointing
checkpoint_handler.do_checkpoint(func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ttoolalias, \
opt_payload=meta_test_failedverdicts_outlog)
return found_a_failure, test_error
#~ def tool_parallel_test_exec()
if len(cand_alias_joblib) > 0:
parallel_count_ = min(len(cand_alias_joblib), parallel_test_count)
joblib.Parallel(n_jobs=parallel_count_, require='sharedmem')\
(joblib.delayed(tool_parallel_test_exec)(ttoolalias) \
for ttoolalias in cand_alias_joblib)
if len(cand_alias_for) > 0:
for tpos, ttoolalias in enumerate(cand_alias_for):
found_a_failure, test_error = \
tool_parallel_test_exec(ttoolalias)
if stop_on_failure and found_a_failure:
# @Checkpoint: Chekpointing for remaining tools
for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]:
checkpoint_handler.do_checkpoint(\
func_name=cp_func_name, \
taskid=cp_task_id, \
tool=rem_tool, \
opt_payload=meta_test_failedverdicts_outlog)
break
if stop_on_failure:
# Make sure the non executed test has the uncertain value (None)
if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases):
for meta_testcase in set(meta_testcases) - \
set(meta_test_failedverdicts_outlog[0]):
meta_test_failedverdicts_outlog[0][meta_testcase] = \
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT
meta_test_failedverdicts_outlog[1][meta_testcase] = \
common_matrices.OutputLogData.\
UNCERTAIN_TEST_OUTLOGDATA
ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \
len(meta_testcases), \
"mismatch between number of tests and reported verdicts:"
+ " Tests without verdict are {};".format(\
set(meta_testcases) - \
set(meta_test_failedverdicts_outlog[0])) \
+ " Test not in testlist are {}.".format(\
set(meta_test_failedverdicts_outlog[0]) - \
set(meta_testcases)), \
__file__)
# For fdupes
if len(self.tests_duplicates_map) > 0:
meta_testcases = meta_testcases_backup
for i in (0, 1):
for mtest in dups_remove_meta_testcases:
# add to results
meta_test_failedverdicts_outlog[i][mtest] = copy.deepcopy(\
meta_test_failedverdicts_outlog[i]\
[self.tests_duplicates_map[mtest]])
for mtest in dup_toadd_test:
# remove from results
del meta_test_failedverdicts_outlog[i][mtest]
if fault_test_execution_matrix_file is not None:
# Load or Create the matrix
fault_test_execution_matrix = common_matrices.ExecutionMatrix( \
filename=fault_test_execution_matrix_file, \
non_key_col_list=meta_testcases)
ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \
"matrix must be empty. Filename is:"
" "+fault_test_execution_matrix_file, __file__)
failverdict2val = {
common_mix.GlobalConstants.FAIL_TEST_VERDICT: \
fault_test_execution_matrix.getActiveCellDefaultVal(),
common_mix.GlobalConstants.PASS_TEST_VERDICT: \
fault_test_execution_matrix.getInactiveCellVal(),
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \
fault_test_execution_matrix.getUncertainCellDefaultVal(),
}
cells_dict = {}
for meta_testcase in meta_test_failedverdicts_outlog[0]:
cells_dict[meta_testcase] = failverdict2val[\
meta_test_failedverdicts_outlog[0][meta_testcase]]
fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \
cells_dict, serialize=True)
if fault_test_execution_execoutput_file is not None:
# Load or Create the data object
fault_test_execution_execoutput = common_matrices.OutputLogData( \
filename=fault_test_execution_execoutput_file)
ERROR_HANDLER.assert_true(\
fault_test_execution_execoutput.is_empty(), \
"outlog data must be empty", __file__)
fault_test_execution_execoutput.add_data(\
{self.PROGRAM_EXECOUTPUT_KEY: \
meta_test_failedverdicts_outlog[1]}, \
serialize=True)
# @Checkpoint: Finished
detailed_exectime = {}
for ttoolalias in testcases_by_tool.keys():
tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY]
detailed_exectime[ttoolalias] = (\
tt.get_checkpointer().get_execution_time(),\
tt.get_checkpointer().get_detailed_execution_time())
checkpoint_handler.set_finished( \
detailed_exectime_obj=detailed_exectime)
if finish_destroy_checkpointer:
checkpoint_handler.destroy()
return meta_test_failedverdicts_outlog
def _call_generation_run(self, runtool, args):
# Delete any klee-out-*
for d in os.listdir(self.tests_working_dir):
if d.startswith('klee-out-'):
shutil.rmtree(os.path.join(self.tests_working_dir, d))
call_shadow_wrapper_file = os.path.join(self.tests_working_dir, \
"shadow_wrap")
devtest_toolalias = self.parent_meta_tool.get_devtest_toolalias()
ERROR_HANDLER.assert_true(devtest_toolalias is not None, \
"devtest must be used when using shadow_se", __file__)
#test_list = list(self.code_builds_factory.repository_manager\
# .get_dev_tests_list())
test_list = []
for meta_test in self.parent_meta_tool.get_testcase_info_object(\
candidate_tool_aliases=[devtest_toolalias])\
.get_tests_list():
toolalias, test = DriversUtils.reverse_meta_element(meta_test)
ERROR_HANDLER.assert_true(toolalias == devtest_toolalias, \
"BUG in above get_testcase_info_object", __file__)
test_list.append(test)
# Get list of klee_change, klee_get_true/false locations.
klee_change_stmts = []
get_lines_callback_obj = self.GetLinesCallbackObject()
get_lines_callback_obj.set_pre_callback_args(self.code_builds_factory\
.repository_manager.revert_src_list_files)
get_lines_callback_obj.set_post_callback_args(klee_change_stmts)
pre_ret, post_ret = self.code_builds_factory.repository_manager\
.custom_read_access(get_lines_callback_obj)
ERROR_HANDLER.assert_true(pre_ret == \
common_mix.GlobalConstants.COMMAND_SUCCESS,\
"pre failed", __file__)
ERROR_HANDLER.assert_true(post_ret == \
common_mix.GlobalConstants.COMMAND_SUCCESS,\
"post failed", __file__)
ERROR_HANDLER.assert_true(len(klee_change_stmts) > 0, \
"No klee_change statement in the sources", __file__)
# Filter only tests that cover those locations,
# if there is stmt coverage matrix
stmt_cov_mat_file = self.head_explorer.get_file_pathname(\
fd_structure.CRITERIA_MATRIX[criteria.TestCriteria\
.STATEMENT_COVERAGE])
cov_tests = None
if os.path.isfile(stmt_cov_mat_file):
stmt_cov_mat = common_matrices.ExecutionMatrix(\
filename=stmt_cov_mat_file)
# due to possible wrapper test splitting we update test_list here
tmp_test_list = []
for mt in stmt_cov_mat.get_nonkey_colname_list():
alias, t = DriversUtils.reverse_meta_element(mt)
if alias == devtest_toolalias:
tmp_test_list.append(t)
test_list = tmp_test_list
meta_stmts = list(stmt_cov_mat.get_keys())
tool_aliases = set()
for meta_stmt in meta_stmts:
alias, stmt = DriversUtils.reverse_meta_element(meta_stmt)
tool_aliases.add(alias)
klee_change_meta_stmts = []
for alias in tool_aliases:
klee_change_meta_stmts += [\
DriversUtils.make_meta_element(e, alias) \
for e in klee_change_stmts]
klee_change_meta_stmts = list(set(meta_stmts) & \
set(klee_change_meta_stmts))
cov_tests = set()
if len(klee_change_meta_stmts) > 0:
for _, t in stmt_cov_mat.query_active_columns_of_rows(\
row_key_list=klee_change_meta_stmts).items():
cov_tests |= set(t)
else:
logging.warning('No test covers the patch (SHADOW)!')
# ERROR_HANDLER.assert_true(len(klee_change_meta_stmts) > 0, \
# "No test covers the patch", __file__)
# tests will be generated in the same dir that has the input .bc file
os.mkdir(self.tests_storage_dir)
# obtain candidate tests
cand_testpair_list = []
for test in test_list:
meta_test = DriversUtils.make_meta_element(test, devtest_toolalias)
if cov_tests is not None and meta_test not in cov_tests:
continue
cand_testpair_list.append((test, meta_test))
# Adjust the max-time in args
## locate max-time
per_test_hard_timeout = None
per_test_timeout = None
if len(cand_testpair_list) > 0:
cur_max_time = float(self.get_value_in_arglist(args, 'max-time'))
if self.driver_config.get_gen_timeout_is_per_test():
per_test_timeout = cur_max_time
else:
per_test_timeout = max(60, \
cur_max_time / len(cand_testpair_list))
self.set_value_in_arglist(args, 'max-time', str(per_test_timeout))
# give time to dump remaning states
per_test_hard_timeout = per_test_timeout + \
self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE
#per_test_hard_timeout = 300 #DBG
# Set the wrapper
with open(call_shadow_wrapper_file, 'w') as wf:
wf.write('#! /bin/bash\n\n')
wf.write('set -u\n')
wf.write('set -o pipefail\n\n')
wf.write('ulimit -s unlimited\n')
# timeout the shadow execution (some test create daemon which)
# are not killed by test timeout. ALSO MAKE SURE TO DESACTIVATE
# IN TEST SCRIPT TIMEOUT
kill_after = 30
wf.write('time_out_cmd="/usr/bin/timeout --kill-after={}s {}"\n'.\
format(kill_after, per_test_hard_timeout))
# kill after and time for timeout to act
per_test_hard_timeout += kill_after + 60
#wf.write(' '.join(['exec', runtool] + args + ['"${@:1}"']) + '\n')
wf.write('\nstdindata="{}/klee-last/{}"\n'.format(\
self.tests_working_dir, \
KTestTestFormat.STDIN_KTEST_DATA_FILE))
wf.write('tmpstdindata="{}/{}"\n\n'.format(self.tests_working_dir,\
KTestTestFormat.STDIN_KTEST_DATA_FILE))
wf.write('if [ -t 0 ] # check if stdin do not exist\n')
wf.write('then\n')
wf.write(' '.join(['\t(', '$time_out_cmd', runtool] + args + \
['"${@:1}"', ') ; EXIT_CODE=$?', '\n']))
wf.write('\t/usr/bin/touch $tmpstdindata\n')
wf.write('else\n')
wf.write('\t(/bin/cat - > $tmpstdindata ) || EXIT_CODE=1\n')
wf.write(' '.join(['\t(', '/bin/cat $tmpstdindata | ', \
'$time_out_cmd', runtool] + args + \
['"${@:1}"', ') ; EXIT_CODE=$?', '\n']))
wf.write('fi\n\n')
wf.write('/bin/mv $tmpstdindata $stdindata || EXIT_CODE=2\n')
wf.write('\n# Provoke "unbound variable" if KLEE fails\n')
wf.write('# Preserve the KLEE exit code\n')
wf.write('exit $EXIT_CODE\n')
os.chmod(call_shadow_wrapper_file, 0o775)
# run test
exes, _ = self.code_builds_factory.repository_manager\
.get_relative_exe_path_map()
ERROR_HANDLER.assert_true(len(exes) == 1, \
"Must have a single exe", __file__)
exe_path_map = {e: call_shadow_wrapper_file for e in exes}
env_vars = {}
self._dir_chmod777(self.tests_storage_dir)
for test, meta_test in cand_testpair_list:
self.parent_meta_tool.execute_testcase(meta_test, exe_path_map, \
env_vars, timeout=per_test_hard_timeout,\
with_output_summary=False)
#logging.debug("DBG: Just executed test '{}'".format(meta_test))
#input(">>>> ") #DBG
# copy the klee out
test_out = os.path.join(self.tests_storage_dir, \
self.get_sorage_name_of_test(test))
os.mkdir(test_out)
for d in glob.glob(self.tests_working_dir + "/klee-out-*"):
# make sure we can do anything with it
self._dir_chmod777(d)
if not self.keep_first_test:
first_test = os.path.join(d, 'test000001.ktest')
if os.path.isfile(first_test):
shutil.move(first_test, first_test + '.disable')
shutil.move(d, test_out)
ERROR_HANDLER.assert_true(len(list(os.listdir(test_out))) > 0, \
"Shadow generated no test for tescase: "+test,\
__file__)
if os.path.islink(os.path.join(self.tests_working_dir, \
'klee-last')):
os.unlink(os.path.join(self.tests_working_dir, 'klee-last'))
# store klee_change locs
common_fs.dumpJSON(klee_change_stmts, self.klee_change_locs_list_file)
def _get_input_bitcode_file(self, code_builds_factory, rel_path_map, \
meta_criteria_tool_obj=None):
meta_mu_src = self.driver_config.get_meta_mutant_source()
# XXX Case of manual annotation
if meta_mu_src == MetaMuSource.ANNOTATION:
with open(self.cand_muts_file, 'w') as f:
# Single mutant (id 1, corresponding to old version)
f.write(str(1) + '\n')
return super(TestcasesToolSemu, self)._get_input_bitcode_file(\
code_builds_factory, rel_path_map, \
meta_criteria_tool_obj=meta_criteria_tool_obj)
if type(meta_mu_src) == str:
# XXX: The actual path to the meta is specified
return meta_mu_src
# XXX: Case of other mutation tools like Mart
# get the meta criterion file from MART or any compatible tool.
mutant_gen_tool_name = meta_mu_src.get_field_value()
mut_tool_alias_to_obj = \
meta_criteria_tool_obj.get_criteria_tools_by_name(\
mutant_gen_tool_name)
if len(mut_tool_alias_to_obj) == 0:
logging.warning(\
'SEMu requires {} to generate mutants but none used'.format(\
mutant_gen_tool_name))
ERROR_HANDLER.assert_true(len(mut_tool_alias_to_obj) == 1, \
"SEMu supports tests generation from"
"a single .bc file for now (todo).", __file__)
t_alias2metamu_bc = {}
t_alias2mutantInfos = {}
for alias, obj in mut_tool_alias_to_obj.items():
dest_bc = rel_path_map[list(rel_path_map)[0]] + '.bc'
shutil.copy2(obj.get_test_gen_metamutant_bc(), dest_bc)
t_alias2metamu_bc[alias] = dest_bc
t_alias2mutantInfos[alias] = obj.get_criterion_info_object(None)
# XXX: get mutants ids by functions
self.mutants_by_funcs = {}
single_alias = list(t_alias2mutantInfos)[0]
single_tool_obj = t_alias2mutantInfos[single_alias]
cand_muts = list(single_tool_obj.get_elements_list())
for mut in single_tool_obj.get_elements_list():
func = single_tool_obj.get_element_data(mut)[\
'mutant_function_name']
#meta_mut = DriversUtils.make_meta_element(mut, single_alias)
if func not in self.mutants_by_funcs:
self.mutants_by_funcs[func] = set()
self.mutants_by_funcs[func].add(mut) #meta_mut)
# XXX: get candidate mutants list
if self.driver_config.get_target_only_live_mutants() \
and os.path.isfile(self.sm_mat_file):
sm_mat = common_matrices.ExecutionMatrix(\
filename=self.sm_mat_file)
mut2killing_tests = sm_mat.query_active_columns_of_rows()
alive_muts = [m for m, k_t in mut2killing_tests.items() \
if len(k_t) == 0]
cand_muts = []
for meta_m in alive_muts:
t_alias, m = DriversUtils.reverse_meta_element(meta_m)
if t_alias in t_alias2metamu_bc: # There is a single one
cand_muts.append(m)
with open(self.cand_muts_file, 'w') as f:
for m in cand_muts:
f.write(str(m) + '\n')
return t_alias2metamu_bc[list(t_alias2metamu_bc)[0]]
def runtests(self, meta_testcases=None, exe_path_map=None, env_vars=None, \
stop_on_failure=False, \
per_test_timeout=None, \
use_recorded_timeout_times=None, \
recalculate_execution_times=False, \
fault_test_execution_matrix_file=None, \
fault_test_execution_execoutput_file=None, \
with_outlog_hash=True, \
test_prioritization_module=None, \
parallel_test_count=1, \
parallel_test_scheduler=None, \
restart_checkpointer=False,
finish_destroy_checkpointer=True):
'''
Execute the list of test cases with the given executable and
say, for each test case, whether it failed
:param meta_testcases: list of test cases to execute
:param exe_path_map: string representing the file system path to
the executable to execute with the tests
:param env_vars: dict of environment variables to set before
executing each test ({<variable>: <value>})
:param stop_on_failure: decide whether to stop the test
execution once a test fails
:param fault_test_execution_matrix_file: Optional matrix file
to store the tests' pass fail execution data
:param fault_test_execution_execoutput_file: Optional output log file
to store the tests' execution actual output (hashed)
:param with_outlog_hash: decide whether to return outlog hash
:param test_prioritization_module: Specify the test prioritization
module.
(TODO: Implement support)
:param parallel_test_count: Specify the number of parallel test
Execution. must be an integer >= 1
:param parallel_test_scheduler: Specify the function that will
handle parallel test scheduling by tool, using
the test execution optimizer.
(TODO: Implement support)
:type restart_checkointer: bool
:param restart_checkointer: Decide whether to discard checkpoint
and restart anew.
:type finish_destroy_checkpointer: bool
:param finish_destroy_checkpointer: Decide whether to automatically
destroy the checkpointer when done or not
Useful is caller has a checkpointer to update.
:returns: dict of testcase and their failed verdict.
{<test case name>: <True if failed, False if passed,
UNCERTAIN_TEST_VERDICT if uncertain>}
If stop_on_failure is True, only return the tests that
have been executed until the failure
'''
# FIXME: Make sure that the support are implemented for
# parallelism and test prioritization. Remove the code bellow
# once supported:
ERROR_HANDLER.assert_true(test_prioritization_module is None, \
"Must implement test prioritization support here", \
__file__)
ERROR_HANDLER.assert_true(parallel_test_count <= 1, \
"Must implement parallel tests execution support here", \
__file__)
ERROR_HANDLER.assert_true(parallel_test_scheduler is None, \
"Must implement parallel tests execution support here", \
__file__)
#~FIXMEnd
# Check arguments Validity
if exe_path_map is None:
exe_path_map = self._get_default_exe_path_map()
ERROR_HANDLER.assert_true(parallel_test_count > 0, \
"invalid parallel test execution count: {}. {}".format( \
parallel_test_count, "must be >= 1"))
# @Checkpoint: create a checkpoint handler
cp_func_name = "runtests"
cp_task_id = 1
checkpoint_handler = \
CheckPointHandler(self.get_checkpoint_state_object())
if restart_checkpointer:
checkpoint_handler.restart()
if checkpoint_handler.is_finished():
logging.warning("%s %s" %("The function 'runtests' is finished", \
"according to checkpoint, but called again. None returned"))
if common_mix.confirm_execution("%s %s" % ( \
"Function 'runtests' is already", \
"finished, do you want to restart?")):
checkpoint_handler.restart()
logging.info("Restarting the finished 'runtests'")
else:
ERROR_HANDLER.error_exit(err_string="%s %s %s" % (\
"Execution halted. Cannot continue because no value", \
" can be returned. Check the results of the", \
"finished execution"), call_location=__file__)
# @Checkpoint: Get the saved payload (data kapt for each tool)
# pair list of testfailed verdict and execution output
meta_test_failedverdicts_outlog = \
checkpoint_handler.get_optional_payload()
if meta_test_failedverdicts_outlog is None:
meta_test_failedverdicts_outlog = [{}, {}]
# Make sure the tests are unique
ERROR_HANDLER.assert_true(len(meta_testcases) == \
len(set(meta_testcases)), \
"not all tests are unique", __file__)
testcases_by_tool = {}
for meta_testcase in meta_testcases:
ttoolalias, testcase = \
DriversUtils.reverse_meta_element(meta_testcase)
if ttoolalias not in testcases_by_tool:
testcases_by_tool[ttoolalias] = []
testcases_by_tool[ttoolalias].append(testcase)
found_a_failure = False
for tpos, ttoolalias in enumerate(testcases_by_tool.keys()):
# @Checkpoint: Check whether already executed
if not checkpoint_handler.is_to_execute(func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ttoolalias):
continue
# Actual execution
ttool = \
self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY]
test_failed_verdicts, test_execoutput = ttool.runtests( \
testcases_by_tool[ttoolalias], \
exe_path_map, env_vars, \
stop_on_failure, \
per_test_timeout=per_test_timeout,
use_recorded_timeout_times=\
use_recorded_timeout_times, \
recalculate_execution_times=\
recalculate_execution_times, \
with_outlog_hash=with_outlog_hash)
for testcase in test_failed_verdicts:
meta_testcase = \
DriversUtils.make_meta_element(testcase, ttoolalias)
meta_test_failedverdicts_outlog[0][meta_testcase] = \
test_failed_verdicts[testcase]
meta_test_failedverdicts_outlog[1][meta_testcase] = \
test_execoutput[testcase]
if test_failed_verdicts[testcase] == \
common_mix.GlobalConstants.COMMAND_UNCERTAIN:
found_a_failure = True
# @Checkpoint: Chekpointing
checkpoint_handler.do_checkpoint(func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ttoolalias, \
opt_payload=meta_test_failedverdicts_outlog)
if stop_on_failure and found_a_failure:
# @Checkpoint: Chekpointing for remaining tools
for rem_tool in list(testcases_by_tool.keys())[tpos + 1:]:
checkpoint_handler.do_checkpoint(func_name=cp_func_name, \
taskid=cp_task_id, \
tool=rem_tool, \
opt_payload=meta_test_failedverdicts_outlog)
break
if stop_on_failure:
# Make sure the non executed test has the uncertain value (None)
if len(meta_test_failedverdicts_outlog[0]) < len(meta_testcases):
for meta_testcase in set(meta_testcases) - \
set(meta_test_failedverdicts_outlog[0]):
meta_test_failedverdicts_outlog[0][meta_testcase] = \
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT
meta_test_failedverdicts_outlog[1][meta_testcase] = \
common_matrices.OutputLogData.\
UNCERTAIN_TEST_OUTLOGDATA
ERROR_HANDLER.assert_true(len(meta_test_failedverdicts_outlog[0]) == \
len(meta_testcases), \
"Not all tests have a verdict reported", __file__)
if fault_test_execution_matrix_file is not None:
# Load or Create the matrix
fault_test_execution_matrix = common_matrices.ExecutionMatrix( \
filename=fault_test_execution_matrix_file, \
non_key_col_list=meta_testcases)
ERROR_HANDLER.assert_true(fault_test_execution_matrix.is_empty(), \
"matrix must be empty", __file__)
failverdict2val = {
common_mix.GlobalConstants.FAIL_TEST_VERDICT: \
fault_test_execution_matrix.getActiveCellDefaultVal(),
common_mix.GlobalConstants.PASS_TEST_VERDICT: \
fault_test_execution_matrix.getInactiveCellVal(),
common_mix.GlobalConstants.UNCERTAIN_TEST_VERDICT: \
fault_test_execution_matrix.getUncertainCellDefaultVal(),
}
cells_dict = {}
for meta_testcase in meta_test_failedverdicts_outlog[0]:
cells_dict[meta_testcase] = failverdict2val[\
meta_test_failedverdicts_outlog[0][meta_testcase]]
fault_test_execution_matrix.add_row_by_key(self.FAULT_MATRIX_KEY, \
cells_dict, serialize=True)
if fault_test_execution_execoutput_file is None:
meta_test_failedverdicts_outlog[1] = None
else:
# Load or Create the data object
fault_test_execution_execoutput = common_matrices.OutputLogData( \
filename=fault_test_execution_execoutput_file)
ERROR_HANDLER.assert_true(\
fault_test_execution_execoutput.is_empty(), \
"outlog data must be empty", __file__)
fault_test_execution_execoutput.add_data(\
{self.PROGRAM_EXECOUTPUT_KEY: \
meta_test_failedverdicts_outlog[1]}, \
serialize=True)
# @Checkpoint: Finished
detailed_exectime = {}
for ttoolalias in testcases_by_tool.keys():
tt = self.testcases_configured_tools[ttoolalias][self.TOOL_OBJ_KEY]
detailed_exectime[ttoolalias] = (\
tt.get_checkpointer().get_execution_time(),\
tt.get_checkpointer().get_detailed_execution_time())
checkpoint_handler.set_finished( \
detailed_exectime_obj=detailed_exectime)
if finish_destroy_checkpointer:
checkpoint_handler.destroy()
return meta_test_failedverdicts_outlog
def runtests_criteria_coverage (self, testcases, criterion_to_matrix, \
criterion_to_executionoutput=None,
criteria_element_list_by_criteria=None, \
re_instrument_code=False, \
cover_criteria_elements_once=False,
prioritization_module_by_criteria=None,
parallel_count=1, \
parallel_criteria_test_scheduler=None,\
restart_checkpointer=False, \
finish_destroy_checkpointer=True):
'''
Executes the instrumented executable code with testscases and
returns the different code coverage matrices.
:param testcases: list of testcases to execute
:param criterion_to_matrix: dict of <criterion, Matrix file
where to store coverage>.
:param criterion_to_executionoutput: dict of <criterion, execoutput
file where to store coverage>.
:param criteria_element_list_by_criteria: dictionary representing the
list of criteria elements (stmts, branches, mutants)
to consider in the test execution matices.
Key is the criterion and the value the list of elements
:param re_instrument_code: Decide whether to instrument code before
running the tests. (Example when instrumentation was
not specifically called. This is True by default)
:param cover_criteria_elements_once: Specify whether to cover criteria
elements once is enough, meaning that we stop
analysing a criterion element once a test covers it.
The remaining test covering verdict will be UNKNOWN.
:param prioritization_module_by_criteria: dict of prioritization module
by criteria. None means no prioritization used.
:type \parallel_count:
:param \parallel_count:
:type \parallel_criteria_test_scheduler:
:param \parallel_criteria_test_scheduler: scheduler that organize
parallelism across criteria tools.
(TODO: Implement support)
:type \restart_checkpointer:
:param \restart_checkpointer:
:type finish_destroy_checkpointer:
:param finish_destroy_checkpointer:
'''
# FIXME: Make sure that the support are implemented for
# parallelism and test prioritization. Remove the code bellow
# once supported:
ERROR_HANDLER.assert_true(parallel_count <= 1, \
"Must implement parallel execution support here", \
__file__)
ERROR_HANDLER.assert_true(parallel_criteria_test_scheduler is None, \
"Must implement parallel codes tests execution support here", \
__file__)
#~FIXMEnd
# Check arguments Validity
ERROR_HANDLER.assert_true(parallel_count > 0, \
"invalid parallel execution count: {}. {}".format( \
parallel_count, "must be >= 1"))
# @Checkpoint: create a checkpoint handler
cp_func_name = "runtests_criteria_coverage"
cp_task_id = 1
checkpoint_handler = CheckPointHandler( \
self.get_checkpoint_state_object())
if restart_checkpointer:
checkpoint_handler.restart()
if checkpoint_handler.is_finished():
return
ERROR_HANDLER.assert_true(len(criterion_to_matrix) > 0, \
"no criterion is enabled", __file__)
ERROR_HANDLER.assert_true(len(set(criterion_to_matrix) - \
set(self.tools_config_by_criterion_dict)) == 0, \
"Passed matrices output are more than tool specified", \
__file__)
if criterion_to_executionoutput is not None:
ERROR_HANDLER.assert_true(set(criterion_to_matrix) == \
set(criterion_to_executionoutput), \
"criteria mismatch between matrix and output", \
__file__)
tool2criteria = self._get_tool2criteria(criterion_to_matrix.keys())
matrices_dir_tmp = os.path.join(self.criteria_working_dir, \
"codecov_dir.tmp")
if os.path.isdir(matrices_dir_tmp):
if restart_checkpointer:
shutil.rmtree(matrices_dir_tmp)
os.mkdir(matrices_dir_tmp)
else:
os.mkdir(matrices_dir_tmp)
if criteria_element_list_by_criteria is None:
criteria_element_list_by_criteria = \
{c: None for c in criterion_to_matrix}
# get criteria elements by tools
criteria_elem_list_by_tool = {}
for criterion in criteria_element_list_by_criteria:
if criteria_element_list_by_criteria[criterion] is None:
for t_conf in self.tools_config_by_criterion_dict[criterion]:
toolalias = t_conf.get_tool_config_alias()
if toolalias not in criteria_elem_list_by_tool:
criteria_elem_list_by_tool[toolalias] = {}
criteria_elem_list_by_tool[toolalias][criterion] = None
continue
ERROR_HANDLER.assert_true(\
len(criteria_element_list_by_criteria[criterion]) != 0, \
"Empty criteria element list for criterion "\
+criterion.get_str(), __file__)
for crit_elem in criteria_element_list_by_criteria[criterion]:
toolalias, elem = DriversUtils.reverse_meta_element(crit_elem)
if toolalias not in criteria_elem_list_by_tool:
criteria_elem_list_by_tool[toolalias] = {}
if criterion not in criteria_elem_list_by_tool[toolalias]:
criteria_elem_list_by_tool[toolalias][criterion] = []
criteria_elem_list_by_tool[toolalias][criterion].append(elem)
ERROR_HANDLER.assert_true(len(set(criteria_elem_list_by_tool) - \
set(self.criteria_configured_tools)) == 0, \
"some tools in data not registered", __file__)
crit2tool2matrixfile = {cv: {} for cv in criterion_to_matrix}
crit2tool2outhashfile = {cv: {} for cv in criterion_to_executionoutput}
for ctoolalias in tool2criteria:
_criteria2matrix = {}
_criteria2outhash = {}
for criterion in tool2criteria[ctoolalias]:
_criteria2matrix[criterion] = os.path.join(matrices_dir_tmp, \
criterion.get_field_value()
+ '-'
+ ctoolalias
+ '.csv')
if criterion_to_executionoutput is None or \
criterion_to_executionoutput[criterion] is None:
_criteria2outhash[criterion] = None
else:
_criteria2outhash[criterion] = \
os.path.join(matrices_dir_tmp, \
criterion.get_field_value()
+ '-'
+ ctoolalias
+ '.outloghash.json')
crit2tool2matrixfile[criterion][ctoolalias] = \
_criteria2matrix[criterion]
crit2tool2outhashfile[criterion][ctoolalias] = \
_criteria2outhash[criterion]
# @Checkpoint: Check whether already executed
if checkpoint_handler.is_to_execute( \
func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ctoolalias):
for criterion in _criteria2matrix:
_criteria2matrix[criterion] = \
common_matrices.ExecutionMatrix( \
filename=_criteria2matrix[criterion], \
non_key_col_list=testcases)
if _criteria2outhash[criterion] is not None:
_criteria2outhash[criterion] = \
common_matrices.OutputLogData( \
filename=_criteria2outhash[criterion])
# Actual execution
ctool = self.criteria_configured_tools[ctoolalias][\
self.TOOL_OBJ_KEY]
ctool.runtests_criteria_coverage(testcases, \
criteria_element_list_by_criteria=\
criteria_elem_list_by_tool[toolalias],\
criterion_to_matrix=_criteria2matrix, \
criterion_to_executionoutput=\
_criteria2outhash,\
re_instrument_code=re_instrument_code, \
cover_criteria_elements_once=\
cover_criteria_elements_once, \
prioritization_module_by_criteria=\
prioritization_module_by_criteria)
# Checkpointing
checkpoint_handler.do_checkpoint( \
func_name=cp_func_name, \
taskid=cp_task_id, \
tool=ctoolalias)
# Aggregate the matrices and out hashes
## Create reult matrices and out hashes
result_matrices = {}
result_outloghashes = {}
for criterion in criterion_to_matrix:
result_matrices[criterion] = common_matrices.ExecutionMatrix( \
filename=criterion_to_matrix[criterion], \
non_key_col_list=testcases)
if criterion_to_executionoutput[criterion] is None:
result_outloghashes[criterion] = None
else:
result_outloghashes[criterion] = \
common_matrices.OutputLogData(filename=\
criterion_to_executionoutput[criterion])
ERROR_HANDLER.assert_true(\
crit2tool2outhashfile[criterion] is not None,
"Bug: log enabled but hidden from tool", __file__)
## Actual aggregate
logging.debug("saving results ...")
for criterion in result_matrices:
result_matrix = result_matrices[criterion]
result_outloghash = result_outloghashes[criterion]
for mtoolalias in crit2tool2matrixfile[criterion]:
tool_matrix = common_matrices.ExecutionMatrix(\
filename=crit2tool2matrixfile[criterion][mtoolalias])
# Check columns
ERROR_HANDLER.assert_true(tool_matrix.get_key_colname() == \
result_matrix.get_key_colname(), \
"mismatch on key column name", __file__)
ERROR_HANDLER.assert_true( \
set(tool_matrix.get_nonkey_colname_list()) == \
set(result_matrix.get_nonkey_colname_list()), \
"mismatch on non key column names", __file__)
# bring in the data
key2nonkeydict = tool_matrix.to_pandas_df().\
set_index(tool_matrix.get_key_colname(), drop=True).\
to_dict(orient="index")
for c_key in key2nonkeydict:
meta_c_key = DriversUtils.make_meta_element(\
str(c_key), mtoolalias)
result_matrix.add_row_by_key(meta_c_key,
key2nonkeydict[c_key],
serialize=False)
# out log hash
if crit2tool2outhashfile[criterion] is not None:
tool_outloghash = common_matrices.OutputLogData(\
filename=\
crit2tool2outhashfile[criterion][mtoolalias])
for objective, objective_data in \
tool_outloghash.get_zip_objective_and_data():
meta_objective = DriversUtils.make_meta_element(\
str(objective), mtoolalias)
result_outloghash.add_data(
{meta_objective: objective_data}, \
serialize=False)
# @Checkpoint: Check whether already executed
if checkpoint_handler.is_to_execute( \
func_name=cp_func_name, \
taskid=cp_task_id + 1,
tool=criterion.get_str()):
# Serialized the computed matrix
result_matrix.serialize()
if result_outloghash is not None:
result_outloghash.serialize()
# @Checkpoint: Checkpointing
checkpoint_handler.do_checkpoint( \
func_name=cp_func_name, \
taskid=cp_task_id + 1,
tool=criterion.get_str())
# Delete the temporary tool matrix's directory
if os.path.isdir(matrices_dir_tmp):
shutil.rmtree(matrices_dir_tmp)
# @Checkpoint: Finished
detailed_exectime = {}
for ctoolalias in tool2criteria:
ct = self.criteria_configured_tools[ctoolalias][self.TOOL_OBJ_KEY]
detailed_exectime[ctoolalias] = (\
ct.get_checkpointer().get_execution_time(),\
ct.get_checkpointer().get_detailed_execution_time())
checkpoint_handler.set_finished(\
detailed_exectime_obj=detailed_exectime)
if finish_destroy_checkpointer:
checkpoint_handler.destroy()
def summarize_data(outdir, summarized_data_dir, mutant_exec_res, \
mut_ex_prepa_data_dir):
other_rel_to_reltests_file = os.path.join(
mut_ex_prepa_data_dir, 'relevantmuts_to_relevanttests.json')
other_rel_to_reltests = common_fs.loadJSON(other_rel_to_reltests_file)
stored_map_file = os.path.join(mut_ex_prepa_data_dir,
'stored_test_map.json')
stored_map = common_fs.loadJSON(stored_map_file)
def get_too2relmuts(relmuts_to_reltests, toollist=None):
res = {tool: set()
for tool in toollist} if toollist is not None else {}
for relmut, t_list in relmuts_to_reltests.items():
for meta_t in t_list:
toolalias, test = DriversUtils.reverse_meta_element(meta_t)
if toollist is None:
if toolalias not in res:
res[toolalias] = set()
else:
assert toolalias in toollist, "PB: toolalias ({}) not in toollist ({})".format(
toolalias, toollist)
res[toolalias].add(relmut)
for toolalias in res:
res[toolalias] = list(res[toolalias])
return res
#~ def get_too2relmuts()
other_tool_to_relmuts = get_too2relmuts(other_rel_to_reltests)
all_tests, fail_tests, relevant_mutants_to_relevant_tests, \
mutants_to_killingtests, tests_to_killed_mutants = load.load (mutant_exec_res, fault_revealing=False)
if not os.path.isdir(summarized_data_dir):
os.mkdir(summarized_data_dir)
cmp_result_file = os.path.join(summarized_data_dir, "compare_results.json")
initial_relmuts_file = os.path.join(summarized_data_dir,
"initial_relevant.json")
rMS_file = os.path.join(summarized_data_dir, "rMS.json")
# get relevant muts by tools
# get tool list
tinf = common_fs.loadJSON(
os.path.join(
mutant_exec_res,
"post/RESULTS_DATA/other_copied_results/testcasesInfos.json"))
tool2relmuts = get_too2relmuts(
relevant_mutants_to_relevant_tests,
toollist=list(set(tinf["CUSTOM"]) - {"custom_devtests"}))
# update with stored map
for relmut, t_list in other_rel_to_reltests.items():
for other_meta_t in t_list:
if other_meta_t in stored_map:
for meta_t in stored_map[other_meta_t]:
toolalias, test = DriversUtils.reverse_meta_element(meta_t)
assert toolalias in tool2relmuts, "PB2: "
tool2relmuts[toolalias].append(relmut)
for toolalias in tool2relmuts:
tool2relmuts[toolalias] = list(set(tool2relmuts[toolalias]))
common_fs.dumpJSON(tool2relmuts, cmp_result_file, pretty=True)
common_fs.dumpJSON(other_tool_to_relmuts,
initial_relmuts_file,
pretty=True)
# Compute and print the relevant mutation score per tool
all_rel_muts = set()
devtest_rel = set()
for talias, rellist in tool2relmuts.items():
all_rel_muts |= set(rellist)
for talias, rellist in other_tool_to_relmuts.items():
all_rel_muts |= set(rellist)
if talias.startswith("custom_devtests"):
devtest_rel |= set(rellist)
rMS = {"ALL-GENONLY": {}, "ALL-ALL": {}, "ADDITIONAL": {}}
for talias, rellist in tool2relmuts.items():
rMS["ALL-ALL"][talias] = len(set(rellist)
| devtest_rel) * 100.0 / len(all_rel_muts)
rMS["ALL-GENONLY"][talias] = len(rellist) * 100.0 / len(all_rel_muts)
rMS["ADDITIONAL"][talias] = len(set(
rellist) - devtest_rel) * 100.0 / len(all_rel_muts - devtest_rel)
common_fs.dumpJSON(rMS, rMS_file, pretty=True)
def prepare_mutant_execution(outdir, muteria_output, original_conf, \
prepare_data_dir, \
avoid_meta_tests_list_file, avoid_meta_mutants_list_file):
if not os.path.isdir(prepare_data_dir):
os.mkdir(prepare_data_dir)
# Store the test list per technique
test_list_file = os.path.join(muteria_output, 'latest', 'RESULTS_DATA', \
'other_copied_results', 'testcasesInfos.json')
shutil.copy2(test_list_file, os.path.join(prepare_data_dir, \
'gentests_'+os.path.basename(test_list_file)))
relevant_exec_outfolder = os.path.join(os.path.dirname(original_conf), \
'output', 'latest')
# Do fdupes with relevant output tests of semu and shadow and store map
r_shadow_tests_src = os.path.join(relevant_exec_outfolder, \
'testscases_workdir', 'shadow_se', 'tests_files')
r_shadow_tests_src_tar = r_shadow_tests_src + '.tar.gz'
r_semu_tests_src = os.path.join(relevant_exec_outfolder, \
'testscases_workdir', 'semu', 'tests_files')
r_semu_tests_src_tar = r_semu_tests_src + '.tar.gz'
cur_exec_outfolder = os.path.join(muteria_output, 'latest')
cur_exec_tg_folder = os.path.join(cur_exec_outfolder, 'testscases_workdir')
cur_shadow_tests_src = None
cur_semu_tests_srcs = []
cur_semu_tests_src_tars = []
for f in os.listdir(cur_exec_tg_folder):
if f.startswith('semu'):
cur_semu_tests_srcs.append(
os.path.join(cur_exec_tg_folder, f, 'tests_files'))
cur_semu_tests_src_tars.append(cur_semu_tests_srcs[-1] + '.tar.gz')
elif f.startswith('shadow_se'):
if cur_shadow_tests_src is not None:
error_exit("Multiple shadow_se folders exist")
cur_shadow_tests_src = os.path.join(cur_exec_tg_folder, f,
'tests_files')
cur_shadow_tests_src_tar = cur_shadow_tests_src + '.tar.gz'
# use folder fdupes from KtestFormat in muteria
## decompress the folders
common_fs.TarGz.decompressDir(r_shadow_tests_src_tar)
common_fs.TarGz.decompressDir(r_semu_tests_src_tar)
common_fs.TarGz.decompressDir(cur_shadow_tests_src_tar)
for i in range(len(cur_semu_tests_src_tars)):
common_fs.TarGz.decompressDir(cur_semu_tests_src_tars[i])
## apply fdupes
def post_test_dup(stored_map, tests_to_avoid, origin,
kepttest2duptest_map):
for kept, duplist in kepttest2duptest_map.items():
alias, _ = DriversUtils.reverse_meta_element(kept)
if alias == origin:
stored_map[kept] = []
for dup in duplist:
dalias, dtest = DriversUtils.reverse_meta_element(dup)
if dalias == origin:
continue
stored_map[kept].append(dup)
tests_to_avoid.append(dup)
#~ def post_test_dup()
stored_map = {}
stored_map_file = os.path.join(prepare_data_dir, 'stored_test_map.json')
tests_to_avoid = []
### Shadow
custom_bin = '/home/shadowvm/shadow/klee-change/Release+Asserts/bin/'
folders = [r_shadow_tests_src, cur_shadow_tests_src]
folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders}
origin = folder2alias[r_shadow_tests_src]
kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\
custom_bin, folders, folder2alias)
post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map)
## remove the untar dirs
for d in folders:
shutil.rmtree(d)
### Semu
custom_bin = None
folders = [r_semu_tests_src] + cur_semu_tests_srcs
folder2alias = {d: os.path.basename(os.path.dirname(d)) for d in folders}
origin = folder2alias[r_semu_tests_src]
kepttest2duptest_map, test2keptdup = KTestTestFormat.cross_folder_fdupes(\
custom_bin, folders, folder2alias)
post_test_dup(stored_map, tests_to_avoid, origin, kepttest2duptest_map)
for d in folders:
shutil.rmtree(d)
## store dup map and tests to avoid
common_fs.dumpJSON(stored_map, stored_map_file, pretty=True)
common_fs.dumpJSON(tests_to_avoid, avoid_meta_tests_list_file, pretty=True)
# XXX TODO: modify tg_conf to use tests to avoid
# Get the criteria dir and add to the muteria out
r_criteria_dir = os.path.join(relevant_exec_outfolder, 'criteria_workdir')
dest_crit_work = os.path.join(muteria_output, 'latest', 'criteria_workdir')
if os.path.isdir(os.path.join(dest_crit_work, 'mart_0')):
shutil.rmtree(os.path.join(dest_crit_work, 'mart_0'))
shutil.copytree(os.path.join(r_criteria_dir, 'mart_0'),
os.path.join(dest_crit_work, 'mart_0'))
# Get the selected mutants (mutants that are Not relevant w.r.t dev tests)
r_res_top = os.path.dirname(os.path.dirname(original_conf))
r_res = os.path.join(r_res_top, 'res')
r_res_tar = glob.glob(r_res_top + '/*res.tar.gz')[0]
## untar res
if os.system('cd {} && tar -xzf {} --exclude {} --exclude {} && test -d res'.format(\
r_res_top, os.path.basename(r_res_tar), \
'post/RESULTS_DATA/other_copied_results/Flakiness', \
'pre/RESULTS_DATA/other_copied_results/Flakiness')) != 0:
error_exit("untar re failed")
## get relevant mutants to relevant tests
all_tests, fail_tests, relevant_mutants_to_relevant_tests, \
mutants_to_killingtests, tests_to_killed_mutants = load.load (r_res, fault_revealing=False)
## remove untar
shutil.rmtree(r_res)
## filter mutants (remove those relevant w.r.t. dev tests)
rel_to_reltests_file = os.path.join(prepare_data_dir,
'relevantmuts_to_relevanttests.json')
mutants_to_avoid = []
### mutants relevant w.r.t. devtests added to avoid
for mut, r_tests in relevant_mutants_to_relevant_tests.items():
for rt in r_tests:
alias, _ = DriversUtils.reverse_meta_element(rt)
if alias.startswith("custom_devtests"):
mutants_to_avoid.append(mut)
break
## save filter mutants as to avoid
common_fs.dumpJSON(relevant_mutants_to_relevant_tests,
rel_to_reltests_file,
pretty=True)
common_fs.dumpJSON(mutants_to_avoid,
avoid_meta_mutants_list_file,
pretty=True)
def _execute_task(self, task):
""" TODO:
1. implement the todos here
"""
# @Checkpointing: see whether the task is untouch to clear tmp
task_untouched = self.cp_data.tasks_obj.task_is_untouched(task)
# Execute base on the task: ---------------------------------------
if task == checkpoint_tasks.Tasks.STARTING:
pass
elif task == checkpoint_tasks.Tasks.FINISHED:
pass
elif task == checkpoint_tasks.Tasks.TESTS_GENERATION_GUIDANCE:
pass #TODO: Generate focus criteria and tests and store in json file
elif task == checkpoint_tasks.Tasks.TESTS_GENERATION:
# @Checkpointing
if task_untouched:
self.meta_testcase_tool.clear_working_dir()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Generate the tests
self.meta_testcase_tool.generate_tests(\
test_tool_type_list=self.cp_data.test_types)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_testcase_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.\
TESTS_EXECUTION_SELECTION_PRIORITIZATION:
out_file_key = outdir_struct.TMP_SELECTED_TESTS_LIST
out_file = self.head_explorer.get_file_pathname(out_file_key)
# @Checkpointing
if task_untouched:
self.head_explorer.remove_file_and_get(out_file_key)
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
all_tests = self.meta_testcase_tool.\
get_testcase_info_object().get_tests_list()
candidate_aliases = \
self.meta_testcase_tool.get_candidate_tools_aliases(\
test_tool_type_list=self.cp_data.test_types)
selected_tests = []
for meta_test in all_tests:
toolalias, _ = DriversUtils.reverse_meta_element(meta_test)
if toolalias in candidate_aliases:
selected_tests.append(meta_test)
# TODO: use the actual selector. For now just use all tests
common_fs.dumpJSON(list(selected_tests), out_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
#self.meta_testexec_optimization_tool.get_checkpoint_state_object()\
# .destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.PASS_FAIL_TESTS_EXECUTION:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_MATRICES_DIR)
matrix_file_key = outdir_struct.TMP_TEST_PASS_FAIL_MATRIX
matrix_file = self.head_explorer.get_file_pathname(matrix_file_key)
execoutput_file_key = \
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT
if self.config.GET_PASSFAIL_OUTPUT_SUMMARY:
execoutput_file = self.head_explorer.get_file_pathname(\
execoutput_file_key)
else:
execoutput_file = None
# @Checkpointing
if task_untouched:
self.head_explorer.remove_file_and_get(matrix_file_key)
self.head_explorer.remove_file_and_get(execoutput_file_key)
self.meta_testcase_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Execute tests
test_list_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_SELECTED_TESTS_LIST)
meta_testcases = common_fs.loadJSON(test_list_file)
# Set test oracle
self.test_oracle_manager.set_oracle(passfail=True)
self.meta_testcase_tool.runtests(meta_testcases=meta_testcases, \
stop_on_failure=\
self.config.STOP_TESTS_EXECUTION_ON_FAILURE, \
recalculate_execution_times=True, \
fault_test_execution_matrix_file=matrix_file, \
fault_test_execution_execoutput_file=execoutput_file, \
test_prioritization_module=\
self.meta_testexec_optimization_tool, \
finish_destroy_checkpointer=False)
# Unset test oracle
self.test_oracle_manager.set_oracle(passfail=False)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_testcase_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION_GUIDANCE:
pass #TODO (Maybe could be used someday. for now, just skip it)
elif task == checkpoint_tasks.Tasks.CRITERIA_GENERATION:
# @Checkpointing
if task_untouched:
self.meta_criteria_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
self.meta_criteria_tool.instrument_code(criteria_enabled_list=\
self.config.ENABLED_CRITERIA.get_val(), \
finish_destroy_checkpointer=False)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Destroy meta test checkpointer
self.meta_criteria_tool.get_checkpoint_state_object()\
.destroy_checkpoint()
elif task == checkpoint_tasks.Tasks.\
CRITERIA_EXECUTION_SELECTION_PRIORITIZATION:
pass #TODO (Maybe could be used someday. for now, just skip it)
elif task == checkpoint_tasks.Tasks.CRITERIA_TESTS_EXECUTION:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_MATRICES_DIR)
matrix_files_keys = {}
matrix_files = {}
execoutput_files_keys = {}
execoutput_files = {}
for criterion in self.config.ENABLED_CRITERIA.get_val():
matrix_files_keys[criterion] = \
outdir_struct.TMP_CRITERIA_MATRIX[criterion]
matrix_files[criterion] = \
self.head_explorer.get_file_pathname(\
matrix_files_keys[criterion])
execoutput_files_keys[criterion] = \
outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion]
if criterion in self.config.CRITERIA_WITH_OUTPUT_SUMMARY:
execoutput_files[criterion] = \
self.head_explorer.get_file_pathname(\
execoutput_files_keys[criterion])
else:
execoutput_files[criterion] = None
# @Checkpointing
if task_untouched:
for _, matrix_file_key in list(matrix_files_keys.items()):
self.head_explorer.remove_file_and_get(matrix_file_key)
for _, execoutput_file_key in list(\
execoutput_files_keys.items()):
self.head_explorer.remove_file_and_get(execoutput_file_key)
self.meta_criteria_tool.get_checkpoint_state_object()\
.restart_task()
self.cp_data.tasks_obj.set_task_executing(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
criteria_set_sequence = self.config.CRITERIA_SEQUENCE.get_val()
#if criteria_set_sequence is None:
# criteria_set_sequence = criteria_pkg.CRITERIA_SEQUENCE
for cs_pos, criteria_set in enumerate(criteria_set_sequence):
criteria_set &= set(matrix_files)
if len(criteria_set) == 0:
continue
# Was it already checkpointed w.r.t criteria set seq
if self.cp_data.criteria_set_is_executed(cs_pos, criteria_set):
continue
# If we have a new criteria set id
self.cp_data.switchto_new_criteria_set(cs_pos, criteria_set)
# get matrices by criteria
test_list_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_SELECTED_TESTS_LIST)
meta_testcases = common_fs.loadJSON(test_list_file)
criterion_to_matrix = {\
c: matrix_files[c] for c in criteria_set}
# TODO: chack set based on the criteria for which execout is enabled
criterion_to_execoutput = {\
c: execoutput_files[c] for c in criteria_set}
# Set test oracle
self.test_oracle_manager.set_oracle(criteria_on=criteria_set)
# execute
self.meta_criteria_tool.runtests_criteria_coverage( \
testcases=meta_testcases, \
criterion_to_matrix=criterion_to_matrix, \
criterion_to_executionoutput=\
criterion_to_execoutput, \
cover_criteria_elements_once=self.config.\
COVER_CRITERIA_ELEMENTS_ONCE.get_val(),\
prioritization_module_by_criteria=\
self.meta_criteriaexec_optimization_tools,\
finish_destroy_checkpointer=True)
# Update matrix if needed to have output diff or such
# TODO: add CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM to config
for crit in criteria_set & set(self.config.\
CRITERIA_REQUIRING_OUTDIFF_WITH_PROGRAM.\
get_val()):
pf_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
if self.config.GET_PASSFAIL_OUTPUT_SUMMARY.get_val():
pf_execoutput_file = \
self.head_explorer.get_file_pathname(\
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT)
else:
pf_execoutput_file = None
DriversUtils.update_matrix_to_cover_when_diference(\
criterion_to_matrix[crit], \
criterion_to_execoutput[crit], \
pf_matrix_file, pf_execoutput_file)
# Unset test oracle
self.test_oracle_manager.set_oracle(criteria_on=None)
# @Checkpointing
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
elif task == checkpoint_tasks.Tasks.PASS_FAIL_STATS:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_STATS_DIR)
tmp_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TEST_PASS_FAIL_MATRIX)
tmp_execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_PROGRAM_TESTEXECUTION_OUTPUT)
execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.PROGRAM_TESTEXECUTION_OUTPUT)
# Merge TMP pass fail and potentially existing pass fail
StatsComputer.merge_lmatrix_into_right(\
tmp_matrix_file, matrix_file)
StatsComputer.merge_lexecoutput_into_right(\
tmp_execoutput_file, execoutput_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Cleanup
self.head_explorer.remove_file_and_get(\
outdir_struct.TMP_TEST_PASS_FAIL_MATRIX)
elif task == checkpoint_tasks.Tasks.CRITERIA_STATS:
# Make sure that the Matrices dir exists
self.head_explorer.get_or_create_and_get_dir(\
outdir_struct.RESULTS_STATS_DIR)
# Merge TMP criteria and potentially existing criteria
for criterion in self.config.ENABLED_CRITERIA.get_val():
tmp_matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_CRITERIA_MATRIX[criterion])
matrix_file = self.head_explorer.get_file_pathname(\
outdir_struct.CRITERIA_MATRIX[criterion])
tmp_execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.TMP_CRITERIA_EXECUTION_OUTPUT[criterion])
execoutput_file = self.head_explorer.get_file_pathname(\
outdir_struct.CRITERIA_EXECUTION_OUTPUT[criterion])
StatsComputer.merge_lmatrix_into_right(tmp_matrix_file, \
matrix_file)
StatsComputer.merge_lexecoutput_into_right(\
tmp_execoutput_file, execoutput_file)
# @Checkpointing
self.cp_data.tasks_obj.set_task_completed(task)
self.checkpointer.write_checkpoint(self.cp_data.get_json_obj())
# Cleanup
for criterion in self.config.ENABLED_CRITERIA.get_val():
self.head_explorer.remove_file_and_get(\
outdir_struct.TMP_CRITERIA_MATRIX[criterion])
elif task == checkpoint_tasks.Tasks.AGGREGATED_STATS:
# Compute the final stats (MS, ...)
StatsComputer.compute_stats(self.config, self.head_explorer)
#-------------------------------------------------------------------
if not self.cp_data.tasks_obj.task_is_complete(task):
# @Checkpoint: set task as done
self.cp_data.tasks_obj.set_task_completed(task)
# @Checkpoint: write checkpoint
self.checkpointer.write_checkpoint(\
json_obj=self.cp_data.get_json_obj())
