def system_test_runner(prog, args_list, test_filename, repo_root_dir, \
exe_path_map=None, env_vars=None, timeout=None, \
collected_output=None, using_wrapper=False, \
dbg_log_execution_out=False):
try:
tmp_env = os.environ.copy()
if env_vars is not None:
#for e, v in env_vars.items():
# tmp_env[e] = v
tmp_env.update(env_vars)
if test_filename is not None:
tmp_env[TEST_FILE_NAME_ENV_VAR] = test_filename
if using_wrapper and timeout is not None:
tmp_env[TEST_EXECUTION_TIMEOUT_ENV_VAR] = str(timeout)
timeout = None
if collected_output is None:
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=args_list, env=tmp_env,\
timeout=timeout, out_on=dbg_log_execution_out,\
err_on=dbg_log_execution_out, \
merge_err_to_out=True)
else:
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=args_list, env=tmp_env,\
timeout=timeout, merge_err_to_out=True)
collected_output.append(retcode)
collected_output.append(out)
collected_output.append(
retcode in DriversUtils.EXEC_TIMED_OUT_RET_CODE)
if dbg_log_execution_out:
logging.debug("(DBG - Test Output):\n" + out)
#except (ValueError, OSError) as os_e:
except OSError as os_e:
# ERROR
logging.warning(
"\ntest execution error in system_test_runner (bellow)")
logging.warning(str(os_e))
logging.warning("COMMAND: " + " ".join([prog] + args_list))
if collected_output is not None:
collected_output.append(None)
collected_output.append(str(os_e))
collected_output.append(False)
return GlobalConstants.TEST_EXECUTION_ERROR
except ValueError as v_e:
logging.warning(
"\ntest execution valueerror in system_test_runner (bellow)")
logging.warning(str(v_e))
logging.warning("COMMAND: " + " ".join([prog] + args_list))
return GlobalConstants.TEST_EXECUTION_ERROR
# Parse the result
hasfail = False
hasfail |= (retcode != 0)
return GlobalConstants.FAIL_TEST_VERDICT if hasfail else \
GlobalConstants.PASS_TEST_VERDICT
def _execute_a_test (self, testcase, exe_path_map, env_vars, \
callback_object=None, timeout=None, collect_output=False):
""" Execute a test given that the executables have been set
properly
"""
prog = 'klee-replay'
if timeout is None:
timeout = self.config.ONE_TEST_EXECUTION_TIMEOUT
ERROR_HANDLER.assert_true(len(exe_path_map) == 1, \
"support a single exe for now", __file__)
ERROR_HANDLER.assert_true(callback_object is None, \
'TODO: handle callback_obj', __file__)
repo_exe = list(exe_path_map.keys())[0]
local_exe = os.path.join(self.klee_used_tmp_build_dir, repo_exe)
if repo_exe not in self.repo_exe_to_local_to_remote:
if not os.path.isdir(os.path.dirname(local_exe)):
os.makedirs(os.path.dirname(local_exe))
self.repo_exe_to_local_to_remote[repo_exe] = {local_exe: None}
remote_exe = exe_path_map[repo_exe]
if remote_exe is None:
remote_exe = repo_exe
if remote_exe != self.repo_exe_to_local_to_remote[repo_exe][local_exe]:
if remote_exe == repo_exe:
self.code_builds_factory.repository_manager.\
set_repo_to_build_default(\
also_copy_to_map={repo_exe: local_exe})
else:
shutil.copy2(remote_exe, local_exe)
args = [local_exe, os.path.join(self.tests_storage_dir, testcase)]
tmp_env = os.environ.copy()
#tmp_env.update(env_vars)
tmp_env['KLEE_REPLAY_TIMEOUT'] = str(timeout)
if collect_output:
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=args, env=tmp_env, \
merge_err_to_out=True)
output_err = (retcode, out)
else:
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=args, env=tmp_env, \
out_on=False, err_on=False)
output_err = None
if retcode in (DriversUtils.EXEC_TIMED_OUT_RET_CODE, \
DriversUtils.EXEC_SEGFAULT_OUT_RET_CODE):
verdict = common_mix.GlobalConstants.FAIL_TEST_VERDICT
else:
verdict = common_mix.GlobalConstants.PASS_TEST_VERDICT
return verdict, output_err
def make_build_func(repo_root_dir, exe_rel_paths, compiler, flags_list, clean,\
reconfigure):
""" Helper for GNU make
"""
def print_err(out, msg):
out = out.splitlines()
print(out, msg)
#~ def print_err()
cwd = os.getcwd()
os.chdir(repo_root_dir)
#try:
tmp_env = os.environ.copy()
if compiler is not None:
tmp_env["CC"] = compiler
if flags_list is not None:
tmp_env["CFLAGS"] = " ".join(flags_list)
if reconfigure:
args_list = ['clean']
retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\
prog='make', args_list=args_list, \
env=tmp_env, merge_err_to_out=True)
if retcode != 0:
print_err(out, "reconfigure failed")
os.chdir(cwd)
return GlobalConstants.COMMAND_FAILURE
if clean:
args_list = ['clean']
retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\
prog='make', args_list=args_list, \
env=tmp_env, merge_err_to_out=True)
if retcode != 0:
print_err(out, "clean failed")
os.chdir(cwd)
return GlobalConstants.COMMAND_FAILURE
retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(\
prog='make', env=tmp_env, merge_err_to_out=True)
if retcode != 0:
print_err(out, "make")
os.chdir(cwd)
return GlobalConstants.COMMAND_FAILURE
#except:
# os.chdir(cwd)
# assert False, "Build Unexpected Error in "+__file__
#return GlobalConstants.COMMAND_FAILURE
os.chdir(cwd)
return GlobalConstants.COMMAND_SUCCESS
#~ def make_build_func()
def _do_instrument_code (self, outputdir, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.gc_files_dir):
shutil.rmtree(self.gc_files_dir)
os.mkdir(self.gc_files_dir)
prog = 'gcc'
flags = ['--coverage', '-fprofile-dir='+self.gc_files_dir, '-O0']
additionals = ["-fkeep-inline-functions"]
# get gcc version
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, ['-dumpversion'])
ERROR_HANDLER.assert_true(ret == 0, "'gcc -dumpversion' failed'")
# if version > 6.5
if int(out.split('.')[0]) >= 6:
if int(out.split('.')[0]) > 6 or int(out.split('.')[1]) > 5:
additionals += ["-fkeep-static-functions"]
flags += additionals
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(\
self.instrumented_code_storage_dir, filename)
self.instrument_callback_obj = self.InstrumentCallbackObject()
self.instrument_callback_obj.set_post_callback_args(\
(self.gc_files_dir, rel_path_map))
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.NATIVE_CODE,\
src_dest_files_paths_map=None,\
compiler=prog, flags_list=flags, clean_tmp=True, \
reconfigure=True, \
callback_object=self.instrument_callback_obj)
# Check
if ret == common_mix.GlobalConstants.COMMAND_FAILURE:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'built problematic'), __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
common_fs.dumpJSON(rel_path_map, self.instrumentation_details)
def _do_instrument_code (self, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.mutant_data):
shutil.rmtree(self.mutant_data)
os.mkdir(self.mutant_data)
prog = 'gpmutation'
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
ERROR_HANDLER.assert_true(len(exes) == 1, \
"Support only a singe exe", __file__)
gpmutation_subj = os.path.basename(exes[0])
args=[gpmutation_subj, self.separate_muts_dir]
# Execute GPMutation
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args_list=args)
if (ret != 0):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([prog]+args) + '\n')
ERROR_HANDLER.error_exit("gpmutation failed!", __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
with open (self.instrumentation_details, 'w') as f:
for exe in exes:
f.write(exe+'\n')
# Archive separated if on
if self.archive_separated:
err_msg = common_fs.TarGz.compressDir(self.separate_muts_dir, \
remove_in_directory=True)
ERROR_HANDLER.assert_true(err_msg is None,\
"Compression failed: "+str(err_msg), __file__)
def _dir_chmod777(self, dirpath):
try:
for root_, dirs_, files_ in os.walk(dirpath):
for sub_d in dirs_:
if os.path.isdir(os.path.join(root_, sub_d)):
os.chmod(os.path.join(root_, sub_d), 0o777)
for f_ in files_:
if os.path.isfile(os.path.join(root_, f_)):
os.chmod(os.path.join(root_, f_), 0o777)
except PermissionError:
ret,_,_ = DriversUtils.execute_and_get_retcode_out_err('sudo', \
args_list=['chmod 777 -R {}'.format(dirpath)])
ERROR_HANDLER.assert_true(ret == 0, \
"'sudo chmod 777 -R "+dirpath+"' failed (returned "+\
str(ret)+")", __file__)
def _collect_temporary_coverage_data(self, criteria_name_list, \
test_execution_verdict, \
used_environment_vars, \
result_dir_tmp):
''' get gcov files from gcda files into result_dir_tmp
'''
prog = 'gcov'
cov2flags = {
TestCriteria.STATEMENT_COVERAGE: [],
TestCriteria.BRANCH_COVERAGE: ['-b', '-c'],
TestCriteria.FUNCTION_COVERAGE: ['-f'],
}
args_list = []
for criterion in criteria_name_list:
args_list += cov2flags[criterion]
gcda_files = self._get_gcda_list()
raw_filename_list = [os.path.splitext(f)[0] for f in gcda_files]
args_list += raw_filename_list
if len(gcda_files) > 0:
# TODO: When gcov generate coverage for different files with
# same name filename bu located at diferent dir. Avoid override.
# Go where the gcov will be looked for
cwd = os.getcwd()
os.chdir(self.gc_files_dir)
# collect gcda (gcno)
r, _, _ = DriversUtils.execute_and_get_retcode_out_err(prog=prog, \
args_list=args_list, out_on=False, \
err_on=False)
os.chdir(cwd)
if r != 0:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'error collecting coverage is problematic'), __file__)
# delete gcda
for gcda_f in gcda_files:
os.remove(gcda_f)
common_fs.dumpJSON(self._get_gcov_list(), \
os.path.join(result_dir_tmp,\
self.gcov_files_list_filename))
def python_unittest_runner(test_name, repo_root_dir, exe_path_map, env_vars, \
timeout, collected_output=None):
# TODO: use exe_path_map
def parse_test(s):
return s.split('...')[0].replace(':','/').replace(' ','')
cwd = os.getcwd()
os.chdir(repo_root_dir)
try:
args_list = ['-m', 'unittest', test_name, '-v']
if collected_output is None:
retcode, stdout, _ = DriversUtils.execute_and_get_retcode_out_err(\
prog=sys.executable, args_list=args_list, \
timeout=timeout, merge_err_to_out=True)
stdout = stdout.splitlines()
else:
# collected_output is a list ([retcode, out_err_log])
# TODO: use wrapper? or parse out for particular codes...
assert False, "TO BE Implemented"
except:
# ERROR
os.chdir(cwd)
return GlobalConstants.TEST_EXECUTION_ERROR
# Parse the result
subtests_verdicts = {}
hasfail = False
hasfail |= (retcode != 0)
for s in stdout:
if s.endswith('... FAIL'):
hasfail = True
subtests_verdicts[parse_test(s)] = True
elif s.endswith('... ok'):
subtests_verdicts[parse_test(s)] = False
#print(subtests_verdicts)
os.chdir(cwd)
return GlobalConstants.FAIL_TEST_VERDICT if hasfail else \
GlobalConstants.PASS_TEST_VERDICT
#~ def python_unittest_runner()
def make_build(repo_root_dir, exe_rel_paths, compiler, flags_list, clean,\
reconfigure):
cwd = os.getcwd()
os.chdir(repo_root_dir)
if flags_list is None:
flags_list = ["-DUSING_MUTERIA"]
else:
flags_list.append("-DUSING_MUTERIA")
# try:
tmp_env = os.environ.copy()
if compiler is not None:
tmp_env["CC"] = compiler
if flags_list is not None:
tmp_env["CFLAGS"] = " ".join(flags_list)
args_list = ['build_project.sh']
if reconfigure:
args_list.append('1')
else:
args_list.append('0')
if clean:
args_list.append('1')
else:
args_list.append('0')
retcode, out, _ = DriversUtils.execute_and_get_retcode_out_err(
prog='bash', args_list=args_list, env=tmp_env, merge_err_to_out=True)
def print_err(out, msg):
#out = out.splitlines()
#print(out, msg)
logging.error(str(out) + msg)
if retcode != 0:
print_err(out, "make")
os.chdir(cwd)
return GlobalConstants.COMMAND_FAILURE
os.chdir(cwd)
return GlobalConstants.COMMAND_SUCCESS
def _do_instrument_code (self, outputdir, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.mutant_data):
shutil.rmtree(self.mutant_data)
os.mkdir(self.mutant_data)
prog = 'mart'
# get llvm compiler path
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, ['--version'])
llvm_compiler_path = None
for line in out.splitlines():
line = line.strip()
if line.startswith('LLVM tools dir:'):
llvm_compiler_path = line.split()[3] #[1:-1]
break
ERROR_HANDLER.assert_true(llvm_compiler_path is not None, \
'Problem getting llvm path for mart', __file__)
# Build into LLVM
back_llvm_compiler = 'clang'
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(self.mutant_data, filename)
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.LLVM_BITCODE,\
src_dest_files_paths_map=rel_path_map,\
compiler=back_llvm_compiler, flags_list=['-g'], \
clean_tmp=True, reconfigure=True, \
llvm_compiler_path=llvm_compiler_path)
if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR:
ERROR_HANDLER.error_exit("Program {}.".format(\
'LLVM (clang) built problematic'), __file__)
# Update exe_map to reflect bitcode extension
rel2bitcode = {}
for r_file, b_file in list(rel_path_map.items()):
bc = b_file + '.bc'
ERROR_HANDLER.assert_true(os.path.isfile(bc), \
"Bitcode file not existing: "+bc, __file__)
rel2bitcode[r_file] = bc
ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \
"Support single bitcode module for now", __file__)
bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]]
# mart params
bool_param, k_v_params = self._get_default_params()
if TestCriteria.STRONG_MUTATION in enabled_criteria:
bool_param['-write-mutants'] = True
args = [bp for bp, en in list(bool_param.items()) if en]
for k, v in list(k_v_params.items()):
if v is not None:
args += [k, v]
args.append(bitcode_file)
# Execute Mart
cwd = os.getcwd()
os.chdir(self.mutant_data)
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args)
os.chdir(cwd)
if (ret != 0):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n')
ERROR_HANDLER.error_exit("mart failed'", __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
store_obj = {c.get_str(): {} for c in enabled_criteria}
for k, v in list(rel_path_map.items()):
exe_file = os.path.basename(v)
if TestCriteria.WEAK_MUTATION in enabled_criteria:
crit_str = TestCriteria.WEAK_MUTATION.get_str()
store_obj[crit_str][k] = exe_file + '.WM'
if TestCriteria.MUTANT_COVERAGE in enabled_criteria:
crit_str = TestCriteria.MUTANT_COVERAGE.get_str()
store_obj[crit_str][k] = exe_file + '.COV'
if TestCriteria.STRONG_MUTATION in enabled_criteria:
crit_str = TestCriteria.STRONG_MUTATION.get_str()
store_obj[crit_str][k] = exe_file + '.MetaMu'
common_fs.dumpJSON(store_obj, self.instrumentation_details)
def _do_generate_tests (self, exe_path_map, outputdir, \
code_builds_factory, max_time=None):
# Setup
if os.path.isdir(self.tests_working_dir):
shutil.rmtree(self.tests_working_dir)
os.mkdir(self.tests_working_dir)
if os.path.isdir(self.tests_storage_dir):
shutil.rmtree(self.tests_storage_dir)
prog = 'klee'
default_sym_args = ['-sym-arg', '5']
back_llvm_compiler = None #'clang'
rel_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
for exe in exes:
filename = os.path.basename(exe)
rel_path_map[exe] = os.path.join(self.tests_working_dir, filename)
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.LLVM_BITCODE,\
src_dest_files_paths_map=rel_path_map,\
compiler=back_llvm_compiler, \
clean_tmp=True, reconfigure=True)
if ret == common_mix.GlobalConstants.TEST_EXECUTION_ERROR:
ERROR_HANDLER.error_exit("Program {}.".format(\
'LLVM built problematic'), __file__)
# Update exe_map to reflect bitcode extension
rel2bitcode = {}
for r_file, b_file in list(rel_path_map.items()):
bc = b_file + '.bc'
ERROR_HANDLER.assert_true(os.path.isfile(bc), \
"Bitcode file not existing: "+bc, __file__)
rel2bitcode[r_file] = bc
ERROR_HANDLER.assert_true(len(rel_path_map) == 1, \
"Support single bitcode module for now", __file__)
bitcode_file = rel2bitcode[list(rel2bitcode.keys())[0]]
# klee params
bool_param, k_v_params = self._get_default_params()
if max_time is not None:
k_v_params['-max-time'] = str(max_time)
args = [bp for bp, en in list(bool_param.items()) if en]
for k, v in list(k_v_params.items()):
if v is not None:
args += [k, str(v)]
args.append(bitcode_file)
# sym args
klee_sym_args = default_sym_args
uc = self.config.get_tool_user_custom()
if uc is not None:
post_bc_cmd = uc.POST_TARGET_CMD_ORDERED_FLAGS_LIST
if post_bc_cmd is not None:
klee_sym_args = []
for tup in post_bc_cmd:
klee_sym_args += list(tup)
args += klee_sym_args
# Execute Klee
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args)
if (ret != 0):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([prog] + args) + '\n')
ERROR_HANDLER.error_exit("klee failed'", __file__)
store_obj = {r: os.path.basename(b) for r, b in rel2bitcode.items()}
common_fs.dumpJSON(store_obj, self.test_details_file)
#~ def _do_generate_tests()
#~ class CustomTestcases
def get_instrumented_executable_paths_map(self, enabled_criteria):
using_gdb_wrapper = self.driver_config.get_use_gdb_wrapper()
if using_gdb_wrapper:
# use wrapper if gdb is installed
using_gdb_wrapper = DriversUtils.check_tool(prog='gdb', \
args_list=['--version'], \
expected_exit_codes=[0])
if using_gdb_wrapper:
# XXX: Docker has issues running programs in Docker,
# unless the container is ran with the following arguments:
#
# docker run --cap-add=SYS_PTRACE \
# --security-opt seccomp=unconfined ...
#
# We check that it is fine by testing on echo
ret, o_e, _ = DriversUtils.execute_and_get_retcode_out_err(
prog='gdb', \
args_list=['--batch-silent', \
'--quiet',
'--return-child-result',
'-ex', 'run',
'--args', 'echo'], \
)#out_on=False, err_on=False)
using_gdb_wrapper = (ret == 0)
if not using_gdb_wrapper:
logging.warning("use gdb is enabled but call to gdb fails"
" (retcode {}) with msg: {}".format(
ret, o_e))
else:
logging.warning("use gdb is enabled but gdb is not installed")
crit_to_exes_map = {}
obj = common_fs.loadJSON(self.instrumentation_details)
#exes = [p for _, p in list(obj.items())]
for name in obj:
obj[name] = os.path.join(self.instrumented_code_storage_dir, \
obj[name])
if using_gdb_wrapper:
# Using GDB WRAPPER
with open(self.gcov_gdb_wrapper_template) as f:
template_str = f.read()
single_exe = obj[list(obj)[0]]
template_str = template_str.replace(\
'MUTERIA_GCOV_PROGRAMEXE_PATHNAME', single_exe)
with open(self.gcov_gdb_wrapper_sh, 'w') as f:
f.write(template_str)
shutil.copymode(single_exe, self.gcov_gdb_wrapper_sh)
obj[list(obj)[0]] = self.gcov_gdb_wrapper_sh
exes_map = obj
for criterion in enabled_criteria:
crit_to_exes_map[criterion] = exes_map
#logging.debug("DBG: {} {} {}".format(\
# "Using gdb wrapper is {}.".format(using_gdb_wrapper), \
# "crit_to_exe_map is {}.".format(crit_to_exes_map), \
# "wraper path is {}!".format(self.gcov_gdb_wrapper_sh)))
return crit_to_exes_map
def execute_test(cls, executable_file, test_file, env_vars, stdin=None, \
must_exist_dir_list=None, \
timeout=None, collected_output=None, \
custom_replay_tool_binary_dir=None):
prog, args = cls._get_replay_prog_args(executable_file, test_file, \
custom_replay_tool_binary_dir)
# klee-replay may create files or dir. in KLEE version with LLVM-3.4,
# those are created in a temporary dir set as <cwd>.temps
# XXX XXX. make sure each test has its own
test_work_dir = test_file + ".execdir"
klee_replay_temps = test_work_dir + '.temps'
for d in (test_work_dir, klee_replay_temps):
if os.path.isdir(d):
try:
shutil.rmtree(d)
except PermissionError:
cls._dir_chmod777(d)
shutil.rmtree(d)
if not os.path.isdir(test_work_dir):
os.mkdir(test_work_dir)
if must_exist_dir_list is not None:
for d in must_exist_dir_list:
td = os.path.join(test_work_dir, d)
if not os.path.isdir(td):
os.makedirs(td)
# XXX Execution setup
tmp_env = os.environ.copy()
if env_vars is not None:
#for e, v in env_vars.items():
# tmp_env[e] = v
tmp_env.update(env_vars)
timeout_return_codes = cls.timedout_retcodes + \
DriversUtils.EXEC_TIMED_OUT_RET_CODE
if timeout is not None:
tmp_env['KLEE_REPLAY_TIMEOUT'] = str(timeout)
kt_over = 10 # 1second
timeout += kt_over
else:
# DBG
logging.warning("@KTEST: calling ktest execution without timeout.")
# XXX Get the parsing regexes to use
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=['--help'], \
merge_err_to_out=True)
clean_regex, status_regex = cls._get_regexes(out, \
clean_everything=True)
# XXX Execute the ktest
#logging.debug("DBG: test_work_dir is {}. its content is {}".format(
# test_work_dir, list(os.listdir(test_work_dir))))
#if collected_output is not None:
# XXX: Use stdbuf to line buffer the stderr to avoid mixing or
# err between klee-replay and the executd prog
use_stdbuf = True
if use_stdbuf:
args = ["--output=L", "--error=L", prog] + args
prog = "stdbuf"
# TODO: check that stdbuf is installed
retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, args_list=args, env=tmp_env, \
stdin=stdin, \
timeout=timeout, timeout_grace_period=5, \
merge_err_to_out=True, cwd=test_work_dir)
retcode, out, exit_status = cls._remove_output_noise(retcode, out, \
clean_regex, status_regex)
# In klee-replay, when exit_status here is not None, retcode is 0
# When there is an issue, like timeout, exit_status is None and
# retcode has the ode of the issue
if exit_status is None:
exit_status = retcode
if collected_output is not None:
collected_output.extend((exit_status, out, \
(retcode in timeout_return_codes or \
retcode in DriversUtils.EXEC_TIMED_OUT_RET_CODE)))
#else:
# retcode, out, err = DriversUtils.execute_and_get_retcode_out_err(\
# prog=prog, args_list=args, env=tmp_env, \
# timeout=timeout, timeout_grace_period=5, \
# out_on=False, err_on=False, \
# cwd=test_work_dir)
# XXX: Go back to previous CWD
for d in (test_work_dir, klee_replay_temps):
if os.path.isdir(d):
try:
shutil.rmtree(d)
except PermissionError:
cls._dir_chmod777(d)
shutil.rmtree(d)
#if must_exist_dir_list is not None:
# try:
# shutil.rmtree(test_work_dir)
# except PermissionError:
# cls._dir_chmod777(test_work_dir)
# shutil.rmtree(test_work_dir)
if retcode in timeout_return_codes + \
DriversUtils.EXEC_SEGFAULT_OUT_RET_CODE:
verdict = common_mix.GlobalConstants.FAIL_TEST_VERDICT
else:
verdict = common_mix.GlobalConstants.PASS_TEST_VERDICT
return verdict
def _do_instrument_code (self, exe_path_map, \
code_builds_factory, \
enabled_criteria, parallel_count=1):
# Setup
if os.path.isdir(self.instrumented_code_storage_dir):
shutil.rmtree(self.instrumented_code_storage_dir)
os.mkdir(self.instrumented_code_storage_dir)
if os.path.isdir(self.gc_files_dir):
try:
shutil.rmtree(self.gc_files_dir)
except PermissionError:
self._dir_chmod777(self.gc_files_dir)
shutil.rmtree(self.gc_files_dir)
os.mkdir(self.gc_files_dir, mode=0o777)
prog = 'gcc'
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
flags = [
'-g', '--coverage', '-fprofile-dir=' + self.gc_files_dir, '-O0'
]
additionals = ["-fkeep-inline-functions"]
#additionals.append('-fprofile-abs-path')
#additionals.append('-fprofile-prefix-path='+code_builds_factory\
# .repository_manager.get_repository_dir_path())
# get gcc version
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
prog, args_list=['-dumpversion'])
ERROR_HANDLER.assert_true(ret == 0, "'gcc -dumpversion' failed'")
# if version > 6.5
if int(out.split('.')[0]) >= 6:
if int(out.split('.')[0]) > 6 or int(out.split('.')[1]) > 5:
additionals += ["-fkeep-static-functions"]
flags += additionals
rel_rel_path_map = {}
rel_abs_path_map = {}
exes, _ = code_builds_factory.repository_manager.\
get_relative_exe_path_map()
ERROR_HANDLER.assert_true(len(exes) == 1, \
"Only single exe supported in GCOV", __file__)
for exe in exes:
filename = os.path.basename(exe)
rel_rel_path_map[exe] = filename
rel_abs_path_map[exe] = os.path.join(\
self.instrumented_code_storage_dir, filename)
self.instrument_callback_obj = self.InstrumentCallbackObject()
self.instrument_callback_obj.set_post_callback_args(\
(self.gc_files_dir, rel_abs_path_map))
pre_ret, ret, post_ret = code_builds_factory.transform_src_into_dest(\
src_fmt=CodeFormats.C_SOURCE,\
dest_fmt=CodeFormats.NATIVE_CODE,\
src_dest_files_paths_map=None,\
compiler=prog, flags_list=flags, clean_tmp=True, \
reconfigure=True, \
callback_object=self.instrument_callback_obj)
# Check
if ret == common_mix.GlobalConstants.COMMAND_FAILURE:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'built problematic'), __file__)
# write down the rel_path_map
ERROR_HANDLER.assert_true(not os.path.isfile(\
self.instrumentation_details), "must not exist here", __file__)
common_fs.dumpJSON(rel_rel_path_map, self.instrumentation_details)
def _collect_temporary_coverage_data(self, criteria_name_list, \
test_execution_verdict, \
used_environment_vars, \
result_dir_tmp, \
testcase):
''' get gcov files from gcda files into result_dir_tmp
'''
prog = 'gcov'
if self.custom_binary_dir is not None:
prog = os.path.join(self.custom_binary_dir, prog)
ERROR_HANDLER.assert_true(os.path.isfile(prog), \
"The tool {} is missing from the specified dir {}"\
.format(os.path.basename(prog), \
self.custom_binary_dir), __file__)
cov2flags = {
TestCriteria.STATEMENT_COVERAGE: [],
TestCriteria.BRANCH_COVERAGE: ['-b', '-c'],
TestCriteria.FUNCTION_COVERAGE: ['-f'],
}
args_list = []
for criterion in criteria_name_list:
args_list += cov2flags[criterion]
gcda_files = self._get_gcda_list()
raw_filename_list = [os.path.splitext(f)[0] for f in gcda_files]
args_list += raw_filename_list
if len(gcda_files) > 0:
# TODO: When gcov generate coverage for different files with
# same name filename but located at diferent dir. Avoid override.
# Go where the gcov will be looked for
cwd = os.getcwd()
os.chdir(self.gc_files_dir)
# collect gcda (gcno)
r, _, err_str = DriversUtils.execute_and_get_retcode_out_err(\
prog=prog, \
args_list=args_list, out_on=False, \
err_on=True, merge_err_to_out=False)
os.chdir(cwd)
if r != 0: # or err_str:
ERROR_HANDLER.error_exit("Program {} {}.".format(prog,\
'collecting coverage is problematic. ')+
"The error msg is {}. \nThe command:\n{}".format(\
err_str, " ".join([prog]+args_list)), __file__)
dot_gcov_file_list = self._get_gcov_list()
# Sources of interest
_, src_map = self.code_builds_factory.repository_manager.\
get_relative_exe_path_map()
base_dot_gcov = [os.path.basename(f) for f in dot_gcov_file_list]
base_raw = [os.path.basename(f) for f in raw_filename_list]
interest = [os.path.basename(s) for s, o in src_map.items()]
interest = [s+'.gcov' for s in interest if os.path.splitext(s)[0] \
in base_raw]
missed = set(interest) - set(base_dot_gcov)
# FIXME: Check for partial failure (compare number of gcno and gcov)
ERROR_HANDLER.assert_true(len(missed) == 0,
"{} did not generate the '.gcov' files {}.".format(\
prog, missed)+ "\nIts stderr is {}.\n CMD: {}".format(\
err_str, " ".join([prog]+args_list)), __file__)
# delete gcda
for gcda_f in gcda_files:
os.remove(gcda_f)
else:
if not self.driver_config.get_allow_missing_coverage():
ERROR_HANDLER.error_exit(\
"Testcase '{}' did not generate gcda, {}".format(\
testcase, "when allow missing coverage is disabled"))
dot_gcov_file_list = []
common_fs.dumpJSON(dot_gcov_file_list, \
os.path.join(result_dir_tmp,\
self.gcov_files_list_filename))
def _call_generation_run(self, runtool, args):
## locate max-time
timeout_grace_period = 600
max_time = None
cur_max_time = float(self.get_value_in_arglist(args, 'max-time'))
#to give time to klee add FRAMEWORK GRACE
max_time = cur_max_time + \
self.config.TEST_GEN_TIMEOUT_FRAMEWORK_GRACE
# set stack to unlimited
stack_ulimit_soft, stack_ulimit_hard = \
resource.getrlimit(resource.RLIMIT_STACK)
if stack_ulimit_soft != -1:
resource.setrlimit(resource.RLIMIT_STACK, (-1, stack_ulimit_hard))
# Execute Klee
if self.driver_config.get_suppress_generation_stdout():
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
runtool, args_list=args, timeout=max_time,\
timeout_grace_period=timeout_grace_period, \
out_on=False, err_on=True, \
merge_err_to_out=False)
out, err = err, out
else:
ret, out, err = DriversUtils.execute_and_get_retcode_out_err(\
runtool, args_list=args, timeout=max_time,\
timeout_grace_period=timeout_grace_period)
#out_on=False, err_on=False)
'''o_d_dbg = self.get_value_in_arglist(args, "output-dir") #DBG
if os.path.isdir(o_d_dbg): #DBG
shutil.rmtree(o_d_dbg) #DBG
import subprocess #DBG
p = subprocess.Popen([runtool]+args, env=None, cwd=None, \
#close_fds=True, \
stdin=None, \
stderr=subprocess.STDOUT, \
stdout=None, shell=True, \
preexec_fn=os.setsid) #DBG
try: #DBG
stdout, stderr = p.communicate(timeout=max_time) #DBG
except subprocess.TimeoutExpired: #DBG
stdout, stderr = p.communicate(timeout=max_time) #DBG
#os.system(" ".join([runtool]+args)) #DBG'''
# restore stack
if stack_ulimit_soft != -1:
resource.setrlimit(resource.RLIMIT_STACK, \
(stack_ulimit_soft, stack_ulimit_hard))
if (ret != 0 and ret not in DriversUtils.EXEC_TIMED_OUT_RET_CODE \
and not out.rstrip().endswith(": ctrl-c detected, exiting.")):
logging.error(out)
logging.error(err)
logging.error("\n>> CMD: " + " ".join([runtool] + args) + '\n')
ERROR_HANDLER.error_exit(
"call to klee testgen failed, " +
"error code is {}".format(ret), __file__)
if self.driver_config.get_verbose_generation():
logging.debug(out)
logging.debug(err)
logging.debug("\nCMD: " + " ".join([runtool] + args))
def convert_code(self, src_fmt, dest_fmt, file_src_dest_map, \
repository_manager, **kwargs):
# TODO: add can_fail parameter, in kwarg, for case like mutant
# compilatioon that can fail but should not terminate execution
# but return a specific value
ERROR_HANDLER.assert_true(src_fmt in self.src_formats, \
"Unsupported src format", __file__)
# post build callbacks
class CopyCallbackObject(DefaultCallbackObject):
def after_command(self):
if self.op_retval == \
common_mix.GlobalConstants.COMMAND_FAILURE:
return common_mix.GlobalConstants.COMMAND_FAILURE
for sf, df in list(file_src_dest_map.items()):
abs_sf = repository_manager.repo_abs_path(sf)
if not os.path.isfile(abs_sf):
ERROR_HANDLER.error_exit(\
"an expected file missing after build: "+\
abs_sf, __file__)
if df is not None:
shutil.copy2(abs_sf, df)
return None
#~ def after_command()
#~ class CopyCallbackObject
# Should not have callback_object and file_src_dest_map at the
# same time
callbak_obj_key = 'callback_object'
if callbak_obj_key in kwargs:
ERROR_HANDLER.assert_true(file_src_dest_map is None,\
"file_src_dest_map must be None "+ \
"if callback_object is passed", __file__)
elif file_src_dest_map is not None and len(file_src_dest_map) > 0:
kwargs[callbak_obj_key] = CopyCallbackObject()
else:
kwargs[callbak_obj_key] = None
# Actual Processing
if (dest_fmt == ccs.CodeFormats.C_PREPROCESSED_SOURCE):
if (src_fmt == ccs.CodeFormats.C_SOURCE):
ERROR_HANDLER.error_exit("Must Implement1", __file__)
else:
for src, dest in list(file_src_dest_map.items()):
shutil.copy2(src, dest)
if (dest_fmt == ccs.CodeFormats.LLVM_BITCODE):
# XXX: This build overrides passed clean_tmp and reconfigure
# also overrides Compiler if wllvm if found
# and does not use the callback object
# and require file_src_dest_map to have the place to store
# generated .bc by specifying the corresponding native file.
# EX: {x.c: /path/to/main} passed to have /path/to/main.bc
# generated
spec_compiler = kwargs['compiler'] if 'compiler' in kwargs \
else None
# special kwargs
spec_llvm_compiler_path = None
if 'llvm_compiler_path' in kwargs:
spec_llvm_compiler_path = kwargs['llvm_compiler_path']
del kwargs['llvm_compiler_path']
if spec_compiler is not None:
bak_llvm_compiler = os.environ['LLVM_COMPILER']
os.environ['LLVM_COMPILER'] = spec_compiler
if spec_llvm_compiler_path is not None:
bak_llvm_compiler_path = os.environ['LLVM_COMPILER_PATH']
os.environ['LLVM_COMPILER_PATH'] = spec_llvm_compiler_path
#1. Ensure wllvm is installed (For now use default llvm compiler)
has_wllvm = DriversUtils.check_tool('wllvm', ['--version'])
ERROR_HANDLER.assert_true(has_wllvm, 'wllvm not found', __file__)
# tmp['LLVM_COMPILER_PATH'] = ...
kwargs['compiler'] = 'wllvm'
kwargs['clean_tmp'] = True
kwargs['reconfigure'] = True
# Normal build followed by executable copying
pre_ret, ret, post_ret = repository_manager.build_code(**kwargs)
ERROR_HANDLER.assert_true(\
ret != common_mix.GlobalConstants.COMMAND_FAILURE and\
pre_ret != common_mix.GlobalConstants.COMMAND_FAILURE and\
post_ret != common_mix.GlobalConstants.COMMAND_FAILURE,\
"Build LLVM bitcode failed!", __file__)
# extract bitcode from copied executables and remove non bitcode
if file_src_dest_map is not None:
for src, dest in list(file_src_dest_map.items()):
ret, out, err = \
DriversUtils.execute_and_get_retcode_out_err( \
"extract-bc", [dest])
ERROR_HANDLER.assert_true(ret == 0, \
'{}. \n# OUT: {}\n# ERR: {}'.format(\
'extract-bc failed', out, err), __file__)
os.remove(dest)
if spec_compiler is not None:
os.environ['LLVM_COMPILER'] = bak_llvm_compiler
if spec_llvm_compiler_path is not None:
os.environ['LLVM_COMPILER_PATH'] = bak_llvm_compiler_path
# Clean build
kwargs['compiler'] = None
pre_ret, ret, post_ret = repository_manager.build_code(**kwargs)
if (dest_fmt == ccs.CodeFormats.OBJECT_FILE):
ERROR_HANDLER.error_exit("Must Implement3", __file__)
if (dest_fmt == ccs.CodeFormats.NATIVE_CODE):
pre_ret, ret, post_ret = repository_manager.build_code(**kwargs)
return pre_ret, ret, post_ret
