def setUp():
old_dir = os.path.abspath('.')
try:
os.chdir(module_dir)
utils.execute("make")
print_inputs = utils.get_executable("./print_inputs")
check_inputs = utils.get_executable("./check_inputs")
n.assert_is_not_none(print_inputs)
n.assert_is_not_none(check_inputs)
finally:
os.chdir(old_dir)
def validate(self, program_file, witness_file):
# err_to_output=True is important so that messages to stderr are in correct relation to messages to stdout!
# This may be important for determining the run result.
cmd_result = utils.execute(self._get_cmd(program_file, witness_file),
quiet=True,
err_to_output=True)
returncode = cmd_result.returncode
# Execute returns a negative returncode -N if the process was killed by signal N
if returncode < 0:
returnsignal = -returncode
else:
returnsignal = 0
if cmd_result.stderr:
tool_output = cmd_result.stderr.split('\n')
else:
tool_output = list()
tool_output += cmd_result.stdout.split('\n')
# Remove last line if empty. FShell expects no empty line at the end.
if len(tool_output) >= 1 and not tool_output[-1]:
tool_output = tool_output[:-1]
validation_result = self.tool.determine_result(returncode,
returnsignal,
tool_output,
isTimeout=False)
return validation_result
def generate_input(self, filename, stop_flag):
default_err = "Unknown error"
self.timer_input_gen.start()
try:
file_to_analyze = utils.get_prepared_name(filename,
self.get_name())
self.timer_file_access.start()
with open(filename, 'r') as outp:
filecontent = outp.read()
self.timer_file_access.stop()
if os.path.exists(file_to_analyze):
logging.warning(
"Prepared file already exists. Not preparing again.")
else:
self.timer_prepare.start()
prepared_content = self.prepare0(filecontent)
self.timer_file_access.start()
with open(file_to_analyze, 'w+') as new_file:
new_file.write(prepared_content)
self.timer_file_access.stop()
self.timer_prepare.stop()
cmds = self.create_input_generation_cmds(file_to_analyze)
for cmd in cmds:
self.timer_generator.start()
result = utils.execute(cmd,
env=self.get_run_env(),
quiet=False,
err_to_output=True,
stop_flag=stop_flag,
timelimit=self.timelimit)
self.timer_generator.stop()
if BaseInputGenerator.failed(result) \
and stop_flag and not stop_flag.is_set():
raise utils.InputGenerationError("Failed at command: " +
' '.join(cmd))
return self._get_success_and_stats()
except utils.CompileError as e:
logging.error("Compile error: %s", e.msg if e.msg else default_err)
return self._get_failed_and_stats()
except utils.InputGenerationError as e:
logging.error("Input generation error: %s",
e.msg if e.msg else default_err)
return self._get_failed_and_stats()
except utils.ParseError as e:
logging.error("Parse error: %s", e.msg if e.msg else default_err)
return self._get_failed_and_stats()
finally:
self.timer_input_gen.stop()
for n, s in self.statistics.stats:
if type(s) is utils.Stopwatch and s.is_running():
s.stop()
self.number_generated_tests.value = len(self.get_test_cases())
def compile(self, program_file, harness_file, output_file):
compile_cmd = self._get_compile_cmd(program_file, harness_file,
output_file)
compile_result = utils.execute(compile_cmd, quiet=True)
if compile_result.returncode != 0:
compile_cmd = self._get_compile_cmd(program_file, harness_file,
output_file, 'gnu90')
compile_result = utils.execute(compile_cmd,
quiet=True,
err_to_output=False)
if compile_result.returncode != 0:
raise utils.CompileError(
"Compilation failed for harness {}".format(harness_file))
return output_file
def get_test_vector(self, test):
def _get_value(single_line):
var_name = single_line.split(':')[2].strip()
prefix_end = var_name.find("'")
var_name = var_name[prefix_end + 1:-1]
return var_name
def _convert_bytestring_to_hex(bytestring):
bytes = bytestring.split(r'\x')
value = ''.join(reversed(bytes))
value = '0x' + value
return value
def _get_var_number(test_info_line):
assert 'object' in test_info_line
# Object number should be at end, e.g. 'object 1: ...'
return test_info_line.split(':')[0].split(' ')[-1]
ktest_tool = [os.path.join(bin_dir, 'ktest-tool')]
exec_output = utils.execute(ktest_tool + [test.origin],
err_to_output=False,
quiet=True)
test_info = exec_output.stdout.split('\n')
vector = utils.TestVector(test.name, test.origin)
last_number = -1
last_nondet_method = None
last_value = None
for line in [l for l in test_info if l.startswith('object')]:
logging.debug("Looking at line: %s", line)
if 'name:' in line:
# assert len(line.split(':')) == 3
var_number = int(_get_var_number(line))
assert var_number > last_number
last_number = var_number
var_name = _get_value(line)
assert last_nondet_method is None, \
"Last nondet method already or still assigned: %s" % last_nondet_method
assert "'" not in var_name, \
"Variable name contains \"'\": %s" % var_name
last_nondet_method = utils.get_corresponding_method_name(
var_name)
elif 'data:' in line:
# assert len(line.split(':')) == 3
var_number = _get_var_number(line)
assert last_nondet_method is not None
# we get the value in bytestring notation, so we have to convert it afterwards
value = _get_value(line)
value = _convert_bytestring_to_hex(value)
assert last_value is None
last_value = str(value)
if last_nondet_method is not None and last_value is not None:
vector.add(last_value, last_nondet_method)
last_nondet_method = None
last_value = None
return vector
def test_generation_and_parsing_consistent():
print_inputs = utils.get_executable(
os.path.join(module_dir, "print_inputs"))
check_inputs = utils.get_executable(
os.path.join(module_dir, "check_inputs"))
if os.path.exists(VECTOR_FILE):
os.remove(VECTOR_FILE)
input_result = utils.execute(print_inputs, quiet=True)
expected = input_result.stdout
with open(VECTOR_FILE, 'r') as inp:
created_values = "\n".join(
l.split(":")[1].strip() for l in inp.readlines())
parse_result = utils.execute(check_inputs,
input_str=created_values,
quiet=True)
actual = parse_result.stdout
n.assert_equal(expected, actual)
def run(self, program_file, test_case):
from tbf.tools import klee
klee_prepared_file = utils.get_prepared_name(program_file, klee.name)
c_version = 'gnu11'
if not self.executable:
compile_cmd = ['gcc']
compile_cmd += [
'-std={}'.format(c_version), "-L", klee.lib_dir,
'-D__alias__(x)=', '-o', self.executable_name,
klee_prepared_file, '-lkleeRuntest', '-lm'
]
result = utils.execute(compile_cmd)
if result.returncode != 0:
c_version = 'gnu90'
compile_cmd = ['gcc']
compile_cmd += [
'-std={}'.format(c_version), "-L", klee.lib_dir,
'-D__alias__(x)=', '-o', self.executable_name,
klee_prepared_file, '-lkleeRuntest', '-lm'
]
self.executable = self.executable_name
if not os.path.exists(self.executable_name):
return [ERROR]
curr_env = utils.get_env()
curr_env['KTEST_FILE'] = test_case.origin
result = utils.execute([self.executable],
env=curr_env,
err_to_output=False)
if utils.found_err(result):
return [FALSE]
else:
return [UNKNOWN]
def get_coverage(self, program_file):
cmd = ['gcov', '-bc', self.harness_file]
res = utils.execute(cmd, quiet=False, err_to_output=False)
full_cov = res.stdout.splitlines()
program_name = os.path.basename(program_file)
lines_executed = None
branches_executed = None
branches_taken = None
for number, line in enumerate(full_cov):
if line.startswith('File') and program_name in line:
lines_executed = self._get_gcov_val(full_cov[number + 1])
branches_executed = self._get_gcov_val(full_cov[number + 2])
branches_taken = self._get_gcov_val(full_cov[number + 3])
break
return lines_executed, branches_executed, branches_taken
def run(self, program_file, test_vector):
executable = self.get_executable_harness(program_file)
input_vector = utils.get_input_vector(test_vector)
if executable:
run_cmd = self._get_run_cmd(executable)
run_result = utils.execute(run_cmd,
quiet=True,
err_to_output=False,
input_str=input_vector,
timelimit=5)
if utils.found_err(run_result):
return [FALSE]
else:
return [UNKNOWN]
else:
return [ERROR]
def _get_test_vector(self, test):
def _get_value(single_line):
var_name = single_line.split(':')[2].strip()
prefix_end = var_name.find("'")
var_name = var_name[prefix_end + 1:-1]
return var_name
ktest_tool = [os.path.join(bin_dir, 'ktest-tool')]
exec_output = utils.execute(ktest_tool + [test.origin],
err_to_output=False,
quiet=True)
test_info = exec_output.stdout.split('\n')
vector = utils.TestVector(test.name, test.origin)
last_number = -1
last_nondet_method = None
last_value = None
for line in [l for l in test_info if l.startswith('object')]:
logging.debug("Looking at line: %s", line)
if 'name:' in line:
#assert len(line.split(':')) == 3
var_number = int(self._get_var_number(line))
assert var_number > last_number
last_number = var_number
var_name = _get_value(line)
assert last_nondet_method is None, \
"Last nondet method already or still assigned: %s" % last_nondet_method
assert "'" not in var_name, \
"Variable name contains \"'\": %s" % var_name
last_nondet_method = utils.get_corresponding_method_name(
var_name)
elif 'data:' in line:
#assert len(line.split(':')) == 3
var_number = self._get_var_number(line)
assert last_nondet_method is not None
value = _get_value(line)
value, = utils.convert_to_int(value, last_nondet_method)
assert last_value is None
last_value = str(value)
if last_nondet_method is not None and last_value is not None:
vector.add(last_value, last_nondet_method)
last_nondet_method = None
last_value = None
return vector