def run_gtest_with_filter(executable_path, gtest_filters, **kwargs):
if kwargs is None:
kwargs = dict()
output_result = kwargs.get('output_result', None)
separate_result = kwargs.get('separate_result', True)
filter_arg = '--gtest_filter='
temp_log_file = 'temp_gtest_log.txt'
result = []
for filtered_test in gtest_filters:
full_filter = [filter_arg + filtered_test]
extra_arguments = []
if output_result is not None:
if separate_result:
separate_result = output_result + filtered_test + '.txt'
extra_arguments = ['>', separate_result]
else:
extra_arguments = ['>', temp_log_file]
extra_arguments_filtered = full_filter + extra_arguments
kwargs['extra_arguments'] = extra_arguments_filtered
killed, _, _ = exe.run_executable(
exe.fix_executable_path(executable_path), **kwargs)
result.append(killed)
if output_result is not None and not separate_result:
sysio.write_file_contents(output_result, temp_log_file)
sysio.write_message(output_result, '\n\n')
sysio.clean_file(temp_log_file)
return result
def write_formatted_line(formatted_file,
test_name,
result,
vresult,
runtime=None):
if runtime is not None:
formatted_line = '{} {} {} ({:.3f})\n'.format(result, vresult,
test_name, runtime)
else:
formatted_line = '{} {} {}\n'.format(result, vresult, test_name)
sysio.write_message(formatted_file, formatted_line)
def check_test_output(test_names,
solution_list,
output_list,
result_file,
ordered_compare,
**kwargs
):
if kwargs is None:
kwargs = dict()
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
all_failed = set()
all_passed = set()
if len(test_names) != len(output_list):
log.log_error('Found {} test names but {} test output'.format(
str(len(test_names)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
if len(solution_list) != len(output_list):
log.log_error('Found {} test solution but {} test output'.format(
str(len(solution_list)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
for i, test_name in enumerate(test_names):
t_solution = solution_list[i]
t_output = output_list[i]
if not sysio.exist_file(t_solution):
log.log_error('Cannot find solution file ' + t_solution, logging_level)
all_failed.add(test_name)
else:
_, missing, extra = comp.compare_files(
t_solution,
t_output,
result_file,
ordered_compare=ordered_compare,
**kwargs
)
sysio.write_message(result_file, '\n\n')
if missing == 0 and extra == 0:
all_passed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_PASS)
else:
all_failed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_FAIL)
return all_failed, all_passed
def write_test_result(result_file,
test_names,
norun,
timeout,
crashed,
failed,
passed,
final_log=False,
logging_level=log.LOGLEVEL_ERROR):
all_failed = list()
for test_name in test_names:
if test_name in passed:
if not final_log:
write_result_line(result_file, test_name, ERESULT_PASS)
else:
all_failed.append(test_name)
if not (test_name in norun or test_name in timeout
or test_name in crashed or test_name in failed):
log.log_error(
'{} not in any execution result set'.format(test_name),
logging_level)
elif not final_log:
if test_name in norun:
write_result_line(result_file, test_name, ERESULT_NORUN)
elif test_name in timeout:
write_result_line(result_file, test_name, ERESULT_TIMEOUT)
elif test_name in crashed:
write_result_line(result_file, test_name, ERESULT_CRASH)
elif test_name in failed:
write_result_line(result_file, test_name, ERESULT_FAIL)
sysio.write_message(result_file, '\n')
if not all_failed:
sysio.write_message(result_file, 'All Test Cases Passed')
else:
sysio.write_message(result_file,
'Failed ' + str(len(all_failed)) + ' tests:\n')
for ftest in all_failed:
sysio.write_message(result_file, '\tTest ' + ftest + '\n')
def write_result_line(result_file, test_name, result):
sysio.write_message(
result_file, 'Test {} {}\n'.format(test_name,
ERESULT_OUTPUT.get(result, '')))
def run_executable(executable_path, **kwargs):
if kwargs is None:
kwargs = dict()
extra_arguments = kwargs.get('extra_arguments', list())
use_valgrind = kwargs.get('use_valgrind', False)
valgrind_file = kwargs.get('valgrind_file', None)
timeout = kwargs.get('timeout', None)
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
logging_force_suppressed = kwargs.get('logging_force_suppressed', False)
killed = EXE_ERROR
utime = EXE_ERROR
retcode = EXE_ERROR
error = False
args = []
if use_valgrind:
if valgrind_file is None:
log.log_warning(
'valgrind turned on but no valgrind log file speficied',
logging_level)
temp_valgrind_log = 'temp_valgrind_log.txt'
args.extend(VALGRIND)
args.append('--log-file=' + temp_valgrind_log)
redirected_stdin_file = None
redirected_stdout_mode = None
redirected_stdout_file = None
redirected_stderr_mode = None
redirected_stderr_file = None
i = 0
while i < len(extra_arguments):
if extra_arguments[i] == '<':
# redirect input
if i + 1 >= len(extra_arguments):
log.log_error('Found input redirection with no input file',
logging_level)
error = True
break
else:
redirected_stdin_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '>' or extra_arguments[i] == '>>':
# redirect output
if extra_arguments[i] == '>':
redirected_stdout_mode = 'w'
else:
redirected_stdout_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found output redirection with no output file',
logging_level)
error = True
break
else:
redirected_stdout_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '2>' or extra_arguments[i] == '2>>':
# redirect output
if extra_arguments[i] == '2>':
redirected_stderr_mode = 'w'
else:
redirected_stderr_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found error redirection with no output file',
logging_level)
error = True
break
else:
redirected_stderr_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
else:
i += 1
if not error:
extra_arguments = [executable_path] + extra_arguments
if not logging_force_suppressed:
log.log_info('Running ' + ' '.join(extra_arguments),
log.LOGLEVEL_INFO)
args.extend(extra_arguments)
if redirected_stdout_file is not None:
redirected_stdout_file = open(redirected_stdout_file,
redirected_stdout_mode)
if redirected_stderr_file is not None:
redirected_stderr_file = open(redirected_stderr_file,
redirected_stderr_mode)
if redirected_stdin_file is not None:
redirected_stdin_file = open(redirected_stdin_file, 'r')
try:
start_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
proc = subprocess.Popen(args,
stdout=redirected_stdout_file,
stderr=redirected_stderr_file,
stdin=redirected_stdin_file)
proc.communicate(timeout=timeout)
killed = EXE_SUCCESS
retcode = proc.returncode
except subprocess.TimeoutExpired:
log.log_warning('Executable {} timed out'.format(executable_path),
logging_level)
killed = EXE_TIMEOUT
proc.kill()
except OSError:
log.log_warning('Executable {} not found'.format(executable_path),
logging_level)
killed = EXE_ERROR
end_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
utime = end_time - start_time
if redirected_stdin_file is not None:
redirected_stdin_file.close()
if redirected_stdout_file is not None:
redirected_stdout_file.close()
if redirected_stderr_file is not None:
redirected_stderr_file.close()
if use_valgrind:
if valgrind_file is not None:
sysio.write_file_contents(valgrind_file,
temp_valgrind_log,
logging_level=logging_level)
sysio.write_message(valgrind_file, '\n\n')
sysio.clean_file(temp_valgrind_log)
return killed, utime, retcode