def write_test_result(result_file,
test_names,
norun,
timeout,
crashed,
failed,
passed,
final_log=False,
logging_level=log.LOGLEVEL_ERROR):
all_failed = list()
for test_name in test_names:
if test_name in passed:
if not final_log:
write_result_line(result_file, test_name, ERESULT_PASS)
else:
all_failed.append(test_name)
if not (test_name in norun or test_name in timeout
or test_name in crashed or test_name in failed):
log.log_error(
'{} not in any execution result set'.format(test_name),
logging_level)
elif not final_log:
if test_name in norun:
write_result_line(result_file, test_name, ERESULT_NORUN)
elif test_name in timeout:
write_result_line(result_file, test_name, ERESULT_TIMEOUT)
elif test_name in crashed:
write_result_line(result_file, test_name, ERESULT_CRASH)
elif test_name in failed:
write_result_line(result_file, test_name, ERESULT_FAIL)
sysio.write_message(result_file, '\n')
if not all_failed:
sysio.write_message(result_file, 'All Test Cases Passed')
else:
sysio.write_message(result_file,
'Failed ' + str(len(all_failed)) + ' tests:\n')
for ftest in all_failed:
sysio.write_message(result_file, '\tTest ' + ftest + '\n')
def generate_grade_report(homework,
grader,
report_dir,
overwrite=False,
logging_level=log.LOGLEVEL_ERROR):
report_filename = report_dir + 'GR{}_hw-username.md'.format(
str(homework.number))
if sysio.exist_file(report_filename):
if not overwrite:
log.log_error('Report {} already exists'.format(report_filename),
logging_level)
return
else:
log.log_warning(
'Overwriting existing report {}'.format(report_filename),
logging_level)
report_file = open(report_filename, mode='w')
title = 'HW {num:02d} Test Case Grade Report'.format(num=homework.number)
md.write_header(report_file, title, GRHLEVEL_TITLE)
homework.write_score_breakdown(report_file)
md.write_paragraph(report_file, GRFOOTER)
report_file.close()
def read_formatted_line(formatted_line, logging_level=log.LOGLEVEL_ERROR):
result = None
vresult = None
test_name = ''
runtime = None
stripped_line = formatted_line.strip()
for result_code in ERESULT_OUTPUT:
if stripped_line.find(result_code) != -1:
if result is not None:
log.log_error(
'Multiple test result code in line ' + formatted_line,
logging_level)
else:
result = result_code
for result_code in VRESULT_OUTPUT:
if stripped_line.find(result_code) != -1:
if vresult is not None:
log.log_error(
'Multiple valgrind result code in line ' + formatted_line,
logging_level)
else:
vresult = result_code
if result is None:
log.log_error(
'Failed to identify test result in line ' + formatted_line,
logging_level)
if vresult is None:
log.log_error(
'Failed to identify valgrind result in line ' + formatted_line,
logging_level)
if result is not None and vresult is not None:
stripped_line = stripped_line.replace(result, '').strip()
stripped_line = stripped_line.replace(vresult, '').strip()
runtime_start = stripped_line.find('(')
if runtime_start == -1:
test_name = stripped_line
else:
test_name = stripped_line[:runtime_start].strip()
runtime_end = stripped_line.find(')')
runtime = float(stripped_line[runtime_start + 1:runtime_end])
return test_name, result, vresult, runtime
def check_test_output(test_names,
solution_list,
output_list,
result_file,
ordered_compare,
**kwargs
):
if kwargs is None:
kwargs = dict()
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
all_failed = set()
all_passed = set()
if len(test_names) != len(output_list):
log.log_error('Found {} test names but {} test output'.format(
str(len(test_names)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
if len(solution_list) != len(output_list):
log.log_error('Found {} test solution but {} test output'.format(
str(len(solution_list)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
for i, test_name in enumerate(test_names):
t_solution = solution_list[i]
t_output = output_list[i]
if not sysio.exist_file(t_solution):
log.log_error('Cannot find solution file ' + t_solution, logging_level)
all_failed.add(test_name)
else:
_, missing, extra = comp.compare_files(
t_solution,
t_output,
result_file,
ordered_compare=ordered_compare,
**kwargs
)
sysio.write_message(result_file, '\n\n')
if missing == 0 and extra == 0:
all_passed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_PASS)
else:
all_failed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_FAIL)
return all_failed, all_passed
def cmake_problem(problem):
# set path to output files
problem.compile_file = os.path.join('compile-logs',
problem.name + '.complog')
stdout_file_path = os.path.join('test-output',
problem.name + '-test-stdout.txt')
stdout_file = open(stdout_file_path, 'w')
# Find tests' output XML file, buried a couple of directories deep
xml_path = glob.glob("Testing/*-*/Test.xml")
if len(xml_path) == 0:
logging_tools.log_error("Cannot find test XML output file!",
problem.logging_level)
return
elif len(xml_path) > 1:
logging_tools.log_error(
"Multiple candidates for test XML file: " + " ".join(xml_path),
problem.logging_level)
return
logging_tools.log_info("Found XML output file: " + xml_path[0],
problem.logging_level)
# parse XML file
test_xml = open(xml_path[0])
test_results = xmltodict.parse(test_xml.read())
test_xml.close()
test_list = []
didnt_run_tests = set()
crashed_tests = set()
timed_out_tests = set()
failed_tests = {}
passed_tests = {}
# Valgrind exit codes, indexed by test
valgrind_exit_codes = []
# now, go through all tests
test_results_list_element = test_results['Site']['Testing']['Test']
for test_results_element in test_results_list_element:
if problem.name in test_results_element['Path']:
test_name = test_results_element['Name']
#print("\n>> Processing test: " + test_name)
# write test results to output file
stdout_file.write("""
------------------------------------------------------------------------------
OUTPUT OF TEST %s:
------------------------------------------------------------------------------
""" % test_name)
stdout_file.write(
test_results_element['Results']['Measurement']['Value'])
# detect Valgrind failues
# note: we want to assign seperate deductions for Valgrind failures and actual test case failures. So,
# we can't use Valgrind's --error-exitcode option, since that would make CTest think all the tests had failed.
valgrind_error = False
match_list = valgrind_results_re.findall(
test_results_element['Results']['Measurement']['Value'])
if match_list is None or len(match_list) < 1:
# program may have died before it got to even producing the valgrind output, or it's a test that doesn't use Valgrind
pass
else:
# make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary
definitely_lost = int(match_list[-1][0].replace(",", ""))
indirectly_lost = int(match_list[-1][1].replace(",", ""))
possibly_lost = int(match_list[-1][2].replace(",", ""))
still_reachable = int(match_list[-1][3].replace(",", ""))
suppressed = int(match_list[-1][4].replace(",", ""))
# print("Valgrind Results: definitely_lost: %d, indirectly_lost: %d, possibly_lost: %d, still_reachable: %d, suppressed: %d" % (definitely_lost, indirectly_lost, possibly_lost, still_reachable, suppressed))
if definitely_lost > 0 or indirectly_lost > 0 or possibly_lost > 0 or still_reachable > 0:
valgrind_error = True
# now look for "X errors in X contexts"
error_match_list = valgrind_errors_re.findall(
test_results_element['Results']['Measurement']['Value'])
if error_match_list is None or len(error_match_list) < 1:
# program may have died before it got to even producing the valgrind output, or it's a test that doesn't use Valgrind
pass
else:
# make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary
num_errors = int(error_match_list[-1].replace(",", ""))
#print("%d valgrind errors" % num_errors)
if num_errors > 0:
valgrind_error = True
if subprocess_valgrind_failure_re.search(
test_results_element['Results']['Measurement']
['Value']) != None:
# print("Valgrind errors found in subprocess execution!")
valgrind_error = True
# now, parse out the test status
passed = test_results_element['@Status'] == 'passed'
# true if the test was not run (as in, it failed to build)
didnt_run = False
# true if the test dies with a segfault/sigfpe/etc
crashed = False
# true if test timed out
timed_out = False
test_time = -1.0
if type(test_results_element['Results']
['NamedMeasurement']) != list:
# if the test did not run because the excutable was not found, there might not be the usual set of NamedMeasurements
didnt_run = True
elif test_results_element["@Status"] == "notrun":
didnt_run = True
else:
# iterate through namedmeasurements
for measurement_element in test_results_element['Results'][
'NamedMeasurement']:
if measurement_element['@name'] == 'Execution Time':
test_time = float(measurement_element['Value'])
if not passed and measurement_element[
'@name'] == 'Exit Code':
if measurement_element['Value'] == 'Timeout':
timed_out = True
elif measurement_element['Value'] != 'Failed':
# if exit code is something other than Failed it means that the test crashed
crashed = True
if not passed and measurement_element[
'@name'] == 'Exit Value':
if int(measurement_element['Value']) == 127:
# Valgrind returns this to indicate that the test was not run
didnt_run = True
# print("crashed: %r, passed: %r" % (crashed, passed))
# write test data to collections
test_list.append(test_name)
if crashed:
crashed_tests.add(test_name)
elif timed_out:
timed_out_tests.add(test_name)
elif didnt_run:
didnt_run_tests.add(test_name)
elif passed:
passed_tests[test_name] = test_time
else:
failed_tests[test_name] = test_time
# figure out what the Valgrind exit code should have been for this test
if valgrind_error:
valgrind_exit_codes.append(executable_tools.VALGRIND_ERROR)
elif crashed:
valgrind_exit_codes.append(executable_tools.EXE_ERROR)
else:
valgrind_exit_codes.append(executable_tools.VALGRIND_SUCCESS)
stdout_file.close()
# write test result files
cs_grading.write_test_result(problem.result_file,
test_list,
didnt_run_tests,
timed_out_tests,
crashed_tests,
failed_tests,
passed_tests,
logging_level=problem.logging_level)
cs_grading.write_formatted_result(problem.formatted_file,
test_list,
valgrind_exit_codes,
didnt_run_tests,
timed_out_tests,
crashed_tests,
failed_tests,
passed_tests,
logging_level=problem.logging_level)
def run_executable(executable_path, **kwargs):
if kwargs is None:
kwargs = dict()
extra_arguments = kwargs.get('extra_arguments', list())
use_valgrind = kwargs.get('use_valgrind', False)
valgrind_file = kwargs.get('valgrind_file', None)
timeout = kwargs.get('timeout', None)
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
logging_force_suppressed = kwargs.get('logging_force_suppressed', False)
killed = EXE_ERROR
utime = EXE_ERROR
retcode = EXE_ERROR
error = False
args = []
if use_valgrind:
if valgrind_file is None:
log.log_warning(
'valgrind turned on but no valgrind log file speficied',
logging_level)
temp_valgrind_log = 'temp_valgrind_log.txt'
args.extend(VALGRIND)
args.append('--log-file=' + temp_valgrind_log)
redirected_stdin_file = None
redirected_stdout_mode = None
redirected_stdout_file = None
redirected_stderr_mode = None
redirected_stderr_file = None
i = 0
while i < len(extra_arguments):
if extra_arguments[i] == '<':
# redirect input
if i + 1 >= len(extra_arguments):
log.log_error('Found input redirection with no input file',
logging_level)
error = True
break
else:
redirected_stdin_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '>' or extra_arguments[i] == '>>':
# redirect output
if extra_arguments[i] == '>':
redirected_stdout_mode = 'w'
else:
redirected_stdout_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found output redirection with no output file',
logging_level)
error = True
break
else:
redirected_stdout_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '2>' or extra_arguments[i] == '2>>':
# redirect output
if extra_arguments[i] == '2>':
redirected_stderr_mode = 'w'
else:
redirected_stderr_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found error redirection with no output file',
logging_level)
error = True
break
else:
redirected_stderr_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
else:
i += 1
if not error:
extra_arguments = [executable_path] + extra_arguments
if not logging_force_suppressed:
log.log_info('Running ' + ' '.join(extra_arguments),
log.LOGLEVEL_INFO)
args.extend(extra_arguments)
if redirected_stdout_file is not None:
redirected_stdout_file = open(redirected_stdout_file,
redirected_stdout_mode)
if redirected_stderr_file is not None:
redirected_stderr_file = open(redirected_stderr_file,
redirected_stderr_mode)
if redirected_stdin_file is not None:
redirected_stdin_file = open(redirected_stdin_file, 'r')
try:
start_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
proc = subprocess.Popen(args,
stdout=redirected_stdout_file,
stderr=redirected_stderr_file,
stdin=redirected_stdin_file)
proc.communicate(timeout=timeout)
killed = EXE_SUCCESS
retcode = proc.returncode
except subprocess.TimeoutExpired:
log.log_warning('Executable {} timed out'.format(executable_path),
logging_level)
killed = EXE_TIMEOUT
proc.kill()
except OSError:
log.log_warning('Executable {} not found'.format(executable_path),
logging_level)
killed = EXE_ERROR
end_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
utime = end_time - start_time
if redirected_stdin_file is not None:
redirected_stdin_file.close()
if redirected_stdout_file is not None:
redirected_stdout_file.close()
if redirected_stderr_file is not None:
redirected_stderr_file.close()
if use_valgrind:
if valgrind_file is not None:
sysio.write_file_contents(valgrind_file,
temp_valgrind_log,
logging_level=logging_level)
sysio.write_message(valgrind_file, '\n\n')
sysio.clean_file(temp_valgrind_log)
return killed, utime, retcode