def load_other_rubric(rubric_file, logging_level=log.LOGLEVEL_ERROR):
deduct = list()
if not sysio.exist_file(rubric_file):
log.log_warning('Rubric file {} not found'.format(rubric_file),
logging_level)
else:
with open(rubric_file, 'r') as rfile:
rubric_reader = csv.reader(rfile, delimiter='\t')
for deduct_row in rubric_reader:
if len(deduct_row) != 2:
log.log_error(
"Row should have two columns: " + str(deduct_row),
logging_level)
elif not misc.is_num(deduct_row[0].strip()):
log.log_error(
"First column needs to be number: " + str(deduct_row),
logging_level)
else:
deduct.append({
RUBRIC_OTHER_POINT:
float(deduct_row[0].strip()),
RUBRIC_OTHER_COMMENT:
deduct_row[1].strip()
})
return deduct
def generate_grade_report(homework,
grader,
report_dir,
overwrite=False,
logging_level=log.LOGLEVEL_ERROR):
report_filename = report_dir + 'GR{}_hw-username.md'.format(
str(homework.number))
if sysio.exist_file(report_filename):
if not overwrite:
log.log_error('Report {} already exists'.format(report_filename),
logging_level)
return
else:
log.log_warning(
'Overwriting existing report {}'.format(report_filename),
logging_level)
report_file = open(report_filename, mode='w')
title = '## HW {num:02d} Test Case Grade Report'.format(
num=homework.number)
md.write_header(report_file, title, GRHLEVEL_TITLE)
homework.write_score_breakdown(report_file)
grader_text = 'HW Graded by: {}'.format(grader.get_grader_info())
md.write_header(report_file, grader_text, GRHLEVEL_OTHER)
md.write_paragraph(report_file, GRFOOTER)
report_file.close()
def run_tests_in_list(executable_path, test_names, test_args, **kwargs):
if kwargs is None:
kwargs = dict()
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
all_norun = set()
all_timeout = set()
all_finished = dict()
if len(test_names) != len(test_args):
log.log_error(
'Found {} tests but {} test arguments'.format(
str(len(test_names)), str(len(test_args))), logging_level)
return set(x for x in test_names), all_timeout, all_finished
for i, test_name in enumerate(test_names):
if not sysio.exist_file(executable_path):
log.log_warning('Executable {} not found'.format(executable_path),
logging_level)
all_norun.add(test_name)
else:
kwargs['extra_arguments'] = test_args[i]
killed, runtime, _ = exe.run_executable(executable_path, **kwargs)
if killed == exe.EXE_ERROR:
all_norun.add(test_name)
elif killed == exe.EXE_TIMEOUT:
all_timeout.add(test_name)
else:
all_finished[test_name] = runtime
return all_norun, all_timeout, all_finished
def load_grader_info(self, grader_info, logging_level=log.LOGLEVEL_ERROR):
if sysio.exist_file(grader_info):
grader_file = open(grader_info, mode='r', errors='ignore')
lines = grader_file.readlines()
grader_file.close()
if len(lines) < 2:
log.log_error(
'Grader info needs 2 lines but found {}'.format(
str(len(lines))), logging_level)
else:
if len(lines) > 2:
log.log_warning(
'Grader info needs 2 lines but found {}'.format(
str(len(lines))), logging_level)
self.name = lines[0].strip()
self.github = lines[1].strip()
else:
log.log_error('Grader info file {} not found'.format(grader_info),
logging_level)
def write_formatted_result(formatted_file,
test_names,
valgrind,
norun,
timeout,
crashed,
failed,
passed,
logging_level=log.LOGLEVEL_ERROR):
for idx, test_name in enumerate(test_names):
result = None
vcode = valgrind[idx] if idx < len(valgrind) else exe.EXE_ERROR
runtime = None
if test_name in passed:
result = ERESULT_PASS
runtime = passed[test_name]
elif test_name in norun:
result = ERESULT_NORUN
elif test_name in timeout:
result = ERESULT_TIMEOUT
elif test_name in crashed:
result = ERESULT_CRASH
elif test_name in failed:
result = ERESULT_FAIL
runtime = failed[test_name]
else:
log.log_error(
'{} not in any execution result set'.format(test_name),
logging_level)
if vcode == exe.EXE_ERROR:
vresult = VRESULT_IGNORE
elif vcode == exe.VALGRIND_SUCCESS:
vresult = VRESULT_PASS
elif vcode == exe.VALGRIND_ERROR or vcode == exe.VALGRIND_SIGSEGV:
vresult = VRESULT_FAIL
else:
log.log_warning('Unknown return code ' + str(vcode), logging_level)
vresult = VRESULT_IGNORE
write_formatted_line(formatted_file, test_name, result, vresult,
runtime)
def load_config(config, config_file, logging_level=log.LOGLEVEL_ERROR):
if not sysio.exist_file(config_file):
log.log_warning('Rubric file {} not found'.format(config_file),
logging_level)
else:
config.read(config_file)
def cmake_problem(problem):
# set path to output files
problem.compile_file = os.path.join('compile-logs',
problem.name + '.complog')
stdout_file_path = os.path.join('test-output',
problem.name + '-test-stdout.txt')
stdout_file = open(stdout_file_path, 'w')
# Find tests' output XML file, buiried a couple of directories deep
xml_path = glob.glob("Testing/*-*/Test.xml")
if len(xml_path) == 0:
logging_tools.log_error("Cannot find test XML output file!",
problem.logging_level)
return
elif len(xml_path) > 1:
logging_tools.log_error(
"Multiple candidates for test XML file: " + " ".join(xml_path),
problem.logging_level)
return
logging_tools.log_info("Found XML output file: " + xml_path[0],
problem.logging_level)
# parse XML file
test_xml = open(xml_path[0])
test_results = xmltodict.parse(test_xml.read())
test_xml.close()
test_list = []
didnt_run_tests = set()
crashed_tests = set()
timed_out_tests = set()
failed_tests = {}
passed_tests = {}
# Valgrind exit codes, indexed by test
valgrind_exit_codes = []
# now, go through all tests
test_results_list_element = test_results['Site']['Testing']['Test']
for test_results_element in test_results_list_element:
if problem.name in test_results_element['Path']:
test_name = test_results_element['Name']
#print("\n>> Processing test: " + test_name)
# write test results to output file
stdout_file.write("""
------------------------------------------------------------------------------
OUTPUT OF TEST %s:
------------------------------------------------------------------------------
""" % test_name)
stdout_file.write(
test_results_element['Results']['Measurement']['Value'])
# detect Valgrind failues
# note: we want to assign seperate deductions for Valgrind failures and actual test case failures. So,
# we can't use Valgrind's --error-exitcode option, since that would make CTest think all the tests had failed.
valgrind_error = False
match_list = valgrind_results_re.findall(
test_results_element['Results']['Measurement']['Value'])
if match_list is None or len(match_list) < 1:
# program may have died before it got to even producing the valgrind output
logging_tools.log_warning(
"cmake_problem.py: could not find valgrind results for test "
+ test_name, problem.logging_level)
else:
# make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary
definitely_lost = int(match_list[-1][0].replace(",", ""))
indirectly_lost = int(match_list[-1][1].replace(",", ""))
possibly_lost = int(match_list[-1][2].replace(",", ""))
still_reachable = int(match_list[-1][3].replace(",", ""))
suppressed = int(match_list[-1][4].replace(",", ""))
#print("Valgrind Results: definitely_lost: %d, indirectly_lost: %d, possibly_lost: %d, still_reachable: %d, suppressed: %d" % (definitely_lost, indirectly_lost, possibly_lost, still_reachable, suppressed))
if definitely_lost > 0 or indirectly_lost > 0 or possibly_lost > 0 or still_reachable > 0:
valgrind_error = True
if subprocess_valgrind_failure_re.search(
test_results_element['Results']['Measurement']
['Value']) != None:
#print("Valgrind errors found in subprocess execution!")
valgrind_error = True
# now, parse out the test status
passed = test_results_element['@Status'] == 'passed'
# true if the test was not run (as in, it failed to build)
didnt_run = False
# true if the test dies with a segfault/sigfpe/etc
crashed = False
# true if test timed out
timed_out = False
# iterate through namedmeasurements
test_time = -1.0
for measurement_element in test_results_element['Results'][
'NamedMeasurement']:
if measurement_element['@name'] == 'Execution Time':
test_time = float(measurement_element['Value'])
if not passed and measurement_element['@name'] == 'Exit Code':
if measurement_element['Value'] == 'Timeout':
timed_out = True
elif measurement_element['Value'] != 'Failed':
# if exit code is something other than Failed it means that the test crashed
crashed = True
if not passed and measurement_element['@name'] == 'Exit Value':
if int(measurement_element['Value']) == 127:
# this exit code appears to indicate that the test was not run
didnt_run = True
#print("crashed: %r, passed: %r" % (crashed, passed))
# write test data to collections
test_list.append(test_name)
if crashed:
crashed_tests.add(test_name)
elif timed_out:
timed_out_tests.add(test_name)
elif didnt_run:
didnt_run_tests.add(test_name)
elif passed:
passed_tests[test_name] = test_time
else:
failed_tests[test_name] = test_time
# figure out what the Valgrind exit code should have been for this test
if valgrind_error:
valgrind_exit_codes.append(executable_tools.VALGRIND_ERROR)
elif crashed:
valgrind_exit_codes.append(executable_tools.EXE_ERROR)
else:
valgrind_exit_codes.append(executable_tools.VALGRIND_SUCCESS)
stdout_file.close()
# write test result files
cs_grading.write_test_result(problem.result_file,
test_list,
didnt_run_tests,
timed_out_tests,
crashed_tests,
failed_tests,
passed_tests,
logging_level=problem.logging_level)
print("Writing formatted file: %s " % problem.formatted_file)
cs_grading.write_formatted_result(problem.formatted_file,
test_list,
valgrind_exit_codes,
didnt_run_tests,
timed_out_tests,
crashed_tests,
failed_tests,
passed_tests,
logging_level=problem.logging_level)
def run_executable(executable_path, **kwargs):
if kwargs is None:
kwargs = dict()
extra_arguments = kwargs.get('extra_arguments', list())
use_valgrind = kwargs.get('use_valgrind', False)
valgrind_file = kwargs.get('valgrind_file', None)
timeout = kwargs.get('timeout', None)
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
logging_force_suppressed = kwargs.get('logging_force_suppressed', False)
killed = EXE_ERROR
utime = EXE_ERROR
retcode = EXE_ERROR
error = False
args = []
if use_valgrind:
if valgrind_file is None:
log.log_warning(
'valgrind turned on but no valgrind log file speficied',
logging_level)
temp_valgrind_log = 'temp_valgrind_log.txt'
args.extend(VALGRIND)
args.append('--log-file=' + temp_valgrind_log)
redirected_stdin_file = None
redirected_stdout_mode = None
redirected_stdout_file = None
redirected_stderr_mode = None
redirected_stderr_file = None
i = 0
while i < len(extra_arguments):
if extra_arguments[i] == '<':
# redirect input
if i + 1 >= len(extra_arguments):
log.log_error('Found input redirection with no input file',
logging_level)
error = True
break
else:
redirected_stdin_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '>' or extra_arguments[i] == '>>':
# redirect output
if extra_arguments[i] == '>':
redirected_stdout_mode = 'w'
else:
redirected_stdout_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found output redirection with no output file',
logging_level)
error = True
break
else:
redirected_stdout_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
elif extra_arguments[i] == '2>' or extra_arguments[i] == '2>>':
# redirect output
if extra_arguments[i] == '2>':
redirected_stderr_mode = 'w'
else:
redirected_stderr_mode = 'a'
if i + 1 >= len(extra_arguments):
log.log_error('Found error redirection with no output file',
logging_level)
error = True
break
else:
redirected_stderr_file = extra_arguments[i + 1]
del extra_arguments[i + 1]
del extra_arguments[i]
else:
i += 1
if not error:
extra_arguments = [executable_path] + extra_arguments
if not logging_force_suppressed:
log.log_info('Running ' + ' '.join(extra_arguments),
log.LOGLEVEL_INFO)
args.extend(extra_arguments)
if redirected_stdout_file is not None:
redirected_stdout_file = open(redirected_stdout_file,
redirected_stdout_mode)
if redirected_stderr_file is not None:
redirected_stderr_file = open(redirected_stderr_file,
redirected_stderr_mode)
if redirected_stdin_file is not None:
redirected_stdin_file = open(redirected_stdin_file, 'r')
try:
start_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
proc = subprocess.Popen(args,
stdout=redirected_stdout_file,
stderr=redirected_stderr_file,
stdin=redirected_stdin_file)
proc.communicate(timeout=timeout)
killed = EXE_SUCCESS
retcode = proc.returncode
except subprocess.TimeoutExpired:
log.log_warning('Executable {} timed out'.format(executable_path),
logging_level)
killed = EXE_TIMEOUT
proc.kill()
except OSError:
log.log_warning('Executable {} not found'.format(executable_path),
logging_level)
killed = EXE_ERROR
end_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
utime = end_time - start_time
if redirected_stdin_file is not None:
redirected_stdin_file.close()
if redirected_stdout_file is not None:
redirected_stdout_file.close()
if redirected_stderr_file is not None:
redirected_stderr_file.close()
if use_valgrind:
if valgrind_file is not None:
sysio.write_file_contents(valgrind_file,
temp_valgrind_log,
logging_level=logging_level)
sysio.write_message(valgrind_file, '\n\n')
sysio.clean_file(temp_valgrind_log)
return killed, utime, retcode
