def load_other_rubric(rubric_file, logging_level=log.LOGLEVEL_ERROR):
deduct = list()
if not sysio.exist_file(rubric_file):
log.log_warning('Rubric file {} not found'.format(rubric_file),
logging_level)
else:
with open(rubric_file, 'r') as rfile:
rubric_reader = csv.reader(rfile, delimiter='\t')
for deduct_row in rubric_reader:
if len(deduct_row) != 2:
log.log_error(
"Row should have two columns: " + str(deduct_row),
logging_level)
elif not misc.is_num(deduct_row[0].strip()):
log.log_error(
"First column needs to be number: " + str(deduct_row),
logging_level)
else:
deduct.append({
RUBRIC_OTHER_POINT:
float(deduct_row[0].strip()),
RUBRIC_OTHER_COMMENT:
deduct_row[1].strip()
})
return deduct
def load_gtest_names(gtest_file, logging_level=log.LOGLEVEL_ERROR):
def parse_test_name(line, logging_level=log.LOGLEVEL_ERROR):
suite_name = ''
test_name = ''
if (line.find('(') == -1 or line.find(')') == -1
or line.find(',') == -1):
log.log_error('Malformed gtest macro ' + line, logging_level)
else:
line = line.replace(GTEST_MACRO, '')
line = line.replace('(', '')
line = line.replace(')', '')
separator = line.find(',')
suite_name = line[:separator].strip()
test_name = line[separator + 1:].strip()
return suite_name + '.' + test_name
### load_test_case body ###
test_names = list()
if not sysio.exist_file(gtest_file):
log.log_error('Cannot find ' + gtest_file, logging_level)
return test_names
test_file = open(gtest_file, 'r')
for line in test_file.readlines():
if line.find(GTEST_MACRO) != -1:
test_names.append(parse_test_name(line))
test_file.close()
return test_names
def generate_grade_report(homework,
grader,
report_dir,
overwrite=False,
logging_level=log.LOGLEVEL_ERROR):
report_filename = report_dir + 'GR{}_hw-username.md'.format(
str(homework.number))
if sysio.exist_file(report_filename):
if not overwrite:
log.log_error('Report {} already exists'.format(report_filename),
logging_level)
return
else:
log.log_warning(
'Overwriting existing report {}'.format(report_filename),
logging_level)
report_file = open(report_filename, mode='w')
title = '## HW {num:02d} Test Case Grade Report'.format(
num=homework.number)
md.write_header(report_file, title, GRHLEVEL_TITLE)
homework.write_score_breakdown(report_file)
grader_text = 'HW Graded by: {}'.format(grader.get_grader_info())
md.write_header(report_file, grader_text, GRHLEVEL_OTHER)
md.write_paragraph(report_file, GRFOOTER)
report_file.close()
def run_tests_in_list(executable_path, test_names, test_args, **kwargs):
if kwargs is None:
kwargs = dict()
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
all_norun = set()
all_timeout = set()
all_finished = dict()
if len(test_names) != len(test_args):
log.log_error(
'Found {} tests but {} test arguments'.format(
str(len(test_names)), str(len(test_args))), logging_level)
return set(x for x in test_names), all_timeout, all_finished
for i, test_name in enumerate(test_names):
if not sysio.exist_file(executable_path):
log.log_warning('Executable {} not found'.format(executable_path),
logging_level)
all_norun.add(test_name)
else:
kwargs['extra_arguments'] = test_args[i]
killed, runtime, _ = exe.run_executable(executable_path, **kwargs)
if killed == exe.EXE_ERROR:
all_norun.add(test_name)
elif killed == exe.EXE_TIMEOUT:
all_timeout.add(test_name)
else:
all_finished[test_name] = runtime
return all_norun, all_timeout, all_finished
def compare_files(source_file, comp_file, write_file, ordered_compare,
**kwargs):
if kwargs is None:
kwargs = dict()
detailed_results = kwargs.get('detailed_results', True)
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
correct = -1
missing = -1
extra = -1
if not sysio.exist_file(source_file):
log.log_error('Source file {} not found'.format(source_file),
logging_level)
else:
write = open(write_file, 'a')
if not sysio.exist_file(comp_file):
write.write(
'\n\tERROR: Compare file {} not found'.format(comp_file))
else:
source = open(source_file, mode='r', errors='ignore')
comp = open(comp_file, mode='r', errors='ignore')
source_lines = source.readlines()
comp_lines = comp.readlines()
source.close()
comp.close()
source_lines = misc.cleanup_lines(source_lines, **kwargs)
comp_lines = misc.cleanup_lines(comp_lines, **kwargs)
if ordered_compare:
correct, missing, extra = compare_files_with_order(
source_lines,
comp_lines,
write,
detailed_results=detailed_results)
else:
correct, missing, extra = compare_files_without_order(
source_lines,
comp_lines,
write,
detailed_results=detailed_results)
write.close()
return correct, missing, extra
def read_student_list(config, logging_level=log.LOGLEVEL_ERROR):
student_list = get_student_list(config)
students = list()
if not sysio.exist_file(student_list):
log.log_error('Cannot open student list ' + student_list,
logging_level)
return students
with open(student_list, 'r') as student_file:
students = student_file.readlines()
students = misc.cleanup_lines(students, skip_white_space=True)
return students
def read_valgrind_result(valgrind_result, logging_level=log.LOGLEVEL_ERROR):
zero_error_count = 0
if not sysio.exist_file(valgrind_result):
log.log_error('Cannot find valgrind result ' + valgrind_result,
logging_level)
else:
comp = open(valgrind_result, mode='r', errors='ignore')
lines = comp.readlines()
comp.close()
for output in lines:
if output.find(VALGRIND_ERROR_ZERO) != -1:
zero_error_count += 1
return zero_error_count
def read_compile_result(compile_result, logging_level=log.LOGLEVEL_ERROR):
warnings = set()
if not sysio.exist_file(compile_result):
log.log_error('Cannot find compile ' + compile_result, logging_level)
else:
comp = open(compile_result, mode='r', errors='ignore')
lines = comp.readlines()
comp.close()
for output in lines:
for warning in CWARNING_OUTPUT:
if output.find(warning) != -1:
warnings.add(warning)
return warnings
def read_formatted_result(formatted_file, logging_level=log.LOGLEVEL_ERROR):
test_names = list()
if not sysio.exist_file(formatted_file):
log.log_error('Cannot find formatted result ' + formatted_file,
logging_level)
else:
form = open(formatted_file, mode='r', errors='ignore')
lines = form.readlines()
form.close()
for formatted_line in lines:
test_name, result, vresult, _ = read_formatted_line(
formatted_line, logging_level=logging_level)
test_names.append((test_name, result, vresult))
return test_names
def read_gtest_result(comp_file, logging_level=log.LOGLEVEL_ERROR):
def parse_test_name(test_output, result):
test_output = test_output.replace(result, '')
test_name = None
runtime = None
space = test_output.find(' (')
if space != -1:
test_name = test_output[:space]
test_output = test_output[space + 2:]
space = test_output.find(' ms)')
if space != -1:
runtime = int(test_output[:space]) / 1000.0
else:
test_name = test_output
return test_name, runtime
### read_gtest_result body ###
if not sysio.exist_file(comp_file):
log.log_error('Cannot find ' + comp_file, logging_level)
return
run = set()
no_terminate = set()
failed = set()
passed = set()
failed_runtime = dict()
passed_runtime = dict()
with open(comp_file, mode='r', errors='ignore') as comp:
lines = comp.readlines()
for output in lines:
output = output.replace('\n', '')
if output.find(GRESULT_RUN) != -1:
test, _ = parse_test_name(output, GRESULT_RUN)
run.add(test)
elif output.find(GRESULT_FAILED) != -1:
test, runtime = parse_test_name(output, GRESULT_FAILED)
if test != '' and runtime is not None:
failed.add(test)
failed_runtime[test] = runtime
elif output.find(GRESULT_PASSED) != -1:
test, runtime = parse_test_name(output, GRESULT_PASSED)
if test != '' and runtime is not None:
passed.add(test)
passed_runtime[test] = runtime
no_terminate = run - failed - passed
return no_terminate, failed_runtime, passed_runtime
def check_test_output(test_names,
solution_list,
output_list,
result_file,
ordered_compare,
**kwargs
):
if kwargs is None:
kwargs = dict()
logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR)
all_failed = set()
all_passed = set()
if len(test_names) != len(output_list):
log.log_error('Found {} test names but {} test output'.format(
str(len(test_names)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
if len(solution_list) != len(output_list):
log.log_error('Found {} test solution but {} test output'.format(
str(len(solution_list)), str(len(output_list))), logging_level)
return set(x for x in test_names), all_passed
for i, test_name in enumerate(test_names):
t_solution = solution_list[i]
t_output = output_list[i]
if not sysio.exist_file(t_solution):
log.log_error('Cannot find solution file ' + t_solution, logging_level)
all_failed.add(test_name)
else:
_, missing, extra = comp.compare_files(
t_solution,
t_output,
result_file,
ordered_compare=ordered_compare,
**kwargs
)
sysio.write_message(result_file, '\n\n')
if missing == 0 and extra == 0:
all_passed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_PASS)
else:
all_failed.add(test_name)
res.write_result_line(result_file, test_name, res.ERESULT_FAIL)
return all_failed, all_passed
def load_grader_info(self, grader_info, logging_level=log.LOGLEVEL_ERROR):
if sysio.exist_file(grader_info):
grader_file = open(grader_info, mode='r', errors='ignore')
lines = grader_file.readlines()
grader_file.close()
if len(lines) < 2:
log.log_error(
'Grader info needs 2 lines but found {}'.format(
str(len(lines))), logging_level)
else:
if len(lines) > 2:
log.log_warning(
'Grader info needs 2 lines but found {}'.format(
str(len(lines))), logging_level)
self.name = lines[0].strip()
self.github = lines[1].strip()
else:
log.log_error('Grader info file {} not found'.format(grader_info),
logging_level)
def load_config(config, config_file, logging_level=log.LOGLEVEL_ERROR):
if not sysio.exist_file(config_file):
log.log_warning('Rubric file {} not found'.format(config_file),
logging_level)
else:
config.read(config_file)
def open_result(self, text_editor=sysio.TEXT_EDITOR):
if sysio.exist_file(self.result_file):
sysio.open_file(self.result_file, text_editor)
if sysio.exist_file(self.valgrind_file):
sysio.open_file(self.valgrind_file, text_editor)
