def load_other_rubric(rubric_file, logging_level=log.LOGLEVEL_ERROR): deduct = list() if not sysio.exist_file(rubric_file): log.log_warning('Rubric file {} not found'.format(rubric_file), logging_level) else: with open(rubric_file, 'r') as rfile: rubric_reader = csv.reader(rfile, delimiter='\t') for deduct_row in rubric_reader: if len(deduct_row) != 2: log.log_error( "Row should have two columns: " + str(deduct_row), logging_level) elif not misc.is_num(deduct_row[0].strip()): log.log_error( "First column needs to be number: " + str(deduct_row), logging_level) else: deduct.append({ RUBRIC_OTHER_POINT: float(deduct_row[0].strip()), RUBRIC_OTHER_COMMENT: deduct_row[1].strip() }) return deduct
def load_gtest_names(gtest_file, logging_level=log.LOGLEVEL_ERROR): def parse_test_name(line, logging_level=log.LOGLEVEL_ERROR): suite_name = '' test_name = '' if (line.find('(') == -1 or line.find(')') == -1 or line.find(',') == -1): log.log_error('Malformed gtest macro ' + line, logging_level) else: line = line.replace(GTEST_MACRO, '') line = line.replace('(', '') line = line.replace(')', '') separator = line.find(',') suite_name = line[:separator].strip() test_name = line[separator + 1:].strip() return suite_name + '.' + test_name ### load_test_case body ### test_names = list() if not sysio.exist_file(gtest_file): log.log_error('Cannot find ' + gtest_file, logging_level) return test_names test_file = open(gtest_file, 'r') for line in test_file.readlines(): if line.find(GTEST_MACRO) != -1: test_names.append(parse_test_name(line)) test_file.close() return test_names
def generate_grade_report(homework, grader, report_dir, overwrite=False, logging_level=log.LOGLEVEL_ERROR): report_filename = report_dir + 'GR{}_hw-username.md'.format( str(homework.number)) if sysio.exist_file(report_filename): if not overwrite: log.log_error('Report {} already exists'.format(report_filename), logging_level) return else: log.log_warning( 'Overwriting existing report {}'.format(report_filename), logging_level) report_file = open(report_filename, mode='w') title = '## HW {num:02d} Test Case Grade Report'.format( num=homework.number) md.write_header(report_file, title, GRHLEVEL_TITLE) homework.write_score_breakdown(report_file) grader_text = 'HW Graded by: {}'.format(grader.get_grader_info()) md.write_header(report_file, grader_text, GRHLEVEL_OTHER) md.write_paragraph(report_file, GRFOOTER) report_file.close()
def run_tests_in_list(executable_path, test_names, test_args, **kwargs): if kwargs is None: kwargs = dict() logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR) all_norun = set() all_timeout = set() all_finished = dict() if len(test_names) != len(test_args): log.log_error( 'Found {} tests but {} test arguments'.format( str(len(test_names)), str(len(test_args))), logging_level) return set(x for x in test_names), all_timeout, all_finished for i, test_name in enumerate(test_names): if not sysio.exist_file(executable_path): log.log_warning('Executable {} not found'.format(executable_path), logging_level) all_norun.add(test_name) else: kwargs['extra_arguments'] = test_args[i] killed, runtime, _ = exe.run_executable(executable_path, **kwargs) if killed == exe.EXE_ERROR: all_norun.add(test_name) elif killed == exe.EXE_TIMEOUT: all_timeout.add(test_name) else: all_finished[test_name] = runtime return all_norun, all_timeout, all_finished
def compare_files(source_file, comp_file, write_file, ordered_compare, **kwargs): if kwargs is None: kwargs = dict() detailed_results = kwargs.get('detailed_results', True) logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR) correct = -1 missing = -1 extra = -1 if not sysio.exist_file(source_file): log.log_error('Source file {} not found'.format(source_file), logging_level) else: write = open(write_file, 'a') if not sysio.exist_file(comp_file): write.write( '\n\tERROR: Compare file {} not found'.format(comp_file)) else: source = open(source_file, mode='r', errors='ignore') comp = open(comp_file, mode='r', errors='ignore') source_lines = source.readlines() comp_lines = comp.readlines() source.close() comp.close() source_lines = misc.cleanup_lines(source_lines, **kwargs) comp_lines = misc.cleanup_lines(comp_lines, **kwargs) if ordered_compare: correct, missing, extra = compare_files_with_order( source_lines, comp_lines, write, detailed_results=detailed_results) else: correct, missing, extra = compare_files_without_order( source_lines, comp_lines, write, detailed_results=detailed_results) write.close() return correct, missing, extra
def read_student_list(config, logging_level=log.LOGLEVEL_ERROR): student_list = get_student_list(config) students = list() if not sysio.exist_file(student_list): log.log_error('Cannot open student list ' + student_list, logging_level) return students with open(student_list, 'r') as student_file: students = student_file.readlines() students = misc.cleanup_lines(students, skip_white_space=True) return students
def read_valgrind_result(valgrind_result, logging_level=log.LOGLEVEL_ERROR): zero_error_count = 0 if not sysio.exist_file(valgrind_result): log.log_error('Cannot find valgrind result ' + valgrind_result, logging_level) else: comp = open(valgrind_result, mode='r', errors='ignore') lines = comp.readlines() comp.close() for output in lines: if output.find(VALGRIND_ERROR_ZERO) != -1: zero_error_count += 1 return zero_error_count
def read_compile_result(compile_result, logging_level=log.LOGLEVEL_ERROR): warnings = set() if not sysio.exist_file(compile_result): log.log_error('Cannot find compile ' + compile_result, logging_level) else: comp = open(compile_result, mode='r', errors='ignore') lines = comp.readlines() comp.close() for output in lines: for warning in CWARNING_OUTPUT: if output.find(warning) != -1: warnings.add(warning) return warnings
def read_formatted_result(formatted_file, logging_level=log.LOGLEVEL_ERROR): test_names = list() if not sysio.exist_file(formatted_file): log.log_error('Cannot find formatted result ' + formatted_file, logging_level) else: form = open(formatted_file, mode='r', errors='ignore') lines = form.readlines() form.close() for formatted_line in lines: test_name, result, vresult, _ = read_formatted_line( formatted_line, logging_level=logging_level) test_names.append((test_name, result, vresult)) return test_names
def read_gtest_result(comp_file, logging_level=log.LOGLEVEL_ERROR): def parse_test_name(test_output, result): test_output = test_output.replace(result, '') test_name = None runtime = None space = test_output.find(' (') if space != -1: test_name = test_output[:space] test_output = test_output[space + 2:] space = test_output.find(' ms)') if space != -1: runtime = int(test_output[:space]) / 1000.0 else: test_name = test_output return test_name, runtime ### read_gtest_result body ### if not sysio.exist_file(comp_file): log.log_error('Cannot find ' + comp_file, logging_level) return run = set() no_terminate = set() failed = set() passed = set() failed_runtime = dict() passed_runtime = dict() with open(comp_file, mode='r', errors='ignore') as comp: lines = comp.readlines() for output in lines: output = output.replace('\n', '') if output.find(GRESULT_RUN) != -1: test, _ = parse_test_name(output, GRESULT_RUN) run.add(test) elif output.find(GRESULT_FAILED) != -1: test, runtime = parse_test_name(output, GRESULT_FAILED) if test != '' and runtime is not None: failed.add(test) failed_runtime[test] = runtime elif output.find(GRESULT_PASSED) != -1: test, runtime = parse_test_name(output, GRESULT_PASSED) if test != '' and runtime is not None: passed.add(test) passed_runtime[test] = runtime no_terminate = run - failed - passed return no_terminate, failed_runtime, passed_runtime
def check_test_output(test_names, solution_list, output_list, result_file, ordered_compare, **kwargs ): if kwargs is None: kwargs = dict() logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR) all_failed = set() all_passed = set() if len(test_names) != len(output_list): log.log_error('Found {} test names but {} test output'.format( str(len(test_names)), str(len(output_list))), logging_level) return set(x for x in test_names), all_passed if len(solution_list) != len(output_list): log.log_error('Found {} test solution but {} test output'.format( str(len(solution_list)), str(len(output_list))), logging_level) return set(x for x in test_names), all_passed for i, test_name in enumerate(test_names): t_solution = solution_list[i] t_output = output_list[i] if not sysio.exist_file(t_solution): log.log_error('Cannot find solution file ' + t_solution, logging_level) all_failed.add(test_name) else: _, missing, extra = comp.compare_files( t_solution, t_output, result_file, ordered_compare=ordered_compare, **kwargs ) sysio.write_message(result_file, '\n\n') if missing == 0 and extra == 0: all_passed.add(test_name) res.write_result_line(result_file, test_name, res.ERESULT_PASS) else: all_failed.add(test_name) res.write_result_line(result_file, test_name, res.ERESULT_FAIL) return all_failed, all_passed
def load_grader_info(self, grader_info, logging_level=log.LOGLEVEL_ERROR): if sysio.exist_file(grader_info): grader_file = open(grader_info, mode='r', errors='ignore') lines = grader_file.readlines() grader_file.close() if len(lines) < 2: log.log_error( 'Grader info needs 2 lines but found {}'.format( str(len(lines))), logging_level) else: if len(lines) > 2: log.log_warning( 'Grader info needs 2 lines but found {}'.format( str(len(lines))), logging_level) self.name = lines[0].strip() self.github = lines[1].strip() else: log.log_error('Grader info file {} not found'.format(grader_info), logging_level)
def load_config(config, config_file, logging_level=log.LOGLEVEL_ERROR): if not sysio.exist_file(config_file): log.log_warning('Rubric file {} not found'.format(config_file), logging_level) else: config.read(config_file)
def open_result(self, text_editor=sysio.TEXT_EDITOR): if sysio.exist_file(self.result_file): sysio.open_file(self.result_file, text_editor) if sysio.exist_file(self.valgrind_file): sysio.open_file(self.valgrind_file, text_editor)