def load_other_rubric(rubric_file, logging_level=log.LOGLEVEL_ERROR): deduct = list() if not sysio.exist_file(rubric_file): log.log_warning('Rubric file {} not found'.format(rubric_file), logging_level) else: with open(rubric_file, 'r') as rfile: rubric_reader = csv.reader(rfile, delimiter='\t') for deduct_row in rubric_reader: if len(deduct_row) != 2: log.log_error( "Row should have two columns: " + str(deduct_row), logging_level) elif not misc.is_num(deduct_row[0].strip()): log.log_error( "First column needs to be number: " + str(deduct_row), logging_level) else: deduct.append({ RUBRIC_OTHER_POINT: float(deduct_row[0].strip()), RUBRIC_OTHER_COMMENT: deduct_row[1].strip() }) return deduct
def generate_grade_report(homework, grader, report_dir, overwrite=False, logging_level=log.LOGLEVEL_ERROR): report_filename = report_dir + 'GR{}_hw-username.md'.format( str(homework.number)) if sysio.exist_file(report_filename): if not overwrite: log.log_error('Report {} already exists'.format(report_filename), logging_level) return else: log.log_warning( 'Overwriting existing report {}'.format(report_filename), logging_level) report_file = open(report_filename, mode='w') title = '## HW {num:02d} Test Case Grade Report'.format( num=homework.number) md.write_header(report_file, title, GRHLEVEL_TITLE) homework.write_score_breakdown(report_file) grader_text = 'HW Graded by: {}'.format(grader.get_grader_info()) md.write_header(report_file, grader_text, GRHLEVEL_OTHER) md.write_paragraph(report_file, GRFOOTER) report_file.close()
def run_tests_in_list(executable_path, test_names, test_args, **kwargs): if kwargs is None: kwargs = dict() logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR) all_norun = set() all_timeout = set() all_finished = dict() if len(test_names) != len(test_args): log.log_error( 'Found {} tests but {} test arguments'.format( str(len(test_names)), str(len(test_args))), logging_level) return set(x for x in test_names), all_timeout, all_finished for i, test_name in enumerate(test_names): if not sysio.exist_file(executable_path): log.log_warning('Executable {} not found'.format(executable_path), logging_level) all_norun.add(test_name) else: kwargs['extra_arguments'] = test_args[i] killed, runtime, _ = exe.run_executable(executable_path, **kwargs) if killed == exe.EXE_ERROR: all_norun.add(test_name) elif killed == exe.EXE_TIMEOUT: all_timeout.add(test_name) else: all_finished[test_name] = runtime return all_norun, all_timeout, all_finished
def load_grader_info(self, grader_info, logging_level=log.LOGLEVEL_ERROR): if sysio.exist_file(grader_info): grader_file = open(grader_info, mode='r', errors='ignore') lines = grader_file.readlines() grader_file.close() if len(lines) < 2: log.log_error( 'Grader info needs 2 lines but found {}'.format( str(len(lines))), logging_level) else: if len(lines) > 2: log.log_warning( 'Grader info needs 2 lines but found {}'.format( str(len(lines))), logging_level) self.name = lines[0].strip() self.github = lines[1].strip() else: log.log_error('Grader info file {} not found'.format(grader_info), logging_level)
def write_formatted_result(formatted_file, test_names, valgrind, norun, timeout, crashed, failed, passed, logging_level=log.LOGLEVEL_ERROR): for idx, test_name in enumerate(test_names): result = None vcode = valgrind[idx] if idx < len(valgrind) else exe.EXE_ERROR runtime = None if test_name in passed: result = ERESULT_PASS runtime = passed[test_name] elif test_name in norun: result = ERESULT_NORUN elif test_name in timeout: result = ERESULT_TIMEOUT elif test_name in crashed: result = ERESULT_CRASH elif test_name in failed: result = ERESULT_FAIL runtime = failed[test_name] else: log.log_error( '{} not in any execution result set'.format(test_name), logging_level) if vcode == exe.EXE_ERROR: vresult = VRESULT_IGNORE elif vcode == exe.VALGRIND_SUCCESS: vresult = VRESULT_PASS elif vcode == exe.VALGRIND_ERROR or vcode == exe.VALGRIND_SIGSEGV: vresult = VRESULT_FAIL else: log.log_warning('Unknown return code ' + str(vcode), logging_level) vresult = VRESULT_IGNORE write_formatted_line(formatted_file, test_name, result, vresult, runtime)
def load_config(config, config_file, logging_level=log.LOGLEVEL_ERROR): if not sysio.exist_file(config_file): log.log_warning('Rubric file {} not found'.format(config_file), logging_level) else: config.read(config_file)
def cmake_problem(problem): # set path to output files problem.compile_file = os.path.join('compile-logs', problem.name + '.complog') stdout_file_path = os.path.join('test-output', problem.name + '-test-stdout.txt') stdout_file = open(stdout_file_path, 'w') # Find tests' output XML file, buiried a couple of directories deep xml_path = glob.glob("Testing/*-*/Test.xml") if len(xml_path) == 0: logging_tools.log_error("Cannot find test XML output file!", problem.logging_level) return elif len(xml_path) > 1: logging_tools.log_error( "Multiple candidates for test XML file: " + " ".join(xml_path), problem.logging_level) return logging_tools.log_info("Found XML output file: " + xml_path[0], problem.logging_level) # parse XML file test_xml = open(xml_path[0]) test_results = xmltodict.parse(test_xml.read()) test_xml.close() test_list = [] didnt_run_tests = set() crashed_tests = set() timed_out_tests = set() failed_tests = {} passed_tests = {} # Valgrind exit codes, indexed by test valgrind_exit_codes = [] # now, go through all tests test_results_list_element = test_results['Site']['Testing']['Test'] for test_results_element in test_results_list_element: if problem.name in test_results_element['Path']: test_name = test_results_element['Name'] #print("\n>> Processing test: " + test_name) # write test results to output file stdout_file.write(""" ------------------------------------------------------------------------------ OUTPUT OF TEST %s: ------------------------------------------------------------------------------ """ % test_name) stdout_file.write( test_results_element['Results']['Measurement']['Value']) # detect Valgrind failues # note: we want to assign seperate deductions for Valgrind failures and actual test case failures. So, # we can't use Valgrind's --error-exitcode option, since that would make CTest think all the tests had failed. valgrind_error = False match_list = valgrind_results_re.findall( test_results_element['Results']['Measurement']['Value']) if match_list is None or len(match_list) < 1: # program may have died before it got to even producing the valgrind output logging_tools.log_warning( "cmake_problem.py: could not find valgrind results for test " + test_name, problem.logging_level) else: # make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary definitely_lost = int(match_list[-1][0].replace(",", "")) indirectly_lost = int(match_list[-1][1].replace(",", "")) possibly_lost = int(match_list[-1][2].replace(",", "")) still_reachable = int(match_list[-1][3].replace(",", "")) suppressed = int(match_list[-1][4].replace(",", "")) #print("Valgrind Results: definitely_lost: %d, indirectly_lost: %d, possibly_lost: %d, still_reachable: %d, suppressed: %d" % (definitely_lost, indirectly_lost, possibly_lost, still_reachable, suppressed)) if definitely_lost > 0 or indirectly_lost > 0 or possibly_lost > 0 or still_reachable > 0: valgrind_error = True if subprocess_valgrind_failure_re.search( test_results_element['Results']['Measurement'] ['Value']) != None: #print("Valgrind errors found in subprocess execution!") valgrind_error = True # now, parse out the test status passed = test_results_element['@Status'] == 'passed' # true if the test was not run (as in, it failed to build) didnt_run = False # true if the test dies with a segfault/sigfpe/etc crashed = False # true if test timed out timed_out = False # iterate through namedmeasurements test_time = -1.0 for measurement_element in test_results_element['Results'][ 'NamedMeasurement']: if measurement_element['@name'] == 'Execution Time': test_time = float(measurement_element['Value']) if not passed and measurement_element['@name'] == 'Exit Code': if measurement_element['Value'] == 'Timeout': timed_out = True elif measurement_element['Value'] != 'Failed': # if exit code is something other than Failed it means that the test crashed crashed = True if not passed and measurement_element['@name'] == 'Exit Value': if int(measurement_element['Value']) == 127: # this exit code appears to indicate that the test was not run didnt_run = True #print("crashed: %r, passed: %r" % (crashed, passed)) # write test data to collections test_list.append(test_name) if crashed: crashed_tests.add(test_name) elif timed_out: timed_out_tests.add(test_name) elif didnt_run: didnt_run_tests.add(test_name) elif passed: passed_tests[test_name] = test_time else: failed_tests[test_name] = test_time # figure out what the Valgrind exit code should have been for this test if valgrind_error: valgrind_exit_codes.append(executable_tools.VALGRIND_ERROR) elif crashed: valgrind_exit_codes.append(executable_tools.EXE_ERROR) else: valgrind_exit_codes.append(executable_tools.VALGRIND_SUCCESS) stdout_file.close() # write test result files cs_grading.write_test_result(problem.result_file, test_list, didnt_run_tests, timed_out_tests, crashed_tests, failed_tests, passed_tests, logging_level=problem.logging_level) print("Writing formatted file: %s " % problem.formatted_file) cs_grading.write_formatted_result(problem.formatted_file, test_list, valgrind_exit_codes, didnt_run_tests, timed_out_tests, crashed_tests, failed_tests, passed_tests, logging_level=problem.logging_level)
def run_executable(executable_path, **kwargs): if kwargs is None: kwargs = dict() extra_arguments = kwargs.get('extra_arguments', list()) use_valgrind = kwargs.get('use_valgrind', False) valgrind_file = kwargs.get('valgrind_file', None) timeout = kwargs.get('timeout', None) logging_level = kwargs.get('logging_level', log.LOGLEVEL_ERROR) logging_force_suppressed = kwargs.get('logging_force_suppressed', False) killed = EXE_ERROR utime = EXE_ERROR retcode = EXE_ERROR error = False args = [] if use_valgrind: if valgrind_file is None: log.log_warning( 'valgrind turned on but no valgrind log file speficied', logging_level) temp_valgrind_log = 'temp_valgrind_log.txt' args.extend(VALGRIND) args.append('--log-file=' + temp_valgrind_log) redirected_stdin_file = None redirected_stdout_mode = None redirected_stdout_file = None redirected_stderr_mode = None redirected_stderr_file = None i = 0 while i < len(extra_arguments): if extra_arguments[i] == '<': # redirect input if i + 1 >= len(extra_arguments): log.log_error('Found input redirection with no input file', logging_level) error = True break else: redirected_stdin_file = extra_arguments[i + 1] del extra_arguments[i + 1] del extra_arguments[i] elif extra_arguments[i] == '>' or extra_arguments[i] == '>>': # redirect output if extra_arguments[i] == '>': redirected_stdout_mode = 'w' else: redirected_stdout_mode = 'a' if i + 1 >= len(extra_arguments): log.log_error('Found output redirection with no output file', logging_level) error = True break else: redirected_stdout_file = extra_arguments[i + 1] del extra_arguments[i + 1] del extra_arguments[i] elif extra_arguments[i] == '2>' or extra_arguments[i] == '2>>': # redirect output if extra_arguments[i] == '2>': redirected_stderr_mode = 'w' else: redirected_stderr_mode = 'a' if i + 1 >= len(extra_arguments): log.log_error('Found error redirection with no output file', logging_level) error = True break else: redirected_stderr_file = extra_arguments[i + 1] del extra_arguments[i + 1] del extra_arguments[i] else: i += 1 if not error: extra_arguments = [executable_path] + extra_arguments if not logging_force_suppressed: log.log_info('Running ' + ' '.join(extra_arguments), log.LOGLEVEL_INFO) args.extend(extra_arguments) if redirected_stdout_file is not None: redirected_stdout_file = open(redirected_stdout_file, redirected_stdout_mode) if redirected_stderr_file is not None: redirected_stderr_file = open(redirected_stderr_file, redirected_stderr_mode) if redirected_stdin_file is not None: redirected_stdin_file = open(redirected_stdin_file, 'r') try: start_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime proc = subprocess.Popen(args, stdout=redirected_stdout_file, stderr=redirected_stderr_file, stdin=redirected_stdin_file) proc.communicate(timeout=timeout) killed = EXE_SUCCESS retcode = proc.returncode except subprocess.TimeoutExpired: log.log_warning('Executable {} timed out'.format(executable_path), logging_level) killed = EXE_TIMEOUT proc.kill() except OSError: log.log_warning('Executable {} not found'.format(executable_path), logging_level) killed = EXE_ERROR end_time = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime utime = end_time - start_time if redirected_stdin_file is not None: redirected_stdin_file.close() if redirected_stdout_file is not None: redirected_stdout_file.close() if redirected_stderr_file is not None: redirected_stderr_file.close() if use_valgrind: if valgrind_file is not None: sysio.write_file_contents(valgrind_file, temp_valgrind_log, logging_level=logging_level) sysio.write_message(valgrind_file, '\n\n') sysio.clean_file(temp_valgrind_log) return killed, utime, retcode