def rotate(problem): output_dir = '../rotate_output/' source_dir = 'rotate_test/' target_dir = '../rotate_target/' cs.mkdir_empty(output_dir, logging_level=problem.logging_level) cs.mkdir_empty(target_dir, logging_level=problem.logging_level) test_sorted = sorted(os.listdir(source_dir)) test_sorted = sorted(test_sorted, key=lambda fn: len(fn)) test_sorted = [ x.replace('.cpp', '') for x in test_sorted if x.find('.cpp') != -1 ] norun, timeout, crashed, failed, passed = cs.run_gtest_in_dir( test_sorted, source_dir, output_dir, target_dir, compile_flags=problem.compile_flags, compile_file=problem.compile_file, use_valgrind=problem.use_valgrind, valgrind_file=problem.valgrind_file, timeout=problem.timeout, logging_level=problem.logging_level) cs.write_test_result(problem.result_file, test_sorted, norun, timeout, crashed, failed, passed, logging_level=problem.logging_level) cs.write_formatted_result(problem.formatted_file, test_sorted, norun, timeout, crashed, failed, passed, logging_level=problem.logging_level) if problem.remove_output: cs.mkdir_empty(output_dir, logging_level=problem.logging_level) cs.mkdir_empty(target_dir, logging_level=problem.logging_level) cs.remove_dir(output_dir) cs.remove_dir(target_dir)
def cmake_problem(problem): # set path to output files problem.compile_file = os.path.join('compile-logs', problem.name + '.complog') stdout_file_path = os.path.join('test-output', problem.name + '-test-stdout.txt') stdout_file = open(stdout_file_path, 'w') # Find tests' output XML file, buried a couple of directories deep xml_path = glob.glob("Testing/*-*/Test.xml") if len(xml_path) == 0: logging_tools.log_error("Cannot find test XML output file!", problem.logging_level) return elif len(xml_path) > 1: logging_tools.log_error( "Multiple candidates for test XML file: " + " ".join(xml_path), problem.logging_level) return logging_tools.log_info("Found XML output file: " + xml_path[0], problem.logging_level) # parse XML file test_xml = open(xml_path[0]) test_results = xmltodict.parse(test_xml.read()) test_xml.close() test_list = [] didnt_run_tests = set() crashed_tests = set() timed_out_tests = set() failed_tests = {} passed_tests = {} # Valgrind exit codes, indexed by test valgrind_exit_codes = [] # now, go through all tests test_results_list_element = test_results['Site']['Testing']['Test'] for test_results_element in test_results_list_element: if problem.name in test_results_element['Path']: test_name = test_results_element['Name'] #print("\n>> Processing test: " + test_name) # write test results to output file stdout_file.write(""" ------------------------------------------------------------------------------ OUTPUT OF TEST %s: ------------------------------------------------------------------------------ """ % test_name) stdout_file.write( test_results_element['Results']['Measurement']['Value']) # detect Valgrind failues # note: we want to assign seperate deductions for Valgrind failures and actual test case failures. So, # we can't use Valgrind's --error-exitcode option, since that would make CTest think all the tests had failed. valgrind_error = False match_list = valgrind_results_re.findall( test_results_element['Results']['Measurement']['Value']) if match_list is None or len(match_list) < 1: # program may have died before it got to even producing the valgrind output, or it's a test that doesn't use Valgrind pass else: # make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary definitely_lost = int(match_list[-1][0].replace(",", "")) indirectly_lost = int(match_list[-1][1].replace(",", "")) possibly_lost = int(match_list[-1][2].replace(",", "")) still_reachable = int(match_list[-1][3].replace(",", "")) suppressed = int(match_list[-1][4].replace(",", "")) # print("Valgrind Results: definitely_lost: %d, indirectly_lost: %d, possibly_lost: %d, still_reachable: %d, suppressed: %d" % (definitely_lost, indirectly_lost, possibly_lost, still_reachable, suppressed)) if definitely_lost > 0 or indirectly_lost > 0 or possibly_lost > 0 or still_reachable > 0: valgrind_error = True # now look for "X errors in X contexts" error_match_list = valgrind_errors_re.findall( test_results_element['Results']['Measurement']['Value']) if error_match_list is None or len(error_match_list) < 1: # program may have died before it got to even producing the valgrind output, or it's a test that doesn't use Valgrind pass else: # make sure to grab the last match in case a student tries to defeat this by printing a fake valgrind summary num_errors = int(error_match_list[-1].replace(",", "")) #print("%d valgrind errors" % num_errors) if num_errors > 0: valgrind_error = True if subprocess_valgrind_failure_re.search( test_results_element['Results']['Measurement'] ['Value']) != None: # print("Valgrind errors found in subprocess execution!") valgrind_error = True # now, parse out the test status passed = test_results_element['@Status'] == 'passed' # true if the test was not run (as in, it failed to build) didnt_run = False # true if the test dies with a segfault/sigfpe/etc crashed = False # true if test timed out timed_out = False test_time = -1.0 if type(test_results_element['Results'] ['NamedMeasurement']) != list: # if the test did not run because the excutable was not found, there might not be the usual set of NamedMeasurements didnt_run = True elif test_results_element["@Status"] == "notrun": didnt_run = True else: # iterate through namedmeasurements for measurement_element in test_results_element['Results'][ 'NamedMeasurement']: if measurement_element['@name'] == 'Execution Time': test_time = float(measurement_element['Value']) if not passed and measurement_element[ '@name'] == 'Exit Code': if measurement_element['Value'] == 'Timeout': timed_out = True elif measurement_element['Value'] != 'Failed': # if exit code is something other than Failed it means that the test crashed crashed = True if not passed and measurement_element[ '@name'] == 'Exit Value': if int(measurement_element['Value']) == 127: # Valgrind returns this to indicate that the test was not run didnt_run = True # print("crashed: %r, passed: %r" % (crashed, passed)) # write test data to collections test_list.append(test_name) if crashed: crashed_tests.add(test_name) elif timed_out: timed_out_tests.add(test_name) elif didnt_run: didnt_run_tests.add(test_name) elif passed: passed_tests[test_name] = test_time else: failed_tests[test_name] = test_time # figure out what the Valgrind exit code should have been for this test if valgrind_error: valgrind_exit_codes.append(executable_tools.VALGRIND_ERROR) elif crashed: valgrind_exit_codes.append(executable_tools.EXE_ERROR) else: valgrind_exit_codes.append(executable_tools.VALGRIND_SUCCESS) stdout_file.close() # write test result files cs_grading.write_test_result(problem.result_file, test_list, didnt_run_tests, timed_out_tests, crashed_tests, failed_tests, passed_tests, logging_level=problem.logging_level) cs_grading.write_formatted_result(problem.formatted_file, test_list, valgrind_exit_codes, didnt_run_tests, timed_out_tests, crashed_tests, failed_tests, passed_tests, logging_level=problem.logging_level)
def hypercube(problem): test = 'permitted' start = 'hypercube_test/start.txt' input_dir = 'hypercube_test/' sol_dir = 'hypercube_sol/' output_dir = '../hypercube_output/' target = 'hypercube' makefile_dir = '../../' if not cs.exist_file(start): cs.log_error('Failed to open start node file', problem.logging_level) return cs.mkdir_empty(output_dir, logging_level=problem.logging_level) test_names = sorted(os.listdir(input_dir)) test_names = sorted(test_names, key=lambda fn: len(fn)) test_names = [ x.replace('.txt', '') for x in test_names if x.find(test) != -1 ] with open(start) as start_file: start_nodes = [x.strip() for x in start_file.readlines()] if len(test_names) != len(start_nodes): cs.log_error('Found {} permitted files but {} start nodes'.format( str(len(test_names)), str(len(start_nodes))), logging_level=problem.logging_level) return input_files = [input_dir + x + '.txt' for x in test_names] output_files = [output_dir + x + '.txt' for x in test_names] solution_files = [sol_dir + x + '.txt' for x in test_names] test_args = list() for i in range(len(test_names)): cs.clean_file(output_files[i]) test_args.append( [start_nodes[i], input_files[i], '>', output_files[i]]) cs.make_code(make_command=target, makefile_dir=makefile_dir, compile_file=problem.compile_file) cs.move_file(makefile_dir + target, './') target = cs.fix_executable_path(target) norun, timeout, finished = cs.run_tests_in_list( target, test_names, test_args, use_valgrind=problem.use_valgrind, valgrind_file=problem.valgrind_file, timeout=problem.timeout, logging_level=problem.logging_level) failed, passed = cs.check_test_output( test_names, solution_files, output_files, problem.result_file, ordered_compare=True, skip_white_space=True, detailed_results=problem.detailed_results, logging_level=problem.logging_level) failed, passed = cs.get_test_runtime(finished, failed, passed) crashed = set() cs.write_test_result(problem.result_file, test_names, norun, timeout, crashed, failed, passed, final_log=True, logging_level=problem.logging_level) cs.write_formatted_result(problem.formatted_file, test_names, norun, timeout, crashed, failed, passed, logging_level=problem.logging_level) if problem.remove_output: cs.mkdir_empty(output_dir, logging_level=problem.logging_level) cs.remove_dir(output_dir) cs.remove_file(target)