def test(locations, test_obj, source, file_paths, env, alterations, keyval, n): OK = 0 from modules.compilation import compile_single import modules.altersource as altersource import difflib import os import shutil harness_dir = locations[1] files_to_remove = [] # check if working directory exists, if not make one working_dir = os.path.join(harness_dir, env["working_dir"]) if not os.path.exists(working_dir): os.mkdir(working_dir) made_working = True else: made_working = False # solution path sol_dir = os.path.join(harness_dir, env["solution_dir"]) sol_path = os.path.join(sol_dir, env["solution_name"]) sol_work = os.path.join(working_dir, env["solution_name"]) # student working file testing_file = env["src_name"] + env["src_extension"] stud_file = os.path.join(working_dir, testing_file) files_to_remove.append(stud_file) # executables exe_path = os.path.join(working_dir, env["exe_name"]) files_to_remove.append(exe_path) sol_exe = os.path.join(working_dir, env["solution_exe"]) files_to_remove.append(sol_exe) # attempt alterations to code stud_contents = open(source.file_loc).read().split('\n') sol_contents = open(sol_path).read().split('\n') for (i, value) in enumerate(alterations["from"]): replaced = altersource.replace_source(stud_contents, value, alterations["to"][i]) if not replaced: test_obj.score = 0 test_obj.message = "Could not find " + value + " in source code." # clean up if made_working: shutil.rmtree(working_dir) elif len(files_to_remove) > 0: for f in files_to_remove: if os.path.isfile(f): os.remove(f) return OK # apply same alterations to solution replaced = altersource.replace_source(sol_contents, value, alterations["to"][i]) if not replaced: test_obj.score = 0 test_obj.message = "Could not find " + value + " in solution." # clean up if made_working: shutil.rmtree(working_dir) elif len(files_to_remove) > 0: for f in files_to_remove: if os.path.isfile(f): os.remove(f) return OK # open the student working file and write the altered source open(stud_file, 'wb+').write("\n".join(stud_contents)) # open the solution working file and write the altered source open(sol_work, 'wb+').write("\n".join(sol_contents)) # attempt compilation of both working files ret_val = compile_single(stud_file, exe_path, harness_dir, env) sol_ret_val = compile_single(sol_work, sol_exe, harness_dir, env) if not ret_val["success"]: test_obj.score = 0 test_obj.message = "File did not compile, cannot compare output." # clean up if made_working: shutil.rmtree(working_dir) elif len(files_to_remove) > 0: for f in files_to_remove: if os.path.isfile(f): os.remove(f) return OK if not sol_ret_val["success"]: test_obj.score = 0 test_obj.message = "Solution did not compile, cannot compare output." # clean up if made_working: shutil.rmtree(working_dir) elif len(files_to_remove) > 0: for f in files_to_remove: if os.path.isfile(f): os.remove(f) return OK # compilation successful proceed with testing outf = os.path.join(working_dir, "test_program.out") files_to_remove.append(outf) solution_outf = os.path.join(working_dir, "solution.out") files_to_remove.append(solution_outf) FNULL = open(os.devnull, 'w') messages = [] suggestions = [] notifications = {} if len(file_paths) > 0: status = {} for input in file_paths: with open(input, 'r') as inp_file: try: # create a task for execution t = Task(env["timeout"]) with open(outf, 'w+') as out_file: t.check_call([exe_path], stdout=out_file, stderr=FNULL, stdin=inp_file) status[testing_file] = True except SystemError as e: status[testing_file] = False # create a notice about the abort throw and put in message m = "Your program's execution was aborted." notifications["abort"] = m # TODO: this access is ugly and should be cleaned up # if seg fault was detected if e[0][0] == -11: cl = ["return value", "values of each parameter"] s = "A segmentation fault occurred when running your " s += "program." suggestions.append(s) # termination of thread due to timeout elif e[0][0] == -15: cl = ["infinite loops (check your conditions)", "attempting to get input when no input exists"] s = "Your program failed to execute " s += "to completion in the time allotted to it. " s += "This may have been caused by several things " s += "including:\n" s += markup_create_indent(markup_create_unlist(cl), 1) suggestions.append(s) # grab standard output if test was not aborted if len(open(outf).read()) > 0: testfile_output = open(outf).readlines() else: testfile_output = "" # format the lines: # eliminate blank lines, strip trail/lead whitespace # convert all to lowercase tf_output = [] testfile_output_string = "" for line in testfile_output: if len(line.strip()) != 0: if line.lower().find(keyval.lower()) != -1: key_loc = line.lower().find(keyval.lower()) space_aft = line.lower().find(" ", key_loc) if len(testfile_output_string) > 0: testfile_output_string += "\n" if space_aft != -1: testfile_output_string += line[space_aft+1:].strip() tf_output.append(testfile_output_string+'\n') # now execute solution with open(input, 'r') as inp_file: try: # create a task for execution t = Task(env["timeout"]) with open(outf, 'w+') as out_file: t.check_call([sol_exe], stdout=out_file, stderr=FNULL, stdin=inp_file) status[env["solution_name"]] = True except SystemError as e: status[env["solution_name"]] = False key = "abort" if key not in notifications: notifications["abort"] = "Solution was aborted!" else: notifications["abort"] += "\nSolution was aborted!" # grab standard output if test wasn't aborted if len(open(outf).read()) > 0: solution_output = open(outf).readlines() else: solution_output = "" # format the lines: # eliminate blank lines, strip trail/lead whitespace # convert all to lowercase sol_output = [] solution_output_string = "" for line in solution_output: if len(line.strip()) != 0: if line.lower().find(keyval.lower()) != -1: key_loc = line.lower().find(keyval.lower()) space_aft = line.lower().find(" ", key_loc) if len(solution_output_string) > 0: solution_output_string += "\n" if space_aft != -1: solution_output_string += line[space_aft+1:].strip() sol_output.append(solution_output_string+'\n') lines = [] difflines = difflib.unified_diff(tf_output, sol_output, fromfile=source.name, tofile='solution.cpp', n=1) unique = [] for (i, line) in enumerate(difflines): if i > 1 and len(line) > 0: if line[0] == '+': unique.append(line) lines.append(line) diffoutput = "".join(lines) if not status[env["solution_name"]] or not status[testing_file]: diffoutput = "Output comparison was skipped because the program" diffoutput += " was terminated due to an abort " diffoutput += "signal." # make the list of unique lines unique = list(set(unique)) if len(diffoutput.strip()) > 0: if len(testfile_output_string.strip()) > 0: m = "Weighted sum value for digit number " m += sol_output[0].strip("\n") m += " is incorrect, where the first digit is the " m += "farthest left." else: m = "Could not find a value to utilize. Attempted to find " m += " value using a case agnostic key value of: " m += keyval s = "The test harness could not find an output value for " s += "these tests. Check the specification and make sure " s += "you are following the stipulated output requirements." suggestions.append(s) messages.append(m) else: test_obj.score += float(test_obj.max_score) / len(file_paths) # limit the messages if warranted if len(messages) > 0: m = "You failed " + str(len(messages)) + " cases. " m += "When you fix one of the cases, it should disappear from the " m += "feedback on your next submission." suggestions.append(m) # turn current messages into bullet point list header = markup_create_header("Bad Values\n", 2) q = list(set(messages)) q.sort() mlist = markup_create_unlist(q) messages = [] messages.append(header + mlist) # put notifications at top of message if there are any if len(notifications) > 0: top = markup_create_header("Notifications\n", 2) notes = [] for (key, value) in notifications.iteritems(): head = markup_create_bold(key.upper()) head += ": " + value notes.append(head) m = markup_create_unlist(notes) messages = [top + m + "\n"] + messages # add suggestions section as last part of message if len(suggestions) > 0: m = markup_create_header("Suggestions\n", 2) m += markup_create_unlist(list(set(suggestions))) messages.append(m) # add messages if len(messages) != 0: if test_obj.message == "": test_obj.message += "\n".join(messages) else: test_obj.message += "\n" + "\n".join(messages) # clean up if made_working: shutil.rmtree(working_dir) elif len(files_to_remove) > 0: for f in files_to_remove: if os.path.isfile(f): os.remove(f) test_obj.score = int(round(test_obj.score)) return OK
def create_unit(self, inc_stmts, unit_harness, unit_main, merge_loc, source, harness, sol_funcs, alterations): import os self.include_stmts = inc_stmts self.unit_harness = unit_harness self.unit_main = unit_main self.test_file = source.file_loc self.sol_funcs = sol_funcs if not os.path.isfile(self.include_stmts): return False if not os.path.isfile(self.unit_harness): return False if not os.path.isfile(self.unit_main): return False for sol_fn in self.sol_funcs: if not os.path.isfile(sol_fn): return False # grab contents of files includes = open(inc_stmts).read().split('\n') harness_contents = open(unit_harness).read().split('\n') main_contents = open(unit_main).read().split('\n') # indent contents of test file prior to getting contents # TODO: re-enable once indent command exists on the system #from modules.indent import indent_file #ret_val = indent_file(source.file_loc, harness, "BSD", ["-cdw","-nut"]) #if ret_val["success"]: # stud_contents = ret_val["message"].split('\n') #else: # return False # TODO: eliminate module to module dependency import modules.altersource as altersource stud_contents = open(source.file_loc).read().split('\n') # replace the student's main with new name altersource.replace_source(stud_contents, "int main", "int _studVer_main") # perform any replacements from configuration file for (i, value) in enumerate(alterations["from"]): altersource.replace_source(stud_contents, value, alterations["to"][i]) # update the test harness call to student's main if testing main if self.fn_name == "main": altersource.replace_source(harness_contents, "STUDENT_FUNC_NAME", "_studVer_main") else: altersource.replace_source(harness_contents, "STUDENT_FUNC_NAME", self.fn_name) # output all the files to a merged file with open(merge_loc, 'w') as merge_file: for line in includes: merge_file.write(line+"\n") merge_file.write("\n") for p in self.sol_funcs: p_lines = open(p).read().split('\n') for line in p_lines: merge_file.write(line+"\n") merge_file.write("\n") for line in stud_contents: merge_file.write(line+"\n") merge_file.write("\n") for line in harness_contents: merge_file.write(line+"\n") merge_file.write("\n") for line in main_contents: merge_file.write(line+"\n") merge_file.write("\n") self.merged_file = merge_loc return True