def test(locations, test_obj, source, file_paths,
env, alterations, keyval, n):
OK = 0
from modules.compilation import compile_single
import modules.altersource as altersource
import difflib
import os
import shutil
harness_dir = locations[1]
files_to_remove = []
# check if working directory exists, if not make one
working_dir = os.path.join(harness_dir, env["working_dir"])
if not os.path.exists(working_dir):
os.mkdir(working_dir)
made_working = True
else:
made_working = False
# solution path
sol_dir = os.path.join(harness_dir, env["solution_dir"])
sol_path = os.path.join(sol_dir, env["solution_name"])
sol_work = os.path.join(working_dir, env["solution_name"])
# student working file
testing_file = env["src_name"] + env["src_extension"]
stud_file = os.path.join(working_dir, testing_file)
files_to_remove.append(stud_file)
# executables
exe_path = os.path.join(working_dir, env["exe_name"])
files_to_remove.append(exe_path)
sol_exe = os.path.join(working_dir, env["solution_exe"])
files_to_remove.append(sol_exe)
# attempt alterations to code
stud_contents = open(source.file_loc).read().split('\n')
sol_contents = open(sol_path).read().split('\n')
for (i, value) in enumerate(alterations["from"]):
replaced = altersource.replace_source(stud_contents, value,
alterations["to"][i])
if not replaced:
test_obj.score = 0
test_obj.message = "Could not find " + value + " in source code."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# apply same alterations to solution
replaced = altersource.replace_source(sol_contents, value,
alterations["to"][i])
if not replaced:
test_obj.score = 0
test_obj.message = "Could not find " + value + " in solution."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# open the student working file and write the altered source
open(stud_file, 'wb+').write("\n".join(stud_contents))
# open the solution working file and write the altered source
open(sol_work, 'wb+').write("\n".join(sol_contents))
# attempt compilation of both working files
ret_val = compile_single(stud_file, exe_path, harness_dir, env)
sol_ret_val = compile_single(sol_work, sol_exe, harness_dir, env)
if not ret_val["success"]:
test_obj.score = 0
test_obj.message = "File did not compile, cannot compare output."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
if not sol_ret_val["success"]:
test_obj.score = 0
test_obj.message = "Solution did not compile, cannot compare output."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# compilation successful proceed with testing
outf = os.path.join(working_dir, "test_program.out")
files_to_remove.append(outf)
solution_outf = os.path.join(working_dir, "solution.out")
files_to_remove.append(solution_outf)
FNULL = open(os.devnull, 'w')
messages = []
suggestions = []
notifications = {}
if len(file_paths) > 0:
status = {}
for input in file_paths:
with open(input, 'r') as inp_file:
try:
# create a task for execution
t = Task(env["timeout"])
with open(outf, 'w+') as out_file:
t.check_call([exe_path], stdout=out_file, stderr=FNULL,
stdin=inp_file)
status[testing_file] = True
except SystemError as e:
status[testing_file] = False
# create a notice about the abort throw and put in message
m = "Your program's execution was aborted."
notifications["abort"] = m
# TODO: this access is ugly and should be cleaned up
# if seg fault was detected
if e[0][0] == -11:
cl = ["return value", "values of each parameter"]
s = "A segmentation fault occurred when running your "
s += "program."
suggestions.append(s)
# termination of thread due to timeout
elif e[0][0] == -15:
cl = ["infinite loops (check your conditions)",
"attempting to get input when no input exists"]
s = "Your program failed to execute "
s += "to completion in the time allotted to it. "
s += "This may have been caused by several things "
s += "including:\n"
s += markup_create_indent(markup_create_unlist(cl), 1)
suggestions.append(s)
# grab standard output if test was not aborted
if len(open(outf).read()) > 0:
testfile_output = open(outf).readlines()
else:
testfile_output = ""
# format the lines:
# eliminate blank lines, strip trail/lead whitespace
# convert all to lowercase
tf_output = []
testfile_output_string = ""
for line in testfile_output:
if len(line.strip()) != 0:
if line.lower().find(keyval.lower()) != -1:
key_loc = line.lower().find(keyval.lower())
space_aft = line.lower().find(" ", key_loc)
if len(testfile_output_string) > 0:
testfile_output_string += "\n"
if space_aft != -1:
testfile_output_string += line[space_aft+1:].strip()
tf_output.append(testfile_output_string+'\n')
# now execute solution
with open(input, 'r') as inp_file:
try:
# create a task for execution
t = Task(env["timeout"])
with open(outf, 'w+') as out_file:
t.check_call([sol_exe], stdout=out_file, stderr=FNULL,
stdin=inp_file)
status[env["solution_name"]] = True
except SystemError as e:
status[env["solution_name"]] = False
key = "abort"
if key not in notifications:
notifications["abort"] = "Solution was aborted!"
else:
notifications["abort"] += "\nSolution was aborted!"
# grab standard output if test wasn't aborted
if len(open(outf).read()) > 0:
solution_output = open(outf).readlines()
else:
solution_output = ""
# format the lines:
# eliminate blank lines, strip trail/lead whitespace
# convert all to lowercase
sol_output = []
solution_output_string = ""
for line in solution_output:
if len(line.strip()) != 0:
if line.lower().find(keyval.lower()) != -1:
key_loc = line.lower().find(keyval.lower())
space_aft = line.lower().find(" ", key_loc)
if len(solution_output_string) > 0:
solution_output_string += "\n"
if space_aft != -1:
solution_output_string += line[space_aft+1:].strip()
sol_output.append(solution_output_string+'\n')
lines = []
difflines = difflib.unified_diff(tf_output, sol_output,
fromfile=source.name, tofile='solution.cpp', n=1)
unique = []
for (i, line) in enumerate(difflines):
if i > 1 and len(line) > 0:
if line[0] == '+':
unique.append(line)
lines.append(line)
diffoutput = "".join(lines)
if not status[env["solution_name"]] or not status[testing_file]:
diffoutput = "Output comparison was skipped because the program"
diffoutput += " was terminated due to an abort "
diffoutput += "signal."
# make the list of unique lines
unique = list(set(unique))
if len(diffoutput.strip()) > 0:
if len(testfile_output_string.strip()) > 0:
m = "Weighted sum value for digit number "
m += sol_output[0].strip("\n")
m += " is incorrect, where the first digit is the "
m += "farthest left."
else:
m = "Could not find a value to utilize. Attempted to find "
m += " value using a case agnostic key value of: "
m += keyval
s = "The test harness could not find an output value for "
s += "these tests. Check the specification and make sure "
s += "you are following the stipulated output requirements."
suggestions.append(s)
messages.append(m)
else:
test_obj.score += float(test_obj.max_score) / len(file_paths)
# limit the messages if warranted
if len(messages) > 0:
m = "You failed " + str(len(messages)) + " cases. "
m += "When you fix one of the cases, it should disappear from the "
m += "feedback on your next submission."
suggestions.append(m)
# turn current messages into bullet point list
header = markup_create_header("Bad Values\n", 2)
q = list(set(messages))
q.sort()
mlist = markup_create_unlist(q)
messages = []
messages.append(header + mlist)
# put notifications at top of message if there are any
if len(notifications) > 0:
top = markup_create_header("Notifications\n", 2)
notes = []
for (key, value) in notifications.iteritems():
head = markup_create_bold(key.upper())
head += ": " + value
notes.append(head)
m = markup_create_unlist(notes)
messages = [top + m + "\n"] + messages
# add suggestions section as last part of message
if len(suggestions) > 0:
m = markup_create_header("Suggestions\n", 2)
m += markup_create_unlist(list(set(suggestions)))
messages.append(m)
# add messages
if len(messages) != 0:
if test_obj.message == "":
test_obj.message += "\n".join(messages)
else:
test_obj.message += "\n" + "\n".join(messages)
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
test_obj.score = int(round(test_obj.score))
return OK
def logictest(locations, test_obj, vars, source, fn_name,
has_output, has_input, files, inc_sol, env, alterations):
import re
OK = 0
ERROR = -1
# TODO: this probably should go in some sort of external file
exception_msg = {}
exception_handled = {}
# Out of Range exceptions
# string.at(int)
exception_msg["Out of Range"] = {}
s = "At least one exception was thrown by calling the at() member "
s += " function of a string with a value that is either negative or "
s += " greater than or equal to the size of the string. Check the value "
s += " passed with each invocation of the function at() in " + fn_name
s += " as well as any functions " + fn_name + " calls."
exception_msg["Out of Range"]["basic_string::at"] = s
# vector.at(int)
s = "At least one exception was thrown by calling the at() member "
s += " function of a vector. The function was provided a value that is "
s += " greater than or equal to the size of the vector or it was given a "
s += " negative value. Check the value "
s += " passed with each invocation of the function at() in " + fn_name
s += " as well as any functions " + fn_name + " calls."
exception_msg["Out of Range"]["vector::_M_range_check"] = s
# initially no exceptions have been handled
for key in exception_msg:
exception_handled[key] = {}
for what_key in exception_msg[key]:
exception_handled[key][what_key] = False
harness_dir = locations[1]
files_to_remove = []
# check if working directory exists, if not make one
working_dir = os.path.join(harness_dir, env["working_dir"])
if not os.path.exists(working_dir):
os.mkdir(working_dir)
made_working = True
else:
made_working = False
# set up path to executable rename
exe_path = os.path.join(working_dir, env["exe_name"])
files_to_remove.append(exe_path)
# attempt compilation
ret_val = compile_single(source.file_loc, exe_path, harness_dir, env)
if not ret_val["success"]:
test_obj.score = 0
test_obj.message = "File did not compile, cannot execute tests."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# TODO: make sure all the paths and files exist before accessing/using
# set up path to unit tests directory
unit_dir = os.path.join(harness_dir, env["unittest_dir"])
# set up path to #include lines
inc_stmt_path = os.path.join(unit_dir, vars["library_includes"])
# set up path to harness C++ file
unit_test_path = []
if files is None:
unit_filename = fn_name + env["src_extension"]
unit_test_path.append(os.path.join(unit_dir, unit_filename))
else:
for i in files:
unit_test_path.append(os.path.join(unit_dir, i))
unittest_module_dir = os.path.join(harness_dir, "modules/unittesting")
unit_main_path = os.path.join(unittest_module_dir, "unit_main.cpp")
# set up path to testing file
merged_path = []
if files is None:
merged_path.append(os.path.join(working_dir, vars["file_name"]))
files_to_remove.append(merged_path[-1])
else:
for (i, test) in enumerate(unit_test_path):
file_nm = vars["file_name"][0:vars["file_name"].find(env["src_extension"])] +str(i)+ env["src_extension"]
merged_path.append(os.path.join(working_dir, file_nm))
files_to_remove.append(merged_path[-1])
# set up paths for unit test output
unit_out = os.path.join(working_dir, "unit_test_output.txt")
unit_err = os.path.join(working_dir, "unit_test_errors.txt")
errf = os.path.join(working_dir, "test_program_err.txt")
outf = os.path.join(working_dir, "test_program_out.txt")
inpf = os.path.join(working_dir, "test_input.txt")
files_to_remove.append(unit_out)
files_to_remove.append(unit_err)
files_to_remove.append(errf)
files_to_remove.append(outf)
files_to_remove.append(inpf)
solution_func_dir = os.path.join(harness_dir, env["solution_func_dir"])
sol_incs = []
for func in inc_sol:
file = func + ".cpp"
file_path = os.path.join(solution_func_dir, file)
sol_incs.append(file_path)
# create unit testing file
units = []
for (i, unit_test) in enumerate(unit_test_path):
unit = UnitTest(fn_name)
ret = unit.create_unit(inc_stmt_path, unit_test, unit_main_path,
merged_path[i], source, harness_dir, sol_incs,
alterations)
if ret:
units.append(unit)
message = []
suggestions = []
run_messages = {}
notifications = {}
unit_fail = set()
contains_output = False
contains_input = False
# merge success
if len(units) == 0:
test_obj.message = "<font color='red'>Could not create unit test.</font>"
else:
for merge_p in merged_path:
# attempt compilation
ret_val = compile_single(merge_p, exe_path, harness_dir, env)
if ret_val["success"]:
break
if not ret_val["success"]:
test_obj.score = 0
header = markup_create_header("Notice\n", 2)
m = "The unit test did not compile."
message.append(header + m)
header = markup_create_header("Compile Errors\n", 4)
m = markup_create_indent(ret_val["message"], 2)
message.append(markup_create_indent(header + m, 1))
header = markup_create_header("Suggestions\n", 2)
s = "If you have not yet implmented the " + fn_name + " function, "
s += " you may ignore the fact that this test did not compile."
suggestions.append(s)
s = "If you have started implementing your function " + fn_name
s += " and are attempting to test it: Make sure you fix all "
s += "compile errors before submission. If yours is compiling, "
s += "make sure " + fn_name + " has a proper definition "
s += "as we cannot invoke the function from the specification "
s += " if you are defining it differently than we expect. "
s += "Be sure to check its:\n"
l = ["return type", "function name",
"parameter types", "number of parameters"]
l_m = markup_create_indent(markup_create_unlist(l), 2)
s += l_m
suggestions.append(s)
message.append(header + markup_create_unlist(suggestions))
# join all the messages and put in test object message
test_obj.message = "\n".join(message)
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# run the unit test
# TODO: implement termination of process if it takes too long
try:
# create a task for execution
t = Task(env["timeout"])
with open(outf, 'w') as out_file:
with open(errf, 'w') as err_file:
inputFile = open(inpf, 'w+')
inputFile.write("1 a holy cow")
inputFile.close()
with open(inpf, 'r') as inp_file:
if has_input:
t.check_call([exe_path, unit_out, unit_err, inpf],
stdout=out_file, stderr=err_file, stdin=inp_file)
else:
t.check_call([exe_path, unit_out, unit_err],
stdout=out_file, stderr=err_file, stdin=inp_file)
# all tests are sent an input file, read remainder
remains = inp_file.read()
# read entire input file for comparison
with open(inpf, 'r') as inp_file:
entire = inp_file.read()
# if function was not meant to have input and it
# consumed something then make note of it
# TODO: if process has_input then remains length is always 0
# this does not imply that it did or didn't take input
if len(remains) != len(entire):
contains_input = True
else:
contains_input = False
except SystemError as e:
unit_fail.add("SystemError")
# create a notice about the abort throw and put in message
m = "The " + fn_name + " unit test was aborted during execution. "
m += "If any reason or trace was given "
m += "these may exist in the error message of the "
m += "specific call that was aborted. If the type of abort was "
m += "recognized there will be guidance in the Suggestions "
m += "section. The abort may have been caused by this function "
m += "or one that it calls."
notifications["abort"] = m
# TODO: this access is ugly and should be cleaned up
# if seg fault was detected
if e[0][0] == -11:
cl = ["return value", "values of each parameter"]
s = "A segmentation fault was detected when invoking your "
s += "function. Make sure you check the following for "
s += fn_name + " as well as any functions it invokes.\n"
s += markup_create_indent(markup_create_unlist(cl), 1)
suggestions.append(s)
# termination of thread due to timeout
elif e[0][0] == -15:
cl = ["infinite loops (check your conditions)",
"attempting to get input when no input exists"]
s = "The unit test for your function, " + fn_name + " failed "
s += "to execute to completion in the time allotted to it. "
s += "This may have been caused by several things including:\n"
s += markup_create_indent(markup_create_unlist(cl), 1)
suggestions.append(s)
else:
s = "The abort may have been caused by " + fn_name + " or any "
s += "function that it calls. If " + fn_name + " invokes a "
s += "function and that function passes its unit test then "
s += "make sure " + fn_name + " passes proper values to the "
s += "other function."
suggestions.append(s)
# if the unit test had any output then add them to message
if os.path.isfile(unit_err):
unit_err_contents = open(unit_err).read().rstrip('\n')
else:
unit_err_contents = ""
# put unit test errors in the message
if len(unit_err_contents) > 0:
lines = unit_err_contents.rstrip("\n").split("\n")
for fail_test in lines:
splits = fail_test.split("\t")
key = splits[0]
errors = "\n".join(splits[1:])
top = fn_name + " unit test error\n"
single_result = markup_create_header(top, 4)
single_result = markup_create_indent(single_result, 2)
single_result += markup_create_indent(errors, 3)
single_result += "\n"
key = key.strip()
unit_fail.add(key)
if key not in run_messages:
run_messages[key] = single_result
else:
run_messages[key] += single_result
error_split = errors.split("\n")
if len(error_split) > 0:
m = "The " + fn_name + " unit test caught an exception "
m += "thrown by " + fn_name + ". If the exception was "
m += "recognized by our harness then more information "
m += "will exist in the information for the invocation "
m += "of the specific unit test that threw the exception. "
m += "Additionally, there may be guidance in the "
m += "Suggestions section of the feedback."
notifications["exception"] = m
for exception in error_split:
for key in exception_msg:
if exception.find(key) != -1:
for what, sug in exception_msg[key].iteritems():
if exception.find(what) != -1:
if not exception_handled[key][what]:
suggestions.append(sug)
exception_handled[key][what] = True
# if the unit test had any output then add them to message
if os.path.isfile(unit_out):
unit_out_contents = open(unit_out).read().rstrip('\n')
else:
unit_out_contents = ""
if len(unit_out_contents) > 0:
con_list = unit_out_contents.split("\n")
combine = []
for fail_test in con_list:
splits = fail_test.split("\t")
key = splits[0]
results = splits[1:]
rlist = markup_create_unlist(results)
top = fn_name + " unit test output\n"
single_result = markup_create_header(top, 4)
single_result = markup_create_indent(single_result, 2)
single_result += markup_create_indent(rlist, 3)
single_result += "\n"
key = key.strip()
unit_fail.add(key)
if key not in run_messages:
run_messages[key] = single_result
else:
run_messages[key] += single_result
# put standard output in the msg if unit testing had output or error
if len(open(outf).read()) > 0:
line = open(outf).read().rstrip("\n")
split_on = "Calling "
pattern = re.compile(split_on)
per_call = pattern.split(line)
for output in per_call:
lines = output.rstrip("\n").split("\n")
key = "Calling " + lines[0]
if len(lines) > 1:
contains_output = True
act_output = "\n".join(lines[1:])
pref_output = markup_create_codeblock(act_output)
top = fn_name + " standard output\n"
single_result = markup_create_header(top, 4)
single_result = markup_create_indent(single_result, 2)
single_result += markup_create_indent(pref_output, 3)
single_result += "\n"
key = key.strip()
# if the case hasn't been flagged for failure, do so
# if output is not expected for this function
if not has_output and contains_output:
unit_fail.add(key)
# only add output if key already exists since it will
# be created by unit_error or unit_out
if key in unit_fail:
if key in run_messages:
if run_messages[key][-1] == "\n":
run_messages[key] += single_result
else:
run_messages[key] += "\n" + single_result
else:
run_messages[key] = single_result
# create a suggestion about the existence of output if some exists
if not has_output and contains_output:
s = "The " + fn_name + " function should not contain any output."
suggestions.append(s)
elif has_output and not contains_output:
s = "The " + fn_name + " function should contain some output."
suggestions.append(s)
unit_fail.add("OutputProblem")
# create a suggestion about the existence of input
if not has_input and contains_input:
s = "The " + fn_name + " function should not consume any input."
suggestions.append(s)
unit_fail.add("InputProblem")
elif has_input and not contains_input:
s = "The " + fn_name + " function should consume some input."
suggestions.append(s)
unit_fail.add("InputProblem")
# create suggestion if any unit tests failed
if len(run_messages) > 0:
s = "In total, you failed " + str(len(run_messages)) + " test "
s += "cases which executed to completion. "
if vars["cases_output"] != 0 and vars["cases_output"] < len(run_messages):
s += "To avoid clutter we have provided information on "
s += str(min(len(run_messages), vars["cases_output"])) + " of "
s += "these cases. Please utilize this information when "
s += "attempting to fix your function, " +fn_name + ". "
s += "When you submit again, if you pass one "
s += "of the displayed cases it will disappear and may be "
s += "replaced by information for another of the cases you "
s += "failed."
elif vars["cases_output"] == 0:
s += "Please attempt to fix your code for your function, "
s += fn_name + ". We have opted to not "
s += "display information for any of the failed cases to "
s += "encourage development of proper testing skills when "
s += "designign a program or function. This includes coming "
s += "up with the cases."
else:
s += "Information on these test cases has been provided. "
s += "Please utilize this information when "
s += "attempting to fix your function, " +fn_name + ". "
s += "When you submit again, if you pass one "
s += "of the displayed cases it will disapppear."
suggestions.append(s)
# add individual testing output to the message
import operator
m_sorted = sorted(run_messages.iteritems(), key=operator.itemgetter(0))
count = 0
for (key, value) in m_sorted:
if count >= vars["cases_output"]:
break
count = count + 1
header = markup_create_indent(markup_create_header(key+"\n", 3),1)
message.append(header + value)
# put notifications at top of message if there are any
if len(notifications) > 0:
top = markup_create_header("Notifications\n", 2)
notes = []
for (key, value) in notifications.iteritems():
head = markup_create_bold(key.upper())
head += ": " + value
notes.append(head)
m = markup_create_unlist(notes)
message = [top + m + "\n"] + message
# add suggestions section as last part of message
if len(suggestions) > 0:
m = markup_create_header("Suggestions\n", 2)
m += markup_create_unlist(suggestions)
message.append(m)
# if unit fail set is empty
if len(unit_fail) == 0:
test_obj.score = test_obj.max_score
else:
test_obj.score = 0
if test_obj.message == "":
test_obj.message += "\n".join(message)
else:
test_obj.message += "\n" + "\n".join(message)
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
def outputtest(locations, test_obj, vars, source, solution, fn_name, has_output,
has_input, files, inc_sol, env, alterations):
import difflib
import re
OK = 0
ERROR = -1
# TODO: this probably should go in some sort of external file
exception_msg = {}
exception_handled = {}
# Out of Range exceptions
# string.at(int)
exception_msg["Out of Range"] = {}
s = "At least one exception was thrown by calling the at() member "
s += " function of a string with a value that is either negative or "
s += " greater than or equal to the size of the string. Check the value "
s += " passed with each invocation of the function at() in " + fn_name
s += " as well as any functions " + fn_name + " calls."
exception_msg["Out of Range"]["basic_string::at"] = s
# vector.at(int)
s = "At least one exception was thrown by calling the at() member "
s += " function of a vector. The function was provided a value that is "
s += " greater than or equal to the size of the vector or it was given a "
s += " negative value. Check the value "
s += " passed with each invocation of the function at() in " + fn_name
s += " as well as any functions " + fn_name + " calls."
exception_msg["Out of Range"]["vector::_M_range_check"] = s
# initially no exceptions have been handled
for key in exception_msg:
exception_handled[key] = {}
for what_key in exception_msg[key]:
exception_handled[key][what_key] = False
harness_dir = locations[1]
files_to_remove = []
# check if working directory exists, if not make one
working_dir = os.path.join(harness_dir, env["working_dir"])
if not os.path.exists(working_dir):
os.mkdir(working_dir)
made_working = True
else:
made_working = False
# set up path to executable rename
exe_path = os.path.join(working_dir, env["exe_name"])
sol_exe = os.path.join(working_dir, env["solution_exe"])
files_to_remove.append(exe_path)
files_to_remove.append(sol_exe)
# attempt compilation
ret_val = compile_single(source.file_loc, exe_path, harness_dir, env)
if not ret_val["success"]:
test_obj.score = 0
test_obj.message = "File did not compile, cannot execute tests."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# attempt compilation of solution
ret_val = compile_single(solution.file_loc, sol_exe, harness_dir, env)
if not ret_val["success"]:
test_obj.score = 0
test_obj.message = "Solution did not compile, cannot execute tests."
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
# TODO: make sure all the paths and files exist before accessing/using
# set up path to unit tests directory
unit_dir = os.path.join(harness_dir, env["unittest_dir"])
# set up path to #include lines
inc_stmt_path = os.path.join(unit_dir, vars["library_includes"])
# set up path to harness C++ file
unit_test_path = []
if files is None:
unit_filename = fn_name + env["src_extension"]
unit_test_path.append(os.path.join(unit_dir, unit_filename))
else:
for i in files:
unit_test_path.append(os.path.join(unit_dir, i))
unittest_module_dir = os.path.join(harness_dir, "modules/unittesting")
unit_main_path = os.path.join(unittest_module_dir, "unit_main.cpp")
# set up path to testing file
merged_path = []
sol_merged_path = []
if files is None:
merged_path.append(os.path.join(working_dir, vars["file_name"]))
sol_merged_path.append(os.path.join(working_dir, solution.name))
files_to_remove.append(merged_path[-1])
files_to_remove.append(sol_merged_path[-1])
else:
for (i, test) in enumerate(unit_test_path):
file_nm = vars["file_name"][0:vars["file_name"].find(env["src_extension"])] +str(i)+ env["src_extension"]
merged_path.append(os.path.join(working_dir, file_nm))
files_to_remove.append(merged_path[-1])
file_nm = solution.name[0:solution.name.find(env["src_extension"])] +str(i)+ env["src_extension"]
sol_merged_path.append(os.path.join(working_dir, file_nm))
files_to_remove.append(sol_merged_path[-1])
# set up paths for unit test output
unit_out = os.path.join(working_dir, "unit_test_output.txt")
unit_err = os.path.join(working_dir, "unit_test_errors.txt")
errf = os.path.join(working_dir, "test_program_err.txt")
outf = os.path.join(working_dir, "test_program_out.txt")
inpf = os.path.join(working_dir, "test_input.txt")
files_to_remove.append(unit_out)
files_to_remove.append(unit_err)
files_to_remove.append(errf)
files_to_remove.append(outf)
files_to_remove.append(inpf)
solution_func_dir = os.path.join(harness_dir, env["solution_func_dir"])
sol_incs = []
for func in inc_sol:
file = func + env["src_extension"]
file_path = os.path.join(solution_func_dir, file)
sol_incs.append(file_path)
# create unit testing file
units = []
for (i, unit_test) in enumerate(unit_test_path):
unit = UnitTest(fn_name)
ret = unit.create_unit(inc_stmt_path, unit_test, unit_main_path,
merged_path[i], source, harness_dir, sol_incs,
alterations)
if ret:
units.append(unit)
# create unit solution files
sol_units = []
for (i, unit_test) in enumerate(unit_test_path):
unit = UnitTest(fn_name)
ret = unit.create_unit(inc_stmt_path, unit_test, unit_main_path,
sol_merged_path[i], solution, harness_dir, sol_incs,
alterations)
if ret:
sol_units.append(unit)
message = []
suggestions = []
run_messages = {}
sol_messages = {}
notifications = {}
unit_fail = set()
contains_output = False
contains_input = False
# merge success
if len(units) == 0 or len(sol_units) == 0:
test_obj.message = "Could not create unit test."
else:
for merge_p in merged_path:
# attempt compilation
ret_val = compile_single(merge_p, exe_path, harness_dir, env)
if ret_val["success"]:
break
if not ret_val["success"]:
test_obj.score = 0
header = markup_create_header("Notice\n", 2)
m = "The unit test did not compile."
message.append(header + m)
header = markup_create_header("Compile Errors\n", 4)
m = markup_create_indent(ret_val["message"], 2)
message.append(markup_create_indent(header + m, 1))
header = markup_create_header("Suggestions\n", 2)
s = "If you have not yet implmented the " + fn_name + " function, "
s += " you may ignore the fact that this test did not compile."
suggestions.append(s)
s = "If you have started implementing your function " + fn_name
s += " and are attempting to test it: Make sure you fix all "
s += "compile errors before submission. If yours is compiling, "
s += "make sure " + fn_name + " has a proper definition "
s += "as we cannot invoke the function from the specification "
s += " if you are defining it differently than we expect. "
s += "Be sure to check its:\n"
l = ["return type", "function name",
"parameter types", "number of parameters"]
l_m = markup_create_indent(markup_create_unlist(l), 2)
s += l_m
suggestions.append(s)
message.append(header + markup_create_unlist(suggestions))
# join all the messages and put in test object message
test_obj.message = "\n".join(message)
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
for sol_p in sol_merged_path:
# attempt compilation
ret_val = compile_single(sol_p, sol_exe, harness_dir, env)
if ret_val["success"]:
break
if not ret_val["success"]:
test_obj.score = 0
header = markup_create_header("Notice\n", 2)
m = "The solution's unit test did not compile."
message.append(header + m)
# run the unit test
try:
# create a task for execution
t = Task(env["timeout"])
with open(outf, 'w+') as out_file:
with open(errf, 'w+') as err_file:
inputFile = open(inpf, 'w+')
inputFile.write("1 a holy cow")
inputFile.close()
with open(inpf, 'r') as inp_file:
if has_input:
t.check_call([exe_path, unit_out, unit_err, inpf],
stdout=out_file, stderr=err_file, stdin=inp_file)
else:
t.check_call([exe_path, unit_out, unit_err],
stdout=out_file, stderr=err_file, stdin=inp_file)
except SystemError as e:
unit_fail.add("SystemError")
# create a notice about the abort throw and put in message
m = "The " + fn_name + " unit test was aborted during execution. "
notifications["abort"] = m
# TODO: this access is ugly and should be cleaned up
# if seg fault was detected
if e[0][0] == -11:
cl = ["return value", "values of each parameter"]
s = "A segmentation fault was detected when invoking your "
s += "function. Make sure you check the following for "
s += fn_name + " as well as any functions it invokes.\n"
s += markup_create_indent(markup_create_unlist(cl), 1)
suggestions.append(s)
# termination of thread due to timeout
elif e[0][0] == -15:
cl = ["infinite loops (check your conditions)",
"attempting to get input when no input exists"]
s = "The unit test for your function, " + fn_name + " failed "
s += "to execute to completion in the time allotted to it. "
s += "This may have been caused by several things including:\n"
s += markup_create_indent(markup_create_unlist(cl), 1)
suggestions.append(s)
else:
s = "The abort may have been caused by " + fn_name + " or any "
s += "function that it calls. If " + fn_name + " invokes a "
s += "function and that function passes its unit test then "
s += "make sure " + fn_name + " passes proper values to the "
s += "other function."
suggestions.append(s)
# put standard output in the msg if unit testing had output or error
if len(open(outf).read()) > 0:
line = open(outf).read().rstrip("\n")
split_on = "Calling "
pattern = re.compile(split_on)
per_call = pattern.split(line)
for output in per_call:
lines = output.rstrip("\n").split("\n")
# if something beyond the call line exists, output exists
if len(lines) > 1:
key = "Calling " + lines[0]
act_output = "\n".join(lines[1:])
run_messages[key] = act_output
# run the solution unit test
try:
# create a task for execution
t = Task(env["timeout"])
with open(outf, 'w+') as out_file:
with open(errf, 'w+') as err_file:
inputFile = open(inpf, 'w+')
inputFile.write("1 a holy cow")
inputFile.close()
with open(inpf, 'r') as inp_file:
if has_input:
t.check_call([sol_exe, unit_out, unit_err, inpf],
stdout=out_file, stderr=err_file, stdin=inp_file)
else:
t.check_call([sol_exe, unit_out, unit_err],
stdout=out_file, stderr=err_file, stdin=inp_file)
except SystemError as e:
unit_fail.add("SystemError")
# creatk a notice about the abort throw and put in message
m = "The " + fn_name + " unit test for solution was aborted."
notifications["abort"] = m
# put standard output in the msg if unit testing had output or error
if len(open(outf).read()) > 0:
line = open(outf).read().rstrip("\n")
split_on = "Calling "
pattern = re.compile(split_on)
per_call = pattern.split(line)
for output in per_call:
lines = output.rstrip("\n").split("\n")
# if something beyond the call line exists, output exists
if len(lines) > 1:
key = "Calling " + lines[0]
sol_output = "\n".join(lines[1:])
sol_messages[key] = sol_output
diff_messages = {}
for (key, value) in sol_messages.iteritems():
if key not in run_messages:
run_messages[key] = ""
# split the solution output into lines and eliminate blanks
output_temp = value.split('\n')
sol_output = []
for line in output_temp:
if len(line.strip()) != 0:
sol_output.append(line.strip().lower()+'\n')
# split the test file output into lines and eliminate blanks
output_temp = run_messages[key].split('\n')
tf_output = []
for line in output_temp:
if len(line.strip()) != 0:
tf_output.append(line.strip().lower()+'\n')
lines = []
difflines = difflib.unified_diff(tf_output, sol_output,
fromfile=source.name, tofile=solution.name, n=1)
unique = []
for (i, line) in enumerate(difflines):
if i > 1 and len(line) > 0:
if line[0] == '+':
unique.append(line)
lines.append(line)
diffoutput = "".join(lines)
# make the list of unique lines
unique = list(set(unique))
if len(diffoutput) > 0:
header = "Differences (" + str(len(unique)) + " unique)\n"
diffoutput = markup_create_codeblock(diffoutput)
diffoutput = markup_create_indent(diffoutput, 3)
diff_messages[key] = header + diffoutput
# create suggestion if any unit diff tests failed
if len(diff_messages) > 0:
s = "In total, you failed " + str(len(run_messages)) + " test "
s += "cases which executed to completion. "
if vars["cases_output"] != 0 and vars["cases_output"] < len(run_messages):
s += "To avoid clutter we have provided information on "
s += str(min(len(run_messages), vars["cases_output"])) + " of "
s += "these cases. Please utilize this information when "
s += "attempting to fix your function, " +fn_name + ". "
s += "When you submit again, if you pass one "
s += "of the displayed cases it will disappear and may be "
s += "replaced by information for another of the cases you "
s += "failed."
elif vars["cases_output"] == 0:
s += "Please attempt to fix your code for your function, "
s += fn_name + ". We have opted to not "
s += "display information for any of the failed cases to "
s += "encourage development of proper testing skills when "
s += "designign a program or function. This includes coming "
s += "up with the cases."
else:
s += "Information on these test cases has been provided. "
s += "Please utilize this information when "
s += "attempting to fix your function, " +fn_name + ". "
s += "When you submit again, if you pass one "
s += "of the displayed cases it will disapppear."
suggestions.append(s)
# add individual testing output to the message
import operator
m_sorted = sorted(diff_messages.iteritems(), key=operator.itemgetter(0))
count = 0
for (key, value) in m_sorted:
if count >= vars["cases_output"]:
break
count = count + 1
header = markup_create_indent(markup_create_header(key+"\n", 3),1)
message.append(header + value)
# put notifications at top of message if there are any
if len(notifications) > 0:
top = markup_create_header("Notifications\n", 2)
notes = []
for (key, value) in notifications.iteritems():
head = markup_create_bold(key.upper())
head += ": " + value
notes.append(head)
m = markup_create_unlist(notes)
message = [top + m + "\n"] + message
# add suggestions section as last part of message
if len(suggestions) > 0:
m = markup_create_header("Suggestions\n", 2)
m += markup_create_unlist(suggestions)
message.append(m)
# if no messages have been added to list to display, all tests passed
if len(message) == 0:
test_obj.score = test_obj.max_score
else:
test_obj.score = 0
if test_obj.message == "":
test_obj.message += "\n".join(message)
else:
test_obj.message += "\n" + "\n".join(message)
# clean up
if made_working:
shutil.rmtree(working_dir)
elif len(files_to_remove) > 0:
for f in files_to_remove:
if os.path.isfile(f):
os.remove(f)
return OK
def test(locations, result, vars, source, env):
from modules.compilation import compile_single
import copy
import shutil
# files to remove on finish
files_to_remove = []
harness_dir = locations[1]
# set up working area
working_dir = os.path.join(harness_dir, env["working_dir"])
if not os.path.exists(working_dir):
os.mkdir(working_dir)
made_working = True
else:
made_working = False
# set up exe path for compilation
exe_path = os.path.join(working_dir, env["exe_name"])
files_to_remove.append(exe_path)
# set up path for inputs
in_path = os.path.join(working_dir, "input_file.txt")
files_to_remove.append(in_path)
# set up path for output
out_path = os.path.join(working_dir, "program_output.txt")
files_to_remove.append(out_path)
# compile the program
ret_val = compile_single(source.file_loc, exe_path, harness_dir, env)
# grab / generate items specific to this test
ins = genIntStrings(100, vars["input_length"], vars["input_seed"])
with open(in_path, 'w+') as inFile:
for i in ins:
inFile.write(i + "\n")
if ret_val["success"]:
# execute the program grabbing its output
try:
t = Task(env["timeout"])
with open(out_path, 'w') as outFile:
with open(in_path, 'r') as inFile:
t.check_call([exe_path], stdout=outFile, stdin=inFile)
notification = ""
except:
notification = "Program did not execute to completion.\n"
notification = markup_create_bold(notification)
notification = markup_create_indent(notification, 1)
# acquire the story portion of the output
storyOutput = getStory(out_path, vars["separator"],
vars["separator_size"])
# run individual tests on the story output
ind_test = "Inputs Utilized"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
test_vars = copy.deepcopy(vars[ind_test])
inputsUsed(t, test_vars, storyOutput, in_path)
t.message = notification + t.message
result.add_test(t)
ind_test = "Paragraph Count"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
test_vars = copy.deepcopy(vars[ind_test])
paragraphCount(t, test_vars, storyOutput)
t.message = notification + t.message
result.add_test(t)
ind_test = "Formatting Check"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
test_vars = copy.deepcopy(vars[ind_test])
formattingCheck(t, test_vars, storyOutput, env)
t.message = notification + t.message
result.add_test(t)
else:
# Did not compile
ind_test = "Inputs Utilized"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
m = "Could not run test due to comiplation failure."
t.message = markup_create_indent(m, 1)
result.add_test(t)
ind_test = "Paragraph Count"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
m = "Could not run test due to comiplation failure."
t.message = markup_create_indent(m, 1)
result.add_test(t)
ind_test = "Formatting Check"
if ind_test in vars:
t = GalahTest()
t.name = ind_test
t.max_score = vars[ind_test]["max_score"]
m = "Could not run test due to comiplation failure."
t.message = markup_create_indent(m, 1)
result.add_test(t)
# remove the working directory if made it
if made_working:
shutil.rmtree(working_dir)
elif os.path.isfile(exe_path):
for f in files_to_remove:
os.remove(f)