def test(self, command_line_arg, input_array, expected_output, points):
self.current_max_points += points
success = self._run_tests_with_runner(command_line_arg, input_array, expected_output)
if success:
self.current_points += points
Logger.log_points(points, points)
else:
Logger.log_points(0, points)
def run(self, command_line_arg, input_array):
if not self.prerequirements_run and not self.prerequirements():
Logger.log("STATUS", "Prerequirements failed." \
" The tests will not be executed and you will be awarded 0 points." \
" Please check if there is either no Makefile or the make command" \
" terminates with a none-zero exit code when calling it without any" \
" arguments in your base directory.", "FAIL")
return 1, "", ""
return self._run(command_line_arg, input_array)
def evaluate(self):
Logger.log("POINTS", f"========== Test evaluation for '{self.testname}' ==========", "POINTS")
sumpoint = 0
summaxpoint = 0
for i in self.points:
Logger.log("POINTS", f"Points for testgroup: '{i}': {Logger.points_fmt(self.points[i], self.max_points[i])}", "POINTS")
sumpoint += self.points[i]
summaxpoint += self.max_points[i]
Logger.log("POINTS", f"Overall result: {Logger.points_fmt(sumpoint, summaxpoint)}", "POINTS")
if sumpoint == summaxpoint:
Logger.log("INFO", "Well done!", "PRAISE")
def run_all_testgroups(self):
self.start_tests()
for testgroup in self.testgroups:
Logger.log("INFO", f"Running testgroup: {testgroup['name']}")
self.reset_points()
testgroup["function"]()
# allow the 'point_id' value to be missing, in this case the 'name' value will also be used for for the 'point_id'
try:
testgroup["point_id"]
except KeyError:
testgroup["point_id"] = testgroup["name"]
if testgroup["point_id"] not in self.points:
self.points[testgroup["point_id"]] = 0
self.points[testgroup["point_id"]] += self.current_points
if testgroup["point_id"] not in self.max_points:
self.max_points[testgroup["point_id"]] = 0
self.max_points[testgroup["point_id"]] += self.current_max_points
Logger.log("POINTS", f"Testgroup points: {Logger.points_fmt(self.current_points, self.current_max_points)}" ,"POINTS")
def build_docker_image(self):
Logger.log("DEBUG", "moving files into place")
shutil.copyfile(
os.path.join(os.path.abspath(os.path.dirname(__file__)),
"Dockerfile"), self.build_path + "/Dockerfile")
self.copytree(self.solution_base_dir, self.build_path)
Logger.log("DEBUG", "building docker image")
self.docker_image, _ = self.docker_client.images.build(
path=self.build_path, tag=self.image_name)
Logger.log(
"DEBUG",
f"creating docker container from image {self.docker_image.short_id}"
)
self.docker_container = self.docker_client.containers.create(
self.docker_image, command='sleep 100000', detach=True) #
Logger.log(
"DEBUG",
f"starting docker container {self.docker_container.short_id}")
if self.docker_container is None:
return False
self.docker_container.start()
return True
def _run_tests_with_runner(self, command_line_arg, input_array, expected_output):
self.next_test()
returncode, output, stderr = self.runner.run(command_line_arg, input_array)
actual_out = self.parseOutput(output)
solution = self.getSolution(actual_out)
if returncode != 0:
Logger.log("STATUS", "Test failed, non-zero exit code", "FAIL")
if stderr is not None:
Logger.log("INFO", f"StdErr: {stderr}")
if output is not None:
Logger.log("INFO", f"StdOut: {output}")
return False
if solution != expected_output:
Logger.log("STATUS", "Test failed", "FAIL")
Logger.log("INFO", f"command line arg: {' '.join(command_line_arg)}")
Logger.log("INFO", "input:")
for line in input_array:
Logger.log("INFO", " ".join(line))
Logger.log("INFO", "expected output:")
for line in expected_output:
Logger.log("INFO", " ".join(line))
Logger.log("INFO", "parsed solution output:")
for line in solution:
Logger.log("INFO", " ".join(line))
if output is not None and output != "":
Logger.log("DEBUG", f"Full StdOut output: {output}", "SUB_OUT")
else:
Logger.log("DEBUG", "StdOut was empty")
if stderr is not None and stderr != "":
Logger.log("DEBUG", f"Full StdErr output: {stderr}", "SUB_OUT_ERR")
else:
Logger.log("DEBUG", "StdErr was empty")
return False
else:
Logger.log("STATUS", "Test successful", "OK")
return True
def next_test(self):
Logger.log("INFO", f"Running test: #{self.current_test_num}", "NEW_TEST")
self.current_test_num += 1
def start_tests(self):
Logger.log("INFO", f"Started Tests: '{self.testname}'")
def prepare(self):
if not self.prepared:
Logger.log("DEBUG", "started preparing")
self.prepared = self.prerequirements() and self.setup(
) and self.run_make_file()
return self.prepared
def setup(self):
if self.skip_setup:
Logger.log("STATUS", "skipping setup", "WARNING")
return True
Logger.log("DEBUG", "started setup")
if not self.check_for_file("setup.sh"):
Logger.log("STATUS", "No setup.sh found, skipping", "WARNING")
return True
returncode, output, stderr = self.run_command('./setup.sh')
if returncode != 0:
Logger.log("STATUS", "Prerequirements failed, non-zero exit code",
"FAIL")
if output is not None:
Logger.log("INFO", "StdOut:")
Logger.log("INFO", output, "STD_OUT")
if stderr is not None:
Logger.log("INFO", "StdErr:")
Logger.log("INFO", stderr, "STD_OUT_ERR")
return False
else:
Logger.log("STATUS", "setup.sh ran successfully", "OK")
return True
def run_make_file(self):
Logger.log("DEBUG", "running makefile")
if not self.check_for_file("makefile"):
Logger.log("STATUS", "No makefile found, skipping", "WARNING")
return True
returncode, output, stderr = self.run_command('make')
if returncode != 0:
Logger.log("STATUS", "Prerequirements failed, non-zero exit code",
"FAIL")
if output is not None:
Logger.log("INFO", "StdOut:")
Logger.log("INFO", output, "STD_OUT")
if stderr is not None:
Logger.log("INFO", "StdErr:")
Logger.log("INFO", stderr, "STD_OUT_ERR")
return False
else:
Logger.log("STATUS", "Makefile ran successfully", "OK")
return True
def prerequirements(self):
Logger.log("DEBUG", "Starting prerequirements")
self.prerequirements_run = self.build_docker_image()
Logger.log("DEBUG", "prerequirements finished")
return self.prerequirements_run