def test(self, command_line_arg, input_array, expected_output, points): self.current_max_points += points success = self._run_tests_with_runner(command_line_arg, input_array, expected_output) if success: self.current_points += points Logger.log_points(points, points) else: Logger.log_points(0, points)
def run(self, command_line_arg, input_array): if not self.prerequirements_run and not self.prerequirements(): Logger.log("STATUS", "Prerequirements failed." \ " The tests will not be executed and you will be awarded 0 points." \ " Please check if there is either no Makefile or the make command" \ " terminates with a none-zero exit code when calling it without any" \ " arguments in your base directory.", "FAIL") return 1, "", "" return self._run(command_line_arg, input_array)
def evaluate(self): Logger.log("POINTS", f"========== Test evaluation for '{self.testname}' ==========", "POINTS") sumpoint = 0 summaxpoint = 0 for i in self.points: Logger.log("POINTS", f"Points for testgroup: '{i}': {Logger.points_fmt(self.points[i], self.max_points[i])}", "POINTS") sumpoint += self.points[i] summaxpoint += self.max_points[i] Logger.log("POINTS", f"Overall result: {Logger.points_fmt(sumpoint, summaxpoint)}", "POINTS") if sumpoint == summaxpoint: Logger.log("INFO", "Well done!", "PRAISE")
def run_all_testgroups(self): self.start_tests() for testgroup in self.testgroups: Logger.log("INFO", f"Running testgroup: {testgroup['name']}") self.reset_points() testgroup["function"]() # allow the 'point_id' value to be missing, in this case the 'name' value will also be used for for the 'point_id' try: testgroup["point_id"] except KeyError: testgroup["point_id"] = testgroup["name"] if testgroup["point_id"] not in self.points: self.points[testgroup["point_id"]] = 0 self.points[testgroup["point_id"]] += self.current_points if testgroup["point_id"] not in self.max_points: self.max_points[testgroup["point_id"]] = 0 self.max_points[testgroup["point_id"]] += self.current_max_points Logger.log("POINTS", f"Testgroup points: {Logger.points_fmt(self.current_points, self.current_max_points)}" ,"POINTS")
def build_docker_image(self): Logger.log("DEBUG", "moving files into place") shutil.copyfile( os.path.join(os.path.abspath(os.path.dirname(__file__)), "Dockerfile"), self.build_path + "/Dockerfile") self.copytree(self.solution_base_dir, self.build_path) Logger.log("DEBUG", "building docker image") self.docker_image, _ = self.docker_client.images.build( path=self.build_path, tag=self.image_name) Logger.log( "DEBUG", f"creating docker container from image {self.docker_image.short_id}" ) self.docker_container = self.docker_client.containers.create( self.docker_image, command='sleep 100000', detach=True) # Logger.log( "DEBUG", f"starting docker container {self.docker_container.short_id}") if self.docker_container is None: return False self.docker_container.start() return True
def _run_tests_with_runner(self, command_line_arg, input_array, expected_output): self.next_test() returncode, output, stderr = self.runner.run(command_line_arg, input_array) actual_out = self.parseOutput(output) solution = self.getSolution(actual_out) if returncode != 0: Logger.log("STATUS", "Test failed, non-zero exit code", "FAIL") if stderr is not None: Logger.log("INFO", f"StdErr: {stderr}") if output is not None: Logger.log("INFO", f"StdOut: {output}") return False if solution != expected_output: Logger.log("STATUS", "Test failed", "FAIL") Logger.log("INFO", f"command line arg: {' '.join(command_line_arg)}") Logger.log("INFO", "input:") for line in input_array: Logger.log("INFO", " ".join(line)) Logger.log("INFO", "expected output:") for line in expected_output: Logger.log("INFO", " ".join(line)) Logger.log("INFO", "parsed solution output:") for line in solution: Logger.log("INFO", " ".join(line)) if output is not None and output != "": Logger.log("DEBUG", f"Full StdOut output: {output}", "SUB_OUT") else: Logger.log("DEBUG", "StdOut was empty") if stderr is not None and stderr != "": Logger.log("DEBUG", f"Full StdErr output: {stderr}", "SUB_OUT_ERR") else: Logger.log("DEBUG", "StdErr was empty") return False else: Logger.log("STATUS", "Test successful", "OK") return True
def next_test(self): Logger.log("INFO", f"Running test: #{self.current_test_num}", "NEW_TEST") self.current_test_num += 1
def start_tests(self): Logger.log("INFO", f"Started Tests: '{self.testname}'")
def prepare(self): if not self.prepared: Logger.log("DEBUG", "started preparing") self.prepared = self.prerequirements() and self.setup( ) and self.run_make_file() return self.prepared
def setup(self): if self.skip_setup: Logger.log("STATUS", "skipping setup", "WARNING") return True Logger.log("DEBUG", "started setup") if not self.check_for_file("setup.sh"): Logger.log("STATUS", "No setup.sh found, skipping", "WARNING") return True returncode, output, stderr = self.run_command('./setup.sh') if returncode != 0: Logger.log("STATUS", "Prerequirements failed, non-zero exit code", "FAIL") if output is not None: Logger.log("INFO", "StdOut:") Logger.log("INFO", output, "STD_OUT") if stderr is not None: Logger.log("INFO", "StdErr:") Logger.log("INFO", stderr, "STD_OUT_ERR") return False else: Logger.log("STATUS", "setup.sh ran successfully", "OK") return True
def run_make_file(self): Logger.log("DEBUG", "running makefile") if not self.check_for_file("makefile"): Logger.log("STATUS", "No makefile found, skipping", "WARNING") return True returncode, output, stderr = self.run_command('make') if returncode != 0: Logger.log("STATUS", "Prerequirements failed, non-zero exit code", "FAIL") if output is not None: Logger.log("INFO", "StdOut:") Logger.log("INFO", output, "STD_OUT") if stderr is not None: Logger.log("INFO", "StdErr:") Logger.log("INFO", stderr, "STD_OUT_ERR") return False else: Logger.log("STATUS", "Makefile ran successfully", "OK") return True
def prerequirements(self): Logger.log("DEBUG", "Starting prerequirements") self.prerequirements_run = self.build_docker_image() Logger.log("DEBUG", "prerequirements finished") return self.prerequirements_run