def __diff_test(self, test): """ Runs a specified command and compares the result to the expected output """ tr = TestResult( name=test["name"], test_type=test["type"], cmd=test["command"], maxpoints=test.get("points"), ) inp = test.get("stdin") if inp is None: with (Path(".config") / test["stdinFile"]).absolute().open() as f: inp = f.read() out = test.get("stdout") if out is None: filename = test.get("stdoutFile") if filename: with (Path(".config") / filename).absolute().open() as f: out = f.read() err = test.get("stderr") if err is None: filename = test.get("stderrFile") if filename: with (Path(".config") / filename).absolute().open() as f: err = f.read() cmd = shlex.split(tr.cmd) # perform globbing of executable name if test.get("glob_command"): executable = list(self.workdir.glob(cmd[0])) if len(executable) > 1: tr.warning = f"WARNING: multiple executables matched for glob {cmd[0]}" elif len(executable) > 0: cmd[0] = str(executable[0].absolute()) tr.cmd = "".join(cmd) else: tr.warning = f"WARNING: no files matched with glob {cmd[0]}" tr.retval, tr.stdout, tr.stderr = utils.run_command( cmd, cwd=self.workdir, sinput=inp, timeout=test.get("timeout"), memory_limit=self.config.get("memory_limit"), process_limit=self.config.get("process_limit"), ) tr.diffout = "" if out: tr.diffout += utils.diff_output(out, tr.stdout) if err: tr.diffout += utils.diff_output(err, tr.stderr) # diff is blank if matches perfectly if len(tr.diffout) == 0: tr.points = tr.maxpoints tr.diffout = "Output is identical" else: tr.points = 0 self.results.append(tr)