def __diff_test(self, test):
""" Runs a specified command and compares the result to the expected output """
tr = TestResult(
name=test["name"],
test_type=test["type"],
cmd=test["command"],
maxpoints=test.get("points"),
)
inp = test.get("stdin")
if inp is None:
with (Path(".config") / test["stdinFile"]).absolute().open() as f:
inp = f.read()
out = test.get("stdout")
if out is None:
filename = test.get("stdoutFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
out = f.read()
err = test.get("stderr")
if err is None:
filename = test.get("stderrFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
err = f.read()
cmd = shlex.split(tr.cmd)
# perform globbing of executable name
if test.get("glob_command"):
executable = list(self.workdir.glob(cmd[0]))
if len(executable) > 1:
tr.warning = f"WARNING: multiple executables matched for glob {cmd[0]}"
elif len(executable) > 0:
cmd[0] = str(executable[0].absolute())
tr.cmd = "".join(cmd)
else:
tr.warning = f"WARNING: no files matched with glob {cmd[0]}"
tr.retval, tr.stdout, tr.stderr = utils.run_command(
cmd,
cwd=self.workdir,
sinput=inp,
timeout=test.get("timeout"),
memory_limit=self.config.get("memory_limit"),
process_limit=self.config.get("process_limit"),
)
tr.diffout = ""
if out:
tr.diffout += utils.diff_output(out, tr.stdout)
if err:
tr.diffout += utils.diff_output(err, tr.stderr)
# diff is blank if matches perfectly
if len(tr.diffout) == 0:
tr.points = tr.maxpoints
tr.diffout = "Output is identical"
else:
tr.points = 0
self.results.append(tr)
