def __junit_test(self, test):
""" Runs a junit test .class file """
tr = TestResult(name=test["name"], test_type=test["type"])
junit_runner = "junit-platform-console-standalone-1.3.1.jar"
cmd = shlex.split(
(f"""java -jar {utils.libdir() / junit_runner}"""
f""" -cp "{self.outdir}" -c {test["classname"]} --reports-dir="""
f"""{self.outdir} --disable-ansi-colors"""),
posix=("win" not in sys.platform),
)
tr.retval, tr.stdout, tr.stderr = utils.run_command(cmd,
cwd=self.workdir)
tr.cmd = " ".join(cmd)
tr.stdout, tr.points, tr.maxpoints = junit.parse_xml(self.workdir /
self.outdir)
self.results.append(tr)
def __custom_test(self, test):
""" Runs any arbitrary command and saves output. """
tr = TestResult(name=test["name"],
test_type=test["type"],
cmd=test["command"])
cmd = shlex.split(tr.cmd)
tr.retval, tr.stdout, tr.stderr = utils.run_command(
cmd,
cwd=self.workdir,
sinput="",
timeout=test.get("timeout"),
memory_limit=self.config.get("memory_limit"),
process_limit=self.config.get("process_limit"),
)
self.results.append(tr)
def runtest(config: Config, workdir: Path, results_dir: Path):
print(f"========== Grading {workdir.name}")
(workdir / config["output_dir"]).mkdir(exist_ok=True, parents=True)
secret_files = []
with (results_dir / f"{workdir.name}.txt").open("w", encoding="utf-8") as logfile:
if "build" in config:
logfile.write(utils.box_text("Build Step") + "\n")
# copy files from project root to build location
if "required_files" in config["build"]:
for file in config["build"]["required_files"]:
(workdir / file["dest"]).mkdir(exist_ok=True, parents=True)
try:
copy(
Path(".config") / file["file"], Path(workdir / file["dest"])
)
except FileNotFoundError as ex:
raise click.FileError(ex.filename, "are you sure it exists?")
if file.get("secret"):
secret_files.append(Path(workdir / file["dest"] / file["file"]))
if "commands" in config["build"]:
for command in config["build"]["commands"]:
br = TestResult(test_type="build", cmd=command)
command = shlex.split(command)
br.retval, br.stdout, br.stderr = utils.run_command(
command, cwd=workdir
)
logfile.write(br.log(config["output"]["build"]))
# loop through and run all tests
test_runner = TestRunner(logfile, workdir, config)
test_runner.run_all()
test_runner.log()
for file in secret_files:
file.unlink()
def __diff_test(self, test):
""" Runs a specified command and compares the result to the expected output """
tr = TestResult(
name=test["name"],
test_type=test["type"],
cmd=test["command"],
maxpoints=test.get("points"),
)
inp = test.get("stdin")
if inp is None:
with (Path(".config") / test["stdinFile"]).absolute().open() as f:
inp = f.read()
out = test.get("stdout")
if out is None:
filename = test.get("stdoutFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
out = f.read()
err = test.get("stderr")
if err is None:
filename = test.get("stderrFile")
if filename:
with (Path(".config") / filename).absolute().open() as f:
err = f.read()
cmd = shlex.split(tr.cmd)
# perform globbing of executable name
if test.get("glob_command"):
executable = list(self.workdir.glob(cmd[0]))
if len(executable) > 1:
tr.warning = f"WARNING: multiple executables matched for glob {cmd[0]}"
elif len(executable) > 0:
cmd[0] = str(executable[0].absolute())
tr.cmd = "".join(cmd)
else:
tr.warning = f"WARNING: no files matched with glob {cmd[0]}"
tr.retval, tr.stdout, tr.stderr = utils.run_command(
cmd,
cwd=self.workdir,
sinput=inp,
timeout=test.get("timeout"),
memory_limit=self.config.get("memory_limit"),
process_limit=self.config.get("process_limit"),
)
tr.diffout = ""
if out:
tr.diffout += utils.diff_output(out, tr.stdout)
if err:
tr.diffout += utils.diff_output(err, tr.stderr)
# diff is blank if matches perfectly
if len(tr.diffout) == 0:
tr.points = tr.maxpoints
tr.diffout = "Output is identical"
else:
tr.points = 0
self.results.append(tr)
