def test_cannot_infer_filename(self):
""" What if there is no file to infer from? """
with self.assertRaises(ValueError):
AssertStdoutMatches()(Run("echo hello world", shell=True)())
results = Run(["echo", "hello world"])()
with self.assertRaises(AssertionError):
AssertStdoutMatches()(results)
def test_valgrind():
results = Run(["ls"])()
AssertValgrindSuccess()(results)
results = Run("echo hello", shell=True)()
AssertValgrindSuccess()(results)
results = Run("grep pip < README.md", shell=True)()
AssertValgrindSuccess()(results)
def test_json_output(self):
Pipeline(
Run([PYTHON, "-m", "grade", "run", "-p", "test_mixins.py"]),
Run([PYTHON, "-m", "grade", "report", "--format", "json"]),
AssertExitSuccess(),
AssertStdoutContains(["tests"]),
Lambda(lambda results: self.assertGreater(
len(json.loads(results.stdout)["tests"]), 3)),
)()
def test_not(self):
Pipeline(
Run(["ls"]),
# Essentially, assert not contains, which should fail.
Not(AssertStdoutContains(strings=["imaginary.file"])),
)()
with self.assertRaises(AssertionError):
Pipeline(Run(["echo", "hello world"]),
Not(AssertStdoutContains(strings=["hello"])))()
def test_or(self):
Pipeline(
Run(["echo", "hello world"]),
Or(AssertStdoutContains(["goodbye"]),
AssertStdoutContains(["hello"])),
)()
with self.assertRaises(AssertionError):
Pipeline(
Run(["echo", "hello world"]),
Or(AssertStdoutContains(["goodbye"]),
AssertStderrContains(["goodbye"])),
)()
def test_input():
Pipeline(
Run(["cat", "README.md"]),
AssertExitSuccess(),
Run(["grep", "Setup"], input=lambda r: r.stdout),
AssertStdoutMatches("## Setup"),
)()
Pipeline(
Run(["grep", "hello", "-"],
input="hello world\nhear me test things!"),
AssertStdoutMatches("hello world"),
)()
Pipeline(Run(["python", "-c", "x = input(); print(x)"], input="5"),
AssertStdoutMatches("5"))()
def test_iteration(self):
pipeline = Pipeline(
Run(["ls"]), AssertExitSuccess(), AssertValgrindSuccess(), WriteOutputs("temp"),
)
[self.assertTrue(callable(callback)) for callback in pipeline]
self.assertIsInstance(pipeline[1], AssertExitSuccess)
self.assertEqual(len(pipeline), 4)
def test_partial_credit(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void"])
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertIn("ERROR:root:", logs.output[0])
self.assertEqual(results.score, 5)
def setUpClass(cls):
""" Here, we can perform any long-duration pre-requisite task.
The calls in this block are only called once, at the very beginning.
A common use-case for this block is compilation.
"""
Run(["ls"])()
def test_successful_no_output():
Pipeline(
Run([PYTHON, "-m", "grade", "run", "-p", "test_mixins.py"]),
AssertExitSuccess(),
AssertStdoutMatches(""),
AssertStderrMatches(""),
)()
def test_value_list_some_fail(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void", "ls", "void"])
values = [10, 1] # 10, 1
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 20)
def test_inferring_filename(self):
""" Can we infer filename if conventions are followed? """
results = Run(["grep", "-h"])()
results = WriteOutputs("-h")(results)
results = AssertStderrMatches()(results)
self.assertIsInstance(results, CompletedProcess)
os.remove("-h.stdout")
os.remove("-h.stderr")
def test_value_list_all_fail(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(5))
values = [10, 1]
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 0)
def test_no_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(10))
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertEqual(10, len(logs.output))
self.assertIn("ERROR:root:['ls'] should have exited unsuccessfully.",
logs.output)
self.assertEqual(results.score, 0)
def test_executable(self):
""" Checking executable files with a Pipeline.
Pipelines are designed around testing executable files.
They are comprised of "layers" which pass CompletedProcess objects to
each other. You can stack layers however you'd like, but they must start
with a call to Run(), which generates the initial CompletedProcess.
"""
Pipeline(Run(["ls"]), AssertExitSuccess(), AssertValgrindSuccess())()
def test_stderr(self):
results = Run(">&2 echo error", shell=True)()
results = WriteStderr()(results)
with self.assertRaises(FileExistsError):
WriteStderr(overwrite=False)(results)
self.assertIsInstance(results, CompletedProcess)
with open("temp", "r") as f:
self.assertEqual(results.stderr, f.read())
os.remove("temp")
def test_outputs(self):
results = Run(["ls"])()
WriteOutputs("temp")(results)
with open("temp.stdout", "r") as f:
self.assertEqual(results.stdout, f.read())
os.remove("temp.stdout")
with open("temp.stderr", "r") as f:
self.assertEqual(results.stderr, f.read())
os.remove("temp.stderr")
def test_stdout(self):
results = Run(["echo", "hello"])()
results = WriteStdout()(results)
with self.assertRaises(FileExistsError):
WriteStdout(overwrite=False)(results)
self.assertIsInstance(results, CompletedProcess)
with open("temp", "r") as f:
self.assertEqual(results.stdout, f.read())
os.remove("temp")
def test_collecting_runtimes(self):
""" CompletedProcess objects all know their duration.
Duration can be used to allocate points based on speed, or add to a
leaderboard!
"""
# To setup a leaderboard, you have to at least set it's name.
self.leaderboardTitle = "Runtime"
# (Desc)ending or (asc)ending order (desc default)
self.leaderboardOrder = "desc"
self.weight = 10
results = Pipeline(Run(["echo", "hello world"]), AssertExitSuccess())()
# Here we set the leaderboard score as the duration.
self.leaderboardScore = results.duration
if results.duration < 1_000:
self.score = 10
elif results.duration < 10_000:
self.score = 5
def test_assert_exit_status():
Pipeline(
Run(["echo", "hello world"]),
AssertExitStatus(0),
Not(AssertExitStatus(1)),
)()
def test_no_arguments():
Pipeline(
Run([PYTHON, "-m", "grade"]),
AssertExitSuccess(),
)
def test_timeout(self):
with self.assertRaises(TimeoutError):
Run(["sleep", "30"], timeout=1)()
def test_exit_success():
results = Run(["ls"])()
AssertExitSuccess()(results)
def test_nonexistant(self):
with self.assertRaises(FileNotFoundError):
Run(["idonotexist"])()
def test_shell_command(self):
results = Run("echo test | grep test", shell=True)()
self.assertEqual(results.returncode, 0)
self.assertGreater(results.duration, 0)
def test_valid_program(self):
results = Run(["ls"])()
self.assertEqual(results.returncode, 0)
self.assertGreater(results.duration, 0)
def test_stderr_contains(self):
results = Run(">&2 echo hello", shell=True)()
AssertStderrContains(["hello"])(results)
with self.assertRaises(AssertionError):
AssertStderrContains(["world"])(results)
def test_stdout_contains(self):
results = Run(["ls"])()
AssertStdoutContains(["README.md", "pyproject.toml"])(results)
with self.assertRaises(AssertionError):
AssertStdoutContains(["pickleRick"])(results)
def test_faster():
Pipeline(
Run(["echo", "hello world"]),
AssertFaster(10),
Not(AssertFaster(0)),
)()
def test_exit_failure():
results = Run(["ls", "--iamnotanarg"])()
AssertExitFailure()(results)
