def test_not(self):
Pipeline(
Run(["ls"]),
# Essentially, assert not contains, which should fail.
Not(AssertStdoutContains(strings=["imaginary.file"])),
)()
with self.assertRaises(AssertionError):
Pipeline(Run(["echo", "hello world"]),
Not(AssertStdoutContains(strings=["hello"])))()
def test_or(self):
Pipeline(
Run(["echo", "hello world"]),
Or(AssertStdoutContains(["goodbye"]),
AssertStdoutContains(["hello"])),
)()
with self.assertRaises(AssertionError):
Pipeline(
Run(["echo", "hello world"]),
Or(AssertStdoutContains(["goodbye"]),
AssertStderrContains(["goodbye"])),
)()
def test_input():
Pipeline(
Run(["cat", "README.md"]),
AssertExitSuccess(),
Run(["grep", "Setup"], input=lambda r: r.stdout),
AssertStdoutMatches("## Setup"),
)()
Pipeline(
Run(["grep", "hello", "-"],
input="hello world\nhear me test things!"),
AssertStdoutMatches("hello world"),
)()
Pipeline(Run(["python", "-c", "x = input(); print(x)"], input="5"),
AssertStdoutMatches("5"))()
def test_iteration(self):
pipeline = Pipeline(
Run(["ls"]), AssertExitSuccess(), AssertValgrindSuccess(), WriteOutputs("temp"),
)
[self.assertTrue(callable(callback)) for callback in pipeline]
self.assertIsInstance(pipeline[1], AssertExitSuccess)
self.assertEqual(len(pipeline), 4)
def test_partial_credit(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void"])
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertIn("ERROR:root:", logs.output[0])
self.assertEqual(results.score, 5)
def test_successful_no_output():
Pipeline(
Run([PYTHON, "-m", "grade", "run", "-p", "test_mixins.py"]),
AssertExitSuccess(),
AssertStdoutMatches(""),
AssertStderrMatches(""),
)()
def test_value_list_all_fail(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(5))
values = [10, 1]
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 0)
def test_value_list_some_fail(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void", "ls", "void"])
values = [10, 1] # 10, 1
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 20)
def test_no_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(10))
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertEqual(10, len(logs.output))
self.assertIn("ERROR:root:['ls'] should have exited unsuccessfully.",
logs.output)
self.assertEqual(results.score, 0)
def test_executable(self):
""" Checking executable files with a Pipeline.
Pipelines are designed around testing executable files.
They are comprised of "layers" which pass CompletedProcess objects to
each other. You can stack layers however you'd like, but they must start
with a call to Run(), which generates the initial CompletedProcess.
"""
Pipeline(Run(["ls"]), AssertExitSuccess(), AssertValgrindSuccess())()
def test_json_output(self):
Pipeline(
Run([PYTHON, "-m", "grade", "run", "-p", "test_mixins.py"]),
Run([PYTHON, "-m", "grade", "report", "--format", "json"]),
AssertExitSuccess(),
AssertStdoutContains(["tests"]),
Lambda(lambda results: self.assertGreater(
len(json.loads(results.stdout)["tests"]), 3)),
)()
def test_collecting_runtimes(self):
""" CompletedProcess objects all know their duration.
Duration can be used to allocate points based on speed, or add to a
leaderboard!
"""
# To setup a leaderboard, you have to at least set it's name.
self.leaderboardTitle = "Runtime"
# (Desc)ending or (asc)ending order (desc default)
self.leaderboardOrder = "desc"
self.weight = 10
results = Pipeline(Run(["echo", "hello world"]), AssertExitSuccess())()
# Here we set the leaderboard score as the duration.
self.leaderboardScore = results.duration
if results.duration < 1_000:
self.score = 10
elif results.duration < 10_000:
self.score = 5
def pipeline(testcase):
""" Create a pipeline for the given testcase. """
return Pipeline(Run(["echo", testcase]), AssertExitSuccess())
def test_fail_multiple(self):
with self.assertRaises(AssertionError):
tests = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()), range(10))
[test() for test in tests]
def test_full_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(10))
results = PartialCredit(pipelines, 10)()
self.assertEqual(results.score, 10)
def test_no_arguments():
Pipeline(
Run([PYTHON, "-m", "grade"]),
AssertExitSuccess(),
)
def test_value_list_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(2))
values = [10, 1]
results = PartialCredit(pipelines, values)()
self.assertEqual(results.score, 11)
def test_faster():
Pipeline(
Run(["echo", "hello world"]),
AssertFaster(10),
Not(AssertFaster(0)),
)()
def test_score_rotation(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(5))
values = [100, 10, 1] # 100, 10
results = PartialCredit(pipelines, values)()
self.assertEqual(results.score, 221)
def test_assert_exit_status():
Pipeline(
Run(["echo", "hello world"]),
AssertExitStatus(0),
Not(AssertExitStatus(1)),
)()
def test_fails(self):
with self.assertRaises(AssertionError):
Pipeline(Run(["ls"]), AssertExitFailure())()
def test_pass_multiple():
tests = map(lambda t: Pipeline(Run(["ls"]), AssertExitSuccess()), range(10))
[test() for test in tests]
def test_passes():
Pipeline(Run(["ls"]), AssertExitSuccess())