def test_partial_credit(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void"])
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertIn("ERROR:root:", logs.output[0])
self.assertEqual(results.score, 5)
def test_executable_multiple_times(self):
""" Checking multiple runs of an executable.
We can also use pipelines in comprehensions and maps,
to test multiple targets at once.
Say you wanted to test that a students code returns zero
for all valid test inputs; this would be a perfect usecase for
multiple targets.
"""
self.weight = 10
# noinspection PyShadowingNames
def pipeline(testcase):
""" Create a pipeline for the given testcase. """
return Pipeline(Run(["echo", testcase]), AssertExitSuccess())
# Now make an iterable (ex: list[str], glob, etc.)
testcases = [c for c in "hello world"]
# You can execute them all at once (all or nothing credit)
[pipeline(testcase)() for testcase in testcases]
# Or, you can also award partial credit for each testcase:
results = PartialCredit(map(pipeline, testcases), 10)()
self.score = results.score
def test_value_list_all_fail(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(5))
values = [10, 1]
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 0)
def test_value_list_some_fail(self):
pipelines = map(lambda t: Pipeline(Run([t]), AssertExitSuccess()),
["ls", "void", "ls", "void"])
values = [10, 1] # 10, 1
logging.disable(logging.CRITICAL)
results = PartialCredit(pipelines, values)()
logging.disable(logging.NOTSET)
self.assertEqual(results.score, 20)
def test_no_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"]), AssertExitFailure()),
range(10))
with self.assertLogs() as logs:
results = PartialCredit(pipelines, 10)()
self.assertEqual(10, len(logs.output))
self.assertIn("ERROR:root:['ls'] should have exited unsuccessfully.",
logs.output)
self.assertEqual(results.score, 0)
def test_score_rotation(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(5))
values = [100, 10, 1] # 100, 10
results = PartialCredit(pipelines, values)()
self.assertEqual(results.score, 221)
def test_value_list_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(2))
values = [10, 1]
results = PartialCredit(pipelines, values)()
self.assertEqual(results.score, 11)
def test_full_credit(self):
pipelines = map(lambda t: Pipeline(Run(["ls"])), range(10))
results = PartialCredit(pipelines, 10)()
self.assertEqual(results.score, 10)