def test_check_output_no_feedback(sys, get_feedback):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.dst_dir = './'
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
data = {}
data['isCorrect'] = False
data['not-feedback'] = 'garbage'
with open(path.join(args.dst_dir, 'feedback.json'), 'w') as outfile:
json.dump(data, outfile)
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Start of standard error:'),
('root', 'INFO', 'End of standard error'),
('root', 'ERROR', "Field 'feedback' not present in parsed output."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
grade.sys.exit.assert_called_with(1)
def test_check_good_output_fractional_score_zero_point_oh(sys, get_feedback):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.dst_dir = './'
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
data = {}
data['fractionalScore'] = 0.0
data['feedback'] = 'Helpful comment!'
with open(path.join(args.dst_dir, 'feedback.json'), 'w') as outfile:
json.dump(data, outfile)
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Start of standard error:'),
('root', 'INFO', 'End of standard error'),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
assert not grade.sys.exit.called, "sys.exit should not be called!"
def test_check_output_bad_return_code(sys, get_feedback):
with LogCapture():
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.dst_dir = './'
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 1
data = {}
data['isCorrect'] = True
data['feedback'] = 'You win!'
with open(path.join(args.dst_dir, 'feedback.json'), 'w') as outfile:
json.dump(data, outfile)
# Run the function under test
grade.run_container(docker_mock, container, args)
grade.sys.exit.assert_called_with(1)
def test_check_output_malformed_output(sys, get_feedback):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.dst_dir = './'
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
outfile = open(path.join(args.dst_dir, 'feedback.json'), 'w')
outfile.write('{"isCorrect":false, "not-feedback": "garbageeeeeeee')
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Start of standard error:'),
('root', 'INFO', 'End of standard error'),
('root', 'ERROR', "The output was not a valid JSON document."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
grade.sys.exit.assert_called_with(1)