def test_check_output_bad_return_code(sys):
with LogCapture():
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 1
docker_mock.logs.side_effect = [
'debug output', '{"isCorrect":true,"feedback":"You win!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
sys.exit.assert_called_with(1)
def test_check_output_missing_grade(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = ['', '{"feedback": "wheeeee"}']
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Required field 'fractionalScore' is missing."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
sys.exit.assert_called_with(1)
def test_check_output_bad_return_code(sys):
with LogCapture():
docker_mock = MagicMock()
container = {
"Id": "myContainerId"
}
args = argparse.Namespace()
args.timeout = 300
docker_mock.wait.return_value = 1
docker_mock.logs.side_effect = [
'debug output',
'{"isCorrect":true,"feedback":"You win!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
sys.exit.assert_called_with(1)
def test_check_output_good_output_fractional_score_zero_point_oh(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'', '{"fractionalScore":0.0, "feedback": "Helpful comment!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
assert not sys.exit.called, "sys.exit should not be called!"
def test_check_output_no_feedback(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'', '{"isCorrect":false, "not-feedback": "garbage"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Field 'feedback' not present in parsed output."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
sys.exit.assert_called_with(1)
def test_check_output_bad_isCorrect(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {"Id": "myimageId"}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'', '{"isCorrect":"true-string-is-not-true","feedback":"You win!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Field 'isCorrect' is not a boolean value."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}"))
sys.exit.assert_called_with(1)
def test_check_output_good_output_fractional_score_zero_point_oh(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myContainerId"
}
args = argparse.Namespace()
args.timeout = 300
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"fractionalScore":0.0, "feedback": "Helpful comment!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
)
assert not sys.exit.called, "sys.exit should not be called!"
def test_check_output_fractional_score_boolean(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myContainerId"
}
args = argparse.Namespace()
args.timeout = 300
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"fractionalScore": false, "feedback": "wheeeee"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Field 'fractionalScore' must be a decimal.")
)
sys.exit.assert_called_with(1)
def test_check_output_no_feedback(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myContainerId"
}
args = argparse.Namespace()
args.timeout = 300
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"isCorrect":false, "not-feedback": "garbage"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Field 'feedback' not present in parsed output.")
)
sys.exit.assert_called_with(1)
def test_check_output_bad_isCorrect(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myContainerId"
}
args = argparse.Namespace()
args.timeout = 300
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"isCorrect":"true-string-is-not-true","feedback":"You win!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Field 'isCorrect' is not a boolean value.")
)
sys.exit.assert_called_with(1)
def test_check_output_good_output_fractional_score_one(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myimageId"
}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"fractionalScore":1, "feedback": "Helpful comment!"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}")
)
assert not sys.exit.called, "sys.exit should not be called!"
def test_check_output_malformed_output(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myimageId"
}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"isCorrect":false, "not-feedback": "garbageeeeeeee'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "The output was not a valid JSON document."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}")
)
sys.exit.assert_called_with(1)
def test_check_output_missing_grade(sys):
with LogCapture() as logs:
docker_mock = MagicMock()
container = {
"Id": "myimageId"
}
args = argparse.Namespace()
args.timeout = 300
args.no_rm = False
docker_mock.wait.return_value = 0
docker_mock.logs.side_effect = [
'',
'{"feedback": "wheeeee"}'
]
# Run the function under test
grade.run_container(docker_mock, container, args)
logs.check(
('root', 'INFO', 'Debug log:'),
('root', 'ERROR', "Required field 'fractionalScore' is missing."),
('root', 'DEBUG', "About to remove container: {'Id': 'myimageId'}")
)
sys.exit.assert_called_with(1)
