def test_single_command_failed_timeout(self):
self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "TO")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024,
Sandbox.EXIT_TIMEOUT))
def test_single_command_nonzero_return(self):
self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "RE")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024,
Sandbox.EXIT_NONZERO_RETURN))
def test_single_command_failed_timeout(self):
self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "TO")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(
stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT))
def test_single_command_nonzero_return(self):
self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "RE")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(
stats, get_stats(0.1, 0.5, 1000 * 1024,
Sandbox.EXIT_NONZERO_RETURN))
def test_single_command_failed_signal(self):
self.sandbox.fake_execute_data(
True, b"o", b"e", 0.1, 0.5, 1000, "SG", signal=11)
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024,
Sandbox.EXIT_SIGNAL, signal=11))
def test_multiple_commands_success(self):
self.sandbox.fake_execute_data(
True, b"o1", b"e1", 0.1, 0.5, 1000, "OK")
self.sandbox.fake_execute_data(
True, b"o2", b"e2", 1.0, 5.0, 10_000, "OK")
stats = generic_step(self.sandbox, TWO_COMMANDS, "name",
collect_output=True)
# 2 commands executed, with exec_num 0 and 1
self.assertEquals(self.sandbox.exec_num, 1)
# Stats are the combination of the two.
self.assertEqual(stats, get_stats(1.1, # sum
5.5, # sum
10_000 * 1024, # max
Sandbox.EXIT_OK,
stdout="o1\n===\no2",
stderr="e1\n===\ne2"))
def test_single_command_failed_timeout_wall(self):
self.sandbox.fake_execute_data(
True, b"o", b"e", 0.1, 0.5, 1000, "TO",
message="Time limit exceeded (wall clock)")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024,
Sandbox.EXIT_TIMEOUT_WALL))
def test_single_command_success_no_collect_output(self):
self.sandbox.fake_execute_data(
True, b"o", "你好".encode("utf-8"), 0.1, 0.5, 1000, "OK")
stats = generic_step(self.sandbox, ONE_COMMAND, "name",
collect_output=False)
# No output collected on stats.
self.assertEqual(
stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK))
# Generic step always redirects stdout and stderr.
self.assertEqual(self.sandbox.stdout_file, "name_stdout_0.txt")
self.assertEqual(self.sandbox.stderr_file, "name_stderr_0.txt")
def test_single_commands_trusted_failed_nonzero_return(self):
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN,
stdout="o", stderr="e")
with patch("cms.grading.steps.trusted.generic_step",
return_value=expected_stats):
success, trusted_success, stats = trusted_step(
self.sandbox, ONE_COMMAND)
# Trusted steps should always succeed, if not, should notify the admin.
self.assertLoggedError()
self.assertTrue(success)
self.assertFalse(trusted_success)
self.assertEqual(stats, expected_stats)
def test_multiple_commands_success(self):
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好")
with patch("cms.grading.steps.trusted.generic_step",
return_value=expected_stats) as mock_generic_step:
success, trusted_success, stats = trusted_step(
self.sandbox, TWO_COMMANDS)
self.assertLoggedError(False)
mock_generic_step.assert_called_once_with(
self.sandbox, TWO_COMMANDS, "trusted")
self.assertTrue(success)
self.assertTrue(trusted_success)
self.assertEqual(stats, expected_stats)
def test_multiple_commands_failure_terminates_early(self):
self.sandbox.fake_execute_data(
True, b"o1", b"e1", 0.1, 0.5, 1000, "RE")
self.sandbox.fake_execute_data(
True, b"o2", b"e2", 1.0, 5.0, 10_000, "OK")
stats = generic_step(self.sandbox, TWO_COMMANDS, "name",
collect_output=True)
# 1 command executed (generic terminates early), with exec_num 0.
self.assertEquals(self.sandbox.exec_num, 0)
# Stats are only for the first command.
self.assertEqual(stats, get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN,
stdout="o1", stderr="e1"))
def test_single_command_failed_signal(self):
self.sandbox.fake_execute_data(True,
b"o",
b"e",
0.1,
0.5,
1000,
"SG",
signal=11)
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(
stats,
get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_SIGNAL, signal=11))
def test_single_command_success(self):
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好")
with patch("cms.grading.steps.compilation.generic_step",
return_value=expected_stats) as mock_generic_step:
success, compilation_success, text, stats = compilation_step(
self.sandbox, ONE_COMMAND)
mock_generic_step.assert_called_once_with(
self.sandbox, ONE_COMMAND, "compilation", collect_output=True)
self.assertLoggedError(False)
self.assertTrue(success)
self.assertTrue(compilation_success)
self.assertEqual(text, [COMPILATION_MESSAGES.get("success").message])
self.assertEqual(stats, expected_stats)
def test_single_command_success(self):
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好")
with patch("cms.grading.steps.compilation.generic_step",
return_value=expected_stats) as mock_generic_step:
success, compilation_success, text, stats = compilation_step(
self.sandbox, ONE_COMMAND)
mock_generic_step.assert_called_once_with(
self.sandbox, ONE_COMMAND, "compilation", collect_output=True)
self.assertLoggedError(False)
self.assertTrue(success)
self.assertTrue(compilation_success)
self.assertEqual(text, [COMPILATION_MESSAGES.get("success").message])
self.assertEqual(stats, expected_stats)
def test_single_command_success_no_collect_output(self):
self.sandbox.fake_execute_data(True, b"o", "你好".encode("utf-8"), 0.1,
0.5, 1000, "OK")
stats = generic_step(self.sandbox,
ONE_COMMAND,
"name",
collect_output=False)
# No output collected on stats.
self.assertEqual(stats,
get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK))
# Generic step always redirects stdout and stderr.
self.assertEqual(self.sandbox.stdout_file, "name_stdout_0.txt")
self.assertEqual(self.sandbox.stderr_file, "name_stderr_0.txt")
def test_single_command_failed_timeout_wall(self):
self.sandbox.fake_execute_data(
True,
b"o",
b"e",
0.1,
0.5,
1000,
"TO",
message="Time limit exceeded (wall clock)")
stats = generic_step(self.sandbox, ONE_COMMAND, "name")
self.assertEqual(
stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT_WALL))
def test_single_commands_trusted_failed_signal(self):
# This case is a "success" for the sandbox (it's the user's fault),
# but trusted is unsuccessful (no executable).
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_SIGNAL, signal=11,
stdout="o", stderr="e")
with patch("cms.grading.steps.trusted.generic_step",
return_value=expected_stats):
success, trusted_success, stats = trusted_step(
self.sandbox, ONE_COMMAND)
# Trusted steps should always succeed, if not, should notify the admin.
self.assertLoggedError()
self.assertTrue(success)
self.assertFalse(trusted_success)
self.assertEqual(stats, expected_stats)
def test_single_command_compilation_failed_timeout(self):
# This case is a "success" for the sandbox (it's the user's fault),
# but compilation is unsuccessful (no executable).
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT,
stdout="o", stderr="e")
with patch("cms.grading.steps.compilation.generic_step",
return_value=expected_stats):
success, compilation_success, text, stats = compilation_step(
self.sandbox, ONE_COMMAND)
# User's fault, no error needs to be logged.
self.assertLoggedError(False)
self.assertTrue(success)
self.assertFalse(compilation_success)
self.assertEqual(text, [COMPILATION_MESSAGES.get("timeout").message])
self.assertEqual(stats, expected_stats)
def test_single_command_compilation_failed_timeout(self):
# This case is a "success" for the sandbox (it's the user's fault),
# but compilation is unsuccessful (no executable).
expected_stats = get_stats(
0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT,
stdout="o", stderr="e")
with patch("cms.grading.steps.compilation.generic_step",
return_value=expected_stats):
success, compilation_success, text, stats = compilation_step(
self.sandbox, ONE_COMMAND)
# User's fault, no error needs to be logged.
self.assertLoggedError(False)
self.assertTrue(success)
self.assertFalse(compilation_success)
self.assertEqual(text, [COMPILATION_MESSAGES.get("timeout").message])
self.assertEqual(stats, expected_stats)
def test_multiple_commands_failure_terminates_early(self):
self.sandbox.fake_execute_data(True, b"o1", b"e1", 0.1, 0.5, 1000,
"RE")
self.sandbox.fake_execute_data(True, b"o2", b"e2", 1.0, 5.0, 10000,
"OK")
stats = generic_step(self.sandbox,
TWO_COMMANDS,
"name",
collect_output=True)
# 1 command executed (generic terminates early), with exec_num 0.
self.assertEquals(self.sandbox.exec_num, 0)
# Stats are only for the first command.
self.assertEqual(
stats,
get_stats(0.1,
0.5,
1000 * 1024,
Sandbox.EXIT_NONZERO_RETURN,
stdout="o1",
stderr="e1"))
def test_multiple_commands_success(self):
self.sandbox.fake_execute_data(True, b"o1", b"e1", 0.1, 0.5, 1000,
"OK")
self.sandbox.fake_execute_data(True, b"o2", b"e2", 1.0, 5.0, 10000,
"OK")
stats = generic_step(self.sandbox,
TWO_COMMANDS,
"name",
collect_output=True)
# 2 commands executed, with exec_num 0 and 1
self.assertEquals(self.sandbox.exec_num, 1)
# Stats are the combination of the two.
self.assertEqual(
stats,
get_stats(
1.1, # sum
5.5, # sum
10000 * 1024, # max
Sandbox.EXIT_OK,
stdout="o1\n===\no2",
stderr="e1\n===\ne2"))
