def test_single_command_failed_timeout(self): self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "TO") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT))
def test_single_command_nonzero_return(self): self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "RE") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN))
def test_single_command_failed_timeout(self): self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "TO") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT))
def test_single_command_nonzero_return(self): self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "RE") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN))
def test_single_command_failed_signal(self): self.sandbox.fake_execute_data( True, b"o", b"e", 0.1, 0.5, 1000, "SG", signal=11) stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_SIGNAL, signal=11))
def test_multiple_commands_success(self): self.sandbox.fake_execute_data( True, b"o1", b"e1", 0.1, 0.5, 1000, "OK") self.sandbox.fake_execute_data( True, b"o2", b"e2", 1.0, 5.0, 10_000, "OK") stats = generic_step(self.sandbox, TWO_COMMANDS, "name", collect_output=True) # 2 commands executed, with exec_num 0 and 1 self.assertEquals(self.sandbox.exec_num, 1) # Stats are the combination of the two. self.assertEqual(stats, get_stats(1.1, # sum 5.5, # sum 10_000 * 1024, # max Sandbox.EXIT_OK, stdout="o1\n===\no2", stderr="e1\n===\ne2"))
def test_single_command_failed_timeout_wall(self): self.sandbox.fake_execute_data( True, b"o", b"e", 0.1, 0.5, 1000, "TO", message="Time limit exceeded (wall clock)") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT_WALL))
def test_single_command_success_no_collect_output(self): self.sandbox.fake_execute_data( True, b"o", "你好".encode("utf-8"), 0.1, 0.5, 1000, "OK") stats = generic_step(self.sandbox, ONE_COMMAND, "name", collect_output=False) # No output collected on stats. self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK)) # Generic step always redirects stdout and stderr. self.assertEqual(self.sandbox.stdout_file, "name_stdout_0.txt") self.assertEqual(self.sandbox.stderr_file, "name_stderr_0.txt")
def test_single_commands_trusted_failed_nonzero_return(self): expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN, stdout="o", stderr="e") with patch("cms.grading.steps.trusted.generic_step", return_value=expected_stats): success, trusted_success, stats = trusted_step( self.sandbox, ONE_COMMAND) # Trusted steps should always succeed, if not, should notify the admin. self.assertLoggedError() self.assertTrue(success) self.assertFalse(trusted_success) self.assertEqual(stats, expected_stats)
def test_multiple_commands_success(self): expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好") with patch("cms.grading.steps.trusted.generic_step", return_value=expected_stats) as mock_generic_step: success, trusted_success, stats = trusted_step( self.sandbox, TWO_COMMANDS) self.assertLoggedError(False) mock_generic_step.assert_called_once_with( self.sandbox, TWO_COMMANDS, "trusted") self.assertTrue(success) self.assertTrue(trusted_success) self.assertEqual(stats, expected_stats)
def test_multiple_commands_failure_terminates_early(self): self.sandbox.fake_execute_data( True, b"o1", b"e1", 0.1, 0.5, 1000, "RE") self.sandbox.fake_execute_data( True, b"o2", b"e2", 1.0, 5.0, 10_000, "OK") stats = generic_step(self.sandbox, TWO_COMMANDS, "name", collect_output=True) # 1 command executed (generic terminates early), with exec_num 0. self.assertEquals(self.sandbox.exec_num, 0) # Stats are only for the first command. self.assertEqual(stats, get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN, stdout="o1", stderr="e1"))
def test_single_command_failed_signal(self): self.sandbox.fake_execute_data(True, b"o", b"e", 0.1, 0.5, 1000, "SG", signal=11) stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_SIGNAL, signal=11))
def test_single_command_success(self): expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好") with patch("cms.grading.steps.compilation.generic_step", return_value=expected_stats) as mock_generic_step: success, compilation_success, text, stats = compilation_step( self.sandbox, ONE_COMMAND) mock_generic_step.assert_called_once_with( self.sandbox, ONE_COMMAND, "compilation", collect_output=True) self.assertLoggedError(False) self.assertTrue(success) self.assertTrue(compilation_success) self.assertEqual(text, [COMPILATION_MESSAGES.get("success").message]) self.assertEqual(stats, expected_stats)
def test_single_command_success(self): expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK, stdout="o", stderr="你好") with patch("cms.grading.steps.compilation.generic_step", return_value=expected_stats) as mock_generic_step: success, compilation_success, text, stats = compilation_step( self.sandbox, ONE_COMMAND) mock_generic_step.assert_called_once_with( self.sandbox, ONE_COMMAND, "compilation", collect_output=True) self.assertLoggedError(False) self.assertTrue(success) self.assertTrue(compilation_success) self.assertEqual(text, [COMPILATION_MESSAGES.get("success").message]) self.assertEqual(stats, expected_stats)
def test_single_command_success_no_collect_output(self): self.sandbox.fake_execute_data(True, b"o", "你好".encode("utf-8"), 0.1, 0.5, 1000, "OK") stats = generic_step(self.sandbox, ONE_COMMAND, "name", collect_output=False) # No output collected on stats. self.assertEqual(stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_OK)) # Generic step always redirects stdout and stderr. self.assertEqual(self.sandbox.stdout_file, "name_stdout_0.txt") self.assertEqual(self.sandbox.stderr_file, "name_stderr_0.txt")
def test_single_command_failed_timeout_wall(self): self.sandbox.fake_execute_data( True, b"o", b"e", 0.1, 0.5, 1000, "TO", message="Time limit exceeded (wall clock)") stats = generic_step(self.sandbox, ONE_COMMAND, "name") self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT_WALL))
def test_single_commands_trusted_failed_signal(self): # This case is a "success" for the sandbox (it's the user's fault), # but trusted is unsuccessful (no executable). expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_SIGNAL, signal=11, stdout="o", stderr="e") with patch("cms.grading.steps.trusted.generic_step", return_value=expected_stats): success, trusted_success, stats = trusted_step( self.sandbox, ONE_COMMAND) # Trusted steps should always succeed, if not, should notify the admin. self.assertLoggedError() self.assertTrue(success) self.assertFalse(trusted_success) self.assertEqual(stats, expected_stats)
def test_single_command_compilation_failed_timeout(self): # This case is a "success" for the sandbox (it's the user's fault), # but compilation is unsuccessful (no executable). expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT, stdout="o", stderr="e") with patch("cms.grading.steps.compilation.generic_step", return_value=expected_stats): success, compilation_success, text, stats = compilation_step( self.sandbox, ONE_COMMAND) # User's fault, no error needs to be logged. self.assertLoggedError(False) self.assertTrue(success) self.assertFalse(compilation_success) self.assertEqual(text, [COMPILATION_MESSAGES.get("timeout").message]) self.assertEqual(stats, expected_stats)
def test_single_command_compilation_failed_timeout(self): # This case is a "success" for the sandbox (it's the user's fault), # but compilation is unsuccessful (no executable). expected_stats = get_stats( 0.1, 0.5, 1000 * 1024, Sandbox.EXIT_TIMEOUT, stdout="o", stderr="e") with patch("cms.grading.steps.compilation.generic_step", return_value=expected_stats): success, compilation_success, text, stats = compilation_step( self.sandbox, ONE_COMMAND) # User's fault, no error needs to be logged. self.assertLoggedError(False) self.assertTrue(success) self.assertFalse(compilation_success) self.assertEqual(text, [COMPILATION_MESSAGES.get("timeout").message]) self.assertEqual(stats, expected_stats)
def test_multiple_commands_failure_terminates_early(self): self.sandbox.fake_execute_data(True, b"o1", b"e1", 0.1, 0.5, 1000, "RE") self.sandbox.fake_execute_data(True, b"o2", b"e2", 1.0, 5.0, 10000, "OK") stats = generic_step(self.sandbox, TWO_COMMANDS, "name", collect_output=True) # 1 command executed (generic terminates early), with exec_num 0. self.assertEquals(self.sandbox.exec_num, 0) # Stats are only for the first command. self.assertEqual( stats, get_stats(0.1, 0.5, 1000 * 1024, Sandbox.EXIT_NONZERO_RETURN, stdout="o1", stderr="e1"))
def test_multiple_commands_success(self): self.sandbox.fake_execute_data(True, b"o1", b"e1", 0.1, 0.5, 1000, "OK") self.sandbox.fake_execute_data(True, b"o2", b"e2", 1.0, 5.0, 10000, "OK") stats = generic_step(self.sandbox, TWO_COMMANDS, "name", collect_output=True) # 2 commands executed, with exec_num 0 and 1 self.assertEquals(self.sandbox.exec_num, 1) # Stats are the combination of the two. self.assertEqual( stats, get_stats( 1.1, # sum 5.5, # sum 10000 * 1024, # max Sandbox.EXIT_OK, stdout="o1\n===\no2", stderr="e1\n===\ne2"))