def add_files_to_sandbox(sandbox: AutograderSandbox, suite: Union[ag_models.AGTestSuite, ag_models.StudentTestSuite], submission: ag_models.Submission): student_files_to_add = [] for student_file in suite.student_files_needed.all(): matching_files = fnmatch.filter(submission.submitted_filenames, student_file.pattern) student_files_to_add += [ os.path.join(core_ut.get_submission_dir(submission), filename) for filename in matching_files ] if student_files_to_add: sandbox.add_files(*student_files_to_add) project_files_to_add = [ file_.abspath for file_ in suite.instructor_files_needed.all() ] if project_files_to_add: owner_and_read_only = { 'owner': 'root' if suite.read_only_instructor_files else SANDBOX_USERNAME, 'read_only': suite.read_only_instructor_files } sandbox.add_files(*project_files_to_add, **owner_and_read_only)
def test_very_large_io_no_truncate(self): repeat_str = b'a' * 1000 num_repeats = 1000000 # 1 GB for i in range(num_repeats): self.stdin.write(repeat_str) with AutograderSandbox() as sandbox: # type: AutograderSandbox self.stdin.seek(0) start = time.time() result = sandbox.run_command(['cat'], stdin=self.stdin) self.assertFalse(result.stdout_truncated) self.assertFalse(result.stderr_truncated) print('Ran command that read and printed {} bytes to stdout in {}'. format(num_repeats * len(repeat_str), time.time() - start)) stdout_size = os.path.getsize(result.stdout.name) print(stdout_size) self.assertEqual(len(repeat_str) * num_repeats, stdout_size) with AutograderSandbox() as sandbox: # type: AutograderSandbox self.stdin.seek(0) start = time.time() result = sandbox.run_command(['bash', '-c', '>&2 cat'], stdin=self.stdin) print('Ran command that read and printed {} bytes to stderr in {}'. format(num_repeats * len(repeat_str), time.time() - start)) stderr_size = os.path.getsize(result.stderr.name) print(stderr_size) self.assertEqual(len(repeat_str) * num_repeats, stderr_size)
def test_container_create_timeout(self, mock_check_call, *args): print(mock) with AutograderSandbox(debug=True): args, kwargs = mock_check_call.call_args self.assertIsNone(kwargs['timeout']) timeout = 42 with AutograderSandbox(container_create_timeout=timeout): args, kwargs = mock_check_call.call_args self.assertEqual(timeout, kwargs['timeout'])
def test_context_manager(self): with AutograderSandbox( name=self.name) as sandbox: # type: AutograderSandbox self.assertEqual(self.name, sandbox.name) # If the container was created successfully, we # should get an error if we try to create another # container with the same name. with self.assertRaises(subprocess.CalledProcessError): with AutograderSandbox(name=self.name): pass # The container should have been deleted at this point, # so we should be able to create another with the same name. with AutograderSandbox(name=self.name): pass
def test_entire_process_tree_killed_on_timeout(self): for program_str in _PROG_THAT_FORKS, _PROG_WITH_SUBPROCESS_STALL: with AutograderSandbox() as sandbox: ps_result = sandbox.run_command(['ps', '-aux' ]).stdout.read().decode() print(ps_result) num_ps_lines = len(ps_result.split('\n')) print(num_ps_lines) script_file = _add_string_to_sandbox_as_file( program_str, '.py', sandbox) start_time = time.time() result = sandbox.run_command(['python3', script_file], timeout=1) self.assertTrue(result.timed_out) time_elapsed = time.time() - start_time self.assertLess(time_elapsed, _SLEEP_TIME // 2, msg='Killing processes took too long') ps_result_after_restart = sandbox.run_command( ['ps', '-aux']).stdout.read().decode() print(ps_result_after_restart) num_ps_lines_after_restart = len( ps_result_after_restart.split('\n')) self.assertEqual(num_ps_lines, num_ps_lines_after_restart)
def test_sandbox_environment_variables_set(self): print_env_var_script = "echo ${}".format(' $'.join( self.environment_variables)) sandbox = AutograderSandbox( environment_variables=self.environment_variables) with sandbox, typing.cast(typing.TextIO, tempfile.NamedTemporaryFile('w+')) as f: f.write(print_env_var_script) f.seek(0) sandbox.add_files(f.name) result = sandbox.run_command(['bash', os.path.basename(f.name)]) expected_output = ' '.join( str(val) for val in self.environment_variables.values()) expected_output += '\n' self.assertEqual(expected_output, result.stdout.read().decode())
def test_try_to_change_cmd_runner(self): runner_path = '/usr/local/bin/cmd_runner.py' with AutograderSandbox() as sandbox: # type: AutograderSandbox # Make sure the file path above is correct sandbox.run_command(['cat', runner_path], check=True) with self.assertRaises(subprocess.CalledProcessError): sandbox.run_command(['touch', runner_path], check=True)
def grade_ag_test_suite_impl(ag_test_suite: ag_models.AGTestSuite, submission: ag_models.Submission, *ag_test_cases_to_run: ag_models.AGTestCase): @retry_should_recover def get_or_create_suite_result(): return ag_models.AGTestSuiteResult.objects.get_or_create( ag_test_suite=ag_test_suite, submission=submission)[0] suite_result = get_or_create_suite_result() sandbox = AutograderSandbox( name='submission{}-suite{}-{}'.format(submission.pk, ag_test_suite.pk, uuid.uuid4().hex), environment_variables={ 'usernames': ' '.join(submission.group.member_names) }, allow_network_access=ag_test_suite.allow_network_access, docker_image=constants.DOCKER_IMAGE_IDS_TO_URLS[ ag_test_suite.docker_image_to_use]) print(ag_test_suite.docker_image_to_use) print(sandbox.docker_image) with sandbox: add_files_to_sandbox(sandbox, ag_test_suite, submission) print('Running setup for', ag_test_suite.name) _run_suite_setup(sandbox, ag_test_suite, suite_result) if not ag_test_cases_to_run: ag_test_cases_to_run = ag_test_suite.ag_test_cases.all() for ag_test_case in ag_test_cases_to_run: print('Grading test case', ag_test_case.name) grade_ag_test_case_impl(sandbox, ag_test_case, suite_result) update_denormalized_ag_test_results(submission.pk)
def test_default_init(self): sandbox = AutograderSandbox() self.assertIsNotNone(sandbox.name) self.assertFalse(sandbox.allow_network_access) self.assertEqual({}, sandbox.environment_variables) self.assertEqual('jameslp/autograder-sandbox:{}'.format(VERSION), sandbox.docker_image)
def test_copy_files_into_sandbox(self): files = [] try: for i in range(10): f = tempfile.NamedTemporaryFile(mode='w+') f.write('this is file {}'.format(i)) f.seek(0) files.append(f) filenames = [file_.name for file_ in files] with AutograderSandbox() as sandbox: sandbox.add_files(*filenames) ls_result = sandbox.run_command(['ls']).stdout.read().decode() actual_filenames = [ filename.strip() for filename in ls_result.split() ] expected_filenames = [ os.path.basename(filename) for filename in filenames ] self.assertCountEqual(expected_filenames, actual_filenames) for file_ in files: file_.seek(0) expected_content = file_.read() actual_content = sandbox.run_command( ['cat', os.path.basename(file_.name)]).stdout.read().decode() self.assertEqual(expected_content, actual_content) finally: for file_ in files: file_.close()
def test_process_spawn_limit(self): # Make sure that wrapping commands in bash -c doesn't affect # the needed process spawn limit. with AutograderSandbox() as sandbox: ag_command = ag_models.AGCommand.objects.validate_and_create( cmd='echo hello', process_spawn_limit=0) result = tasks.run_command_from_args( ag_command.cmd, sandbox, max_num_processes=ag_command.process_spawn_limit, max_stack_size=ag_command.stack_size_limit, max_virtual_memory=ag_command.virtual_memory_limit, timeout=ag_command.time_limit, ) self.assertEqual(0, result.return_code) print(result.stdout.read()) print(result.stderr.read()) extra_bash_dash_c = ag_models.AGCommand.objects.validate_and_create( cmd='bash -c "echo hello"', process_spawn_limit=0) result = tasks.run_command_from_args( extra_bash_dash_c.cmd, sandbox, max_num_processes=ag_command.process_spawn_limit, max_stack_size=ag_command.stack_size_limit, max_virtual_memory=ag_command.virtual_memory_limit, timeout=ag_command.time_limit, ) self.assertEqual(0, result.return_code) print(result.stdout.read()) print(result.stderr.read())
def test_overwrite_non_read_only_file(self): original_content = "some stuff" overwrite_content = 'some new stuff' with tempfile.NamedTemporaryFile('w+') as f: f.write(original_content) f.seek(0) added_filename = os.path.basename(f.name) with AutograderSandbox() as sandbox: sandbox.add_files(f.name) actual_content = sandbox.run_command( ['cat', added_filename], check=True).stdout.read().decode() self.assertEqual(original_content, actual_content) sandbox.run_command([ 'bash', '-c', "printf '{}' > {}".format(overwrite_content, added_filename) ]) actual_content = sandbox.run_command( ['cat', added_filename], check=True).stdout.read().decode() self.assertEqual(overwrite_content, actual_content)
def _call_function_and_allocate_sandbox_if_needed(func, sandbox): if sandbox is None: sandbox = AutograderSandbox() with sandbox: return func(sandbox) else: return func(sandbox)
def test_error_set_allow_network_access_while_running(self): with AutograderSandbox() as sandbox: with self.assertRaises(ValueError): sandbox.allow_network_access = True self.assertFalse(sandbox.allow_network_access) result = sandbox.run_command(self.google_ping_cmd) self.assertNotEqual(0, result.return_code)
def test_no_stdin_specified_redirects_devnull(self): # If no stdin is redirected, this command will time out. # If /dev/null is redirected it should terminate normally. # This behavior is handled by the autograder_sandbox library. cmd = 'python3 -c "import sys; sys.stdin.read(); print(\'done\')"' # Run command from args with AutograderSandbox() as sandbox: result = tasks.run_command_from_args(cmd, sandbox, max_num_processes=10, max_stack_size=10000000, max_virtual_memory=500000000, timeout=2) self.assertFalse(result.timed_out) self.assertEqual(0, result.return_code) self.assertEqual('done\n', result.stdout.read().decode()) # Run ag command with AutograderSandbox() as sandbox: ag_command = ag_models.AGCommand.objects.validate_and_create( cmd=cmd, process_spawn_limit=10, time_limit=2) result = tasks.run_ag_command(ag_command, sandbox) self.assertFalse(result.timed_out) self.assertEqual(0, result.return_code) self.assertEqual('done\n', result.stdout.read().decode()) project = obj_build.make_project() ag_test_suite = ag_models.AGTestSuite.objects.validate_and_create( name='Suite', project=project) ag_test_case = ag_models.AGTestCase.objects.validate_and_create( name='Case', ag_test_suite=ag_test_suite) # Run ag test command with AutograderSandbox() as sandbox: ag_test_command = ag_models.AGTestCommand.objects.validate_and_create( ag_test_case=ag_test_case, name='Read stdin', cmd=cmd, stdin_source=ag_models.StdinSource.none, time_limit=2, process_spawn_limit=10) result = tasks.run_ag_test_command(ag_test_command, sandbox, ag_test_suite) self.assertFalse(result.timed_out) self.assertEqual(0, result.return_code) self.assertEqual('done\n', result.stdout.read().decode())
def test_non_unicode_chars_in_output_command_timed_out(self): with AutograderSandbox() as sandbox: sandbox.add_files(self.file_to_print) result = sandbox.run_command( ['bash', '-c', 'cat {}; sleep 5'.format(self.file_to_print)], timeout=1) self.assertTrue(result.timed_out) self.assertEqual(self.non_utf, result.stdout.read()) with AutograderSandbox() as sandbox: sandbox.add_files(self.file_to_print) result = sandbox.run_command([ 'bash', '-c', '>&2 cat {}; sleep 5'.format(self.file_to_print) ], timeout=1) self.assertTrue(result.timed_out) self.assertEqual(self.non_utf, result.stderr.read())
def test_truncate_stderr(self): truncate_length = 10 long_output = b'a' * 100 expected_output = long_output[:truncate_length] self._write_and_seek(self.stdin, long_output) with AutograderSandbox() as sandbox: # type: AutograderSandbox result = sandbox.run_command(['bash', '-c', '>&2 cat'], stdin=self.stdin, truncate_stderr=truncate_length) self.assertEqual(expected_output, result.stderr.read()) self.assertTrue(result.stderr_truncated) self.assertFalse(result.stdout_truncated)
def test_non_default_init(self): docker_image = 'waaaaluigi' sandbox = AutograderSandbox( name=self.name, docker_image=docker_image, allow_network_access=True, environment_variables=self.environment_variables) self.assertEqual(self.name, sandbox.name) self.assertEqual(docker_image, sandbox.docker_image) self.assertTrue(sandbox.allow_network_access) self.assertEqual(self.environment_variables, sandbox.environment_variables)
def test_non_unicode_chars_in_output_on_process_error(self): with AutograderSandbox() as sandbox: sandbox.add_files(self.file_to_print) with self.assertRaises(subprocess.CalledProcessError) as cm: sandbox.run_command([ 'bash', '-c', 'cat {}; exit 1'.format(self.file_to_print) ], check=True) self.assertEqual(self.non_utf, cm.exception.stdout.read()) with AutograderSandbox() as sandbox: sandbox.add_files(self.file_to_print) with self.assertRaises(subprocess.CalledProcessError) as cm: sandbox.run_command([ 'bash', '-c', '>&2 cat {}; exit 1'.format( self.file_to_print) ], check=True) self.assertEqual(self.non_utf, cm.exception.stderr.read())
def test_restart_added_files_preserved(self): with AutograderSandbox() as sandbox: file_to_add = os.path.abspath(__file__) sandbox.add_files(file_to_add) ls_result = sandbox.run_command(['ls']).stdout.read().decode() print(ls_result) self.assertEqual(os.path.basename(file_to_add) + '\n', ls_result) sandbox.restart() ls_result = sandbox.run_command(['ls']).stdout.read().decode() self.assertEqual(os.path.basename(file_to_add) + '\n', ls_result)
def test_command_tries_to_read_from_stdin_when_stdin_arg_is_none(self): with AutograderSandbox() as sandbox: result = sandbox.run_command( [ 'python3', '-c', "import sys; sys.stdin.read(); print('done')" ], max_num_processes=10, max_stack_size=10000000, max_virtual_memory=500000000, timeout=2, ) self.assertFalse(result.timed_out) self.assertEqual(0, result.return_code)
def test_non_unicode_chars_in_normal_output(self): with AutograderSandbox() as sandbox: # type: AutograderSandbox sandbox.add_files(self.file_to_print) result = sandbox.run_command(['cat', self.file_to_print]) stdout = result.stdout.read() print(stdout) self.assertEqual(self.non_utf, stdout) result = sandbox.run_command( ['bash', '-c', '>&2 cat ' + self.file_to_print]) stderr = result.stderr.read() print(stderr) self.assertEqual(self.non_utf, stderr)
def test_shell_parse_error(self): with AutograderSandbox() as sandbox: ag_command = ag_models.AGCommand.objects.validate_and_create( cmd='echo hello"') result = tasks.run_command_from_args( ag_command.cmd, sandbox, max_num_processes=ag_command.process_spawn_limit, max_stack_size=ag_command.stack_size_limit, max_virtual_memory=ag_command.virtual_memory_limit, timeout=ag_command.time_limit, ) self.assertNotEqual(0, result.return_code) print(result.stdout.read()) print(result.stderr.read())
def test_shell_output_redirection(self): with AutograderSandbox() as sandbox: ag_command = ag_models.AGCommand.objects.validate_and_create( cmd='printf "spam" > file', process_spawn_limit=0) tasks.run_command_from_args( ag_command.cmd, sandbox, max_num_processes=ag_command.process_spawn_limit, max_stack_size=ag_command.stack_size_limit, max_virtual_memory=ag_command.virtual_memory_limit, timeout=ag_command.time_limit, ) result = sandbox.run_command(['cat', 'file'], check=True) self.assertEqual(0, result.return_code) self.assertEqual('spam', result.stdout.read().decode())
def test_permission_denied(self): with AutograderSandbox() as sandbox: sandbox.run_command(['touch', 'not_executable'], check=True) sandbox.run_command(['chmod', '666', 'not_executable'], check=True) ag_command = ag_models.AGCommand.objects.validate_and_create( cmd='./not_executable') result = tasks.run_command_from_args( ag_command.cmd, sandbox, max_num_processes=ag_command.process_spawn_limit, max_stack_size=ag_command.stack_size_limit, max_virtual_memory=ag_command.virtual_memory_limit, timeout=ag_command.time_limit, ) self.assertNotEqual(0, result.return_code) print(result.stdout.read()) print(result.stderr.read())
def test_add_files_root_owner_and_read_only(self): original_content = "some stuff you shouldn't change" overwrite_content = 'lol I changed it anyway u nub' with tempfile.NamedTemporaryFile('w+') as f: f.write(original_content) f.seek(0) added_filename = os.path.basename(f.name) with AutograderSandbox() as sandbox: sandbox.add_files(f.name, owner='root', read_only=True) actual_content = sandbox.run_command( ['cat', added_filename], check=True).stdout.read().decode() self.assertEqual(original_content, actual_content) with self.assertRaises(subprocess.CalledProcessError): sandbox.run_command(['touch', added_filename], check=True) with self.assertRaises(subprocess.CalledProcessError): sandbox.run_command([ 'bash', '-c', "printf '{}' > {}".format( overwrite_content, added_filename) ], check=True) actual_content = sandbox.run_command( ['cat', added_filename], check=True).stdout.read().decode() self.assertEqual(original_content, actual_content) root_touch_result = sandbox.run_command( ['touch', added_filename], check=True, as_root=True) self.assertEqual(0, root_touch_result.return_code) sandbox.run_command([ 'bash', '-c', "printf '{}' > {}".format( overwrite_content, added_filename) ], as_root=True, check=True) actual_content = sandbox.run_command( ['cat', added_filename]).stdout.read().decode() self.assertEqual(overwrite_content, actual_content)
def run_command_from_args(cmd: str, sandbox: AutograderSandbox, *, max_num_processes: int, max_stack_size: int, max_virtual_memory: int, timeout: int, stdin: Optional[FileIO] = None) -> CompletedCommand: run_result = sandbox.run_command( ['bash', '-c', cmd], stdin=stdin, as_root=False, max_num_processes=max_num_processes, max_stack_size=max_stack_size, max_virtual_memory=max_virtual_memory, timeout=timeout, truncate_stdout=constants.MAX_OUTPUT_LENGTH, truncate_stderr=constants.MAX_OUTPUT_LENGTH) return run_result
class AutograderSandboxBasicRunCommandTestCase(unittest.TestCase): def setUp(self): self.sandbox = AutograderSandbox() self.root_cmd = ["touch", "/"] def test_run_legal_command_non_root(self): stdout_content = "hello world" expected_output = stdout_content.encode() + b'\n' with self.sandbox: cmd_result = self.sandbox.run_command(["echo", stdout_content]) self.assertEqual(0, cmd_result.return_code) self.assertEqual(expected_output, cmd_result.stdout.read()) def test_run_illegal_command_non_root(self): with self.sandbox: cmd_result = self.sandbox.run_command(self.root_cmd) self.assertNotEqual(0, cmd_result.return_code) self.assertNotEqual("", cmd_result.stderr) def test_run_command_as_root(self): with self.sandbox: cmd_result = self.sandbox.run_command(self.root_cmd, as_root=True) self.assertEqual(0, cmd_result.return_code) self.assertEqual(b"", cmd_result.stderr.read()) def test_run_command_raise_on_error(self): """ Tests that an exception is thrown only when check is True and the command exits with nonzero status. """ with self.sandbox: # No exception should be raised. cmd_result = self.sandbox.run_command(self.root_cmd, as_root=True, check=True) self.assertEqual(0, cmd_result.return_code) with self.assertRaises(subprocess.CalledProcessError): self.sandbox.run_command(self.root_cmd, check=True) def test_run_command_executable_does_not_exist_no_error(self): with self.sandbox: cmd_result = self.sandbox.run_command(['not_an_exe']) self.assertNotEqual(0, cmd_result.return_code)
def test_set_allow_network_access(self): sandbox = AutograderSandbox() self.assertFalse(sandbox.allow_network_access) with sandbox: result = sandbox.run_command(self.google_ping_cmd) self.assertNotEqual(0, result.return_code) sandbox.allow_network_access = True self.assertTrue(sandbox.allow_network_access) with sandbox: result = sandbox.run_command(self.google_ping_cmd) self.assertEqual(0, result.return_code) sandbox.allow_network_access = False self.assertFalse(sandbox.allow_network_access) with sandbox: result = sandbox.run_command(self.google_ping_cmd) self.assertNotEqual(0, result.return_code)
def test_copy_and_rename_file_into_sandbox(self): expected_content = 'this is a file' with tempfile.NamedTemporaryFile('w+') as f: f.write(expected_content) f.seek(0) with AutograderSandbox() as sandbox: new_name = 'new_filename.txt' sandbox.add_and_rename_file(f.name, new_name) ls_result = sandbox.run_command(['ls']).stdout.read().decode() actual_filenames = [ filename.strip() for filename in ls_result.split() ] expected_filenames = [new_name] self.assertCountEqual(expected_filenames, actual_filenames) actual_content = sandbox.run_command( ['cat', new_name]).stdout.read().decode() self.assertEqual(expected_content, actual_content)