def add_files_to_sandbox(sandbox: AutograderSandbox,
suite: Union[ag_models.AGTestSuite,
ag_models.StudentTestSuite],
submission: ag_models.Submission):
student_files_to_add = []
for student_file in suite.student_files_needed.all():
matching_files = fnmatch.filter(submission.submitted_filenames,
student_file.pattern)
student_files_to_add += [
os.path.join(core_ut.get_submission_dir(submission), filename)
for filename in matching_files
]
if student_files_to_add:
sandbox.add_files(*student_files_to_add)
project_files_to_add = [
file_.abspath for file_ in suite.instructor_files_needed.all()
]
if project_files_to_add:
owner_and_read_only = {
'owner':
'root' if suite.read_only_instructor_files else SANDBOX_USERNAME,
'read_only': suite.read_only_instructor_files
}
sandbox.add_files(*project_files_to_add, **owner_and_read_only)
def test_very_large_io_no_truncate(self):
repeat_str = b'a' * 1000
num_repeats = 1000000 # 1 GB
for i in range(num_repeats):
self.stdin.write(repeat_str)
with AutograderSandbox() as sandbox: # type: AutograderSandbox
self.stdin.seek(0)
start = time.time()
result = sandbox.run_command(['cat'], stdin=self.stdin)
self.assertFalse(result.stdout_truncated)
self.assertFalse(result.stderr_truncated)
print('Ran command that read and printed {} bytes to stdout in {}'.
format(num_repeats * len(repeat_str),
time.time() - start))
stdout_size = os.path.getsize(result.stdout.name)
print(stdout_size)
self.assertEqual(len(repeat_str) * num_repeats, stdout_size)
with AutograderSandbox() as sandbox: # type: AutograderSandbox
self.stdin.seek(0)
start = time.time()
result = sandbox.run_command(['bash', '-c', '>&2 cat'],
stdin=self.stdin)
print('Ran command that read and printed {} bytes to stderr in {}'.
format(num_repeats * len(repeat_str),
time.time() - start))
stderr_size = os.path.getsize(result.stderr.name)
print(stderr_size)
self.assertEqual(len(repeat_str) * num_repeats, stderr_size)
def test_container_create_timeout(self, mock_check_call, *args):
print(mock)
with AutograderSandbox(debug=True):
args, kwargs = mock_check_call.call_args
self.assertIsNone(kwargs['timeout'])
timeout = 42
with AutograderSandbox(container_create_timeout=timeout):
args, kwargs = mock_check_call.call_args
self.assertEqual(timeout, kwargs['timeout'])
def test_context_manager(self):
with AutograderSandbox(
name=self.name) as sandbox: # type: AutograderSandbox
self.assertEqual(self.name, sandbox.name)
# If the container was created successfully, we
# should get an error if we try to create another
# container with the same name.
with self.assertRaises(subprocess.CalledProcessError):
with AutograderSandbox(name=self.name):
pass
# The container should have been deleted at this point,
# so we should be able to create another with the same name.
with AutograderSandbox(name=self.name):
pass
def test_entire_process_tree_killed_on_timeout(self):
for program_str in _PROG_THAT_FORKS, _PROG_WITH_SUBPROCESS_STALL:
with AutograderSandbox() as sandbox:
ps_result = sandbox.run_command(['ps', '-aux'
]).stdout.read().decode()
print(ps_result)
num_ps_lines = len(ps_result.split('\n'))
print(num_ps_lines)
script_file = _add_string_to_sandbox_as_file(
program_str, '.py', sandbox)
start_time = time.time()
result = sandbox.run_command(['python3', script_file],
timeout=1)
self.assertTrue(result.timed_out)
time_elapsed = time.time() - start_time
self.assertLess(time_elapsed,
_SLEEP_TIME // 2,
msg='Killing processes took too long')
ps_result_after_restart = sandbox.run_command(
['ps', '-aux']).stdout.read().decode()
print(ps_result_after_restart)
num_ps_lines_after_restart = len(
ps_result_after_restart.split('\n'))
self.assertEqual(num_ps_lines, num_ps_lines_after_restart)
def test_sandbox_environment_variables_set(self):
print_env_var_script = "echo ${}".format(' $'.join(
self.environment_variables))
sandbox = AutograderSandbox(
environment_variables=self.environment_variables)
with sandbox, typing.cast(typing.TextIO,
tempfile.NamedTemporaryFile('w+')) as f:
f.write(print_env_var_script)
f.seek(0)
sandbox.add_files(f.name)
result = sandbox.run_command(['bash', os.path.basename(f.name)])
expected_output = ' '.join(
str(val) for val in self.environment_variables.values())
expected_output += '\n'
self.assertEqual(expected_output, result.stdout.read().decode())
def test_try_to_change_cmd_runner(self):
runner_path = '/usr/local/bin/cmd_runner.py'
with AutograderSandbox() as sandbox: # type: AutograderSandbox
# Make sure the file path above is correct
sandbox.run_command(['cat', runner_path], check=True)
with self.assertRaises(subprocess.CalledProcessError):
sandbox.run_command(['touch', runner_path], check=True)
def grade_ag_test_suite_impl(ag_test_suite: ag_models.AGTestSuite,
submission: ag_models.Submission,
*ag_test_cases_to_run: ag_models.AGTestCase):
@retry_should_recover
def get_or_create_suite_result():
return ag_models.AGTestSuiteResult.objects.get_or_create(
ag_test_suite=ag_test_suite, submission=submission)[0]
suite_result = get_or_create_suite_result()
sandbox = AutograderSandbox(
name='submission{}-suite{}-{}'.format(submission.pk, ag_test_suite.pk,
uuid.uuid4().hex),
environment_variables={
'usernames': ' '.join(submission.group.member_names)
},
allow_network_access=ag_test_suite.allow_network_access,
docker_image=constants.DOCKER_IMAGE_IDS_TO_URLS[
ag_test_suite.docker_image_to_use])
print(ag_test_suite.docker_image_to_use)
print(sandbox.docker_image)
with sandbox:
add_files_to_sandbox(sandbox, ag_test_suite, submission)
print('Running setup for', ag_test_suite.name)
_run_suite_setup(sandbox, ag_test_suite, suite_result)
if not ag_test_cases_to_run:
ag_test_cases_to_run = ag_test_suite.ag_test_cases.all()
for ag_test_case in ag_test_cases_to_run:
print('Grading test case', ag_test_case.name)
grade_ag_test_case_impl(sandbox, ag_test_case, suite_result)
update_denormalized_ag_test_results(submission.pk)
def test_default_init(self):
sandbox = AutograderSandbox()
self.assertIsNotNone(sandbox.name)
self.assertFalse(sandbox.allow_network_access)
self.assertEqual({}, sandbox.environment_variables)
self.assertEqual('jameslp/autograder-sandbox:{}'.format(VERSION),
sandbox.docker_image)
def test_copy_files_into_sandbox(self):
files = []
try:
for i in range(10):
f = tempfile.NamedTemporaryFile(mode='w+')
f.write('this is file {}'.format(i))
f.seek(0)
files.append(f)
filenames = [file_.name for file_ in files]
with AutograderSandbox() as sandbox:
sandbox.add_files(*filenames)
ls_result = sandbox.run_command(['ls']).stdout.read().decode()
actual_filenames = [
filename.strip() for filename in ls_result.split()
]
expected_filenames = [
os.path.basename(filename) for filename in filenames
]
self.assertCountEqual(expected_filenames, actual_filenames)
for file_ in files:
file_.seek(0)
expected_content = file_.read()
actual_content = sandbox.run_command(
['cat',
os.path.basename(file_.name)]).stdout.read().decode()
self.assertEqual(expected_content, actual_content)
finally:
for file_ in files:
file_.close()
def test_process_spawn_limit(self):
# Make sure that wrapping commands in bash -c doesn't affect
# the needed process spawn limit.
with AutograderSandbox() as sandbox:
ag_command = ag_models.AGCommand.objects.validate_and_create(
cmd='echo hello', process_spawn_limit=0)
result = tasks.run_command_from_args(
ag_command.cmd,
sandbox,
max_num_processes=ag_command.process_spawn_limit,
max_stack_size=ag_command.stack_size_limit,
max_virtual_memory=ag_command.virtual_memory_limit,
timeout=ag_command.time_limit,
)
self.assertEqual(0, result.return_code)
print(result.stdout.read())
print(result.stderr.read())
extra_bash_dash_c = ag_models.AGCommand.objects.validate_and_create(
cmd='bash -c "echo hello"', process_spawn_limit=0)
result = tasks.run_command_from_args(
extra_bash_dash_c.cmd,
sandbox,
max_num_processes=ag_command.process_spawn_limit,
max_stack_size=ag_command.stack_size_limit,
max_virtual_memory=ag_command.virtual_memory_limit,
timeout=ag_command.time_limit,
)
self.assertEqual(0, result.return_code)
print(result.stdout.read())
print(result.stderr.read())
def test_overwrite_non_read_only_file(self):
original_content = "some stuff"
overwrite_content = 'some new stuff'
with tempfile.NamedTemporaryFile('w+') as f:
f.write(original_content)
f.seek(0)
added_filename = os.path.basename(f.name)
with AutograderSandbox() as sandbox:
sandbox.add_files(f.name)
actual_content = sandbox.run_command(
['cat', added_filename],
check=True).stdout.read().decode()
self.assertEqual(original_content, actual_content)
sandbox.run_command([
'bash', '-c',
"printf '{}' > {}".format(overwrite_content,
added_filename)
])
actual_content = sandbox.run_command(
['cat', added_filename],
check=True).stdout.read().decode()
self.assertEqual(overwrite_content, actual_content)
def _call_function_and_allocate_sandbox_if_needed(func, sandbox):
if sandbox is None:
sandbox = AutograderSandbox()
with sandbox:
return func(sandbox)
else:
return func(sandbox)
def test_error_set_allow_network_access_while_running(self):
with AutograderSandbox() as sandbox:
with self.assertRaises(ValueError):
sandbox.allow_network_access = True
self.assertFalse(sandbox.allow_network_access)
result = sandbox.run_command(self.google_ping_cmd)
self.assertNotEqual(0, result.return_code)
def test_no_stdin_specified_redirects_devnull(self):
# If no stdin is redirected, this command will time out.
# If /dev/null is redirected it should terminate normally.
# This behavior is handled by the autograder_sandbox library.
cmd = 'python3 -c "import sys; sys.stdin.read(); print(\'done\')"'
# Run command from args
with AutograderSandbox() as sandbox:
result = tasks.run_command_from_args(cmd,
sandbox,
max_num_processes=10,
max_stack_size=10000000,
max_virtual_memory=500000000,
timeout=2)
self.assertFalse(result.timed_out)
self.assertEqual(0, result.return_code)
self.assertEqual('done\n', result.stdout.read().decode())
# Run ag command
with AutograderSandbox() as sandbox:
ag_command = ag_models.AGCommand.objects.validate_and_create(
cmd=cmd, process_spawn_limit=10, time_limit=2)
result = tasks.run_ag_command(ag_command, sandbox)
self.assertFalse(result.timed_out)
self.assertEqual(0, result.return_code)
self.assertEqual('done\n', result.stdout.read().decode())
project = obj_build.make_project()
ag_test_suite = ag_models.AGTestSuite.objects.validate_and_create(
name='Suite', project=project)
ag_test_case = ag_models.AGTestCase.objects.validate_and_create(
name='Case', ag_test_suite=ag_test_suite)
# Run ag test command
with AutograderSandbox() as sandbox:
ag_test_command = ag_models.AGTestCommand.objects.validate_and_create(
ag_test_case=ag_test_case,
name='Read stdin',
cmd=cmd,
stdin_source=ag_models.StdinSource.none,
time_limit=2,
process_spawn_limit=10)
result = tasks.run_ag_test_command(ag_test_command, sandbox,
ag_test_suite)
self.assertFalse(result.timed_out)
self.assertEqual(0, result.return_code)
self.assertEqual('done\n', result.stdout.read().decode())
def test_non_unicode_chars_in_output_command_timed_out(self):
with AutograderSandbox() as sandbox:
sandbox.add_files(self.file_to_print)
result = sandbox.run_command(
['bash', '-c', 'cat {}; sleep 5'.format(self.file_to_print)],
timeout=1)
self.assertTrue(result.timed_out)
self.assertEqual(self.non_utf, result.stdout.read())
with AutograderSandbox() as sandbox:
sandbox.add_files(self.file_to_print)
result = sandbox.run_command([
'bash', '-c', '>&2 cat {}; sleep 5'.format(self.file_to_print)
],
timeout=1)
self.assertTrue(result.timed_out)
self.assertEqual(self.non_utf, result.stderr.read())
def test_truncate_stderr(self):
truncate_length = 10
long_output = b'a' * 100
expected_output = long_output[:truncate_length]
self._write_and_seek(self.stdin, long_output)
with AutograderSandbox() as sandbox: # type: AutograderSandbox
result = sandbox.run_command(['bash', '-c', '>&2 cat'],
stdin=self.stdin,
truncate_stderr=truncate_length)
self.assertEqual(expected_output, result.stderr.read())
self.assertTrue(result.stderr_truncated)
self.assertFalse(result.stdout_truncated)
def test_non_default_init(self):
docker_image = 'waaaaluigi'
sandbox = AutograderSandbox(
name=self.name,
docker_image=docker_image,
allow_network_access=True,
environment_variables=self.environment_variables)
self.assertEqual(self.name, sandbox.name)
self.assertEqual(docker_image, sandbox.docker_image)
self.assertTrue(sandbox.allow_network_access)
self.assertEqual(self.environment_variables,
sandbox.environment_variables)
def test_non_unicode_chars_in_output_on_process_error(self):
with AutograderSandbox() as sandbox:
sandbox.add_files(self.file_to_print)
with self.assertRaises(subprocess.CalledProcessError) as cm:
sandbox.run_command([
'bash', '-c', 'cat {}; exit 1'.format(self.file_to_print)
],
check=True)
self.assertEqual(self.non_utf, cm.exception.stdout.read())
with AutograderSandbox() as sandbox:
sandbox.add_files(self.file_to_print)
with self.assertRaises(subprocess.CalledProcessError) as cm:
sandbox.run_command([
'bash', '-c', '>&2 cat {}; exit 1'.format(
self.file_to_print)
],
check=True)
self.assertEqual(self.non_utf, cm.exception.stderr.read())
def test_restart_added_files_preserved(self):
with AutograderSandbox() as sandbox:
file_to_add = os.path.abspath(__file__)
sandbox.add_files(file_to_add)
ls_result = sandbox.run_command(['ls']).stdout.read().decode()
print(ls_result)
self.assertEqual(os.path.basename(file_to_add) + '\n', ls_result)
sandbox.restart()
ls_result = sandbox.run_command(['ls']).stdout.read().decode()
self.assertEqual(os.path.basename(file_to_add) + '\n', ls_result)
def test_command_tries_to_read_from_stdin_when_stdin_arg_is_none(self):
with AutograderSandbox() as sandbox:
result = sandbox.run_command(
[
'python3', '-c',
"import sys; sys.stdin.read(); print('done')"
],
max_num_processes=10,
max_stack_size=10000000,
max_virtual_memory=500000000,
timeout=2,
)
self.assertFalse(result.timed_out)
self.assertEqual(0, result.return_code)
def test_non_unicode_chars_in_normal_output(self):
with AutograderSandbox() as sandbox: # type: AutograderSandbox
sandbox.add_files(self.file_to_print)
result = sandbox.run_command(['cat', self.file_to_print])
stdout = result.stdout.read()
print(stdout)
self.assertEqual(self.non_utf, stdout)
result = sandbox.run_command(
['bash', '-c', '>&2 cat ' + self.file_to_print])
stderr = result.stderr.read()
print(stderr)
self.assertEqual(self.non_utf, stderr)
def test_shell_parse_error(self):
with AutograderSandbox() as sandbox:
ag_command = ag_models.AGCommand.objects.validate_and_create(
cmd='echo hello"')
result = tasks.run_command_from_args(
ag_command.cmd,
sandbox,
max_num_processes=ag_command.process_spawn_limit,
max_stack_size=ag_command.stack_size_limit,
max_virtual_memory=ag_command.virtual_memory_limit,
timeout=ag_command.time_limit,
)
self.assertNotEqual(0, result.return_code)
print(result.stdout.read())
print(result.stderr.read())
def test_shell_output_redirection(self):
with AutograderSandbox() as sandbox:
ag_command = ag_models.AGCommand.objects.validate_and_create(
cmd='printf "spam" > file', process_spawn_limit=0)
tasks.run_command_from_args(
ag_command.cmd,
sandbox,
max_num_processes=ag_command.process_spawn_limit,
max_stack_size=ag_command.stack_size_limit,
max_virtual_memory=ag_command.virtual_memory_limit,
timeout=ag_command.time_limit,
)
result = sandbox.run_command(['cat', 'file'], check=True)
self.assertEqual(0, result.return_code)
self.assertEqual('spam', result.stdout.read().decode())
def test_permission_denied(self):
with AutograderSandbox() as sandbox:
sandbox.run_command(['touch', 'not_executable'], check=True)
sandbox.run_command(['chmod', '666', 'not_executable'], check=True)
ag_command = ag_models.AGCommand.objects.validate_and_create(
cmd='./not_executable')
result = tasks.run_command_from_args(
ag_command.cmd,
sandbox,
max_num_processes=ag_command.process_spawn_limit,
max_stack_size=ag_command.stack_size_limit,
max_virtual_memory=ag_command.virtual_memory_limit,
timeout=ag_command.time_limit,
)
self.assertNotEqual(0, result.return_code)
print(result.stdout.read())
print(result.stderr.read())
def test_add_files_root_owner_and_read_only(self):
original_content = "some stuff you shouldn't change"
overwrite_content = 'lol I changed it anyway u nub'
with tempfile.NamedTemporaryFile('w+') as f:
f.write(original_content)
f.seek(0)
added_filename = os.path.basename(f.name)
with AutograderSandbox() as sandbox:
sandbox.add_files(f.name, owner='root', read_only=True)
actual_content = sandbox.run_command(
['cat', added_filename],
check=True).stdout.read().decode()
self.assertEqual(original_content, actual_content)
with self.assertRaises(subprocess.CalledProcessError):
sandbox.run_command(['touch', added_filename], check=True)
with self.assertRaises(subprocess.CalledProcessError):
sandbox.run_command([
'bash', '-c', "printf '{}' > {}".format(
overwrite_content, added_filename)
],
check=True)
actual_content = sandbox.run_command(
['cat', added_filename],
check=True).stdout.read().decode()
self.assertEqual(original_content, actual_content)
root_touch_result = sandbox.run_command(
['touch', added_filename], check=True, as_root=True)
self.assertEqual(0, root_touch_result.return_code)
sandbox.run_command([
'bash', '-c', "printf '{}' > {}".format(
overwrite_content, added_filename)
],
as_root=True,
check=True)
actual_content = sandbox.run_command(
['cat', added_filename]).stdout.read().decode()
self.assertEqual(overwrite_content, actual_content)
def run_command_from_args(cmd: str,
sandbox: AutograderSandbox,
*,
max_num_processes: int,
max_stack_size: int,
max_virtual_memory: int,
timeout: int,
stdin: Optional[FileIO] = None) -> CompletedCommand:
run_result = sandbox.run_command(
['bash', '-c', cmd],
stdin=stdin,
as_root=False,
max_num_processes=max_num_processes,
max_stack_size=max_stack_size,
max_virtual_memory=max_virtual_memory,
timeout=timeout,
truncate_stdout=constants.MAX_OUTPUT_LENGTH,
truncate_stderr=constants.MAX_OUTPUT_LENGTH)
return run_result
class AutograderSandboxBasicRunCommandTestCase(unittest.TestCase):
def setUp(self):
self.sandbox = AutograderSandbox()
self.root_cmd = ["touch", "/"]
def test_run_legal_command_non_root(self):
stdout_content = "hello world"
expected_output = stdout_content.encode() + b'\n'
with self.sandbox:
cmd_result = self.sandbox.run_command(["echo", stdout_content])
self.assertEqual(0, cmd_result.return_code)
self.assertEqual(expected_output, cmd_result.stdout.read())
def test_run_illegal_command_non_root(self):
with self.sandbox:
cmd_result = self.sandbox.run_command(self.root_cmd)
self.assertNotEqual(0, cmd_result.return_code)
self.assertNotEqual("", cmd_result.stderr)
def test_run_command_as_root(self):
with self.sandbox:
cmd_result = self.sandbox.run_command(self.root_cmd, as_root=True)
self.assertEqual(0, cmd_result.return_code)
self.assertEqual(b"", cmd_result.stderr.read())
def test_run_command_raise_on_error(self):
"""
Tests that an exception is thrown only when check is True
and the command exits with nonzero status.
"""
with self.sandbox:
# No exception should be raised.
cmd_result = self.sandbox.run_command(self.root_cmd,
as_root=True,
check=True)
self.assertEqual(0, cmd_result.return_code)
with self.assertRaises(subprocess.CalledProcessError):
self.sandbox.run_command(self.root_cmd, check=True)
def test_run_command_executable_does_not_exist_no_error(self):
with self.sandbox:
cmd_result = self.sandbox.run_command(['not_an_exe'])
self.assertNotEqual(0, cmd_result.return_code)
def test_set_allow_network_access(self):
sandbox = AutograderSandbox()
self.assertFalse(sandbox.allow_network_access)
with sandbox:
result = sandbox.run_command(self.google_ping_cmd)
self.assertNotEqual(0, result.return_code)
sandbox.allow_network_access = True
self.assertTrue(sandbox.allow_network_access)
with sandbox:
result = sandbox.run_command(self.google_ping_cmd)
self.assertEqual(0, result.return_code)
sandbox.allow_network_access = False
self.assertFalse(sandbox.allow_network_access)
with sandbox:
result = sandbox.run_command(self.google_ping_cmd)
self.assertNotEqual(0, result.return_code)
def test_copy_and_rename_file_into_sandbox(self):
expected_content = 'this is a file'
with tempfile.NamedTemporaryFile('w+') as f:
f.write(expected_content)
f.seek(0)
with AutograderSandbox() as sandbox:
new_name = 'new_filename.txt'
sandbox.add_and_rename_file(f.name, new_name)
ls_result = sandbox.run_command(['ls']).stdout.read().decode()
actual_filenames = [
filename.strip() for filename in ls_result.split()
]
expected_filenames = [new_name]
self.assertCountEqual(expected_filenames, actual_filenames)
actual_content = sandbox.run_command(
['cat', new_name]).stdout.read().decode()
self.assertEqual(expected_content, actual_content)