class TestSandbox(TestCase):
PATH_FIXTURES = os.path.abspath("tests/fixtures/sandbox/")
@classmethod
def setUpClass(cls):
initializer.init()
@classmethod
def tearDownClass(cls):
pass
def setUp(self) -> None:
self.sandbox = None
def tearDown(self) -> None:
if self.sandbox is not None:
self.sandbox.wait(0.1)
del self.sandbox
self.sandbox = None
@staticmethod
def sandbox_helper(sandbox: Sandbox, command, privileged=False):
stdout, stderr = TemporaryFile("wb+"), TemporaryFile("wb+")
sandbox.execute(command=command,
stdin_fd=None,
stdout_fd=stdout,
stderr_fd=stderr,
privileged=privileged)
stdout.flush()
stdout.seek(0)
stdout_text = stdout.read().decode().strip()
stdout.close()
stderr.flush()
stderr.seek(0)
stderr_text = stderr.read().decode().strip()
stderr.close()
# If running java or javac or jar the JVM prints an annoying message:
# "Picked up JAVA_TOOL_OPTIONS: <actual options set by sandbox environment>
# Remove it from the stderr if it is there
if any(java in command for java in ["java", "javac", "jar"]):
stdout_text = "\n".join([
line for line in stdout_text.splitlines()
if not line.startswith("Picked up JAVA_TOOL_OPTIONS")
])
stderr_text = "\n".join([
line for line in stderr_text.splitlines()
if not line.startswith("Picked up JAVA_TOOL_OPTIONS")
])
return stdout_text, stderr_text
# ================================= #
# Chroot Setup #
# ================================= #
def test_working_directory_is_empty(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="ls")
self.assertEqual("", stderr)
self.assertEqual("", stdout)
def test_working_directory_is_home(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="pwd")
self.assertEqual("", stderr)
self.assertEqual("/home", stdout)
def test_chroot_has_proper_fs_tree(self):
# Root directory has a proper chroot structure
# List all entries in "/" folder (but skip "total...", thus not using -la)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ls -ld /* /.*")
self.assertEqual("", stderr)
# All required system directories are present
for mount_dir in [
"bin", "dev", "etc", "lib", "lib64", "proc", "sys", "usr"
]:
self.assertIn(mount_dir, stdout)
# All dirs (".", "..", "/home" and mounted directories) have the correct permissions
self.assertEqual(stdout.count("xr-xr-x"), len(stdout.splitlines()))
def test_sys_structure_is_mounted(self):
# There are files in the mounted directories
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="cat /proc/uptime")
self.assertEqual("", stderr)
self.assertNotEqual("", stdout)
# Sanity check that an error is printed on a missing file
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="cat /proc/foobarbaz")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_chroot_second_time(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="chroot ..")
self.assertIn("Operation not permitted", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="sudo chroot ..")
self.assertIn("is not allowed to execute '/usr/sbin/chroot ..'",
stderr)
# ================================= #
# File System Access #
# ================================= #
def test_cant_touch_this(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="touch /proc/uptime")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_make_directories(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="mkdir foo")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_remove_directories(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="cd .. && rmdir home")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_create_files(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="touch foo.txt")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_redirect_to_files(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="echo bla > foo.txt")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_cannot_rm_rf(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="rm -rf /")
self.assertNotEqual("", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="sudo rm -rf /")
self.assertNotEqual("", stderr)
def test_cp(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="cp /bin/bash .")
self.assertIn("Permission denied", stderr)
def test_mv(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="mv /bin/bash .")
self.assertIn("Permission denied", stderr)
def test_create_symlink(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ln -s /bin/bash bash")
self.assertIn("Permission denied", stderr)
def test_mount(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="mount /bin /usr")
self.assertIn("only root can do that", stderr)
# ================================= #
# Network Access #
# ================================= #
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_no_ping_dns_resolving(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ping www.google.com")
self.assertIn("Temporary failure in name resolution", stderr)
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_no_ping_to_ip_address(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ping 8.8.8.8")
self.assertIn("Operation not permitted", stderr)
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_no_wget_dns_resolving(self):
stdout, stderr = self.sandbox_helper(
sandbox=Sandbox(), command="wget www.google.com/robots.txt")
self.assertIn("Temporary failure in name resolution", stderr)
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_no_wget_from_ip_address(self):
stdout, stderr = self.sandbox_helper(
sandbox=Sandbox(), command="wget 216.58.212.4/robots.txt")
# The process should reach the MAX_EXECUTION_TIME and be killed (being stuck on "Connecting to...")
self.assertEqual("Connecting to 216.58.212.4:80...",
stderr.splitlines()[-1])
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_no_localhost_access(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ping localhost")
self.assertIn("Operation not permitted", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="ping 127.0.0.1")
self.assertIn("Operation not permitted", stderr)
# ================================= #
# User and process info #
# ================================= #
def test_priority_in_boundaries(self):
self.assertGreaterEqual(config.PROCESS_PRIORITY_REAL,
os.sched_get_priority_min(os.SCHED_RR))
self.assertLessEqual(config.PROCESS_PRIORITY_REAL,
os.sched_get_priority_max(os.SCHED_RR))
def test_cannot_run_commands_with_sudo(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="sudo ls -la")
self.assertNotEqual("", stderr)
self.assertEqual("", stdout)
def test_prlimit_output(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="prlimit")
expected = {
"AS": [
config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
"bytes"
],
"CPU":
[config.MAX_EXECUTION_TIME, config.MAX_EXECUTION_TIME, "seconds"],
"DATA": [
config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
"bytes"
],
"FSIZE": [
config.MAX_EXECUTION_OUTPUT, config.MAX_EXECUTION_OUTPUT,
"bytes"
],
"NOFILE": [config.MAX_OPEN_FILES, config.MAX_OPEN_FILES, "files"],
"NPROC": [config.MAX_PROCESSES, config.MAX_PROCESSES, "processes"],
"RSS": [
config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
"bytes"
],
"STACK":
[config.MAX_EXECUTION_STACK, config.MAX_EXECUTION_STACK, "bytes"],
}
lines = stdout.splitlines()
for line in lines:
tokens = line.split()
if tokens[0] in expected:
self.assertEqual(int(tokens[-3]), expected[tokens[0]][0])
self.assertEqual(int(tokens[-2]), expected[tokens[0]][1])
self.assertEqual(tokens[-1], expected[tokens[0]][2])
def test_niceness_level(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="nice")
self.assertEqual(str(config.PROCESS_PRIORITY_NICE), stdout)
def test_worker_id(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="id -u")
self.assertGreaterEqual(int(stdout), 1000)
def test_worker_user(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="whoami")
self.assertTrue("worker" in stdout)
def test_scheduling_algorithm(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="chrt -p $$")
self.assertIn("scheduling policy: SCHED_OTHER",
stdout.splitlines()[0]) # Standard UNIX scheduler
def test_process_info(self):
stdout, stderr = self.sandbox_helper(
sandbox=Sandbox(),
command="ps -o uid,pid,ppid,cls,pri,ni,rtprio -p $$")
process_info = stdout.splitlines()[1].split()
self.assertGreaterEqual(int(process_info[0]), 1000) # User ID (again)
self.assertEqual(
process_info[3], "TS"
) # Scheduling algorithm, RR = real-time round robin, TS = standard
self.assertEqual(process_info[5],
str(config.PROCESS_PRIORITY_NICE)) # Nice level
self.assertEqual(process_info[6], "-") # Priority
# ================================= #
# Sandbox API #
# ================================= #
def test_has_file(self):
self.sandbox = Sandbox()
self.assertFalse(self.sandbox.has_file("foo.txt"))
self.assertTrue(self.sandbox.has_file("../usr/bin/timeout"))
def test_get_file(self):
self.sandbox = Sandbox()
self.assertFalse(os.path.isfile("./time_binary"))
self.sandbox.get_file("../usr/bin/timeout", "./timeout_binary")
self.assertTrue(os.path.isfile("./timeout_binary"))
os.remove("./timeout_binary")
def test_put_file(self):
self.sandbox = Sandbox()
self.assertFalse(self.sandbox.has_file("foo.txt"))
self.sandbox.put_file("install_steps.txt", "foo.txt")
self.assertTrue(self.sandbox.has_file("foo.txt"))
def test_put_file_with_permissions_write(self):
self.sandbox = Sandbox()
self.assertFalse(self.sandbox.has_file("foo.txt"))
self.sandbox.put_file("install_steps.txt", "foo.txt")
self.assertTrue(self.sandbox.has_file("foo.txt"))
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="echo bar > foo.txt")
self.assertNotEqual(stderr, "") # No permissions to write
self.sandbox.put_file("install_steps.txt", "foo.txt", 0o777)
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="echo bar > foo.txt")
self.assertEqual(stderr, "") # This time has permissions
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="cat foo.txt")
self.assertEqual(stdout,
"bar") # Double check by printing the file's contents
def test_put_file_with_permissions_exec(self):
self.sandbox = Sandbox()
self.sandbox.put_file("/bin/ls", "ls_binary")
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="./ls_binary ..")
self.assertEqual(stderr, "")
self.assertIn("bin", stdout) # Should list parent folder
self.sandbox.put_file("/bin/ls", "ls_binary", 0o766)
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="./ls_binary ..")
self.assertIn("Permission denied",
stderr) # Should be unable to execute
self.assertEqual(stdout, "")
def test_del_file(self):
self.sandbox = Sandbox()
self.assertFalse(self.sandbox.has_file("foo.txt"))
self.sandbox.put_file("install_steps.txt", "foo.txt")
self.assertTrue(self.sandbox.has_file("foo.txt"))
self.sandbox.del_file("foo.txt")
self.assertFalse(self.sandbox.has_file("foo.txt"))
def test_read_file(self):
self.sandbox = Sandbox()
self.sandbox.put_file("install_steps.txt", "foo.txt")
contents = self.sandbox.read_file("foo.txt").decode()
self.assertIn("python", contents)
def test_execute_blocking(self):
self.sandbox = Sandbox()
output = TemporaryFile(mode="w+b")
start_time = perf_counter()
self.sandbox.execute(command="sleep 0.2 ; echo foo",
stdin_fd=None,
stdout_fd=output,
stderr_fd=None,
blocking=True)
self.assertGreaterEqual(perf_counter() - start_time, 0.2)
self.assertEqual(output.tell(), 4) # Already printed "foo\n"
def test_execute_non_blocking(self):
self.sandbox = Sandbox()
output = TemporaryFile(mode="w+b")
start_time = perf_counter()
self.sandbox.execute(command="sleep 0.2 ; echo foo",
stdin_fd=None,
stdout_fd=output,
stderr_fd=None,
blocking=False)
self.assertLess(perf_counter() - start_time, 0.1)
self.assertEqual(output.tell(), 0) # Haven't yet printed anything
sleep(0.3)
self.assertEqual(output.tell(), 4) # But printing it eventually
def test_privileged_execution(self):
self.sandbox = Sandbox()
self.sandbox.put_file("/bin/ls", "ls_binary", 0o744)
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="./ls_binary ..")
self.assertIn("Permission denied",
stderr) # Should be unable to execute
self.assertEqual("", stdout)
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="./ls_binary ..",
privileged=True)
self.assertEqual("",
stderr) # But privileged user should be able to do it
self.assertIn("bin", stdout)
def test_privileged_deletion(self):
self.sandbox = Sandbox()
self.sandbox.put_file("/bin/ls", "ls_binary", 0o744)
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="rm ls_binary")
self.assertIn("Permission denied",
stderr) # Should be unable to execute
self.assertTrue(self.sandbox.has_file("ls_binary"))
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command="rm ls_binary",
privileged=True)
self.assertEqual("",
stderr) # But privileged user should be able to do it
self.assertFalse(self.sandbox.has_file("ls_binary"))
# ================================= #
# Applied ulimits #
# ================================= #
def test_output_limit(self):
self.sandbox = Sandbox()
file_size = 1000000 # 1MB
output = NamedTemporaryFile(mode="w+", delete=True)
for i in range(file_size // 10):
output.write("test test\n")
output.flush()
self.sandbox.put_file(output.name, "foo.txt")
target_size = 0
while target_size + file_size <= config.MAX_EXECUTION_OUTPUT:
target_size += file_size
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox,
command="for i in {{1..{}}}; do cat foo.txt; done;".format(
target_size // file_size))
self.assertEqual("", stderr)
self.assertEqual(len(stdout), target_size - 1)
target_size += file_size
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox,
command="for i in {{1..{}}}; do cat foo.txt; done;".format(
target_size // file_size))
self.assertIn("File size limit exceeded", stderr)
self.assertEqual(len(stdout), config.MAX_EXECUTION_OUTPUT)
def test_no_input_limit(self):
self.sandbox = Sandbox()
file_size = 50000000 # 50MB
output = NamedTemporaryFile(mode="w+", delete=True)
message = "Without IT I'm just espr\n"
for i in range(file_size // len(message)):
output.write(message)
output.flush()
self.sandbox.put_file(output.name, "foo.txt")
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox, command="wc -c < foo.txt && wc -l < foo.txt")
self.assertEqual("", stderr)
self.assertEqual(len(stdout.splitlines()), 2)
self.assertEqual(int(stdout.splitlines()[0]), file_size)
self.assertEqual(int(stdout.splitlines()[1]),
file_size // len(message))
@mock.patch("config.MAX_EXECUTION_TIME", 0.5)
def test_hard_timeout(self):
start_time = perf_counter()
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="sleep 3; echo foo")
self.assertEqual("", stdout)
self.assertEqual("", stderr)
exec_time = perf_counter() - start_time
self.assertGreaterEqual(exec_time, 0.5)
self.assertLess(exec_time, 0.7)
@mock.patch("config.MAX_EXECUTION_TIME", 1.0)
def test_fork_bomb(self):
# Run to see if it crashes the system and how many processes it spawns
start_time = perf_counter()
stdout, stderr = TemporaryFile(mode="w+"), TemporaryFile(mode="w+")
self.sandbox = Sandbox()
self.sandbox.execute(command=":(){ :|:& };:",
stdin_fd=None,
stdout_fd=stdout,
stderr_fd=stderr,
blocking=False)
# Check number of processes by this worker and its CPU usage continuously
# (but sleep for 0.01 seconds so we don't do it more than 100 times)
iteration = 0
max_cpu = 0.0
max_processes = 0
while perf_counter() - start_time < config.MAX_EXECUTION_TIME:
if iteration % 2 == 0:
ps_info = os.popen("ps -U {}".format(
self.sandbox._worker.name)).read()
max_processes = max(max_processes,
len(ps_info.splitlines()) - 1)
else:
cpu_info = os.popen(
"top -b -n 1 -u {} | awk 'NR>7 {{ sum += $9; }} END {{ print sum; }}'"
.format(self.sandbox._worker.name)).read()
max_cpu = max(max_cpu, float(cpu_info))
iteration += 1
sleep(0.01)
self.assertLess(perf_counter() - start_time, 1.2)
self.assertLessEqual(max_processes, config.MAX_PROCESSES)
self.assertLessEqual(max_cpu, 100.0)
stdout.seek(0)
self.assertEqual("", stdout.read())
stderr.seek(0)
self.assertIn("fork: retry: Resource temporarily unavailable",
stderr.read())
# At this point the sandbox is still running (as the fork bomb processes are detached)
# Make sure that wait() kills it entirely
self.assertTrue(self.sandbox.is_running())
self.sandbox.wait(0.1)
self.assertFalse(self.sandbox.is_running())
@mock.patch("config.MAX_EXECUTION_TIME", 0.3)
def test_sandbox_wait_kills_sleepers(self):
stdout, stderr = TemporaryFile(mode="w+"), TemporaryFile(mode="w+")
self.sandbox = Sandbox()
self.sandbox.execute(command=":(){ :|:& };:",
stdin_fd=None,
stdout_fd=stdout,
stderr_fd=stderr,
blocking=False)
# While the program is within its time limit it is at max processes
sleep(0.2)
self.assertTrue(self.sandbox.is_running())
ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
self.assertEqual(len(ps_info.splitlines()) - 1, config.MAX_PROCESSES)
# What's worse, even after that they are still alive
# (as they don't use much CPU, so are not affected by MAX_EXECUTION_TIME)
sleep(0.2)
self.assertTrue(self.sandbox.is_running())
ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
self.assertEqual(len(ps_info.splitlines()) - 1, config.MAX_PROCESSES)
# However, wait() should terminate everything
self.sandbox.wait(0.1)
self.assertFalse(self.sandbox.is_running())
ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
self.assertEqual(len(ps_info.splitlines()) - 1, 0)
def test_cpu_usage(self):
self.sandbox = Sandbox()
self.sandbox.put_file(os.path.join(self.PATH_FIXTURES, "factor.py"))
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox,
command="/usr/bin/time --format '%U %P' pypy3 factor.py")
self.assertIn("1000000000000037", stdout)
self.assertEqual(len(stderr.splitlines()), 1)
user_time = float(stderr.split()[0])
percent_cpu = int(stderr.split()[1].split('%')[0])
# Took at least half a second
self.assertGreater(user_time, 0.5)
# CPU usage was around 100%
self.assertTrue(95 < percent_cpu <= 100)
def test_memory_usage_heap(self):
self.sandbox = Sandbox()
self.sandbox.put_file(
os.path.join(self.PATH_FIXTURES, "mem_allocator.cpp"))
self.sandbox_helper(
sandbox=self.sandbox,
command=
"g++ -O2 -std=c++17 -w -s -o mem_allocator mem_allocator.cpp",
privileged=True)
self.assertTrue(self.sandbox.has_file("mem_allocator"))
command = "/usr/bin/time --quiet --format='%M' /bin/bash -c \"./mem_allocator heap {}\"" +\
" ; code=$? ; >&2 echo $code ; exit $code"
targets = [10000000, 100000000, 500000000, 1000000000,
2000000000] # 10MB, 100MB, 500MB, 1GB, 2GB
for target in targets:
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox, command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertEqual(exit_code, 0)
self.assertTrue(target <= exec_memory * 1024 <= target +
5000000) # Up to 5MB overhead for C++ libraries
# Twenty megabytes less than the threshold is okay
target = config.MAX_EXECUTION_MEMORY - 20000000
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertEqual(exit_code, 0)
self.assertTrue(target <= exec_memory * 1024 <= target +
5000000) # Up to 5MB overhead for C++ libraries
# Allocating around the threshold is no longer okay
target = config.MAX_EXECUTION_MEMORY
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertNotEqual(exit_code, 0)
self.assertTrue(
exec_memory * 1024 <= config.MAX_EXECUTION_MEMORY + 1024)
def test_memory_usage_stack(self):
self.sandbox = Sandbox()
self.sandbox.put_file(
os.path.join(self.PATH_FIXTURES, "mem_allocator.cpp"))
self.sandbox_helper(
sandbox=self.sandbox,
command=
"g++ -O2 -std=c++17 -w -s -o mem_allocator mem_allocator.cpp",
privileged=True)
self.assertTrue(self.sandbox.has_file("mem_allocator"))
command = "/usr/bin/time --quiet --format='%M' /bin/bash -c \"./mem_allocator stack {}\"" +\
" ; code=$? ; >&2 echo $code ; exit $code"
# Test different target stack sizes (should all be okay)
targets = [1000000, 10000000, 50000000] # 1MB, 10MB, 50MB
for target in targets:
stdout, stderr = self.sandbox_helper(
sandbox=self.sandbox, command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertEqual(exit_code, 0)
self.assertTrue(target <= exec_memory * 1024 <= target +
5000000) # Up to 5MB overhead for C++ libraries
# Half a megabyte less than the threshold is okay
target = config.MAX_EXECUTION_STACK - 500000
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertEqual(exit_code, 0)
self.assertTrue(target <= exec_memory * 1024 <= target +
5000000) # Up to 5MB overhead for C++ libraries
# Half a megabyte more than the threshold is not okay
target = config.MAX_EXECUTION_STACK + 500000
stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
command=command.format(target))
exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
stderr.splitlines()[-2])
self.assertNotEqual(exit_code, 0)
self.assertTrue(target <= exec_memory * 1024 <= target +
5000000) # Up to 5MB overhead for C++ libraries
# ================================= #
# High-level prerequisites #
# ================================= #
def test_languages_are_available(self):
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="g++")
self.assertIn("g++: fatal error: no input files", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="java")
self.assertIn("Usage: java [options]", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="javac")
self.assertIn("Usage: javac <options> <source files>", stdout)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="jar")
self.assertIn("Usage: jar [OPTION...]", stderr)
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command="pypy3 --version")
self.assertIn("Python 3.", stdout)
def test_time_command_available(self):
stdout, stderr = self.sandbox_helper(
sandbox=Sandbox(), command="ls -la /usr/bin | grep -w time")
self.assertEqual("", stderr)
self.assertTrue(stdout.startswith("-rwx") and stdout.endswith("time"))
command = "/usr/bin/time --quiet --format='%U %S %e %M' /bin/bash -c 'sleep 0.33; echo foo'" + \
" ; code=$? ; >&2 echo $code ; exit $code"
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command=command)
self.assertEqual(stdout, "foo")
self.assertEqual(int(stderr.splitlines()[-1]), 0) # Exit code
self.assertLess(float(stderr.splitlines()[-2].split()[0]),
0.1) # User time
self.assertLess(float(stderr.splitlines()[-2].split()[1]),
0.1) # Kernel time
self.assertAlmostEqual(float(stderr.splitlines()[-2].split()[2]),
0.33,
delta=0.1) # Clock time
def test_timeout_command_available(self):
stdout, stderr = self.sandbox_helper(
sandbox=Sandbox(), command="ls -la /usr/bin | grep -w timeout")
self.assertEqual("", stderr)
self.assertTrue(
stdout.startswith("-rwx") and stdout.endswith("timeout"))
start_time = perf_counter()
command = "/usr/bin/timeout 0.3s /bin/bash -c 'sleep 1.0; echo foo'"
stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
command=command)
self.assertEqual(stdout, "") # No output, killed before that
self.assertLess(perf_counter() - start_time,
0.5) # Killed shortly after the timeout
# Up to MAX_PARALLEL_WORKERS run in parallel.
# We expect if we run less than or equal to MAX_PARALLEL_WORKERS to take clock time equal to the longest
# of them. If we run even a single one more, we expect the clock time to be roughly twice as long.
def dummy_sleep_helper(self):
start_time = perf_counter()
sandbox = Sandbox()
waiting_time = perf_counter() - start_time
self.sandbox_helper(sandbox=sandbox, command="sleep 0.5 ; echo foo")
return waiting_time
def test_workers_under_limit(self):
start_time = perf_counter()
pool = ThreadPoolExecutor(max_workers=config.MAX_PARALLEL_WORKERS)
futures = [
pool.submit(self.dummy_sleep_helper)
for _ in range(config.MAX_PARALLEL_WORKERS)
]
# Each of the processes runs in ~0.5s, but since we schedule them through a thread pool
# we should reach this point much earlier.
self.assertLess(perf_counter() - start_time, 0.1)
# Wait for all workers to complete and get the maximum waiting time for a Sandbox object
max_waiting_time = 0.0
for future in futures:
max_waiting_time = max(max_waiting_time, future.result())
# Expecting none of the workers to wait for another one to finish, this get a Sandbox object immediately
self.assertLess(max_waiting_time, 0.2)
# Waiting all of them to complete takes at least 0.5 seconds
self.assertGreaterEqual(perf_counter() - start_time, 0.5)
# But not much more than 0.5 seconds
self.assertLess(perf_counter() - start_time, 0.7)
def test_workers_over_limit(self):
start_time = perf_counter()
pool = ThreadPoolExecutor(max_workers=config.MAX_PARALLEL_WORKERS + 1)
futures = [
pool.submit(self.dummy_sleep_helper)
for _ in range(config.MAX_PARALLEL_WORKERS + 1)
]
# Each of the processes runs in ~0.5s, but since we schedule them through a thread pool
# we should reach this point much earlier. One of the threads should be blocked on waiting
# for a sandbox, though.
self.assertLess(perf_counter() - start_time, 0.1)
# Wait for all workers to complete and get the maximum waiting time for a Sandbox object
max_waiting_time = 0.0
for future in futures:
max_waiting_time = max(max_waiting_time, future.result())
# Expecting one of the workers to wait for another one to finish before getting a Sandbox object
self.assertGreaterEqual(max_waiting_time, 0.5)
self.assertLess(max_waiting_time, 0.7)
# Waiting all of them to complete takes at least 1 second (twice as much)
self.assertGreaterEqual(perf_counter() - start_time, 1.0)
# But not much more than 1 second
self.assertLess(perf_counter() - start_time, 1.4)