Exemple #1
0
def execute_two_player_game(submit_id, test: TestInfo, run_config: RunConfig,
                            player_one_executable_path,
                            player_two_executable_path) -> RunResult:

    log_file = "interaction.log"
    tester_executable_name = "tester." + run_config.tester_path.split(".")[-1]
    player_one_executable_name = "solution1." + player_one_executable_path.split(
        ".")[-1]
    player_two_executable_name = "solution2." + player_two_executable_path.split(
        ".")[-1]

    args = {
        "tester_run_command":
        Runner.get_run_command(
            language=common.get_language_by_exec_name(tester_executable_name),
            executable=tester_executable_name,
            memory_limit=config.MAX_TESTER_MEMORY),
        "player_one_run_command":
        Runner.get_run_command(language=common.get_language_by_exec_name(
            player_one_executable_name),
                               executable=player_one_executable_name,
                               memory_limit=run_config.memory_limit),
        "player_two_run_command":
        Runner.get_run_command(language=common.get_language_by_exec_name(
            player_two_executable_name),
                               executable=player_two_executable_name,
                               memory_limit=run_config.memory_limit),
        "tester_timeout":
        config.MAX_GAME_LENGTH,
        "solution_timeout":
        run_config.timeout,
        "time_limit":
        run_config.time_limit,
        "memory_limit":
        run_config.memory_limit,
        "log_file":
        log_file
    }

    empty_file = NamedTemporaryFile(mode="w+t", delete=False)
    empty_file_path = os.path.abspath(empty_file.name)

    # Copy all the needed files to the sandbox directory:
    #   1) wrapper.py
    #   2) the tester executable
    #   3) player one's solution executable
    #   4) player two's solution executable
    #   5) the input file (read by the tester)
    #   6) game log (empty file with write permissions)

    sandbox = Sandbox()
    sandbox.put_file(os.path.join(config.ROOT_DIR, "wrapper.py"))
    sandbox.put_file(run_config.tester_path,
                     target_name=tester_executable_name)
    sandbox.put_file(player_one_executable_path,
                     target_name=player_one_executable_name)
    sandbox.put_file(player_two_executable_path,
                     target_name=player_two_executable_name)
    sandbox.put_file(test.inpPath, target_name="input.txt", mode=0o777)
    sandbox.put_file(empty_file_path, target_name=log_file, mode=0o777)

    # Run the executable, while measuring CPU and Memory consumption
    run_result = Runner.run_program(
        sandbox=sandbox,
        executable_path=os.path.join(config.ROOT_DIR, "interactor_2P.py"),
        memory_limit=config.MAX_EXECUTION_MEMORY,
        timeout=config.MAX_GAME_LENGTH,
        input_bytes=json.dumps(args, indent=4, sort_keys=True).encode())
    interaction_log = sandbox.read_file(log_file)
    del sandbox

    # Record the game log to the /replays folder
    replay_id = save_replay_log(interaction_log)

    # The interactor crashed or got killed
    # Don't check memory limit, as it can be caused by child processes (including solution)
    if run_result.exec_time >= config.MAX_GAME_LENGTH:
        message = "Interactor took too much time to complete ({:.3f}s).".format(
            run_result.exec_time)
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)
    if run_result.exit_code != 0:
        message = "Interactor exited with non-zero exit code ({}).".format(
            run_result.exit_code)
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # Parse the response from the interactor
    try:
        results = json.loads(run_result.output.decode(config.OUTPUT_ENCODING))
    except ValueError:
        message = "Could not decode interactor's output: {}!".format(
            run_result.output.decode())
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # print("RESULTS:\n{}".format(json.dumps(results, indent=4, sort_keys=True)))

    # Okay, let's assume the interactor was okay. Now check if the tester crashed.
    if results["internal_error"]:
        message = "Tester crashed or some other internal error happened. Result from interactor:\n{}".format(
            json.dumps(results, indent=4, sort_keys=True))
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # Everything with the system seems okay.
    # Leave the caller function decide what the test status will be
    return RunResult(status=TestStatus.ACCEPTED,
                     output=run_result.output,
                     replay_id=replay_id)
Exemple #2
0
class TestSandbox(TestCase):
    PATH_FIXTURES = os.path.abspath("tests/fixtures/sandbox/")

    @classmethod
    def setUpClass(cls):
        initializer.init()

    @classmethod
    def tearDownClass(cls):
        pass

    def setUp(self) -> None:
        self.sandbox = None

    def tearDown(self) -> None:
        if self.sandbox is not None:
            self.sandbox.wait(0.1)
            del self.sandbox
            self.sandbox = None

    @staticmethod
    def sandbox_helper(sandbox: Sandbox, command, privileged=False):
        stdout, stderr = TemporaryFile("wb+"), TemporaryFile("wb+")
        sandbox.execute(command=command,
                        stdin_fd=None,
                        stdout_fd=stdout,
                        stderr_fd=stderr,
                        privileged=privileged)

        stdout.flush()
        stdout.seek(0)
        stdout_text = stdout.read().decode().strip()
        stdout.close()
        stderr.flush()
        stderr.seek(0)
        stderr_text = stderr.read().decode().strip()
        stderr.close()

        # If running java or javac or jar the JVM prints an annoying message:
        # "Picked up JAVA_TOOL_OPTIONS: <actual options set by sandbox environment>
        # Remove it from the stderr if it is there
        if any(java in command for java in ["java", "javac", "jar"]):
            stdout_text = "\n".join([
                line for line in stdout_text.splitlines()
                if not line.startswith("Picked up JAVA_TOOL_OPTIONS")
            ])
            stderr_text = "\n".join([
                line for line in stderr_text.splitlines()
                if not line.startswith("Picked up JAVA_TOOL_OPTIONS")
            ])
        return stdout_text, stderr_text

    # ================================= #
    #           Chroot Setup            #
    # ================================= #
    def test_working_directory_is_empty(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="ls")
        self.assertEqual("", stderr)
        self.assertEqual("", stdout)

    def test_working_directory_is_home(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="pwd")
        self.assertEqual("", stderr)
        self.assertEqual("/home", stdout)

    def test_chroot_has_proper_fs_tree(self):
        # Root directory has a proper chroot structure
        # List all entries in "/" folder (but skip "total...", thus not using -la)
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ls -ld /* /.*")
        self.assertEqual("", stderr)

        # All required system directories are present
        for mount_dir in [
                "bin", "dev", "etc", "lib", "lib64", "proc", "sys", "usr"
        ]:
            self.assertIn(mount_dir, stdout)

        # All dirs (".", "..", "/home" and mounted directories) have the correct permissions
        self.assertEqual(stdout.count("xr-xr-x"), len(stdout.splitlines()))

    def test_sys_structure_is_mounted(self):
        # There are files in the mounted directories
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="cat /proc/uptime")
        self.assertEqual("", stderr)
        self.assertNotEqual("", stdout)

        # Sanity check that an error is printed on a missing file
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="cat /proc/foobarbaz")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_chroot_second_time(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="chroot ..")
        self.assertIn("Operation not permitted", stderr)
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="sudo chroot ..")
        self.assertIn("is not allowed to execute '/usr/sbin/chroot ..'",
                      stderr)

    # ================================= #
    #         File System Access        #
    # ================================= #
    def test_cant_touch_this(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="touch /proc/uptime")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_make_directories(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="mkdir foo")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_remove_directories(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="cd .. && rmdir home")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_create_files(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="touch foo.txt")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_redirect_to_files(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="echo bla > foo.txt")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_cannot_rm_rf(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="rm -rf /")
        self.assertNotEqual("", stderr)
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="sudo rm -rf /")
        self.assertNotEqual("", stderr)

    def test_cp(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="cp /bin/bash .")
        self.assertIn("Permission denied", stderr)

    def test_mv(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="mv /bin/bash .")
        self.assertIn("Permission denied", stderr)

    def test_create_symlink(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ln -s /bin/bash bash")
        self.assertIn("Permission denied", stderr)

    def test_mount(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="mount /bin /usr")
        self.assertIn("only root can do that", stderr)

    # ================================= #
    #           Network Access          #
    # ================================= #
    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_no_ping_dns_resolving(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ping www.google.com")
        self.assertIn("Temporary failure in name resolution", stderr)

    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_no_ping_to_ip_address(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ping 8.8.8.8")
        self.assertIn("Operation not permitted", stderr)

    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_no_wget_dns_resolving(self):
        stdout, stderr = self.sandbox_helper(
            sandbox=Sandbox(), command="wget www.google.com/robots.txt")
        self.assertIn("Temporary failure in name resolution", stderr)

    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_no_wget_from_ip_address(self):
        stdout, stderr = self.sandbox_helper(
            sandbox=Sandbox(), command="wget 216.58.212.4/robots.txt")
        # The process should reach the MAX_EXECUTION_TIME and be killed (being stuck on "Connecting to...")
        self.assertEqual("Connecting to 216.58.212.4:80...",
                         stderr.splitlines()[-1])

    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_no_localhost_access(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ping localhost")
        self.assertIn("Operation not permitted", stderr)
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="ping 127.0.0.1")
        self.assertIn("Operation not permitted", stderr)

    # ================================= #
    #        User and process info      #
    # ================================= #
    def test_priority_in_boundaries(self):
        self.assertGreaterEqual(config.PROCESS_PRIORITY_REAL,
                                os.sched_get_priority_min(os.SCHED_RR))
        self.assertLessEqual(config.PROCESS_PRIORITY_REAL,
                             os.sched_get_priority_max(os.SCHED_RR))

    def test_cannot_run_commands_with_sudo(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="sudo ls -la")
        self.assertNotEqual("", stderr)
        self.assertEqual("", stdout)

    def test_prlimit_output(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="prlimit")

        expected = {
            "AS": [
                config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
                "bytes"
            ],
            "CPU":
            [config.MAX_EXECUTION_TIME, config.MAX_EXECUTION_TIME, "seconds"],
            "DATA": [
                config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
                "bytes"
            ],
            "FSIZE": [
                config.MAX_EXECUTION_OUTPUT, config.MAX_EXECUTION_OUTPUT,
                "bytes"
            ],
            "NOFILE": [config.MAX_OPEN_FILES, config.MAX_OPEN_FILES, "files"],
            "NPROC": [config.MAX_PROCESSES, config.MAX_PROCESSES, "processes"],
            "RSS": [
                config.MAX_EXECUTION_MEMORY, config.MAX_EXECUTION_MEMORY,
                "bytes"
            ],
            "STACK":
            [config.MAX_EXECUTION_STACK, config.MAX_EXECUTION_STACK, "bytes"],
        }

        lines = stdout.splitlines()
        for line in lines:
            tokens = line.split()
            if tokens[0] in expected:
                self.assertEqual(int(tokens[-3]), expected[tokens[0]][0])
                self.assertEqual(int(tokens[-2]), expected[tokens[0]][1])
                self.assertEqual(tokens[-1], expected[tokens[0]][2])

    def test_niceness_level(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="nice")
        self.assertEqual(str(config.PROCESS_PRIORITY_NICE), stdout)

    def test_worker_id(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="id -u")
        self.assertGreaterEqual(int(stdout), 1000)

    def test_worker_user(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="whoami")
        self.assertTrue("worker" in stdout)

    def test_scheduling_algorithm(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="chrt -p $$")
        self.assertIn("scheduling policy: SCHED_OTHER",
                      stdout.splitlines()[0])  # Standard UNIX scheduler

    def test_process_info(self):
        stdout, stderr = self.sandbox_helper(
            sandbox=Sandbox(),
            command="ps -o uid,pid,ppid,cls,pri,ni,rtprio -p $$")
        process_info = stdout.splitlines()[1].split()
        self.assertGreaterEqual(int(process_info[0]), 1000)  # User ID (again)
        self.assertEqual(
            process_info[3], "TS"
        )  # Scheduling algorithm, RR = real-time round robin, TS = standard
        self.assertEqual(process_info[5],
                         str(config.PROCESS_PRIORITY_NICE))  # Nice level
        self.assertEqual(process_info[6], "-")  # Priority

    # ================================= #
    #            Sandbox API            #
    # ================================= #
    def test_has_file(self):
        self.sandbox = Sandbox()
        self.assertFalse(self.sandbox.has_file("foo.txt"))
        self.assertTrue(self.sandbox.has_file("../usr/bin/timeout"))

    def test_get_file(self):
        self.sandbox = Sandbox()
        self.assertFalse(os.path.isfile("./time_binary"))
        self.sandbox.get_file("../usr/bin/timeout", "./timeout_binary")
        self.assertTrue(os.path.isfile("./timeout_binary"))
        os.remove("./timeout_binary")

    def test_put_file(self):
        self.sandbox = Sandbox()
        self.assertFalse(self.sandbox.has_file("foo.txt"))
        self.sandbox.put_file("install_steps.txt", "foo.txt")
        self.assertTrue(self.sandbox.has_file("foo.txt"))

    def test_put_file_with_permissions_write(self):
        self.sandbox = Sandbox()
        self.assertFalse(self.sandbox.has_file("foo.txt"))
        self.sandbox.put_file("install_steps.txt", "foo.txt")
        self.assertTrue(self.sandbox.has_file("foo.txt"))
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="echo bar > foo.txt")
        self.assertNotEqual(stderr, "")  # No permissions to write
        self.sandbox.put_file("install_steps.txt", "foo.txt", 0o777)
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="echo bar > foo.txt")
        self.assertEqual(stderr, "")  # This time has permissions
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="cat foo.txt")
        self.assertEqual(stdout,
                         "bar")  # Double check by printing the file's contents

    def test_put_file_with_permissions_exec(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file("/bin/ls", "ls_binary")
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="./ls_binary ..")
        self.assertEqual(stderr, "")
        self.assertIn("bin", stdout)  # Should list parent folder

        self.sandbox.put_file("/bin/ls", "ls_binary", 0o766)
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="./ls_binary ..")
        self.assertIn("Permission denied",
                      stderr)  # Should be unable to execute
        self.assertEqual(stdout, "")

    def test_del_file(self):
        self.sandbox = Sandbox()
        self.assertFalse(self.sandbox.has_file("foo.txt"))
        self.sandbox.put_file("install_steps.txt", "foo.txt")
        self.assertTrue(self.sandbox.has_file("foo.txt"))
        self.sandbox.del_file("foo.txt")
        self.assertFalse(self.sandbox.has_file("foo.txt"))

    def test_read_file(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file("install_steps.txt", "foo.txt")
        contents = self.sandbox.read_file("foo.txt").decode()
        self.assertIn("python", contents)

    def test_execute_blocking(self):
        self.sandbox = Sandbox()
        output = TemporaryFile(mode="w+b")
        start_time = perf_counter()
        self.sandbox.execute(command="sleep 0.2 ; echo foo",
                             stdin_fd=None,
                             stdout_fd=output,
                             stderr_fd=None,
                             blocking=True)
        self.assertGreaterEqual(perf_counter() - start_time, 0.2)
        self.assertEqual(output.tell(), 4)  # Already printed "foo\n"

    def test_execute_non_blocking(self):
        self.sandbox = Sandbox()
        output = TemporaryFile(mode="w+b")
        start_time = perf_counter()
        self.sandbox.execute(command="sleep 0.2 ; echo foo",
                             stdin_fd=None,
                             stdout_fd=output,
                             stderr_fd=None,
                             blocking=False)
        self.assertLess(perf_counter() - start_time, 0.1)
        self.assertEqual(output.tell(), 0)  # Haven't yet printed anything
        sleep(0.3)
        self.assertEqual(output.tell(), 4)  # But printing it eventually

    def test_privileged_execution(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file("/bin/ls", "ls_binary", 0o744)

        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="./ls_binary ..")
        self.assertIn("Permission denied",
                      stderr)  # Should be unable to execute
        self.assertEqual("", stdout)
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="./ls_binary ..",
                                             privileged=True)
        self.assertEqual("",
                         stderr)  # But privileged user should be able to do it
        self.assertIn("bin", stdout)

    def test_privileged_deletion(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file("/bin/ls", "ls_binary", 0o744)

        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="rm ls_binary")
        self.assertIn("Permission denied",
                      stderr)  # Should be unable to execute
        self.assertTrue(self.sandbox.has_file("ls_binary"))
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command="rm ls_binary",
                                             privileged=True)
        self.assertEqual("",
                         stderr)  # But privileged user should be able to do it
        self.assertFalse(self.sandbox.has_file("ls_binary"))

    # ================================= #
    #          Applied ulimits          #
    # ================================= #
    def test_output_limit(self):
        self.sandbox = Sandbox()

        file_size = 1000000  # 1MB
        output = NamedTemporaryFile(mode="w+", delete=True)
        for i in range(file_size // 10):
            output.write("test test\n")
        output.flush()
        self.sandbox.put_file(output.name, "foo.txt")

        target_size = 0
        while target_size + file_size <= config.MAX_EXECUTION_OUTPUT:
            target_size += file_size
        stdout, stderr = self.sandbox_helper(
            sandbox=self.sandbox,
            command="for i in {{1..{}}}; do cat foo.txt; done;".format(
                target_size // file_size))
        self.assertEqual("", stderr)
        self.assertEqual(len(stdout), target_size - 1)

        target_size += file_size
        stdout, stderr = self.sandbox_helper(
            sandbox=self.sandbox,
            command="for i in {{1..{}}}; do cat foo.txt; done;".format(
                target_size // file_size))
        self.assertIn("File size limit exceeded", stderr)
        self.assertEqual(len(stdout), config.MAX_EXECUTION_OUTPUT)

    def test_no_input_limit(self):
        self.sandbox = Sandbox()

        file_size = 50000000  # 50MB
        output = NamedTemporaryFile(mode="w+", delete=True)
        message = "Without IT I'm just espr\n"
        for i in range(file_size // len(message)):
            output.write(message)
        output.flush()
        self.sandbox.put_file(output.name, "foo.txt")

        stdout, stderr = self.sandbox_helper(
            sandbox=self.sandbox, command="wc -c < foo.txt && wc -l < foo.txt")
        self.assertEqual("", stderr)
        self.assertEqual(len(stdout.splitlines()), 2)
        self.assertEqual(int(stdout.splitlines()[0]), file_size)
        self.assertEqual(int(stdout.splitlines()[1]),
                         file_size // len(message))

    @mock.patch("config.MAX_EXECUTION_TIME", 0.5)
    def test_hard_timeout(self):
        start_time = perf_counter()
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="sleep 3; echo foo")
        self.assertEqual("", stdout)
        self.assertEqual("", stderr)
        exec_time = perf_counter() - start_time
        self.assertGreaterEqual(exec_time, 0.5)
        self.assertLess(exec_time, 0.7)

    @mock.patch("config.MAX_EXECUTION_TIME", 1.0)
    def test_fork_bomb(self):
        # Run to see if it crashes the system and how many processes it spawns
        start_time = perf_counter()

        stdout, stderr = TemporaryFile(mode="w+"), TemporaryFile(mode="w+")
        self.sandbox = Sandbox()
        self.sandbox.execute(command=":(){ :|:& };:",
                             stdin_fd=None,
                             stdout_fd=stdout,
                             stderr_fd=stderr,
                             blocking=False)

        # Check number of processes by this worker and its CPU usage continuously
        # (but sleep for 0.01 seconds so we don't do it more than 100 times)
        iteration = 0
        max_cpu = 0.0
        max_processes = 0
        while perf_counter() - start_time < config.MAX_EXECUTION_TIME:
            if iteration % 2 == 0:
                ps_info = os.popen("ps -U {}".format(
                    self.sandbox._worker.name)).read()
                max_processes = max(max_processes,
                                    len(ps_info.splitlines()) - 1)
            else:
                cpu_info = os.popen(
                    "top -b -n 1 -u {} | awk 'NR>7 {{ sum += $9; }} END {{ print sum; }}'"
                    .format(self.sandbox._worker.name)).read()
                max_cpu = max(max_cpu, float(cpu_info))
            iteration += 1
            sleep(0.01)

        self.assertLess(perf_counter() - start_time, 1.2)
        self.assertLessEqual(max_processes, config.MAX_PROCESSES)
        self.assertLessEqual(max_cpu, 100.0)

        stdout.seek(0)
        self.assertEqual("", stdout.read())
        stderr.seek(0)
        self.assertIn("fork: retry: Resource temporarily unavailable",
                      stderr.read())

        # At this point the sandbox is still running (as the fork bomb processes are detached)
        # Make sure that wait() kills it entirely
        self.assertTrue(self.sandbox.is_running())
        self.sandbox.wait(0.1)
        self.assertFalse(self.sandbox.is_running())

    @mock.patch("config.MAX_EXECUTION_TIME", 0.3)
    def test_sandbox_wait_kills_sleepers(self):
        stdout, stderr = TemporaryFile(mode="w+"), TemporaryFile(mode="w+")
        self.sandbox = Sandbox()
        self.sandbox.execute(command=":(){ :|:& };:",
                             stdin_fd=None,
                             stdout_fd=stdout,
                             stderr_fd=stderr,
                             blocking=False)

        # While the program is within its time limit it is at max processes
        sleep(0.2)
        self.assertTrue(self.sandbox.is_running())
        ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
        self.assertEqual(len(ps_info.splitlines()) - 1, config.MAX_PROCESSES)

        # What's worse, even after that they are still alive
        # (as they don't use much CPU, so are not affected by MAX_EXECUTION_TIME)
        sleep(0.2)
        self.assertTrue(self.sandbox.is_running())
        ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
        self.assertEqual(len(ps_info.splitlines()) - 1, config.MAX_PROCESSES)

        # However, wait() should terminate everything
        self.sandbox.wait(0.1)
        self.assertFalse(self.sandbox.is_running())
        ps_info = os.popen("ps -U {}".format(self.sandbox._worker.name)).read()
        self.assertEqual(len(ps_info.splitlines()) - 1, 0)

    def test_cpu_usage(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file(os.path.join(self.PATH_FIXTURES, "factor.py"))
        stdout, stderr = self.sandbox_helper(
            sandbox=self.sandbox,
            command="/usr/bin/time --format '%U %P' pypy3 factor.py")
        self.assertIn("1000000000000037", stdout)
        self.assertEqual(len(stderr.splitlines()), 1)
        user_time = float(stderr.split()[0])
        percent_cpu = int(stderr.split()[1].split('%')[0])
        # Took at least half a second
        self.assertGreater(user_time, 0.5)
        # CPU usage was around 100%
        self.assertTrue(95 < percent_cpu <= 100)

    def test_memory_usage_heap(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file(
            os.path.join(self.PATH_FIXTURES, "mem_allocator.cpp"))
        self.sandbox_helper(
            sandbox=self.sandbox,
            command=
            "g++ -O2 -std=c++17 -w -s -o mem_allocator mem_allocator.cpp",
            privileged=True)
        self.assertTrue(self.sandbox.has_file("mem_allocator"))

        command = "/usr/bin/time --quiet --format='%M' /bin/bash -c \"./mem_allocator heap {}\"" +\
                  " ; code=$? ; >&2 echo $code ; exit $code"

        targets = [10000000, 100000000, 500000000, 1000000000,
                   2000000000]  # 10MB, 100MB, 500MB, 1GB, 2GB
        for target in targets:
            stdout, stderr = self.sandbox_helper(
                sandbox=self.sandbox, command=command.format(target))
            exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
                stderr.splitlines()[-2])
            self.assertEqual(exit_code, 0)
            self.assertTrue(target <= exec_memory * 1024 <= target +
                            5000000)  # Up to 5MB overhead for C++ libraries

        # Twenty megabytes less than the threshold is okay
        target = config.MAX_EXECUTION_MEMORY - 20000000
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command=command.format(target))
        exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
            stderr.splitlines()[-2])
        self.assertEqual(exit_code, 0)
        self.assertTrue(target <= exec_memory * 1024 <= target +
                        5000000)  # Up to 5MB overhead for C++ libraries

        # Allocating around the threshold is no longer okay
        target = config.MAX_EXECUTION_MEMORY
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command=command.format(target))
        exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
            stderr.splitlines()[-2])
        self.assertNotEqual(exit_code, 0)
        self.assertTrue(
            exec_memory * 1024 <= config.MAX_EXECUTION_MEMORY + 1024)

    def test_memory_usage_stack(self):
        self.sandbox = Sandbox()
        self.sandbox.put_file(
            os.path.join(self.PATH_FIXTURES, "mem_allocator.cpp"))
        self.sandbox_helper(
            sandbox=self.sandbox,
            command=
            "g++ -O2 -std=c++17 -w -s -o mem_allocator mem_allocator.cpp",
            privileged=True)
        self.assertTrue(self.sandbox.has_file("mem_allocator"))

        command = "/usr/bin/time --quiet --format='%M' /bin/bash -c \"./mem_allocator stack {}\"" +\
                  " ; code=$? ; >&2 echo $code ; exit $code"

        # Test different target stack sizes (should all be okay)
        targets = [1000000, 10000000, 50000000]  # 1MB, 10MB, 50MB
        for target in targets:
            stdout, stderr = self.sandbox_helper(
                sandbox=self.sandbox, command=command.format(target))
            exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
                stderr.splitlines()[-2])
            self.assertEqual(exit_code, 0)
            self.assertTrue(target <= exec_memory * 1024 <= target +
                            5000000)  # Up to 5MB overhead for C++ libraries

        # Half a megabyte less than the threshold is okay
        target = config.MAX_EXECUTION_STACK - 500000
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command=command.format(target))
        exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
            stderr.splitlines()[-2])
        self.assertEqual(exit_code, 0)
        self.assertTrue(target <= exec_memory * 1024 <= target +
                        5000000)  # Up to 5MB overhead for C++ libraries

        # Half a megabyte more than the threshold is not okay
        target = config.MAX_EXECUTION_STACK + 500000
        stdout, stderr = self.sandbox_helper(sandbox=self.sandbox,
                                             command=command.format(target))
        exit_code, exec_memory = int(stderr.splitlines()[-1]), int(
            stderr.splitlines()[-2])
        self.assertNotEqual(exit_code, 0)
        self.assertTrue(target <= exec_memory * 1024 <= target +
                        5000000)  # Up to 5MB overhead for C++ libraries

    # ================================= #
    #      High-level prerequisites     #
    # ================================= #
    def test_languages_are_available(self):
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="g++")
        self.assertIn("g++: fatal error: no input files", stderr)

        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="java")
        self.assertIn("Usage: java [options]", stderr)

        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="javac")
        self.assertIn("Usage: javac <options> <source files>", stdout)

        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(), command="jar")
        self.assertIn("Usage: jar [OPTION...]", stderr)

        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command="pypy3 --version")
        self.assertIn("Python 3.", stdout)

    def test_time_command_available(self):
        stdout, stderr = self.sandbox_helper(
            sandbox=Sandbox(), command="ls -la /usr/bin | grep -w time")
        self.assertEqual("", stderr)
        self.assertTrue(stdout.startswith("-rwx") and stdout.endswith("time"))

        command = "/usr/bin/time --quiet --format='%U %S %e %M' /bin/bash -c 'sleep 0.33; echo foo'" + \
                  " ; code=$? ; >&2 echo $code ; exit $code"
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command=command)
        self.assertEqual(stdout, "foo")
        self.assertEqual(int(stderr.splitlines()[-1]), 0)  # Exit code
        self.assertLess(float(stderr.splitlines()[-2].split()[0]),
                        0.1)  # User time
        self.assertLess(float(stderr.splitlines()[-2].split()[1]),
                        0.1)  # Kernel time
        self.assertAlmostEqual(float(stderr.splitlines()[-2].split()[2]),
                               0.33,
                               delta=0.1)  # Clock time

    def test_timeout_command_available(self):
        stdout, stderr = self.sandbox_helper(
            sandbox=Sandbox(), command="ls -la /usr/bin | grep -w timeout")
        self.assertEqual("", stderr)
        self.assertTrue(
            stdout.startswith("-rwx") and stdout.endswith("timeout"))

        start_time = perf_counter()
        command = "/usr/bin/timeout 0.3s /bin/bash -c 'sleep 1.0; echo foo'"
        stdout, stderr = self.sandbox_helper(sandbox=Sandbox(),
                                             command=command)
        self.assertEqual(stdout, "")  # No output, killed before that
        self.assertLess(perf_counter() - start_time,
                        0.5)  # Killed shortly after the timeout

    # Up to MAX_PARALLEL_WORKERS run in parallel.
    # We expect if we run less than or equal to MAX_PARALLEL_WORKERS to take clock time equal to the longest
    # of them. If we run even a single one more, we expect the clock time to be roughly twice as long.
    def dummy_sleep_helper(self):
        start_time = perf_counter()
        sandbox = Sandbox()
        waiting_time = perf_counter() - start_time
        self.sandbox_helper(sandbox=sandbox, command="sleep 0.5 ; echo foo")
        return waiting_time

    def test_workers_under_limit(self):
        start_time = perf_counter()

        pool = ThreadPoolExecutor(max_workers=config.MAX_PARALLEL_WORKERS)
        futures = [
            pool.submit(self.dummy_sleep_helper)
            for _ in range(config.MAX_PARALLEL_WORKERS)
        ]

        # Each of the processes runs in ~0.5s, but since we schedule them through a thread pool
        # we should reach this point much earlier.
        self.assertLess(perf_counter() - start_time, 0.1)

        # Wait for all workers to complete and get the maximum waiting time for a Sandbox object
        max_waiting_time = 0.0
        for future in futures:
            max_waiting_time = max(max_waiting_time, future.result())

        # Expecting none of the workers to wait for another one to finish, this get a Sandbox object immediately
        self.assertLess(max_waiting_time, 0.2)
        # Waiting all of them to complete takes at least 0.5 seconds
        self.assertGreaterEqual(perf_counter() - start_time, 0.5)
        # But not much more than 0.5 seconds
        self.assertLess(perf_counter() - start_time, 0.7)

    def test_workers_over_limit(self):
        start_time = perf_counter()

        pool = ThreadPoolExecutor(max_workers=config.MAX_PARALLEL_WORKERS + 1)
        futures = [
            pool.submit(self.dummy_sleep_helper)
            for _ in range(config.MAX_PARALLEL_WORKERS + 1)
        ]

        # Each of the processes runs in ~0.5s, but since we schedule them through a thread pool
        # we should reach this point much earlier. One of the threads should be blocked on waiting
        # for a sandbox, though.
        self.assertLess(perf_counter() - start_time, 0.1)

        # Wait for all workers to complete and get the maximum waiting time for a Sandbox object
        max_waiting_time = 0.0
        for future in futures:
            max_waiting_time = max(max_waiting_time, future.result())

        # Expecting one of the workers to wait for another one to finish before getting a Sandbox object
        self.assertGreaterEqual(max_waiting_time, 0.5)
        self.assertLess(max_waiting_time, 0.7)
        # Waiting all of them to complete takes at least 1 second (twice as much)
        self.assertGreaterEqual(perf_counter() - start_time, 1.0)
        # But not much more than 1 second
        self.assertLess(perf_counter() - start_time, 1.4)
Exemple #3
0
def execute_interactive(submit_id, test: TestInfo,
                        run_config: RunConfig) -> RunResult:
    log_file = "interaction.log"
    tester_executable_name = "tester." + run_config.tester_path.split(".")[-1]
    solution_executable_name = "solution." + run_config.executable_path.split(
        ".")[-1]

    # Both the tester and the solution get extra time in order to account for time tester
    # processes input/output. The tester gets additional time to make sure it can print
    # the log and results properly after the solution completes its execution.
    tester_communication_time = min(1.0, run_config.time_limit * 2)
    tester_postprocessing_time = 1.0
    solution_timeout = run_config.timeout + tester_communication_time
    tester_timeout = solution_timeout + tester_postprocessing_time

    args = {
        "tester_timeout":
        tester_timeout,
        "solution_timeout":
        solution_timeout,
        "tester_run_command":
        Runner.get_run_command(
            language=common.get_language_by_exec_name(tester_executable_name),
            executable=tester_executable_name,
            memory_limit=config.MAX_TESTER_MEMORY),
        "solution_run_command":
        Runner.get_run_command(language=common.get_language_by_exec_name(
            solution_executable_name),
                               executable=solution_executable_name,
                               memory_limit=run_config.memory_limit),
        "log_file":
        log_file
    }
    # print(json.dumps(args, indent=4, sort_keys=True))

    empty_file = NamedTemporaryFile(mode="w+t", delete=False)
    empty_file_path = os.path.abspath(empty_file.name)

    # Copy all the needed files to the sandbox directory:
    #   1) wrapper.py
    #   2) the tester executable
    #   3) the solution executable
    #   4) the input file (read by the tester)
    #   5) game log (empty file with write permissions)

    sandbox = Sandbox()
    sandbox.put_file(os.path.join(config.ROOT_DIR, "wrapper.py"))
    sandbox.put_file(run_config.tester_path,
                     target_name=tester_executable_name)
    sandbox.put_file(run_config.executable_path,
                     target_name=solution_executable_name)
    sandbox.put_file(test.inpPath, target_name="input.txt", mode=0o777)
    sandbox.put_file(empty_file_path, target_name=log_file, mode=0o777)

    # Run the executable, while measuring CPU and Memory consumption
    run_result = Runner.run_program(
        sandbox=sandbox,
        executable_path=os.path.join(config.ROOT_DIR, "interactor_1P.py"),
        memory_limit=config.MAX_EXECUTION_MEMORY,
        timeout=config.MAX_GAME_LENGTH,
        input_bytes=json.dumps(args, indent=4, sort_keys=True).encode())
    interaction_log = sandbox.read_file(log_file)
    del sandbox

    # Record the game log to the /replays folder
    replay_id = save_replay_log(interaction_log)

    # The interactor crashed or got killed
    # Don't check memory limit, as it can be caused by child processes (including solution)
    if run_result.exec_time >= config.MAX_GAME_LENGTH:
        message = "Interactor took too much time to complete ({:.3f}s).".format(
            run_result.exec_time)
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)
    if run_result.exit_code != 0:
        message = "Interactor exited with non-zero exit code ({}).".format(
            run_result.exit_code)
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # Parse the response from the interactor
    try:
        results = json.loads(run_result.output.decode(config.OUTPUT_ENCODING))
    except ValueError:
        message = "Could not decode interactor's output: {}!".format(
            run_result.output.decode())
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # print("RESULTS:\n{}".format(json.dumps(results, indent=4, sort_keys=True)))

    # Okay, let's assume the interactor was okay. Now check if the tester crashed.
    if results["internal_error"] or results["tester_exit_code"] != 0:
        message = "Tester crashed or some other internal error happened. Result from interactor:\n{}".format(
            json.dumps(results, indent=4, sort_keys=True))
        logger.error("Submit {id} | {message}".format(id=submit_id,
                                                      message=message))
        return RunResult(status=TestStatus.INTERNAL_ERROR,
                         error=message,
                         replay_id=replay_id)

    # Everything with the system seems okay.
    # Get the score and the solution's exit_code, exec_time, and exec_memory
    exit_code = int(results["solution_exit_code"])
    # Calculate final time and memory (offset for VM start-up time)
    solution_language = common.get_language_by_exec_name(
        run_config.executable_path)
    exec_time = max(
        0,
        float(results["solution_exec_time"]) -
        common.get_time_offset(solution_language))
    exec_memory = max(
        0,
        float(results["solution_exec_memory"]) -
        common.get_memory_offset(solution_language))
    # Get tester's output
    output = ("" if "tester_message" not in results else
              results["tester_message"]).encode()

    # Leave the caller function decide what the test status will be
    return RunResult(exit_code=exit_code,
                     exec_time=exec_time,
                     exec_memory=exec_memory,
                     output=output,
                     replay_id=replay_id)