def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        # f stand for first, s for second.
        first_sandbox = create_sandbox(self)
        second_sandbox = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)

        # First step: we start the first manager.
        first_filename = "manager"
        first_command = ["./%s" % first_filename, "0", fifo]
        first_executables_to_get = {
            first_filename:
            self.job.executables[first_filename].digest
            }
        first_files_to_get = {
            "input.txt": self.job.testcases[test_number].input
            }
        first_allow_path = ["input.txt", fifo]

        # Put the required files into the sandbox
        for filename, digest in first_executables_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename, digest, executable=True)
        for filename, digest in first_files_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(
            first_sandbox,
            first_command,
            self.job.time_limit,
            self.job.memory_limit,
            first_allow_path,
            stdin_redirect="input.txt",
            wait=False)

        # Second step: we start the second manager.
        second_filename = "manager"
        second_command = ["./%s" % second_filename, "1", fifo]
        second_executables_to_get = {
            second_filename:
            self.job.executables[second_filename].digest
            }
        second_files_to_get = {}
        second_allow_path = [fifo, "output.txt"]

        # Put the required files into the second sandbox
        for filename, digest in second_executables_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename, digest, executable=True)
        for filename, digest in second_files_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(
            second_sandbox,
            second_command,
            self.job.time_limit,
            self.job.memory_limit,
            second_allow_path,
            stdout_redirect="output.txt",
            wait=False)

        # Consume output.
        wait_without_std([second, first])
        # TODO: check exit codes with translate_box_exitcode.

        success_first, first_plus = \
            evaluation_step_after_run(first_sandbox)
        success_second, second_plus = \
            evaluation_step_after_run(second_sandbox)

        self.job.evaluations[test_number] = {'sandboxes': [first_sandbox.path,
                                                           second_sandbox.path],
                                             'plus': second_plus}
        outcome = None
        text = None
        evaluation = self.job.evaluations[test_number]
        success = True

        # Error in the sandbox: report failure!
        if not success_first or not success_second:
            success = False

        # Contestant's error: the marks won't be good
        elif not is_evaluation_passed(first_plus) or \
                not is_evaluation_passed(second_plus):
            outcome = 0.0
            if not is_evaluation_passed(first_plus):
                text = human_evaluation_message(first_plus)
            else:
                text = human_evaluation_message(second_plus)
            if self.job.get_output:
                evaluation['output'] = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists('output.txt'):
                outcome = 0.0
                text = "Execution didn't produce file output.txt"
                if self.job.get_output:
                    evaluation['output'] = None

            else:
                # If asked so, put the output file into the storage
                if self.job.get_output:
                    evaluation['output'] = second_sandbox.get_file_to_storage(
                        "output.txt",
                        "Output file for testcase %d in job %s" %
                        (test_number, self.job.info))

                # If not asked otherwise, evaluate the output file
                if not self.job.only_execution:
                    # Put the reference solution into the sandbox
                    second_sandbox.create_file_from_storage(
                        "res.txt",
                        self.job.testcases[test_number].output)

                    outcome, text = white_diff_step(
                        second_sandbox, "output.txt", "res.txt")

        # Whatever happened, we conclude.
        evaluation['success'] = success
        evaluation['outcome'] = outcome
        evaluation['text'] = text

        delete_sandbox(first_sandbox)
        delete_sandbox(second_sandbox)

        return success
Example #2
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(file_cacher)
        sandbox_user = create_sandbox(file_cacher)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_in = os.path.join(fifo_dir, "in")
        fifo_out = os.path.join(fifo_dir, "out")
        os.mkfifo(fifo_in)
        os.mkfifo(fifo_out)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo_in, 0o666)
        os.chmod(fifo_out, 0o666)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
        manager_executables_to_get = {
            manager_filename: job.managers[manager_filename].digest
        }
        manager_files_to_get = {"input.txt": job.input}
        manager_allow_dirs = [fifo_dir]
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename,
                                                 digest,
                                                 executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(sandbox_mgr,
                                             manager_command,
                                             job.time_limit,
                                             0,
                                             allow_dirs=manager_allow_dirs,
                                             writable_files=["output.txt"],
                                             stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = job.executables.keys()[0]
        command = ["./%s" % executable_filename, fifo_out, fifo_in]
        executables_to_get = {
            executable_filename: job.executables[executable_filename].digest
        }
        user_allow_dirs = [fifo_dir]
        for filename, digest in executables_to_get.iteritems():
            sandbox_user.create_file_from_storage(filename,
                                                  digest,
                                                  executable=True)
        process = evaluation_step_before_run(sandbox_user,
                                             command,
                                             job.time_limit,
                                             job.memory_limit,
                                             allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std([process, manager])
        # TODO: check exit codes with translate_box_exitcode.

        success_user, plus_user = \
            evaluation_step_after_run(sandbox_user)
        success_mgr, unused_plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        job.sandboxes = [sandbox_user.path, sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    "output.txt", "Output file in job %s" % job.info)
            else:
                job.user_output = None

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr)
        delete_sandbox(sandbox_user)
        if not config.keep_sandbox:
            rmtree(fifo_dir)
Example #3
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if not check_executables_number(job, 1):
            return

        executable_filename = next(iterkeys(job.executables))
        executable_digest = job.executables[executable_filename].digest

        first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
        second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
        job.sandboxes.append(first_sandbox.get_root_path())
        job.sandboxes.append(second_sandbox.get_root_path())

        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo, 0o666)

        # First step: we start the first manager.
        first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
        first_executables_to_get = {executable_filename: executable_digest}
        first_files_to_get = {
            TwoSteps.INPUT_FILENAME: job.input
            }

        # Put the required files into the sandbox
        for filename, digest in iteritems(first_executables_to_get):
            first_sandbox.create_file_from_storage(filename,
                                                   digest,
                                                   executable=True)
        for filename, digest in iteritems(first_files_to_get):
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(
            first_sandbox,
            first_command,
            job.time_limit,
            job.memory_limit,
            dirs_map={fifo_dir: ("/fifo", "rw")},
            stdin_redirect=TwoSteps.INPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox,
            wait=False)

        # Second step: we start the second manager.
        second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
        second_executables_to_get = {executable_filename: executable_digest}
        second_files_to_get = {}

        # Put the required files into the second sandbox
        for filename, digest in iteritems(second_executables_to_get):
            second_sandbox.create_file_from_storage(filename,
                                                    digest,
                                                    executable=True)
        for filename, digest in iteritems(second_files_to_get):
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(
            second_sandbox,
            second_command,
            job.time_limit,
            job.memory_limit,
            dirs_map={fifo_dir: ("/fifo", "rw")},
            stdout_redirect=TwoSteps.OUTPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox,
            wait=False)

        # Consume output.
        wait_without_std([second, first])

        box_success_first, evaluation_success_first, first_stats = \
            evaluation_step_after_run(first_sandbox)
        box_success_second, evaluation_success_second, second_stats = \
            evaluation_step_after_run(second_sandbox)

        box_success = box_success_first and box_success_second
        evaluation_success = \
            evaluation_success_first and evaluation_success_second
        stats = merge_execution_stats(first_stats, second_stats)

        outcome = None
        text = None

        # Error in the sandbox: nothing to do!
        if not box_success:
            pass

        # Contestant's error: the marks won't be good
        elif not evaluation_success:
            outcome = 0.0
            text = human_evaluation_message(stats)
            if job.get_output:
                job.user_output = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
                outcome = 0.0
                text = [N_("Evaluation didn't produce file %s"),
                        TwoSteps.OUTPUT_FILENAME]
                if job.get_output:
                    job.user_output = None

            else:
                # If asked so, put the output file into the storage
                if job.get_output:
                    job.user_output = second_sandbox.get_file_to_storage(
                        TwoSteps.OUTPUT_FILENAME,
                        "Output file in job %s" % job.info,
                        trunc_len=100 * 1024)

                # If just asked to execute, fill text and set dummy outcome.
                if job.only_execution:
                    outcome = 0.0
                    text = [N_("Execution completed successfully")]

                # Otherwise evaluate the output file.
                else:
                    box_success, outcome, text = eval_output(
                        file_cacher, job,
                        TwoSteps.CHECKER_CODENAME
                        if self._uses_checker() else None,
                        user_output_path=second_sandbox.relative_path(
                            TwoSteps.OUTPUT_FILENAME))

        # Fill in the job with the results.
        job.success = box_success
        job.outcome = str(outcome) if outcome is not None else None
        job.text = text
        job.plus = stats

        delete_sandbox(first_sandbox, job.success)
        delete_sandbox(second_sandbox, job.success)
Example #4
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""

        if len(self.parameters) <= 0:
            num_processes = 1
        else:
            num_processes = self.parameters[0]
        indices = range(num_processes)
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
        sandbox_user = [
            create_sandbox(file_cacher, job.multithreaded_sandbox)
            for i in indices
        ]
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
        fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
        for i in indices:
            os.mkfifo(fifo_in[i])
            os.mkfifo(fifo_out[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_in[i], 0o666)
            os.chmod(fifo_out[i], 0o666)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename]
        for i in indices:
            manager_command.append(fifo_in[i])
            manager_command.append(fifo_out[i])
        manager_executables_to_get = {
            manager_filename: job.managers[manager_filename].digest
        }
        manager_files_to_get = {"input.txt": job.input}
        manager_allow_dirs = fifo_dir
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename,
                                                 digest,
                                                 executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            num_processes * job.time_limit,
            0,
            allow_dirs=manager_allow_dirs,
            writable_files=["output.txt"],
            stdin_redirect="input.txt",
            stdout_redirect="output.txt",
        )

        # Second step: we start the user submission compiled with the
        # stub.
        language = get_language(job.language)
        executable_filename = job.executables.keys()[0]
        executables_to_get = {
            executable_filename: job.executables[executable_filename].digest
        }
        processes = [None for i in indices]
        for i in indices:
            args = [fifo_out[i], fifo_in[i]]
            if num_processes != 1:
                args.append(str(i))
            commands = language.get_evaluation_commands(executable_filename,
                                                        main="grader",
                                                        args=args)
            user_allow_dirs = [fifo_dir[i]]
            for filename, digest in executables_to_get.iteritems():
                sandbox_user[i].create_file_from_storage(filename,
                                                         digest,
                                                         executable=True)
            # Assumes that the actual execution of the user solution
            # is the last command in commands, and that the previous
            # are "setup" that doesn't need tight control.
            if len(commands) > 1:
                evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                commands[-1],
                job.time_limit,
                job.memory_limit,
                allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std(processes + [manager])
        # TODO: check exit codes with translate_box_exitcode.

        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        success_user = all(r[0] for r in user_results)
        plus_user = reduce(merge_evaluation_results,
                           [r[1] for r in user_results])
        success_mgr, unused_plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        if plus_user['exit_status'] == Sandbox.EXIT_OK and \
                plus_user["execution_time"] >= job.time_limit:
            plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        # Merge results.
        job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
            if job.get_output:
                job.user_output = None
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome = None
            text = None

            input_filename = "input.txt"
            output_filename = "output.txt"
            # Check that the output file was created
            if not sandbox_mgr.file_exists(output_filename):
                outcome = 0.0
                text = [
                    N_("Evaluation didn't produce file %s"), output_filename
                ]
                if job.get_output:
                    job.user_output = None

            else:
                # If asked so, put the output file into the storage
                if job.get_output:
                    job.user_output = sandbox_mgr.get_file_to_storage(
                        output_filename,
                        "Output file in job %s" % job.info,
                        trunc_len=1024 * 1024 * 10)

                # If just asked to execute, fill text and set dummy
                # outcome.
                if job.only_execution:
                    outcome = 0.0
                    text = [N_("Execution completed successfully")]

                # Otherwise evaluate the output file.
                else:

                    # Put the reference solution into the sandbox
                    sandbox_mgr.create_file_from_storage("res.txt", job.output)

                    # Check the solution with white_diff
                    if self.parameters[1] == "diff":
                        outcome, text = white_diff_step(
                            sandbox_mgr, output_filename, "res.txt")

                    # Check the solution with a comparator
                    elif self.parameters[1] == "comparator":
                        manager_filename = "checker"

                        if manager_filename not in job.managers:
                            logger.error(
                                "Configuration error: missing or "
                                "invalid comparator (it must be "
                                "named 'checker')",
                                extra={"operation": job.info})
                            success = False

                        else:
                            sandbox_mgr.create_file_from_storage(
                                manager_filename,
                                job.managers[manager_filename].digest,
                                executable=True)
                            # Rewrite input file. The untrusted
                            # contestant program should not be able to
                            # modify it; however, the grader may
                            # destroy the input file to prevent the
                            # contestant's program from directly
                            # accessing it. Since we cannot create
                            # files already existing in the sandbox,
                            # we try removing the file first.
                            try:
                                sandbox_mgr.remove_file(input_filename)
                            except OSError as e:
                                # Let us be extra sure that the file
                                # was actually removed and we did not
                                # mess up with permissions.
                                assert not sandbox_mgr.file_exists(
                                    input_filename)
                            sandbox_mgr.create_file_from_storage(
                                input_filename, job.input)

                            # Allow using any number of processes (because e.g.
                            # one may want to write a bash checker who calls
                            # other processes). Set to a high number because
                            # to avoid fork-bombing the worker.
                            sandbox_mgr.max_processes = 1000

                            success, _ = evaluation_step(
                                sandbox_mgr, [[
                                    "./%s" % manager_filename, input_filename,
                                    "res.txt", output_filename
                                ]])
                        if success:
                            try:
                                outcome, text = \
                                    extract_outcome_and_text(sandbox_mgr)
                            except ValueError as e:
                                logger.error(
                                    "Invalid output from "
                                    "comparator: %s",
                                    e.message,
                                    extra={"operation": job.info})
                                success = False

                    else:
                        raise ValueError("Unrecognized second parameter"
                                         " `%s' for Communication tasktype." %
                                         self.parameters[2])

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr, job.success)
        for s in sandbox_user:
            delete_sandbox(s, job.success)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
Example #5
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(file_cacher)
        sandbox_user = create_sandbox(file_cacher)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_in = os.path.join(fifo_dir, "in")
        fifo_out = os.path.join(fifo_dir, "out")
        os.mkfifo(fifo_in)
        os.mkfifo(fifo_out)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo_in, 0o666)
        os.chmod(fifo_out, 0o666)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
        manager_executables_to_get = {
            manager_filename:
            job.managers[manager_filename].digest
            }
        manager_files_to_get = {
            "input.txt": job.input
            }
        manager_allow_dirs = [fifo_dir]
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(
                filename, digest, executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            job.time_limit,
            0,
            allow_dirs=manager_allow_dirs,
            stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = job.executables.keys()[0]
        command = ["./%s" % executable_filename, fifo_out, fifo_in]
        executables_to_get = {
            executable_filename:
            job.executables[executable_filename].digest
            }
        user_allow_dirs = [fifo_dir]
        for filename, digest in executables_to_get.iteritems():
            sandbox_user.create_file_from_storage(
                filename, digest, executable=True)
        process = evaluation_step_before_run(
            sandbox_user,
            command,
            job.time_limit,
            job.memory_limit,
            allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std([process, manager])
        # TODO: check exit codes with translate_box_exitcode.

        success_user, plus_user = \
            evaluation_step_after_run(sandbox_user)
        success_mgr, plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        job.sandboxes = [sandbox_user.path,
                         sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    "output.txt",
                    "Output file in job %s" % job.info)
            else:
                job.user_output = None

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = str(outcome) if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr)
        delete_sandbox(sandbox_user)
        shutil.rmtree(fifo_dir)
Example #6
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if len(self.parameters) <= 0:
            num_processes = 1
        else:
            num_processes = self.parameters[0]
        indices = range(num_processes)
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
        sandbox_user = [
            create_sandbox(file_cacher, job.multithreaded_sandbox)
            for i in indices
        ]
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
        fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
        for i in indices:
            os.mkfifo(fifo_in[i])
            os.mkfifo(fifo_out[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_in[i], 0o666)
            os.chmod(fifo_out[i], 0o666)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename]
        for i in indices:
            manager_command.append(fifo_in[i])
            manager_command.append(fifo_out[i])
        manager_executables_to_get = {
            manager_filename: job.managers[manager_filename].digest
        }
        manager_files_to_get = {"input.txt": job.input}
        manager_allow_dirs = fifo_dir
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename,
                                                 digest,
                                                 executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(sandbox_mgr,
                                             manager_command,
                                             num_processes * job.time_limit,
                                             0,
                                             allow_dirs=manager_allow_dirs,
                                             writable_files=["output.txt"],
                                             stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        language = get_language(job.language)
        executable_filename = job.executables.keys()[0]
        executables_to_get = {
            executable_filename: job.executables[executable_filename].digest
        }
        processes = [None for i in indices]
        for i in indices:
            args = [fifo_out[i], fifo_in[i]]
            if num_processes != 1:
                args.append(str(i))
            commands = language.get_evaluation_commands(executable_filename,
                                                        main="stub",
                                                        args=args)
            user_allow_dirs = [fifo_dir[i]]
            for filename, digest in executables_to_get.iteritems():
                sandbox_user[i].create_file_from_storage(filename,
                                                         digest,
                                                         executable=True)
            # Assumes that the actual execution of the user solution
            # is the last command in commands, and that the previous
            # are "setup" that doesn't need tight control.
            if len(commands) > 1:
                evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                commands[-1],
                job.time_limit,
                job.memory_limit,
                allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std(processes + [manager])
        # TODO: check exit codes with translate_box_exitcode.

        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        success_user = all(r[0] for r in user_results)
        plus_user = reduce(merge_evaluation_results,
                           [r[1] for r in user_results])
        success_mgr, unused_plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        if plus_user['exit_status'] == Sandbox.EXIT_OK and \
                plus_user["execution_time"] >= job.time_limit:
            plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        # Merge results.
        job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    "output.txt", "Output file in job %s" % job.info)
            else:
                job.user_output = None

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr, job.success)
        for s in sandbox_user:
            delete_sandbox(s, job.success)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
Example #7
0
    def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        sandbox_mgr = create_sandbox(self)
        sandbox_user = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_in = os.path.join(fifo_dir, "in")
        fifo_out = os.path.join(fifo_dir, "out")
        os.mkfifo(fifo_in)
        os.mkfifo(fifo_out)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
        manager_executables_to_get = {
            manager_filename:
            self.submission.task.managers[manager_filename].digest
            }
        manager_files_to_get = {
            "input.txt": self.submission.task.testcases[test_number].input
            }
        manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
        manager = self.evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            manager_executables_to_get,
            manager_files_to_get,
            self.submission.task.time_limit,
            0,
            manager_allow_path,
            stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = self.submission.executables.keys()[0]
        command = ["./%s" % executable_filename, fifo_out, fifo_in]
        executables_to_get = {
            executable_filename:
            self.submission.executables[executable_filename].digest
            }
        allow_path = [fifo_in, fifo_out]
        process = self.evaluation_step_before_run(
            sandbox_user,
            command,
            executables_to_get,
            {},
            self.submission.task.time_limit,
            self.submission.task.memory_limit,
            allow_path)

        # Consume output.
        wait_without_std([process, manager])
        # TODO: check exit codes with translate_box_exitcode.

        success_user, outcome_user, text_user, plus = \
                      self.evaluation_step_after_run(sandbox_user, final=False)
        success_mgr, outcome_mgr, text_mgr, _ = \
                     self.evaluation_step_after_run(sandbox_mgr, final=True)

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, outcome_user, text_user
        # If outcome_user is not None, it is 0.0 and it means that
        # there has been some errors in the user solution, and outcome
        # and text are meaningful, so we use them.
        elif outcome_user is not None:
            success, outcome, text = success_user, outcome_user, text_user
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success, outcome, text = success_mgr, outcome_mgr, text_mgr

        delete_sandbox(sandbox_mgr)
        delete_sandbox(sandbox_user)
        return self.finish_evaluation_testcase(
            test_number, success, outcome, text, plus)
Example #8
0
    def evaluate(self, job, file_cacher):
        if not check_executables_number(job, 1):
            return
        executable_filename = next(iter(job.executables.keys()))
        executable_digest = job.executables[executable_filename].digest

        # Make sure the required manager is among the job managers.
        if not check_manager_present(job, self.MANAGER_FILENAME):
            return
        manager_digest = job.managers[self.MANAGER_FILENAME].digest

        # Create FIFO dirs and first batch of FIFos
        fifo_dir_base = tempfile.mkdtemp(dir=config.temp_dir)

        def fifo_dir(i, j):
            p = os.path.join(fifo_dir_base, f"fifo{i}_{j}")
            if not os.path.exists(p):
                os.mkdir(p)
            return p

        abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq")
        fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq")
        os.mkfifo(fifo_solution_quitter)
        os.mkfifo(fifo_manager_quitter)
        os.chmod(abortion_control_fifo_dir, 0o755)
        os.chmod(fifo_solution_quitter, 0o666)
        os.chmod(fifo_manager_quitter, 0o666)

        sandbox_abortion_control_fifo_dir = "/abort"
        sandbox_fifo_solution_quitter = \
            os.path.join(sandbox_abortion_control_fifo_dir, "sq")
        sandbox_fifo_manager_quitter = \
            os.path.join(sandbox_abortion_control_fifo_dir, "mq")

        # Start the manager. Redirecting to stdin is unnecessary, but for
        # historical reasons the manager can choose to read from there
        # instead than from INPUT_FILENAME.
        manager_command = ["./%s" % self.MANAGER_FILENAME]
        manager_command += [
            sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter
        ]

        # Create the manager sandbox and copy manager and input and
        # reference output.
        sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
        job.sandboxes.append(sandbox_mgr.get_root_path())
        sandbox_mgr.create_file_from_storage(self.MANAGER_FILENAME,
                                             manager_digest,
                                             executable=True)
        sandbox_mgr.create_file_from_storage(self.INPUT_FILENAME, job.input)
        sandbox_mgr.create_file_from_storage(self.OK_FILENAME, job.output)

        # We could use trusted_step for the manager, since it's fully
        # admin-controlled. But trusted_step is only synchronous at the moment.
        # Thus we use evaluation_step, and we set a time limit generous enough
        # to prevent user programs from sending the manager in timeout.
        # This means that:
        # - the manager wall clock timeout must be greater than the sum of all
        #     wall clock timeouts of the user programs;
        # - with the assumption that the work the manager performs is not
        #     greater than the work performed by the user programs, the manager
        #     user timeout must be greater than the maximum allowed total time
        #     of the user programs; in theory, this is the task's time limit,
        #     but in practice is num_processes times that because the
        #     constraint on the total time can only be enforced after all user
        #     programs terminated.
        sandbox_fifo_dir_base = "/fifo"

        def sandbox_fifo_dir(i, j):
            return f"{sandbox_fifo_dir_base}/fifo{i}_{j}"

        manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
                                 config.trusted_sandbox_max_time_s)
        manager_dirs_map = {
            abortion_control_fifo_dir:
            (sandbox_abortion_control_fifo_dir, "rw")
        }

        # TODO: can we avoid creating all these directories?
        MAX_NUM_INSTANCES = 42

        list_of_fifo_dirs = []

        for pr in range(0, self.num_processes):
            for i in range(0, MAX_NUM_INSTANCES):
                d = fifo_dir(i, pr)
                list_of_fifo_dirs.append(d)
                manager_dirs_map[d] = (sandbox_fifo_dir(i, pr), "rw")

        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            manager_time_limit,
            config.trusted_sandbox_max_memory_kib * 1024,
            dirs_map=manager_dirs_map,
            writable_files=[self.OUTPUT_FILENAME],
            stdin_redirect=self.INPUT_FILENAME,
            multiprocess=True)

        solution_quitter = open(fifo_solution_quitter, "r")
        manager_quitter = open(fifo_manager_quitter, "w")

        def finish_run():
            wait_without_std(processes)
            L = [finish_run_single(i) for i in indices]
            return all(L)

        def finish_run_single(i):
            nonlocal wall_clock_acc
            nonlocal num_runs

            user_results.append(evaluation_step_after_run(sandbox_user[i]))
            wall_clock_acc += user_results[-1][2]["execution_wall_clock_time"]
            num_runs += 1
            runtimes[i].append(user_results[-1][2]["execution_time"])

            # Convert tuple to list for write access to entries
            L = list(user_results[-1])
            L[2]["execution_time"] = runtimes[i][-1] / max_num_runs

            if (L[2]["execution_time"] >= job.time_limit):
                L[2]["exit_status"] = Sandbox.EXIT_TIMEOUT

            user_results[-1] = tuple(L)

            if not self._uses_stub():
                # It can happen that the submission runs out of memory and then
                # gets killed by the manager while it is being shut down, in
                # which case isolate does not report a signal as the exit
                # status. To catch this, we look for cg-oom-killed in the logs
                sandbox_user[i].get_log()
                if user_results[-1][1] and \
                   "cg-oom-killed" in sandbox_user[i].log:
                    # Convert tuple to list for write access to entries
                    r = list(user_results[-1])
                    r[1] = False
                    r[2]["status"] = ["SG"]
                    r[2]["exit_status"] = "signal"
                    r[2]["signal"] = -41  # sit by a lake
                    r[2]["message"] = ["out of memory"]
                    user_results[-1] = tuple(r)

            return user_results[-1][0] and user_results[-1][1]

        def respond(okay=True):
            manager_quitter.write("O" if okay else "X")
            manager_quitter.flush()

        def read_int_from_manager():
            L = []
            while True:
                c = solution_quitter.read(1)
                if c == 'B':
                    break
                else:
                    L.append(c)
            return int("".join(L))

        quit = False

        for pr in range(0, self.num_processes):
            if quit:
                break

            wall_clock_acc = 0
            num_runs = 0

            # Startup message to sync pipes
            manager_quitter.write('S')
            manager_quitter.flush()

            # Ask the manager for the number of processes
            num_instances = read_int_from_manager()
            indices = range(0, num_instances)
            max_num_runs = read_int_from_manager()

            # Create remaining FIFOs
            fifo_user_to_manager = [
                os.path.join(fifo_dir(i, pr), f"u{pr}_{i}_to_m")
                for i in indices
            ]
            fifo_manager_to_user = [
                os.path.join(fifo_dir(i, pr), f"m_to_u{pr}_{i}")
                for i in indices
            ]
            for i in indices:
                os.mkfifo(fifo_user_to_manager[i])
                os.mkfifo(fifo_manager_to_user[i])
                os.chmod(fifo_dir(i, pr), 0o755)
                os.chmod(fifo_user_to_manager[i], 0o666)
                os.chmod(fifo_manager_to_user[i], 0o666)

            # Names of the fifos after being mapped inside the sandboxes.
            sandbox_fifo_user_to_manager = \
                [os.path.join(sandbox_fifo_dir(i, pr),
                              f"u{pr}_{i}_to_m") for i in indices]
            sandbox_fifo_manager_to_user = \
                [os.path.join(sandbox_fifo_dir(i, pr),
                              f"m_to_u{pr}_{i}") for i in indices]

            for i in indices:
                print(sandbox_fifo_user_to_manager[i],
                      file=manager_quitter,
                      flush=True)
                print(sandbox_fifo_manager_to_user[i],
                      file=manager_quitter,
                      flush=True)

            # Create the user sandbox(es) and copy the executable.
            sandbox_user = [
                create_sandbox(file_cacher, name="user_evaluate")
                for i in indices
            ]
            job.sandboxes.extend(s.get_root_path() for s in sandbox_user)

            for i in indices:
                sandbox_user[i].create_file_from_storage(executable_filename,
                                                         executable_digest,
                                                         executable=True)

                # Prepare the user submissions
                language = get_language(job.language)
                main = self.STUB_BASENAME if self._uses_stub() \
                    else os.path.splitext(executable_filename)[0]
                processes = [None for i in indices]
                user_results = []

                args = []
                stdin_redirect = None
                stdout_redirect = None
                if self._uses_fifos():
                    args.extend([
                        sandbox_fifo_manager_to_user[i],
                        sandbox_fifo_user_to_manager[i]
                    ])
                if self.num_processes != 1:
                    args.append(str(i))
                if self._uses_stub():
                    main = self.STUB_BASENAME
                else:
                    main = executable_filename
                commands = language.get_evaluation_commands(
                    executable_filename, main=main, args=args)

                # Assumes that the actual execution of the user solution is the
                # last command in commands, and that the previous are "setup"
                # that don't need tight control.
                if len(commands) > 1:
                    trusted_step(sandbox_user[i], commands[:-1])

            processes = [None for _ in indices]
            runtimes = [[] for _ in indices]

            while True:
                for i in indices:
                    processes[i] = evaluation_step_before_run(
                        sandbox_user[i],
                        commands[-1],
                        job.time_limit * max_num_runs * num_instances,
                        job.memory_limit,
                        dirs_map={
                            fifo_dir(i, pr): (sandbox_fifo_dir(i, pr), "rw")
                        },
                        stdin_redirect=sandbox_fifo_manager_to_user[i],
                        stdout_redirect=sandbox_fifo_user_to_manager[i],
                        multiprocess=job.multithreaded_sandbox)

                response = solution_quitter.read(1)

                if response == "C":  # continue
                    if not finish_run():
                        # this run was not successful, time to call it quits
                        quit = True
                        respond(okay=False)
                        break
                    respond()
                elif response == "N":  # next process
                    if not finish_run():
                        # this run was not successful, time to call it quits
                        quit = True
                        respond(okay=False)
                        break
                    respond()
                    break
                elif response == "Q":
                    if not self._uses_stub():
                        time.sleep(.01)
                        processes[i].send_signal(signal.SIGINT)
                    finish_run()
                    respond()
                    quit = True
                    break
                else:
                    raise RuntimeError("Received '{}' ".format(response) +
                                       "through solution_quitter.")

        # Wait for the manager to conclude, without blocking them on I/O.
        wait_without_std([manager])

        solution_quitter.close()
        manager_quitter.close()

        # Get the results of the manager sandbox.
        box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        # Coalesce the results of the user sandboxes.
        box_success_user = all(r[0] for r in user_results)
        evaluation_success_user = all(r[1] for r in user_results)
        stats_user = reduce(my_merge_execution_stats,
                            [r[2] for r in user_results])
        # The actual running time is the sum of every user process, but each
        # sandbox can only check its own; if the sum is greater than the time
        # limit we adjust the result.
        if box_success_user and evaluation_success_user and \
                stats_user["execution_time"] >= job.time_limit:
            evaluation_success_user = False
            stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        success = box_success_user \
            and box_success_mgr and evaluation_success_mgr
        outcome = None
        text = None

        # If at least one sandbox had problems, or the manager did not
        # terminate correctly, we report an error (and no need for user stats).
        if not success:
            stats_user = None

        # If just asked to execute, fill text and set dummy outcome.
        elif job.only_execution:
            outcome = 0.0
            text = [N_("Execution completed successfully")]

        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not evaluation_success_user:
            outcome = 0.0
            text = human_evaluation_message(stats_user)

        # Otherwise, we use the manager to obtain the outcome.
        else:
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file with additional information,
        # provided that it exists.
        if job.get_output:
            if sandbox_mgr.file_exists(self.OUTPUT_FILENAME):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    self.OUTPUT_FILENAME,
                    "Output file in job %s" % job.info,
                    trunc_len=100 * 1024)
            else:
                job.user_output = None

        # Fill in the job with the results.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text
        job.plus = stats_user

        delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox)
        for s in sandbox_user:
            delete_sandbox(s, job.success, job.keep_sandbox)
        if job.success and not config.keep_sandbox and not job.keep_sandbox:
            rmtree(fifo_dir_base)
            rmtree(abortion_control_fifo_dir)
Example #9
0
 def finish_run():
     wait_without_std(processes)
     L = [finish_run_single(i) for i in indices]
     return all(L)
    def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(self)
        sandbox_user = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_in = os.path.join(fifo_dir, "in")
        fifo_out = os.path.join(fifo_dir, "out")
        os.mkfifo(fifo_in)
        os.mkfifo(fifo_out)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
        manager_executables_to_get = {
            manager_filename:
            self.job.managers[manager_filename].digest
            }
        manager_files_to_get = {
            "input.txt": self.job.testcases[test_number].input
            }
        manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(
                filename, digest, executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            self.job.time_limit,
            0,
            manager_allow_path,
            stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = self.job.executables.keys()[0]
        command = ["./%s" % executable_filename, fifo_out, fifo_in]
        executables_to_get = {
            executable_filename:
            self.job.executables[executable_filename].digest
            }
        allow_path = [fifo_in, fifo_out]
        for filename, digest in executables_to_get.iteritems():
            sandbox_user.create_file_from_storage(
                filename, digest, executable=True)
        process = evaluation_step_before_run(
            sandbox_user,
            command,
            self.job.time_limit,
            self.job.memory_limit,
            allow_path)

        # Consume output.
        wait_without_std([process, manager])
        # TODO: check exit codes with translate_box_exitcode.

        success_user, plus_user = \
            evaluation_step_after_run(sandbox_user)
        success_mgr, plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        self.job.evaluations[test_number] = \
            {'sandboxes': [sandbox_user.path,
                           sandbox_mgr.path],
             'plus': plus_user}
        evaluation = self.job.evaluations[test_number]

        # If at least one evaluation had problems, we report the
        # problems. (TODO: shouldn't outcome and text better be None
        # and None?)
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If outcome_user is not None, it is 0.0 and it means that
        # there has been some errors in the user solution, and outcome
        # and text are meaningful, so we use them.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if self.job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                evaluation['output'] = sandbox_mgr.get_file_to_storage(
                    "output.txt",
                    "Output file for testcase %d in job %s" %
                    (test_number, self.job.info))
            else:
                evaluation['output'] = None

        # Whatever happened, we conclude.
        evaluation['success'] = success
        evaluation['outcome'] = str(outcome) if outcome is not None else None
        evaluation['text'] = text
        delete_sandbox(sandbox_mgr)
        delete_sandbox(sandbox_user)
        return success
Example #11
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        # f stand for first, s for second.
        first_sandbox = create_sandbox(file_cacher,
                                       multithreaded=job.multithreaded_sandbox,
                                       name="first_evaluate")
        second_sandbox = create_sandbox(
            file_cacher,
            multithreaded=job.multithreaded_sandbox,
            name="second_evaluate")
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo, 0o666)

        # First step: we start the first manager.
        first_filename = "manager"
        first_command = ["./%s" % first_filename, "0", fifo]
        first_executables_to_get = {
            first_filename: job.executables[first_filename].digest
        }
        first_files_to_get = {"input.txt": job.input}
        first_allow_path = [fifo_dir]

        # Put the required files into the sandbox
        for filename, digest in iteritems(first_executables_to_get):
            first_sandbox.create_file_from_storage(filename,
                                                   digest,
                                                   executable=True)
        for filename, digest in iteritems(first_files_to_get):
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(first_sandbox,
                                           first_command,
                                           job.time_limit,
                                           job.memory_limit,
                                           first_allow_path,
                                           stdin_redirect="input.txt",
                                           wait=False)

        # Second step: we start the second manager.
        second_filename = "manager"
        second_command = ["./%s" % second_filename, "1", fifo]
        second_executables_to_get = {
            second_filename: job.executables[second_filename].digest
        }
        second_files_to_get = {}
        second_allow_path = [fifo_dir]

        # Put the required files into the second sandbox
        for filename, digest in iteritems(second_executables_to_get):
            second_sandbox.create_file_from_storage(filename,
                                                    digest,
                                                    executable=True)
        for filename, digest in iteritems(second_files_to_get):
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(second_sandbox,
                                            second_command,
                                            job.time_limit,
                                            job.memory_limit,
                                            second_allow_path,
                                            stdout_redirect="output.txt",
                                            wait=False)

        # Consume output.
        wait_without_std([second, first])
        # TODO: check exit codes with translate_box_exitcode.

        success_first, first_plus = \
            evaluation_step_after_run(first_sandbox)
        success_second, second_plus = \
            evaluation_step_after_run(second_sandbox)

        job.sandboxes = [first_sandbox.path, second_sandbox.path]
        job.plus = second_plus

        success = True
        outcome = None
        text = []

        # Error in the sandbox: report failure!
        if not success_first or not success_second:
            success = False

        # Contestant's error: the marks won't be good
        elif not is_evaluation_passed(first_plus) or \
                not is_evaluation_passed(second_plus):
            outcome = 0.0
            if not is_evaluation_passed(first_plus):
                text = human_evaluation_message(first_plus)
            else:
                text = human_evaluation_message(second_plus)
            if job.get_output:
                job.user_output = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists('output.txt'):
                outcome = 0.0
                text = [N_("Evaluation didn't produce file %s"), "output.txt"]
                if job.get_output:
                    job.user_output = None

            else:
                # If asked so, put the output file into the storage
                if job.get_output:
                    job.user_output = second_sandbox.get_file_to_storage(
                        "output.txt", "Output file in job %s" % job.info)

                # If not asked otherwise, evaluate the output file
                if not job.only_execution:
                    # Put the reference solution into the sandbox
                    second_sandbox.create_file_from_storage(
                        "res.txt", job.output)

                    # If a checker is not provided, use white-diff
                    if self.parameters[0] == "diff":
                        outcome, text = white_diff_step(
                            second_sandbox, "output.txt", "res.txt")

                    elif self.parameters[0] == "comparator":
                        if TwoSteps.CHECKER_FILENAME not in job.managers:
                            logger.error(
                                "Configuration error: missing or "
                                "invalid comparator (it must be "
                                "named `checker')",
                                extra={"operation": job.info})
                            success = False
                        else:
                            second_sandbox.create_file_from_storage(
                                TwoSteps.CHECKER_FILENAME,
                                job.managers[TwoSteps.CHECKER_FILENAME].digest,
                                executable=True)
                            # Rewrite input file, as in Batch.py
                            try:
                                second_sandbox.remove_file("input.txt")
                            except OSError as e:
                                assert not second_sandbox.file_exists(
                                    "input.txt")
                            second_sandbox.create_file_from_storage(
                                "input.txt", job.input)
                            success, _ = evaluation_step(
                                second_sandbox, [[
                                    "./%s" % TwoSteps.CHECKER_FILENAME,
                                    "input.txt", "res.txt", "output.txt"
                                ]])
                            if success:
                                try:
                                    outcome, text = extract_outcome_and_text(
                                        second_sandbox)
                                except ValueError as e:
                                    logger.error(
                                        "Invalid output from "
                                        "comparator: %s",
                                        e,
                                        extra={"operation": job.info})
                                    success = False
                    else:
                        raise ValueError("Uncrecognized first parameter"
                                         " `%s' for TwoSteps tasktype." %
                                         self.parameters[0])

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = str(outcome) if outcome is not None else None
        job.text = text

        delete_sandbox(first_sandbox, job.success)
        delete_sandbox(second_sandbox, job.success)
Example #12
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if not check_executables_number(job, 1):
            return
        executable_filename = next(iterkeys(job.executables))
        executable_digest = job.executables[executable_filename].digest

        # Make sure the required manager is among the job managers.
        if not check_manager_present(job, self.MANAGER_FILENAME):
            return
        manager_digest = job.managers[self.MANAGER_FILENAME].digest

        # Indices for the objects related to each user process.
        indices = range(self.num_processes)

        # Create FIFOs.
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        if not self._uses_grader():
            abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_user_to_manager = [
            os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices]
        fifo_manager_to_user = [
            os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices]
        if not self._uses_grader():
            fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq")
            fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq")
        for i in indices:
            os.mkfifo(fifo_user_to_manager[i])
            os.mkfifo(fifo_manager_to_user[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_user_to_manager[i], 0o666)
            os.chmod(fifo_manager_to_user[i], 0o666)
        if not self._uses_grader():
            os.mkfifo(fifo_solution_quitter)
            os.mkfifo(fifo_manager_quitter)
            os.chmod(abortion_control_fifo_dir, 0o755)
            os.chmod(fifo_solution_quitter, 0o666)
            os.chmod(fifo_manager_quitter, 0o666)

        # Names of the fifos after being mapped inside the sandboxes.
        sandbox_fifo_dir = ["/fifo%d" % i for i in indices]
        sandbox_fifo_user_to_manager = [
            os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices]
        sandbox_fifo_manager_to_user = [
            os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices]
        if not self._uses_grader():
            sandbox_abortion_control_fifo_dir = "/abort"
            sandbox_fifo_solution_quitter = \
                os.path.join(sandbox_abortion_control_fifo_dir, "sq")
            sandbox_fifo_manager_quitter = \
                os.path.join(sandbox_abortion_control_fifo_dir, "mq")

        # Create the manager sandbox and copy manager and input and
        # reference output.
        sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
        job.sandboxes.append(sandbox_mgr.get_root_path())
        sandbox_mgr.create_file_from_storage(
            self.MANAGER_FILENAME, manager_digest, executable=True)
        sandbox_mgr.create_file_from_storage(
            self.INPUT_FILENAME, job.input)
        sandbox_mgr.create_file_from_storage(
            self.OK_FILENAME, job.output)

        # Create the user sandbox(es) and copy the executable.
        sandbox_user = [create_sandbox(file_cacher, name="user_evaluate")
                        for i in indices]
        job.sandboxes.extend(s.get_root_path() for s in sandbox_user)
        for i in indices:
            sandbox_user[i].create_file_from_storage(
                executable_filename, executable_digest, executable=True)

        # Start the manager. Redirecting to stdin is unnecessary, but for
        # historical reasons the manager can choose to read from there
        # instead than from INPUT_FILENAME.
        manager_command = ["./%s" % self.MANAGER_FILENAME]
        for i in indices:
            manager_command += [sandbox_fifo_user_to_manager[i],
                                sandbox_fifo_manager_to_user[i]]
        if not self._uses_grader():
            manager_command += [sandbox_fifo_solution_quitter,
                                sandbox_fifo_manager_quitter]
        # We could use trusted_step for the manager, since it's fully
        # admin-controlled. But trusted_step is only synchronous at the moment.
        # Thus we use evaluation_step, and we set a time limit generous enough
        # to prevent user programs from sending the manager in timeout.
        # This means that:
        # - the manager wall clock timeout must be greater than the sum of all
        #     wall clock timeouts of the user programs;
        # - with the assumption that the work the manager performs is not
        #     greater than the work performed by the user programs, the manager
        #     user timeout must be greater than the maximum allowed total time
        #     of the user programs; in theory, this is the task's time limit,
        #     but in practice is num_processes times that because the
        #     constraint on the total time can only be enforced after all user
        #     programs terminated.
        manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
                                 config.trusted_sandbox_max_time_s)
        manager_dirs_map = dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw"))
                                for i in indices)
        if not self._uses_grader():
            manager_dirs_map[abortion_control_fifo_dir] = \
                (sandbox_abortion_control_fifo_dir, "rw")
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            manager_time_limit,
            config.trusted_sandbox_max_memory_kib // 1024,
            dirs_map=manager_dirs_map,
            writable_files=[self.OUTPUT_FILENAME],
            stdin_redirect=self.INPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox)

        if not self._uses_grader():
            solution_quitter = open(fifo_solution_quitter, "r")
            manager_quitter = open(fifo_manager_quitter, "w")
            manager_quitter_open = True

        # Start the user submissions compiled with the stub.
        language = get_language(job.language)
        processes = [None for i in indices]
        for i in indices:
            args = [sandbox_fifo_manager_to_user[i],
                    sandbox_fifo_user_to_manager[i]]
            if self.num_processes != 1:
                args.append(str(i))
            if self._uses_grader():
                main = self.STUB_BASENAME
            else:
                main = executable_filename
            commands = language.get_evaluation_commands(
                executable_filename,
                main=main,
                args=args)
            # Assumes that the actual execution of the user solution is the
            # last command in commands, and that the previous are "setup"
            # that don't need tight control.
            if len(commands) > 1:
                trusted_step(sandbox_user[i], commands[:-1])
            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                commands[-1],
                job.time_limit,
                job.memory_limit,
                dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")},
                stdin_redirect=sandbox_fifo_manager_to_user[i],
                stdout_redirect=sandbox_fifo_user_to_manager[i],
                multiprocess=job.multithreaded_sandbox)

        if not self._uses_grader():
            # Manager still running but wants to quit
            if solution_quitter.read() == "<3":
                for i in indices:
                    processes[i].send_signal(signal.SIGINT)  # Kill user
                wait_without_std(processes)
                manager_quitter.close()
                manager_quitter_open = False

        # Wait for the processes to conclude, without blocking them on I/O.
        wait_without_std(processes + [manager])

        if not self._uses_grader():
            solution_quitter.close()
            if manager_quitter_open:
                manager_quitter.close()

        # Get the results of the manager sandbox.
        box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        # Coalesce the results of the user sandboxes.
        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        box_success_user = all(r[0] for r in user_results)
        evaluation_success_user = all(r[1] for r in user_results)
        stats_user = reduce(merge_execution_stats,
                            [r[2] for r in user_results])
        # The actual running time is the sum of every user process, but each
        # sandbox can only check its own; if the sum is greater than the time
        # limit we adjust the result.
        if box_success_user and evaluation_success_user and \
                stats_user["execution_time"] >= job.time_limit:
            evaluation_success_user = False
            stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        success = box_success_user \
            and box_success_mgr and evaluation_success_mgr
        outcome = None
        text = None

        # If at least one sandbox had problems, or the manager did not
        # terminate correctly, we report an error (and no need for user stats).
        if not success:
            stats_user = None
            pass

        # If just asked to execute, fill text and set dummy outcome.
        elif job.only_execution:
            outcome = 0.0
            text = [N_("Execution completed successfully")]

        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not evaluation_success_user:
            outcome = 0.0
            text = human_evaluation_message(stats_user)

        # Otherwise, we use the manager to obtain the outcome.
        else:
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file with additional information,
        # provided that it exists.
        if job.get_output:
            if sandbox_mgr.file_exists(self.OUTPUT_FILENAME):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    self.OUTPUT_FILENAME,
                    "Output file in job %s" % job.info,
                    trunc_len=100 * 1024)
            else:
                job.user_output = None

        # Fill in the job with the results.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text
        job.plus = stats_user

        delete_sandbox(sandbox_mgr, job.success)
        for s in sandbox_user:
            delete_sandbox(s, job.success)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
            if not self._uses_grader():
                rmtree(abortion_control_fifo_dir)
Example #13
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if len(self.parameters) <= 0:
            num_processes = 1
        else:
            num_processes = self.parameters[0]
        indices = range(num_processes)
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(
            file_cacher,
            multithreaded=job.multithreaded_sandbox,
            name="manager_evaluate")
        sandbox_user = [
            create_sandbox(
                file_cacher,
                multithreaded=job.multithreaded_sandbox,
                name="user_evaluate")
            for i in indices]
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
        fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
        for i in indices:
            os.mkfifo(fifo_in[i])
            os.mkfifo(fifo_out[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_in[i], 0o666)
            os.chmod(fifo_out[i], 0o666)

        # First step: prepare the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename]
        for i in indices:
            manager_command.append(fifo_in[i])
            manager_command.append(fifo_out[i])
        manager_executables_to_get = {
            manager_filename:
            job.managers[manager_filename].digest
            }
        manager_files_to_get = {
            "input.txt": job.input
            }
        manager_allow_dirs = fifo_dir
        for filename, digest in iteritems(manager_executables_to_get):
            sandbox_mgr.create_file_from_storage(
                filename, digest, executable=True)
        for filename, digest in iteritems(manager_files_to_get):
            sandbox_mgr.create_file_from_storage(filename, digest)

        # Second step: load the executables for the user processes
        # (done before launching the manager so that it does not
        # impact its wall clock time).
        assert len(job.executables) == 1
        executable_filename = next(iterkeys(job.executables))
        executables_to_get = {
            executable_filename:
            job.executables[executable_filename].digest
            }
        for i in indices:
            for filename, digest in iteritems(executables_to_get):
                sandbox_user[i].create_file_from_storage(
                    filename, digest, executable=True)

        # Third step: start the manager.
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            num_processes * job.time_limit,
            0,
            allow_dirs=manager_allow_dirs,
            writable_files=["output.txt"],
            stdin_redirect="input.txt")

        # Fourth step: start the user submissions compiled with the stub.
        language = get_language(job.language)
        processes = [None for i in indices]
        for i in indices:
            args = [fifo_out[i], fifo_in[i]]
            if num_processes != 1:
                args.append(str(i))
            commands = language.get_evaluation_commands(
                executable_filename,
                main="stub",
                args=args)
            user_allow_dirs = [fifo_dir[i]]
            # Assumes that the actual execution of the user solution
            # is the last command in commands, and that the previous
            # are "setup" that doesn't need tight control.
            if len(commands) > 1:
                evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                commands[-1],
                job.time_limit,
                job.memory_limit,
                allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std(processes + [manager])
        # TODO: check exit codes with translate_box_exitcode.

        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        success_user = all(r[0] for r in user_results)
        plus_user = reduce(merge_evaluation_results,
                           [r[1] for r in user_results])
        success_mgr, unused_plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        if plus_user['exit_status'] == Sandbox.EXIT_OK and \
                plus_user["execution_time"] >= job.time_limit:
            plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        # Merge results.
        job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, []
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    "output.txt",
                    "Output file in job %s" % job.info)
            else:
                job.user_output = None

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr, job.success)
        for s in sandbox_user:
            delete_sandbox(s, job.success)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
Example #14
0
    def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        # f stand for first, s for second.
        first_sandbox = create_sandbox(self)
        second_sandbox = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)

        # First step: we start the first manager.
        first_filename = "manager"
        first_command = ["./%s" % first_filename, "0", fifo]
        first_executables_to_get = {
            first_filename:
            self.submission.executables[first_filename].digest
            }
        first_files_to_get = {
            "input.txt": self.submission.task.testcases[test_number].input
            }
        first_allow_path = ["input.txt", fifo]
        first = self.evaluation_step_before_run(
            first_sandbox,
            first_command,
            first_executables_to_get,
            first_files_to_get,
            self.submission.task.time_limit,
            self.submission.task.memory_limit,
            first_allow_path,
            stdin_redirect="input.txt")

        # Second step: we start the second manager.
        second_filename = "manager"
        second_command = ["./%s" % second_filename, "1", fifo]
        second_executables_to_get = {
            second_filename:
            self.submission.executables[second_filename].digest
            }
        second_files_to_get = {}
        second_allow_path = [fifo, "output.txt"]
        second = self.evaluation_step_before_run(
            second_sandbox,
            second_command,
            second_executables_to_get,
            second_files_to_get,
            self.submission.task.time_limit,
            self.submission.task.memory_limit,
            second_allow_path)

        # Consume output.
        wait_without_std([second, first])
        # TODO: check exit codes with translate_box_exitcode.

        success_first, outcome_first, text_first, _ = \
            self.evaluation_step_after_run(first_sandbox, final=False)
        success_second, outcome_second, text_second, plus = \
            self.evaluation_step_after_run(second_sandbox, final=True)

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_first:
            success, outcome, text = False, outcome_first, text_first
        elif not success_second:
            success, outcome, text = False, outcome_second, text_second
        # Otherwise, compare the second output to the results
        # (there is no comparator since the manager is expected to turn the
        # output in diffable form)
        else:
            success, outcome, text = self.white_diff_step(
                second_sandbox,
                "output.txt", "res.txt",
                {"res.txt":
                 self.submission.task.testcases[test_number].output})

        delete_sandbox(first_sandbox)
        delete_sandbox(second_sandbox)
        return self.finish_evaluation_testcase(
            test_number, success, outcome, text, plus)
Example #15
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if not check_executables_number(job, 1):
            return

        executable_filename = next(iter(job.executables.keys()))
        executable_digest = job.executables[executable_filename].digest

        first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
        second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
        job.sandboxes.append(first_sandbox.get_root_path())
        job.sandboxes.append(second_sandbox.get_root_path())

        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo, 0o666)

        # First step: we start the first manager.
        first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
        first_executables_to_get = {executable_filename: executable_digest}
        first_files_to_get = {TwoSteps.INPUT_FILENAME: job.input}

        # Put the required files into the sandbox
        for filename, digest in first_executables_to_get.items():
            first_sandbox.create_file_from_storage(filename,
                                                   digest,
                                                   executable=True)
        for filename, digest in first_files_to_get.items():
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(
            first_sandbox,
            first_command,
            job.time_limit,
            job.memory_limit,
            dirs_map={fifo_dir: ("/fifo", "rw")},
            stdin_redirect=TwoSteps.INPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox,
            wait=False)

        # Second step: we start the second manager.
        second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
        second_executables_to_get = {executable_filename: executable_digest}
        second_files_to_get = {}

        # Put the required files into the second sandbox
        for filename, digest in second_executables_to_get.items():
            second_sandbox.create_file_from_storage(filename,
                                                    digest,
                                                    executable=True)
        for filename, digest in second_files_to_get.items():
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(
            second_sandbox,
            second_command,
            job.time_limit,
            job.memory_limit,
            dirs_map={fifo_dir: ("/fifo", "rw")},
            stdout_redirect=TwoSteps.OUTPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox,
            wait=False)

        # Consume output.
        wait_without_std([second, first])

        box_success_first, evaluation_success_first, first_stats = \
            evaluation_step_after_run(first_sandbox)
        box_success_second, evaluation_success_second, second_stats = \
            evaluation_step_after_run(second_sandbox)

        box_success = box_success_first and box_success_second
        evaluation_success = \
            evaluation_success_first and evaluation_success_second
        stats = merge_execution_stats(first_stats, second_stats)

        outcome = None
        text = None

        # Error in the sandbox: nothing to do!
        if not box_success:
            pass

        # Contestant's error: the marks won't be good
        elif not evaluation_success:
            outcome = 0.0
            text = human_evaluation_message(stats)
            if job.get_output:
                job.user_output = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
                outcome = 0.0
                text = [
                    N_("Evaluation didn't produce file %s"),
                    TwoSteps.OUTPUT_FILENAME
                ]
                if job.get_output:
                    job.user_output = None

            else:
                # If asked so, put the output file into the storage
                if job.get_output:
                    job.user_output = second_sandbox.get_file_to_storage(
                        TwoSteps.OUTPUT_FILENAME,
                        "Output file in job %s" % job.info,
                        trunc_len=100 * 1024)

                # If just asked to execute, fill text and set dummy outcome.
                if job.only_execution:
                    outcome = 0.0
                    text = [N_("Execution completed successfully")]

                # Otherwise evaluate the output file.
                else:
                    box_success, outcome, text = eval_output(
                        file_cacher,
                        job,
                        TwoSteps.CHECKER_CODENAME
                        if self._uses_checker() else None,
                        user_output_path=second_sandbox.relative_path(
                            TwoSteps.OUTPUT_FILENAME))

        # Fill in the job with the results.
        job.success = box_success
        job.outcome = str(outcome) if outcome is not None else None
        job.text = text
        job.plus = stats

        delete_sandbox(first_sandbox, job.success, job.keep_sandbox)
        delete_sandbox(second_sandbox, job.success, job.keep_sandbox)
Example #16
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if not check_executables_number(job, 1):
            return
        executable_filename = next(iterkeys(job.executables))
        executable_digest = job.executables[executable_filename].digest

        # Make sure the required manager is among the job managers.
        if not check_manager_present(job, Communication.MANAGER_FILENAME):
            return
        manager_digest = job.managers[Communication.MANAGER_FILENAME].digest

        # Indices for the objects related to each user process.
        indices = range(self.num_processes)

        # Create FIFOs.
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
        fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
        for i in indices:
            os.mkfifo(fifo_in[i])
            os.mkfifo(fifo_out[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_in[i], 0o666)
            os.chmod(fifo_out[i], 0o666)

        # Create the manager sandbox and copy manager and input.
        sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
        job.sandboxes.append(sandbox_mgr.path)
        sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME,
                                             manager_digest,
                                             executable=True)
        sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME,
                                             job.input)

        # Create the user sandbox(es) and copy the executable.
        sandbox_user = [
            create_sandbox(file_cacher, name="user_evaluate") for i in indices
        ]
        job.sandboxes.extend(s.path for s in sandbox_user)
        for i in indices:
            sandbox_user[i].create_file_from_storage(executable_filename,
                                                     executable_digest,
                                                     executable=True)
            if Communication.STUB_PRELOAD_FILENAME in job.managers:
                digest = job.managers[
                    Communication.STUB_PRELOAD_FILENAME].digest
                sandbox_user[i].create_file_from_storage(
                    Communication.STUB_PRELOAD_FILENAME,
                    digest,
                    executable=True)

        # Start the manager. Redirecting to stdin is unnecessary, but for
        # historical reasons the manager can choose to read from there
        # instead than from INPUT_FILENAME.
        manager_command = ["./%s" % Communication.MANAGER_FILENAME]
        for i in indices:
            manager_command += [fifo_in[i], fifo_out[i]]
        # We could use trusted_step for the manager, since it's fully
        # admin-controlled. But trusted_step is only synchronous at the moment.
        # Thus we use evaluation_step, and we set a time limit generous enough
        # to prevent user programs from sending the manager in timeout.
        # This means that:
        # - the manager wall clock timeout must be greater than the sum of all
        #     wall clock timeouts of the user programs;
        # - with the assumption that the work the manager performs is not
        #     greater than the work performed by the user programs, the manager
        #     user timeout must be greater than the maximum allowed total time
        #     of the user programs; in theory, this is the task's time limit,
        #     but in practice is num_processes times that because the
        #     constraint on the total time can only be enforced after all user
        #     programs terminated.
        manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
                                 config.trusted_sandbox_max_time_s)
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            manager_time_limit,
            config.trusted_sandbox_max_memory_kib // 1024,
            allow_dirs=fifo_dir,
            writable_files=[Communication.OUTPUT_FILENAME],
            stdin_redirect=Communication.INPUT_FILENAME,
            multiprocess=job.multithreaded_sandbox)

        # Start the user submissions compiled with the stub.
        language = get_language(job.language)
        processes = [None for i in indices]
        for i in indices:
            args = [fifo_out[i], fifo_in[i]]
            if self.num_processes != 1:
                args.append(str(i))
            commands = language.get_evaluation_commands(executable_filename,
                                                        main="stub",
                                                        args=args)
            # Assumes that the actual execution of the user solution is the
            # last command in commands, and that the previous are "setup"
            # that don't need tight control.
            if len(commands) > 1:
                trusted_step(sandbox_user[i], commands[:-1])

            last_cmd = commands[-1]

            # Inject preload program if needed
            if Communication.STUB_PRELOAD_FILENAME in job.managers:
                last_cmd = [
                    "./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i],
                    fifo_in[i]
                ] + commands[-1]

            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                last_cmd,
                job.time_limit,
                job.memory_limit,
                allow_dirs=[fifo_dir[i]],
                multiprocess=job.multithreaded_sandbox)

        # Wait for the processes to conclude, without blocking them on I/O.
        wait_without_std(processes + [manager])

        # Get the results of the manager sandbox.
        box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        # Coalesce the results of the user sandboxes.
        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        box_success_user = all(r[0] for r in user_results)
        evaluation_success_user = all(r[1] for r in user_results)
        stats_user = reduce(merge_execution_stats,
                            [r[2] for r in user_results])
        # The actual running time is the sum of every user process, but each
        # sandbox can only check its own; if the sum is greater than the time
        # limit we adjust the result.
        if box_success_user and evaluation_success_user and \
                stats_user["execution_time"] >= job.time_limit:
            evaluation_success_user = False
            stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        success = box_success_user \
            and box_success_mgr and evaluation_success_mgr
        outcome = None
        text = None

        # If at least one sandbox had problems, or the manager did not
        # terminate correctly, we report an error (and no need for user stats).
        if not success:
            stats_user = None
            pass

        # If just asked to execute, fill text and set dummy outcome.
        elif job.only_execution:
            outcome = 0.0
            text = [N_("Execution completed successfully")]

        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not evaluation_success_user:
            outcome = 0.0
            text = human_evaluation_message(stats_user, job.feedback_level)

        # Otherwise, we use the manager to obtain the outcome.
        else:
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file with additional information,
        # provided that it exists.
        if job.get_output:
            if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    Communication.OUTPUT_FILENAME,
                    "Output file in job %s" % job.info,
                    trunc_len=100 * 1024)
            else:
                job.user_output = None

        # Fill in the job with the results.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text
        job.plus = stats_user

        delete_sandbox(sandbox_mgr, job.success)
        for s in sandbox_user:
            delete_sandbox(s, job.success)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
Example #17
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        if len(self.parameters) <= 0:
            num_processes = 1
        else:
            num_processes = self.parameters[0]
        indices = range(num_processes)
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(file_cacher)
        sandbox_user = [create_sandbox(file_cacher) for i in indices]
        fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
        fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
        fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
        for i in indices:
            os.mkfifo(fifo_in[i])
            os.mkfifo(fifo_out[i])
            os.chmod(fifo_dir[i], 0o755)
            os.chmod(fifo_in[i], 0o666)
            os.chmod(fifo_out[i], 0o666)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename]
        for i in indices:
            manager_command.append(fifo_in[i])
            manager_command.append(fifo_out[i])
        manager_executables_to_get = {
            manager_filename:
            job.managers[manager_filename].digest
            }
        manager_files_to_get = {
            "input.txt": job.input
            }
        manager_allow_dirs = fifo_dir
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(
                filename, digest, executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(
            sandbox_mgr,
            manager_command,
            num_processes * job.time_limit,
            0,
            allow_dirs=manager_allow_dirs,
            writable_files=["output.txt"],
            stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = job.executables.keys()[0]
        executables_to_get = {
            executable_filename:
            job.executables[executable_filename].digest
            }
        processes = [None for i in indices]
        for i in indices:
            command = ["./%s" % executable_filename, fifo_out[i], fifo_in[i]]
            if num_processes != 1:
                command.append(str(i))
            user_allow_dirs = [fifo_dir[i]]
            for filename, digest in executables_to_get.iteritems():
                sandbox_user[i].create_file_from_storage(
                    filename, digest, executable=True)
            processes[i] = evaluation_step_before_run(
                sandbox_user[i],
                command,
                job.time_limit,
                job.memory_limit,
                allow_dirs=user_allow_dirs)

        # Consume output.
        wait_without_std(processes + [manager])
        # TODO: check exit codes with translate_box_exitcode.

        user_results = [evaluation_step_after_run(s) for s in sandbox_user]
        success_user = all(r[0] for r in user_results)
        plus_user = reduce(merge_evaluation_results,
                           [r[1] for r in user_results])
        success_mgr, unused_plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        if plus_user['exit_status'] == Sandbox.EXIT_OK and \
                plus_user["execution_time"] >= job.time_limit:
            plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT

        # Merge results.
        job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
        job.plus = plus_user

        # If at least one evaluation had problems, we report the
        # problems.
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If the user sandbox detected some problem (timeout, ...),
        # the outcome is 0.0 and the text describes that problem.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                job.user_output = sandbox_mgr.get_file_to_storage(
                    "output.txt",
                    "Output file in job %s" % job.info)
            else:
                job.user_output = None

        # Whatever happened, we conclude.
        job.success = success
        job.outcome = "%s" % outcome if outcome is not None else None
        job.text = text

        delete_sandbox(sandbox_mgr)
        for s in sandbox_user:
            delete_sandbox(s)
        if not config.keep_sandbox:
            for d in fifo_dir:
                rmtree(d)
Example #18
0
    def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        # f stand for first, s for second.
        first_sandbox = create_sandbox(self)
        second_sandbox = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)

        # First step: we start the first manager.
        first_filename = "manager"
        first_command = ["./%s" % first_filename, "0", fifo]
        first_executables_to_get = {
            first_filename: self.job.executables[first_filename].digest
        }
        first_files_to_get = {
            "input.txt": self.job.testcases[test_number].input
        }
        first_allow_path = ["input.txt", fifo]

        # Put the required files into the sandbox
        for filename, digest in first_executables_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename,
                                                   digest,
                                                   executable=True)
        for filename, digest in first_files_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(first_sandbox,
                                           first_command,
                                           self.job.time_limit,
                                           self.job.memory_limit,
                                           first_allow_path,
                                           stdin_redirect="input.txt",
                                           wait=False)

        # Second step: we start the second manager.
        second_filename = "manager"
        second_command = ["./%s" % second_filename, "1", fifo]
        second_executables_to_get = {
            second_filename: self.job.executables[second_filename].digest
        }
        second_files_to_get = {}
        second_allow_path = [fifo, "output.txt"]

        # Put the required files into the second sandbox
        for filename, digest in second_executables_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename,
                                                    digest,
                                                    executable=True)
        for filename, digest in second_files_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(second_sandbox,
                                            second_command,
                                            self.job.time_limit,
                                            self.job.memory_limit,
                                            second_allow_path,
                                            stdout_redirect="output.txt",
                                            wait=False)

        # Consume output.
        wait_without_std([second, first])
        # TODO: check exit codes with translate_box_exitcode.

        success_first, first_plus = \
            evaluation_step_after_run(first_sandbox)
        success_second, second_plus = \
            evaluation_step_after_run(second_sandbox)

        self.job.evaluations[test_number] = {
            'sandboxes': [first_sandbox.path, second_sandbox.path],
            'plus': second_plus
        }
        outcome = None
        text = None
        evaluation = self.job.evaluations[test_number]
        success = True

        # Error in the sandbox: report failure!
        if not success_first or not success_second:
            success = False

        # Contestant's error: the marks won't be good
        elif not is_evaluation_passed(first_plus) or \
                not is_evaluation_passed(second_plus):
            outcome = 0.0
            if not is_evaluation_passed(first_plus):
                text = human_evaluation_message(first_plus)
            else:
                text = human_evaluation_message(second_plus)
            if self.job.get_output:
                evaluation['output'] = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists('output.txt'):
                outcome = 0.0
                text = "Execution didn't produce file output.txt"
                if self.job.get_output:
                    evaluation['output'] = None

            else:
                # If asked so, put the output file into the storage
                if self.job.get_output:
                    evaluation['output'] = second_sandbox.get_file_to_storage(
                        "output.txt", "Output file for testcase %d in job %s" %
                        (test_number, self.job.info))

                # If not asked otherwise, evaluate the output file
                if not self.job.only_execution:
                    # Put the reference solution into the sandbox
                    second_sandbox.create_file_from_storage(
                        "res.txt", self.job.testcases[test_number].output)

                    outcome, text = white_diff_step(second_sandbox,
                                                    "output.txt", "res.txt")

        # Whatever happened, we conclude.
        evaluation['success'] = success
        evaluation['outcome'] = outcome
        evaluation['text'] = text

        delete_sandbox(first_sandbox)
        delete_sandbox(second_sandbox)

        return success
    def evaluate_testcase(self, test_number):
        """See TaskType.evaluate_testcase."""
        # Create sandboxes and FIFOs
        sandbox_mgr = create_sandbox(self)
        sandbox_user = create_sandbox(self)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo_in = os.path.join(fifo_dir, "in")
        fifo_out = os.path.join(fifo_dir, "out")
        os.mkfifo(fifo_in)
        os.mkfifo(fifo_out)

        # First step: we start the manager.
        manager_filename = "manager"
        manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
        manager_executables_to_get = {
            manager_filename: self.job.managers[manager_filename].digest
        }
        manager_files_to_get = {
            "input.txt": self.job.testcases[test_number].input
        }
        manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
        for filename, digest in manager_executables_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename,
                                                 digest,
                                                 executable=True)
        for filename, digest in manager_files_to_get.iteritems():
            sandbox_mgr.create_file_from_storage(filename, digest)
        manager = evaluation_step_before_run(sandbox_mgr,
                                             manager_command,
                                             self.job.time_limit,
                                             0,
                                             manager_allow_path,
                                             stdin_redirect="input.txt")

        # Second step: we start the user submission compiled with the
        # stub.
        executable_filename = self.job.executables.keys()[0]
        command = ["./%s" % executable_filename, fifo_out, fifo_in]
        executables_to_get = {
            executable_filename:
            self.job.executables[executable_filename].digest
        }
        allow_path = [fifo_in, fifo_out]
        for filename, digest in executables_to_get.iteritems():
            sandbox_user.create_file_from_storage(filename,
                                                  digest,
                                                  executable=True)
        process = evaluation_step_before_run(sandbox_user, command,
                                             self.job.time_limit,
                                             self.job.memory_limit, allow_path)

        # Consume output.
        wait_without_std([process, manager])
        # TODO: check exit codes with translate_box_exitcode.

        success_user, plus_user = \
            evaluation_step_after_run(sandbox_user)
        success_mgr, plus_mgr = \
            evaluation_step_after_run(sandbox_mgr)

        self.job.evaluations[test_number] = \
            {'sandboxes': [sandbox_user.path,
                           sandbox_mgr.path],
             'plus': plus_user}
        evaluation = self.job.evaluations[test_number]

        # If at least one evaluation had problems, we report the
        # problems. (TODO: shouldn't outcome and text better be None
        # and None?)
        if not success_user or not success_mgr:
            success, outcome, text = False, None, None
        # If outcome_user is not None, it is 0.0 and it means that
        # there has been some errors in the user solution, and outcome
        # and text are meaningful, so we use them.
        elif not is_evaluation_passed(plus_user):
            success = True
            outcome, text = 0.0, human_evaluation_message(plus_user)
        # Otherwise, we use the manager to obtain the outcome.
        else:
            success = True
            outcome, text = extract_outcome_and_text(sandbox_mgr)

        # If asked so, save the output file, provided that it exists
        if self.job.get_output:
            if sandbox_mgr.file_exists("output.txt"):
                evaluation['output'] = sandbox_mgr.get_file_to_storage(
                    "output.txt", "Output file for testcase %d in job %s" %
                    (test_number, self.job.info))
            else:
                evaluation['output'] = None

        # Whatever happened, we conclude.
        evaluation['success'] = success
        evaluation['outcome'] = str(outcome) if outcome is not None else None
        evaluation['text'] = text
        delete_sandbox(sandbox_mgr)
        delete_sandbox(sandbox_user)
        return success
Example #20
0
    def evaluate(self, job, file_cacher):
        """See TaskType.evaluate."""
        # f stand for first, s for second.
        first_sandbox = create_sandbox(file_cacher)
        second_sandbox = create_sandbox(file_cacher)
        fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
        fifo = os.path.join(fifo_dir, "fifo")
        os.mkfifo(fifo)
        os.chmod(fifo_dir, 0o755)
        os.chmod(fifo, 0o666)

        # First step: we start the first manager.
        first_filename = "manager"
        first_command = ["./%s" % first_filename, "0", fifo]
        first_executables_to_get = {
            first_filename:
            job.executables[first_filename].digest
            }
        first_files_to_get = {
            "input.txt": job.input
            }
        first_allow_path = [fifo_dir]

        # Put the required files into the sandbox
        for filename, digest in first_executables_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename,
                                                   digest,
                                                   executable=True)
        for filename, digest in first_files_to_get.iteritems():
            first_sandbox.create_file_from_storage(filename, digest)

        first = evaluation_step_before_run(
            first_sandbox,
            first_command,
            job.time_limit,
            job.memory_limit,
            first_allow_path,
            stdin_redirect="input.txt",
            wait=False)

        # Second step: we start the second manager.
        second_filename = "manager"
        second_command = ["./%s" % second_filename, "1", fifo]
        second_executables_to_get = {
            second_filename:
            job.executables[second_filename].digest
            }
        second_files_to_get = {}
        second_allow_path = [fifo_dir]

        # Put the required files into the second sandbox
        for filename, digest in second_executables_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename,
                                                    digest,
                                                    executable=True)
        for filename, digest in second_files_to_get.iteritems():
            second_sandbox.create_file_from_storage(filename, digest)

        second = evaluation_step_before_run(
            second_sandbox,
            second_command,
            job.time_limit,
            job.memory_limit,
            second_allow_path,
            stdout_redirect="output.txt",
            wait=False)

        # Consume output.
        wait_without_std([second, first])
        # TODO: check exit codes with translate_box_exitcode.

        success_first, first_plus = \
            evaluation_step_after_run(first_sandbox)
        success_second, second_plus = \
            evaluation_step_after_run(second_sandbox)

        job.sandboxes = [first_sandbox.path,
                         second_sandbox.path]
        job.plus = second_plus

        success = True
        outcome = None
        text = None

        # Error in the sandbox: report failure!
        if not success_first or not success_second:
            success = False

        # Contestant's error: the marks won't be good
        elif not is_evaluation_passed(first_plus) or \
                not is_evaluation_passed(second_plus):
            outcome = 0.0
            if not is_evaluation_passed(first_plus):
                text = human_evaluation_message(first_plus)
            else:
                text = human_evaluation_message(second_plus)
            if job.get_output:
                job.user_output = None

        # Otherwise, advance to checking the solution
        else:

            # Check that the output file was created
            if not second_sandbox.file_exists('output.txt'):
                outcome = 0.0
                text = [N_("Evaluation didn't produce file %s"), "output.txt"]
                if job.get_output:
                    job.user_output = None

            else:
                # If asked so, put the output file into the storage
                if job.get_output:
                    job.user_output = second_sandbox.get_file_to_storage(
                        "output.txt",
                        "Output file in job %s" % job.info)

                # If not asked otherwise, evaluate the output file
                if not job.only_execution:
                    # Put the reference solution into the sandbox
                    second_sandbox.create_file_from_storage(
                        "res.txt",
                        job.output)

                    # If a checker is not provided, use white-diff
                    if self.parameters[0] == "diff":
                        outcome, text = white_diff_step(
                            second_sandbox, "output.txt", "res.txt")

                    elif self.parameters[0] == "comparator":
                        if TwoSteps.CHECKER_FILENAME not in job.managers:
                            logger.error("Configuration error: missing or "
                                         "invalid comparator (it must be "
                                         "named `checker')",
                                         extra={"operation": job.info})
                            success = False
                        else:
                            second_sandbox.create_file_from_storage(
                                TwoSteps.CHECKER_FILENAME,
                                job.managers[TwoSteps.CHECKER_FILENAME].digest,
                                executable=True)
                            # Rewrite input file, as in Batch.py
                            try:
                                second_sandbox.remove_file("input.txt")
                            except OSError as e:
                                assert not second_sandbox.file_exists(
                                    "input.txt")
                            second_sandbox.create_file_from_storage(
                                "input.txt",
                                job.input)
                            success, _ = evaluation_step(
                                second_sandbox,
                                [["./%s" % TwoSteps.CHECKER_FILENAME,
                                  "input.txt", "res.txt", "output.txt"]])
                            if success:
                                try:
                                    outcome, text = extract_outcome_and_text(
                                        second_sandbox)
                                except ValueError, e:
                                    logger.error("Invalid output from "
                                                 "comparator: %s", e.message,
                                                 extra={"operation": job.info})
                                    success = False
                    else:
                        raise ValueError("Uncrecognized first parameter"
                                         " `%s' for TwoSteps tasktype." %
                                         self.parameters[0])