def test_failure_invalid_utf8(self): self.sandbox.fake_file("o", b"0.45") self.sandbox.fake_file("e", INVALID_UTF8) with self.assertRaises(ValueError): extract_outcome_and_text(self.sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, Communication.MANAGER_FILENAME): return manager_digest = job.managers[Communication.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # Create the manager sandbox and copy manager and input. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.path) sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME, job.input) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.path for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) if Communication.STUB_PRELOAD_FILENAME in job.managers: digest = job.managers[ Communication.STUB_PRELOAD_FILENAME].digest sandbox_user[i].create_file_from_storage( Communication.STUB_PRELOAD_FILENAME, digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % Communication.MANAGER_FILENAME] for i in indices: manager_command += [fifo_in[i], fifo_out[i]] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, allow_dirs=fifo_dir, writable_files=[Communication.OUTPUT_FILENAME], stdin_redirect=Communication.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) # Start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if self.num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="stub", args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) last_cmd = commands[-1] # Inject preload program if needed if Communication.STUB_PRELOAD_FILENAME in job.managers: last_cmd = [ "./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i], fifo_in[i] ] + commands[-1] processes[i] = evaluation_step_before_run( sandbox_user[i], last_cmd, job.time_limit, job.memory_limit, allow_dirs=[fifo_dir[i]], multiprocess=job.multithreaded_sandbox) # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None pass # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user, job.feedback_level) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( Communication.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def test_text_is_translated(self): self.sandbox.fake_file("o", b"0.45\n") self.sandbox.fake_file("e", b"translate:success\n") outcome, text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Output is correct"])
def test_failure_not_a_float(self): self.sandbox.fake_file("o", b"not a float\n") self.sandbox.fake_file("e", b"Text to return.\n") with self.assertRaises(ValueError): extract_outcome_and_text(self.sandbox)
def test_works_without_newlines(self): self.sandbox.fake_file("o", b"0.45") self.sandbox.fake_file("e", b"Text to return.") outcome, text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."])
def test_text_is_stripped(self): self.sandbox.fake_file("o", b" 0.45\t \nignored") self.sandbox.fake_file("e", b"\t Text to return.\r\n") outcome, text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."])
def test_following_lines_ignored(self): self.sandbox.fake_file("o", b"0.45\nNothing\n") self.sandbox.fake_file("e", b"Text to return.\nto see here") outcome, text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."])
def test_success(self): self.sandbox.fake_file("o", b"0.45\n") self.sandbox.fake_file("e", "你好.\n".encode("utf-8")) outcome, text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["你好."])
def test_failure_missing_file(self): self.sandbox.fake_file("o", b"0.45\n") with self.assertRaises(FileNotFoundError): extract_outcome_and_text(self.sandbox)
def evaluate(self, job, file_cacher): if not check_executables_number(job, 1): return executable_filename = next(iter(job.executables.keys())) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, self.MANAGER_FILENAME): return manager_digest = job.managers[self.MANAGER_FILENAME].digest # Create FIFO dirs and first batch of FIFos fifo_dir_base = tempfile.mkdtemp(dir=config.temp_dir) def fifo_dir(i, j): p = os.path.join(fifo_dir_base, f"fifo{i}_{j}") if not os.path.exists(p): os.mkdir(p) return p abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq") fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq") os.mkfifo(fifo_solution_quitter) os.mkfifo(fifo_manager_quitter) os.chmod(abortion_control_fifo_dir, 0o755) os.chmod(fifo_solution_quitter, 0o666) os.chmod(fifo_manager_quitter, 0o666) sandbox_abortion_control_fifo_dir = "/abort" sandbox_fifo_solution_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "sq") sandbox_fifo_manager_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "mq") # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % self.MANAGER_FILENAME] manager_command += [ sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter ] # Create the manager sandbox and copy manager and input and # reference output. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.get_root_path()) sandbox_mgr.create_file_from_storage(self.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(self.INPUT_FILENAME, job.input) sandbox_mgr.create_file_from_storage(self.OK_FILENAME, job.output) # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. sandbox_fifo_dir_base = "/fifo" def sandbox_fifo_dir(i, j): return f"{sandbox_fifo_dir_base}/fifo{i}_{j}" manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager_dirs_map = { abortion_control_fifo_dir: (sandbox_abortion_control_fifo_dir, "rw") } # TODO: can we avoid creating all these directories? MAX_NUM_INSTANCES = 42 list_of_fifo_dirs = [] for pr in range(0, self.num_processes): for i in range(0, MAX_NUM_INSTANCES): d = fifo_dir(i, pr) list_of_fifo_dirs.append(d) manager_dirs_map[d] = (sandbox_fifo_dir(i, pr), "rw") manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib * 1024, dirs_map=manager_dirs_map, writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, multiprocess=True) solution_quitter = open(fifo_solution_quitter, "r") manager_quitter = open(fifo_manager_quitter, "w") def finish_run(): wait_without_std(processes) L = [finish_run_single(i) for i in indices] return all(L) def finish_run_single(i): nonlocal wall_clock_acc nonlocal num_runs user_results.append(evaluation_step_after_run(sandbox_user[i])) wall_clock_acc += user_results[-1][2]["execution_wall_clock_time"] num_runs += 1 runtimes[i].append(user_results[-1][2]["execution_time"]) # Convert tuple to list for write access to entries L = list(user_results[-1]) L[2]["execution_time"] = runtimes[i][-1] / max_num_runs if (L[2]["execution_time"] >= job.time_limit): L[2]["exit_status"] = Sandbox.EXIT_TIMEOUT user_results[-1] = tuple(L) if not self._uses_stub(): # It can happen that the submission runs out of memory and then # gets killed by the manager while it is being shut down, in # which case isolate does not report a signal as the exit # status. To catch this, we look for cg-oom-killed in the logs sandbox_user[i].get_log() if user_results[-1][1] and \ "cg-oom-killed" in sandbox_user[i].log: # Convert tuple to list for write access to entries r = list(user_results[-1]) r[1] = False r[2]["status"] = ["SG"] r[2]["exit_status"] = "signal" r[2]["signal"] = -41 # sit by a lake r[2]["message"] = ["out of memory"] user_results[-1] = tuple(r) return user_results[-1][0] and user_results[-1][1] def respond(okay=True): manager_quitter.write("O" if okay else "X") manager_quitter.flush() def read_int_from_manager(): L = [] while True: c = solution_quitter.read(1) if c == 'B': break else: L.append(c) return int("".join(L)) quit = False for pr in range(0, self.num_processes): if quit: break wall_clock_acc = 0 num_runs = 0 # Startup message to sync pipes manager_quitter.write('S') manager_quitter.flush() # Ask the manager for the number of processes num_instances = read_int_from_manager() indices = range(0, num_instances) max_num_runs = read_int_from_manager() # Create remaining FIFOs fifo_user_to_manager = [ os.path.join(fifo_dir(i, pr), f"u{pr}_{i}_to_m") for i in indices ] fifo_manager_to_user = [ os.path.join(fifo_dir(i, pr), f"m_to_u{pr}_{i}") for i in indices ] for i in indices: os.mkfifo(fifo_user_to_manager[i]) os.mkfifo(fifo_manager_to_user[i]) os.chmod(fifo_dir(i, pr), 0o755) os.chmod(fifo_user_to_manager[i], 0o666) os.chmod(fifo_manager_to_user[i], 0o666) # Names of the fifos after being mapped inside the sandboxes. sandbox_fifo_user_to_manager = \ [os.path.join(sandbox_fifo_dir(i, pr), f"u{pr}_{i}_to_m") for i in indices] sandbox_fifo_manager_to_user = \ [os.path.join(sandbox_fifo_dir(i, pr), f"m_to_u{pr}_{i}") for i in indices] for i in indices: print(sandbox_fifo_user_to_manager[i], file=manager_quitter, flush=True) print(sandbox_fifo_manager_to_user[i], file=manager_quitter, flush=True) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.get_root_path() for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) # Prepare the user submissions language = get_language(job.language) main = self.STUB_BASENAME if self._uses_stub() \ else os.path.splitext(executable_filename)[0] processes = [None for i in indices] user_results = [] args = [] stdin_redirect = None stdout_redirect = None if self._uses_fifos(): args.extend([ sandbox_fifo_manager_to_user[i], sandbox_fifo_user_to_manager[i] ]) if self.num_processes != 1: args.append(str(i)) if self._uses_stub(): main = self.STUB_BASENAME else: main = executable_filename commands = language.get_evaluation_commands( executable_filename, main=main, args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) processes = [None for _ in indices] runtimes = [[] for _ in indices] while True: for i in indices: processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit * max_num_runs * num_instances, job.memory_limit, dirs_map={ fifo_dir(i, pr): (sandbox_fifo_dir(i, pr), "rw") }, stdin_redirect=sandbox_fifo_manager_to_user[i], stdout_redirect=sandbox_fifo_user_to_manager[i], multiprocess=job.multithreaded_sandbox) response = solution_quitter.read(1) if response == "C": # continue if not finish_run(): # this run was not successful, time to call it quits quit = True respond(okay=False) break respond() elif response == "N": # next process if not finish_run(): # this run was not successful, time to call it quits quit = True respond(okay=False) break respond() break elif response == "Q": if not self._uses_stub(): time.sleep(.01) processes[i].send_signal(signal.SIGINT) finish_run() respond() quit = True break else: raise RuntimeError("Received '{}' ".format(response) + "through solution_quitter.") # Wait for the manager to conclude, without blocking them on I/O. wait_without_std([manager]) solution_quitter.close() manager_quitter.close() # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(my_merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(self.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( self.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox) for s in sandbox_user: delete_sandbox(s, job.success, job.keep_sandbox) if job.success and not config.keep_sandbox and not job.keep_sandbox: rmtree(fifo_dir_base) rmtree(abortion_control_fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, self.MANAGER_FILENAME): return manager_digest = job.managers[self.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] if not self._uses_grader(): abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_user_to_manager = [ os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices] fifo_manager_to_user = [ os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices] if not self._uses_grader(): fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq") fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq") for i in indices: os.mkfifo(fifo_user_to_manager[i]) os.mkfifo(fifo_manager_to_user[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_user_to_manager[i], 0o666) os.chmod(fifo_manager_to_user[i], 0o666) if not self._uses_grader(): os.mkfifo(fifo_solution_quitter) os.mkfifo(fifo_manager_quitter) os.chmod(abortion_control_fifo_dir, 0o755) os.chmod(fifo_solution_quitter, 0o666) os.chmod(fifo_manager_quitter, 0o666) # Names of the fifos after being mapped inside the sandboxes. sandbox_fifo_dir = ["/fifo%d" % i for i in indices] sandbox_fifo_user_to_manager = [ os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices] sandbox_fifo_manager_to_user = [ os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices] if not self._uses_grader(): sandbox_abortion_control_fifo_dir = "/abort" sandbox_fifo_solution_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "sq") sandbox_fifo_manager_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "mq") # Create the manager sandbox and copy manager and input and # reference output. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.get_root_path()) sandbox_mgr.create_file_from_storage( self.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage( self.INPUT_FILENAME, job.input) sandbox_mgr.create_file_from_storage( self.OK_FILENAME, job.output) # Create the user sandbox(es) and copy the executable. sandbox_user = [create_sandbox(file_cacher, name="user_evaluate") for i in indices] job.sandboxes.extend(s.get_root_path() for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage( executable_filename, executable_digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % self.MANAGER_FILENAME] for i in indices: manager_command += [sandbox_fifo_user_to_manager[i], sandbox_fifo_manager_to_user[i]] if not self._uses_grader(): manager_command += [sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager_dirs_map = dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw")) for i in indices) if not self._uses_grader(): manager_dirs_map[abortion_control_fifo_dir] = \ (sandbox_abortion_control_fifo_dir, "rw") manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, dirs_map=manager_dirs_map, writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) if not self._uses_grader(): solution_quitter = open(fifo_solution_quitter, "r") manager_quitter = open(fifo_manager_quitter, "w") manager_quitter_open = True # Start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [sandbox_fifo_manager_to_user[i], sandbox_fifo_user_to_manager[i]] if self.num_processes != 1: args.append(str(i)) if self._uses_grader(): main = self.STUB_BASENAME else: main = executable_filename commands = language.get_evaluation_commands( executable_filename, main=main, args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")}, stdin_redirect=sandbox_fifo_manager_to_user[i], stdout_redirect=sandbox_fifo_user_to_manager[i], multiprocess=job.multithreaded_sandbox) if not self._uses_grader(): # Manager still running but wants to quit if solution_quitter.read() == "<3": for i in indices: processes[i].send_signal(signal.SIGINT) # Kill user wait_without_std(processes) manager_quitter.close() manager_quitter_open = False # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) if not self._uses_grader(): solution_quitter.close() if manager_quitter_open: manager_quitter.close() # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None pass # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(self.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( self.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d) if not self._uses_grader(): rmtree(abortion_control_fifo_dir)