def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.job.executables[first_filename].digest } first_files_to_get = { "input.txt": self.job.testcases[test_number].input } first_allow_path = ["input.txt", fifo] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, self.job.time_limit, self.job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, self.job.time_limit, self.job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) self.job.evaluations[test_number] = {'sandboxes': [first_sandbox.path, second_sandbox.path], 'plus': second_plus} outcome = None text = None evaluation = self.job.evaluations[test_number] success = True # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = "Execution didn't produce file output.txt" if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = second_sandbox.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = outcome evaluation['text'] = text delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) os.chmod(fifo_dir, 0o755) os.chmod(fifo_in, 0o666) os.chmod(fifo_out, 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = [fifo_dir] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: job.executables[executable_filename].digest } user_allow_dirs = [fifo_dir] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage(filename, digest, executable=True) process = evaluation_step_before_run(sandbox_user, command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) job.sandboxes = [sandbox_user.path, sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) if not config.keep_sandbox: rmtree(fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest first_sandbox = create_sandbox(file_cacher, name="first_evaluate") second_sandbox = create_sandbox(file_cacher, name="second_evaluate") job.sandboxes.append(first_sandbox.get_root_path()) job.sandboxes.append(second_sandbox.get_root_path()) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"] first_executables_to_get = {executable_filename: executable_digest} first_files_to_get = { TwoSteps.INPUT_FILENAME: job.input } # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdin_redirect=TwoSteps.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Second step: we start the second manager. second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"] second_executables_to_get = {executable_filename: executable_digest} second_files_to_get = {} # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdout_redirect=TwoSteps.OUTPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Consume output. wait_without_std([second, first]) box_success_first, evaluation_success_first, first_stats = \ evaluation_step_after_run(first_sandbox) box_success_second, evaluation_success_second, second_stats = \ evaluation_step_after_run(second_sandbox) box_success = box_success_first and box_success_second evaluation_success = \ evaluation_success_first and evaluation_success_second stats = merge_execution_stats(first_stats, second_stats) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), TwoSteps.OUTPUT_FILENAME] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( TwoSteps.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=second_sandbox.relative_path( TwoSteps.OUTPUT_FILENAME)) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox) sandbox_user = [ create_sandbox(file_cacher, job.multithreaded_sandbox) for i in indices ] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt", stdout_redirect="output.txt", ) # Second step: we start the user submission compiled with the # stub. language = get_language(job.language) executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="grader", args=args) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage(filename, digest, executable=True) # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) if job.get_output: job.user_output = None # Otherwise, we use the manager to obtain the outcome. else: success = True outcome = None text = None input_filename = "input.txt" output_filename = "output.txt" # Check that the output file was created if not sandbox_mgr.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox_mgr.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=1024 * 1024 * 10) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox_mgr.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[1] == "diff": outcome, text = white_diff_step( sandbox_mgr, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[1] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox_mgr.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox_mgr.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox_mgr.file_exists( input_filename) sandbox_mgr.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox_mgr.max_processes = 1000 success, _ = evaluation_step( sandbox_mgr, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox_mgr) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized second parameter" " `%s' for Communication tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) os.chmod(fifo_dir, 0o755) os.chmod(fifo_in, 0o666) os.chmod(fifo_out, 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = [fifo_dir] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, job.time_limit, 0, allow_dirs=manager_allow_dirs, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: job.executables[executable_filename].digest } user_allow_dirs = [fifo_dir] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage( filename, digest, executable=True) process = evaluation_step_before_run( sandbox_user, command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) job.sandboxes = [sandbox_user.path, sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = str(outcome) if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) shutil.rmtree(fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox) sandbox_user = [ create_sandbox(file_cacher, job.multithreaded_sandbox) for i in indices ] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. language = get_language(job.language) executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="stub", args=args) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage(filename, digest, executable=True) # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.submission.task.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.submission.task.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] manager = self.evaluation_step_before_run( sandbox_mgr, manager_command, manager_executables_to_get, manager_files_to_get, self.submission.task.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.submission.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.submission.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] process = self.evaluation_step_before_run( sandbox_user, command, executables_to_get, {}, self.submission.task.time_limit, self.submission.task.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, outcome_user, text_user, plus = \ self.evaluation_step_after_run(sandbox_user, final=False) success_mgr, outcome_mgr, text_mgr, _ = \ self.evaluation_step_after_run(sandbox_mgr, final=True) # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, outcome_user, text_user # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif outcome_user is not None: success, outcome, text = success_user, outcome_user, text_user # Otherwise, we use the manager to obtain the outcome. else: success, outcome, text = success_mgr, outcome_mgr, text_mgr delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus)
def evaluate(self, job, file_cacher): if not check_executables_number(job, 1): return executable_filename = next(iter(job.executables.keys())) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, self.MANAGER_FILENAME): return manager_digest = job.managers[self.MANAGER_FILENAME].digest # Create FIFO dirs and first batch of FIFos fifo_dir_base = tempfile.mkdtemp(dir=config.temp_dir) def fifo_dir(i, j): p = os.path.join(fifo_dir_base, f"fifo{i}_{j}") if not os.path.exists(p): os.mkdir(p) return p abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq") fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq") os.mkfifo(fifo_solution_quitter) os.mkfifo(fifo_manager_quitter) os.chmod(abortion_control_fifo_dir, 0o755) os.chmod(fifo_solution_quitter, 0o666) os.chmod(fifo_manager_quitter, 0o666) sandbox_abortion_control_fifo_dir = "/abort" sandbox_fifo_solution_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "sq") sandbox_fifo_manager_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "mq") # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % self.MANAGER_FILENAME] manager_command += [ sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter ] # Create the manager sandbox and copy manager and input and # reference output. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.get_root_path()) sandbox_mgr.create_file_from_storage(self.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(self.INPUT_FILENAME, job.input) sandbox_mgr.create_file_from_storage(self.OK_FILENAME, job.output) # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. sandbox_fifo_dir_base = "/fifo" def sandbox_fifo_dir(i, j): return f"{sandbox_fifo_dir_base}/fifo{i}_{j}" manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager_dirs_map = { abortion_control_fifo_dir: (sandbox_abortion_control_fifo_dir, "rw") } # TODO: can we avoid creating all these directories? MAX_NUM_INSTANCES = 42 list_of_fifo_dirs = [] for pr in range(0, self.num_processes): for i in range(0, MAX_NUM_INSTANCES): d = fifo_dir(i, pr) list_of_fifo_dirs.append(d) manager_dirs_map[d] = (sandbox_fifo_dir(i, pr), "rw") manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib * 1024, dirs_map=manager_dirs_map, writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, multiprocess=True) solution_quitter = open(fifo_solution_quitter, "r") manager_quitter = open(fifo_manager_quitter, "w") def finish_run(): wait_without_std(processes) L = [finish_run_single(i) for i in indices] return all(L) def finish_run_single(i): nonlocal wall_clock_acc nonlocal num_runs user_results.append(evaluation_step_after_run(sandbox_user[i])) wall_clock_acc += user_results[-1][2]["execution_wall_clock_time"] num_runs += 1 runtimes[i].append(user_results[-1][2]["execution_time"]) # Convert tuple to list for write access to entries L = list(user_results[-1]) L[2]["execution_time"] = runtimes[i][-1] / max_num_runs if (L[2]["execution_time"] >= job.time_limit): L[2]["exit_status"] = Sandbox.EXIT_TIMEOUT user_results[-1] = tuple(L) if not self._uses_stub(): # It can happen that the submission runs out of memory and then # gets killed by the manager while it is being shut down, in # which case isolate does not report a signal as the exit # status. To catch this, we look for cg-oom-killed in the logs sandbox_user[i].get_log() if user_results[-1][1] and \ "cg-oom-killed" in sandbox_user[i].log: # Convert tuple to list for write access to entries r = list(user_results[-1]) r[1] = False r[2]["status"] = ["SG"] r[2]["exit_status"] = "signal" r[2]["signal"] = -41 # sit by a lake r[2]["message"] = ["out of memory"] user_results[-1] = tuple(r) return user_results[-1][0] and user_results[-1][1] def respond(okay=True): manager_quitter.write("O" if okay else "X") manager_quitter.flush() def read_int_from_manager(): L = [] while True: c = solution_quitter.read(1) if c == 'B': break else: L.append(c) return int("".join(L)) quit = False for pr in range(0, self.num_processes): if quit: break wall_clock_acc = 0 num_runs = 0 # Startup message to sync pipes manager_quitter.write('S') manager_quitter.flush() # Ask the manager for the number of processes num_instances = read_int_from_manager() indices = range(0, num_instances) max_num_runs = read_int_from_manager() # Create remaining FIFOs fifo_user_to_manager = [ os.path.join(fifo_dir(i, pr), f"u{pr}_{i}_to_m") for i in indices ] fifo_manager_to_user = [ os.path.join(fifo_dir(i, pr), f"m_to_u{pr}_{i}") for i in indices ] for i in indices: os.mkfifo(fifo_user_to_manager[i]) os.mkfifo(fifo_manager_to_user[i]) os.chmod(fifo_dir(i, pr), 0o755) os.chmod(fifo_user_to_manager[i], 0o666) os.chmod(fifo_manager_to_user[i], 0o666) # Names of the fifos after being mapped inside the sandboxes. sandbox_fifo_user_to_manager = \ [os.path.join(sandbox_fifo_dir(i, pr), f"u{pr}_{i}_to_m") for i in indices] sandbox_fifo_manager_to_user = \ [os.path.join(sandbox_fifo_dir(i, pr), f"m_to_u{pr}_{i}") for i in indices] for i in indices: print(sandbox_fifo_user_to_manager[i], file=manager_quitter, flush=True) print(sandbox_fifo_manager_to_user[i], file=manager_quitter, flush=True) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.get_root_path() for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) # Prepare the user submissions language = get_language(job.language) main = self.STUB_BASENAME if self._uses_stub() \ else os.path.splitext(executable_filename)[0] processes = [None for i in indices] user_results = [] args = [] stdin_redirect = None stdout_redirect = None if self._uses_fifos(): args.extend([ sandbox_fifo_manager_to_user[i], sandbox_fifo_user_to_manager[i] ]) if self.num_processes != 1: args.append(str(i)) if self._uses_stub(): main = self.STUB_BASENAME else: main = executable_filename commands = language.get_evaluation_commands( executable_filename, main=main, args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) processes = [None for _ in indices] runtimes = [[] for _ in indices] while True: for i in indices: processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit * max_num_runs * num_instances, job.memory_limit, dirs_map={ fifo_dir(i, pr): (sandbox_fifo_dir(i, pr), "rw") }, stdin_redirect=sandbox_fifo_manager_to_user[i], stdout_redirect=sandbox_fifo_user_to_manager[i], multiprocess=job.multithreaded_sandbox) response = solution_quitter.read(1) if response == "C": # continue if not finish_run(): # this run was not successful, time to call it quits quit = True respond(okay=False) break respond() elif response == "N": # next process if not finish_run(): # this run was not successful, time to call it quits quit = True respond(okay=False) break respond() break elif response == "Q": if not self._uses_stub(): time.sleep(.01) processes[i].send_signal(signal.SIGINT) finish_run() respond() quit = True break else: raise RuntimeError("Received '{}' ".format(response) + "through solution_quitter.") # Wait for the manager to conclude, without blocking them on I/O. wait_without_std([manager]) solution_quitter.close() manager_quitter.close() # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(my_merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(self.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( self.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox) for s in sandbox_user: delete_sandbox(s, job.success, job.keep_sandbox) if job.success and not config.keep_sandbox and not job.keep_sandbox: rmtree(fifo_dir_base) rmtree(abortion_control_fifo_dir)
def finish_run(): wait_without_std(processes) L = [finish_run_single(i) for i in indices] return all(L)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.job.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, self.job.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage( filename, digest, executable=True) process = evaluation_step_before_run( sandbox_user, command, self.job.time_limit, self.job.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) self.job.evaluations[test_number] = \ {'sandboxes': [sandbox_user.path, sandbox_mgr.path], 'plus': plus_user} evaluation = self.job.evaluations[test_number] # If at least one evaluation had problems, we report the # problems. (TODO: shouldn't outcome and text better be None # and None?) if not success_user or not success_mgr: success, outcome, text = False, None, None # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if self.job.get_output: if sandbox_mgr.file_exists("output.txt"): evaluation['output'] = sandbox_mgr.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) else: evaluation['output'] = None # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # f stand for first, s for second. first_sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="first_evaluate") second_sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="second_evaluate") fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: job.executables[first_filename].digest } first_files_to_get = {"input.txt": job.input} first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run(first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run(second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) job.sandboxes = [first_sandbox.path, second_sandbox.path] job.plus = second_plus success = True outcome = None text = [] # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), "output.txt"] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", job.output) # If a checker is not provided, use white-diff if self.parameters[0] == "diff": outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": if TwoSteps.CHECKER_FILENAME not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: second_sandbox.create_file_from_storage( TwoSteps.CHECKER_FILENAME, job.managers[TwoSteps.CHECKER_FILENAME].digest, executable=True) # Rewrite input file, as in Batch.py try: second_sandbox.remove_file("input.txt") except OSError as e: assert not second_sandbox.file_exists( "input.txt") second_sandbox.create_file_from_storage( "input.txt", job.input) success, _ = evaluation_step( second_sandbox, [[ "./%s" % TwoSteps.CHECKER_FILENAME, "input.txt", "res.txt", "output.txt" ]]) if success: try: outcome, text = extract_outcome_and_text( second_sandbox) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e, extra={"operation": job.info}) success = False else: raise ValueError("Uncrecognized first parameter" " `%s' for TwoSteps tasktype." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = str(outcome) if outcome is not None else None job.text = text delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, self.MANAGER_FILENAME): return manager_digest = job.managers[self.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] if not self._uses_grader(): abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_user_to_manager = [ os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices] fifo_manager_to_user = [ os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices] if not self._uses_grader(): fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq") fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq") for i in indices: os.mkfifo(fifo_user_to_manager[i]) os.mkfifo(fifo_manager_to_user[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_user_to_manager[i], 0o666) os.chmod(fifo_manager_to_user[i], 0o666) if not self._uses_grader(): os.mkfifo(fifo_solution_quitter) os.mkfifo(fifo_manager_quitter) os.chmod(abortion_control_fifo_dir, 0o755) os.chmod(fifo_solution_quitter, 0o666) os.chmod(fifo_manager_quitter, 0o666) # Names of the fifos after being mapped inside the sandboxes. sandbox_fifo_dir = ["/fifo%d" % i for i in indices] sandbox_fifo_user_to_manager = [ os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices] sandbox_fifo_manager_to_user = [ os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices] if not self._uses_grader(): sandbox_abortion_control_fifo_dir = "/abort" sandbox_fifo_solution_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "sq") sandbox_fifo_manager_quitter = \ os.path.join(sandbox_abortion_control_fifo_dir, "mq") # Create the manager sandbox and copy manager and input and # reference output. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.get_root_path()) sandbox_mgr.create_file_from_storage( self.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage( self.INPUT_FILENAME, job.input) sandbox_mgr.create_file_from_storage( self.OK_FILENAME, job.output) # Create the user sandbox(es) and copy the executable. sandbox_user = [create_sandbox(file_cacher, name="user_evaluate") for i in indices] job.sandboxes.extend(s.get_root_path() for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage( executable_filename, executable_digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % self.MANAGER_FILENAME] for i in indices: manager_command += [sandbox_fifo_user_to_manager[i], sandbox_fifo_manager_to_user[i]] if not self._uses_grader(): manager_command += [sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager_dirs_map = dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw")) for i in indices) if not self._uses_grader(): manager_dirs_map[abortion_control_fifo_dir] = \ (sandbox_abortion_control_fifo_dir, "rw") manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, dirs_map=manager_dirs_map, writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) if not self._uses_grader(): solution_quitter = open(fifo_solution_quitter, "r") manager_quitter = open(fifo_manager_quitter, "w") manager_quitter_open = True # Start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [sandbox_fifo_manager_to_user[i], sandbox_fifo_user_to_manager[i]] if self.num_processes != 1: args.append(str(i)) if self._uses_grader(): main = self.STUB_BASENAME else: main = executable_filename commands = language.get_evaluation_commands( executable_filename, main=main, args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")}, stdin_redirect=sandbox_fifo_manager_to_user[i], stdout_redirect=sandbox_fifo_user_to_manager[i], multiprocess=job.multithreaded_sandbox) if not self._uses_grader(): # Manager still running but wants to quit if solution_quitter.read() == "<3": for i in indices: processes[i].send_signal(signal.SIGINT) # Kill user wait_without_std(processes) manager_quitter.close() manager_quitter_open = False # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) if not self._uses_grader(): solution_quitter.close() if manager_quitter_open: manager_quitter.close() # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None pass # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(self.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( self.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d) if not self._uses_grader(): rmtree(abortion_control_fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="manager_evaluate") sandbox_user = [ create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="user_evaluate") for i in indices] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: prepare the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = fifo_dir for filename, digest in iteritems(manager_executables_to_get): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in iteritems(manager_files_to_get): sandbox_mgr.create_file_from_storage(filename, digest) # Second step: load the executables for the user processes # (done before launching the manager so that it does not # impact its wall clock time). assert len(job.executables) == 1 executable_filename = next(iterkeys(job.executables)) executables_to_get = { executable_filename: job.executables[executable_filename].digest } for i in indices: for filename, digest in iteritems(executables_to_get): sandbox_user[i].create_file_from_storage( filename, digest, executable=True) # Third step: start the manager. manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Fourth step: start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands( executable_filename, main="stub", args=args) user_allow_dirs = [fifo_dir[i]] # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, [] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.submission.executables[first_filename].digest } first_files_to_get = { "input.txt": self.submission.task.testcases[test_number].input } first_allow_path = ["input.txt", fifo] first = self.evaluation_step_before_run( first_sandbox, first_command, first_executables_to_get, first_files_to_get, self.submission.task.time_limit, self.submission.task.memory_limit, first_allow_path, stdin_redirect="input.txt") # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.submission.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] second = self.evaluation_step_before_run( second_sandbox, second_command, second_executables_to_get, second_files_to_get, self.submission.task.time_limit, self.submission.task.memory_limit, second_allow_path) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, outcome_first, text_first, _ = \ self.evaluation_step_after_run(first_sandbox, final=False) success_second, outcome_second, text_second, plus = \ self.evaluation_step_after_run(second_sandbox, final=True) # If at least one evaluation had problems, we report the # problems. if not success_first: success, outcome, text = False, outcome_first, text_first elif not success_second: success, outcome, text = False, outcome_second, text_second # Otherwise, compare the second output to the results # (there is no comparator since the manager is expected to turn the # output in diffable form) else: success, outcome, text = self.white_diff_step( second_sandbox, "output.txt", "res.txt", {"res.txt": self.submission.task.testcases[test_number].output}) delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iter(job.executables.keys())) executable_digest = job.executables[executable_filename].digest first_sandbox = create_sandbox(file_cacher, name="first_evaluate") second_sandbox = create_sandbox(file_cacher, name="second_evaluate") job.sandboxes.append(first_sandbox.get_root_path()) job.sandboxes.append(second_sandbox.get_root_path()) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"] first_executables_to_get = {executable_filename: executable_digest} first_files_to_get = {TwoSteps.INPUT_FILENAME: job.input} # Put the required files into the sandbox for filename, digest in first_executables_to_get.items(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.items(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdin_redirect=TwoSteps.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Second step: we start the second manager. second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"] second_executables_to_get = {executable_filename: executable_digest} second_files_to_get = {} # Put the required files into the second sandbox for filename, digest in second_executables_to_get.items(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.items(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdout_redirect=TwoSteps.OUTPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Consume output. wait_without_std([second, first]) box_success_first, evaluation_success_first, first_stats = \ evaluation_step_after_run(first_sandbox) box_success_second, evaluation_success_second, second_stats = \ evaluation_step_after_run(second_sandbox) box_success = box_success_first and box_success_second evaluation_success = \ evaluation_success_first and evaluation_success_second stats = merge_execution_stats(first_stats, second_stats) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), TwoSteps.OUTPUT_FILENAME ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( TwoSteps.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=second_sandbox.relative_path( TwoSteps.OUTPUT_FILENAME)) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(first_sandbox, job.success, job.keep_sandbox) delete_sandbox(second_sandbox, job.success, job.keep_sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, Communication.MANAGER_FILENAME): return manager_digest = job.managers[Communication.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # Create the manager sandbox and copy manager and input. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.path) sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME, job.input) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.path for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) if Communication.STUB_PRELOAD_FILENAME in job.managers: digest = job.managers[ Communication.STUB_PRELOAD_FILENAME].digest sandbox_user[i].create_file_from_storage( Communication.STUB_PRELOAD_FILENAME, digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % Communication.MANAGER_FILENAME] for i in indices: manager_command += [fifo_in[i], fifo_out[i]] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, allow_dirs=fifo_dir, writable_files=[Communication.OUTPUT_FILENAME], stdin_redirect=Communication.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) # Start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if self.num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="stub", args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) last_cmd = commands[-1] # Inject preload program if needed if Communication.STUB_PRELOAD_FILENAME in job.managers: last_cmd = [ "./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i], fifo_in[i] ] + commands[-1] processes[i] = evaluation_step_before_run( sandbox_user[i], last_cmd, job.time_limit, job.memory_limit, allow_dirs=[fifo_dir[i]], multiprocess=job.multithreaded_sandbox) # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None pass # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user, job.feedback_level) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( Communication.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = [create_sandbox(file_cacher) for i in indices] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: command = ["./%s" % executable_filename, fifo_out[i], fifo_in[i]] if num_processes != 1: command.append(str(i)) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage( filename, digest, executable=True) processes[i] = evaluation_step_before_run( sandbox_user[i], command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) for s in sandbox_user: delete_sandbox(s) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.job.executables[first_filename].digest } first_files_to_get = { "input.txt": self.job.testcases[test_number].input } first_allow_path = ["input.txt", fifo] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run(first_sandbox, first_command, self.job.time_limit, self.job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run(second_sandbox, second_command, self.job.time_limit, self.job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) self.job.evaluations[test_number] = { 'sandboxes': [first_sandbox.path, second_sandbox.path], 'plus': second_plus } outcome = None text = None evaluation = self.job.evaluations[test_number] success = True # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = "Execution didn't produce file output.txt" if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = second_sandbox.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) outcome, text = white_diff_step(second_sandbox, "output.txt", "res.txt") # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = outcome evaluation['text'] = text delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return success
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.job.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, self.job.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage(filename, digest, executable=True) process = evaluation_step_before_run(sandbox_user, command, self.job.time_limit, self.job.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) self.job.evaluations[test_number] = \ {'sandboxes': [sandbox_user.path, sandbox_mgr.path], 'plus': plus_user} evaluation = self.job.evaluations[test_number] # If at least one evaluation had problems, we report the # problems. (TODO: shouldn't outcome and text better be None # and None?) if not success_user or not success_mgr: success, outcome, text = False, None, None # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if self.job.get_output: if sandbox_mgr.file_exists("output.txt"): evaluation['output'] = sandbox_mgr.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) else: evaluation['output'] = None # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # f stand for first, s for second. first_sandbox = create_sandbox(file_cacher) second_sandbox = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: job.executables[first_filename].digest } first_files_to_get = { "input.txt": job.input } first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) job.sandboxes = [first_sandbox.path, second_sandbox.path] job.plus = second_plus success = True outcome = None text = None # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), "output.txt"] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", job.output) # If a checker is not provided, use white-diff if self.parameters[0] == "diff": outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": if TwoSteps.CHECKER_FILENAME not in job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: second_sandbox.create_file_from_storage( TwoSteps.CHECKER_FILENAME, job.managers[TwoSteps.CHECKER_FILENAME].digest, executable=True) # Rewrite input file, as in Batch.py try: second_sandbox.remove_file("input.txt") except OSError as e: assert not second_sandbox.file_exists( "input.txt") second_sandbox.create_file_from_storage( "input.txt", job.input) success, _ = evaluation_step( second_sandbox, [["./%s" % TwoSteps.CHECKER_FILENAME, "input.txt", "res.txt", "output.txt"]]) if success: try: outcome, text = extract_outcome_and_text( second_sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Uncrecognized first parameter" " `%s' for TwoSteps tasktype." % self.parameters[0])