def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension # Prepare the files to copy in the sandbox and to add to the # compilation command. files_to_get = {} source_filenames = [] # The stub, that must have been provided (copy and add to compilation). stub_filename = "stub%s" % source_ext if not check_manager_present(job, stub_filename): return source_filenames.append(stub_filename) files_to_get[stub_filename] = job.managers[stub_filename].digest # User's submitted file(s) (copy and add to compilation). for codename, file_ in iteritems(job.files): source_filename = codename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Any other useful manager (just copy). for filename, manager in iteritems(job.managers): if is_manager_for_compilation(filename, language): files_to_get[filename] = manager.digest # Prepare the compilation command executable_filename = self._executable_filename(iterkeys(job.files)) commands = language.get_compilation_commands(source_filenames, executable_filename) # Create the sandbox. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.path) # Copy all required files in the sandbox. for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables. job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup. delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension if not check_files_number(job, 1): return user_file_format = next(iterkeys(job.files)) user_source_filename = user_file_format.replace(".%l", source_ext) executable_filename = user_file_format.replace(".%l", "") # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line, and # we check that it exists. source_filenames = [user_source_filename] if self._uses_grader(): grader_source_filename = Batch.GRADER_BASENAME + source_ext if not check_manager_present(job, grader_source_filename): return source_filenames.insert(0, grader_source_filename) # Prepare the compilation command. commands = language.get_compilation_commands(source_filenames, executable_filename) # Create the sandbox. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.path) # Copy required files in the sandbox (includes the grader if present). sandbox.create_file_from_storage(user_source_filename, job.files[user_file_format].digest) for filename, manager in iteritems(job.managers): if is_manager_for_compilation(filename, language): sandbox.create_file_from_storage(filename, manager.digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables. job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup. delete_sandbox(sandbox, job.success)
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.job.language # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.job.files) != 1: self.job.success = True self.job.compilation_success = False self.job.text = "Invalid files in submission" logger.error("Submission contains %d files, expecting 1" % len(self.job.files)) return True # Create the sandbox sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = self.job.files.keys()[0] source_filenames = [] # Stub. source_filenames.append("stub.%s" % language) files_to_get[source_filenames[-1]] = self.job.managers["stub.%s" % language].digest # User's submission. source_filenames.append(format_filename.replace("%l", language)) files_to_get[source_filenames[-1]] = self.job.files[format_filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") command = get_compilation_command(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = compilation_step(sandbox, command) # Retrieve the compiled executables self.job.success = operation_success self.job.compilation_success = compilation_success self.job.plus = plus self.job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, self.job.info) ) self.job.executables[executable_filename] = Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = [] user_output_filename = self._get_user_output_filename(job) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if user_output_filename not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return # First and only one step: diffing (manual or with manager). # Put user output and reference solution into the sandbox. sandbox.create_file_from_storage( OutputOnly.OUTPUT_FILENAME, job.files[user_output_filename].digest) sandbox.create_file_from_storage( OutputOnly.CORRECT_OUTPUT_FILENAME, job.output) if self._uses_checker(): # Checker also requires the input file. sandbox.create_file_from_storage( OutputOnly.INPUT_FILENAME, job.input) success, outcome, text = OutputOnly._run_checker(sandbox, job) else: success = True outcome, text = white_diff_step( sandbox, OutputOnly.OUTPUT_FILENAME, OutputOnly.CORRECT_OUTPUT_FILENAME) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = [] user_output_filename = self._get_user_output_filename(job) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if user_output_filename not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return # First and only one step: diffing (manual or with manager). # Put user output and reference solution into the sandbox. sandbox.create_file_from_storage( OutputOnly.OUTPUT_FILENAME, job.files[user_output_filename].digest) sandbox.create_file_from_storage(OutputOnly.CORRECT_OUTPUT_FILENAME, job.output) if self._uses_checker(): # Checker also requires the input file. sandbox.create_file_from_storage(OutputOnly.INPUT_FILENAME, job.input) success, outcome, text = OutputOnly._run_checker(sandbox, job) else: success = True outcome, text = white_diff_step(sandbox, OutputOnly.OUTPUT_FILENAME, OutputOnly.CORRECT_OUTPUT_FILENAME) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.submission.language # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.submission.files) != 1: return self.finish_compilation( True, False, "Invalid files in submission", to_log="Submission contains %d files, expecting 1" % len(self.submission.files)) # First and only one compilation. sandbox = create_sandbox(self) files_to_get = {} format_filename = self.submission.files.keys()[0] source_filenames = [format_filename.replace("%l", language)] files_to_get[source_filenames[0]] = \ self.submission.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. if self.parameters[0] == "grader": source_filenames.append("grader.%s" % language) files_to_get[source_filenames[1]] = \ self.submission.task.managers["grader.%s" % language].digest executable_filename = format_filename.replace(".%l", "") command = get_compilation_command(language, source_filenames, executable_filename) operation_success, compilation_success, text, _ = \ self.compilation_step( sandbox, command, files_to_get, {executable_filename: "Executable %s for submission %s" % (executable_filename, self.submission.id)}) delete_sandbox(sandbox) # We had only one compilation, hence we pipe directly its # result to the finalization. return self.finish_compilation(operation_success, compilation_success, text)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.submission.files: return self.finish_evaluation_testcase( test_number, True, 0.0, "File not submitted.") # First and only one step: diffing (manual or with manager). output_digest = self.submission.files["output_%03d.txt" % test_number].digest # TODO: this should check self.parameters, not managers. if len(self.submission.task.managers) == 0: # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success, outcome, text = self.white_diff_step( sandbox, "output.txt", "res.txt", {"res.txt": self.submission.task.testcases[test_number].output, "output.txt": output_digest}) else: # Manager present: wonderful, he'll do all the job. manager_filename = self.submission.task.managers.keys()[0] success, outcome, text = self.evaluation_step( sandbox, ["./%s" % manager_filename, "input.txt", "res.txt", "output.txt"], {manager_filename: self.submission.task.managers[manager_filename].digest}, {"output.txt": output_digest, "res.txt": self.submission.task.testcases[test_number].output, "input.txt": self.submission.task.testcases[test_number].input}, allow_path=["input.txt", "output.txt", "res.txt"], final=True) # Whatever happened, we conclude. delete_sandbox(sandbox) return self.finish_evaluation_testcase(test_number, success, outcome, text)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox) sandbox_user = [ create_sandbox(file_cacher, job.multithreaded_sandbox) for i in indices ] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt", stdout_redirect="output.txt", ) # Second step: we start the user submission compiled with the # stub. language = get_language(job.language) executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="grader", args=args) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage(filename, digest, executable=True) # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) if job.get_output: job.user_output = None # Otherwise, we use the manager to obtain the outcome. else: success = True outcome = None text = None input_filename = "input.txt" output_filename = "output.txt" # Check that the output file was created if not sandbox_mgr.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox_mgr.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=1024 * 1024 * 10) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox_mgr.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[1] == "diff": outcome, text = white_diff_step( sandbox_mgr, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[1] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox_mgr.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox_mgr.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox_mgr.file_exists( input_filename) sandbox_mgr.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox_mgr.max_processes = 1000 success, _ = evaluation_step( sandbox_mgr, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox_mgr) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized second parameter" " `%s' for Communication tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.job.language header = HEADERS_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.job.files) != 2: self.job.success = True self.job.compilation_success = False self.job.text = "Invalid files in submission" logger.warning("Submission contains %d files, expecting 2" % len(self.job.files)) return True # First and only one compilation. sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) files_to_get = {} # User's submissions and headers. source_filenames = [] for filename, _file in self.job.files.iteritems(): source_filename = filename.replace("%l", language) source_filenames.append(source_filename) files_to_get[source_filename] = _file.digest # Headers. header_filename = filename.replace("%l", header) source_filenames.append(header_filename) files_to_get[header_filename] = \ self.job.managers[header_filename].digest # Manager. manager_filename = "manager.%s" % language source_filenames.append(manager_filename) files_to_get[manager_filename] = \ self.job.managers[manager_filename].digest # Manager's header. manager_filename = "manager.%s" % header source_filenames.append(manager_filename) files_to_get[manager_filename] = \ self.job.managers[manager_filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Get compilation command and compile. executable_filename = "manager" command = get_compilation_command(language, source_filenames, executable_filename) operation_success, compilation_success, text, plus = \ compilation_step(sandbox, command) # Retrieve the compiled executables self.job.success = operation_success self.job.compilation_success = compilation_success self.job.plus = plus self.job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, self.job.info)) self.job.executables[executable_filename] = \ Executable(digest, executable_filename) # Cleanup delete_sandbox(sandbox)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return # Create the sandbox. sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="compile") job.sandboxes.append(sandbox.path) user_file_format = next(iterkeys(job.files)) user_source_filename = user_file_format.replace(".%l", source_ext) executable_filename = user_file_format.replace(".%l", "") # Copy required files in the sandbox (includes the grader if present). sandbox.create_file_from_storage( user_source_filename, job.files[user_file_format].digest) for filename in iterkeys(job.managers): if Batch._is_manager_for_compilation(filename): sandbox.create_file_from_storage( filename, job.managers[filename].digest) # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line. source_filenames = [user_source_filename] if self._uses_grader(): grader_source_filename = Batch.GRADER_BASENAME + source_ext source_filenames.insert(0, grader_source_filename) # Prepare the compilation command. commands = language.get_compilation_commands( source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return self.job.evaluations[test_number] = {'sandboxes': [sandbox.path], 'plus': {}} evaluation = self.job.evaluations[test_number] outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.job.files: evaluation['success'] = True evaluation['outcome'] = 0.0 evaluation['text'] = "File not submitted." return True # First and only one step: diffing (manual or with manager). output_digest = self.job.files["output_%03d.txt" % test_number].digest # Put the files into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) sandbox.create_file_from_storage( "output.txt", output_digest) # TODO: this should check self.parameters, not managers. if len(self.job.managers) == 0: # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step( sandbox, "output.txt", "res.txt") else: # Manager present: wonderful, he'll do all the job. manager_filename = self.job.managers.keys()[0] sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) input_digest = self.job.testcases[test_number].input sandbox.create_file_from_storage( "input.txt", input_digest) success, _ = evaluation_step( sandbox, ["./%s" % manager_filename, "input.txt", "res.txt", "output.txt"], allow_path=["input.txt", "output.txt", "res.txt"]) if success: outcome, text = extract_outcome_and_text(sandbox) # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox) return success
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} source_filenames = [] # Stub. stub_filename = "stub%s" % source_ext source_filenames.append(stub_filename) files_to_get[stub_filename] = job.managers[stub_filename].digest # User's submission. for filename, fileinfo in job.files.iteritems(): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = fileinfo.digest # Also copy all managers that might be useful during compilation. for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in HEADER_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(source) for source in SOURCE_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(obj) for obj in OBJECT_EXTS): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = \ "_".join(pattern.replace(".%l", "") for pattern in job.files.keys()) commands = language.get_compilation_commands(source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.job.executables[first_filename].digest } first_files_to_get = { "input.txt": self.job.testcases[test_number].input } first_allow_path = ["input.txt", fifo] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, self.job.time_limit, self.job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, self.job.time_limit, self.job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) self.job.evaluations[test_number] = {'sandboxes': [first_sandbox.path, second_sandbox.path], 'plus': second_plus} outcome = None text = None evaluation = self.job.evaluations[test_number] success = True # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = "Execution didn't produce file output.txt" if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = second_sandbox.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = outcome evaluation['text'] = text delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return success
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] header_ext = LANGUAGE_TO_HEADER_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 2: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 2", len(job.files), extra={"operation": job.info}) return True # First and only one compilation. sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) files_to_get = {} # User's submissions and headers. source_filenames = [] for filename, file_ in job.files.iteritems(): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Headers. header_filename = filename.replace(".%l", header_ext) source_filenames.append(header_filename) files_to_get[header_filename] = \ job.managers[header_filename].digest # Manager. manager_filename = "manager%s" % source_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # Manager's header. manager_filename = "manager%s" % header_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Get compilation command and compile. executable_filename = "manager" commands = get_compilation_commands(language, source_filenames, executable_filename) operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return self.job.evaluations[test_number] = {"sandboxes": [sandbox.path], "plus": {}} evaluation = self.job.evaluations[test_number] outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.job.files: evaluation["success"] = True evaluation["outcome"] = 0.0 evaluation["text"] = "File not submitted." return True # First and only one step: diffing (manual or with manager). output_digest = self.job.files["output_%03d.txt" % test_number].digest # Put the files into the sandbox sandbox.create_file_from_storage("res.txt", self.job.testcases[test_number].output) sandbox.create_file_from_storage("output.txt", output_digest) if self.job.task_type_parameters[0] == "diff": # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") elif self.job.task_type_parameters[0] == "comparator": # Manager present: wonderful, he'll do all the job. manager_filename = "checker" if not manager_filename in self.job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')") success = False else: sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True ) input_digest = self.job.testcases[test_number].input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step( sandbox, ["./%s" % manager_filename, "input.txt", "res.txt", "output.txt"], allow_path=["input.txt", "output.txt", "res.txt"], ) if success: outcome, text = extract_outcome_and_text(sandbox) else: raise ValueError( "Unrecognized first parameter " "`%s' for OutputOnly tasktype. " "Should be `diff' or `comparator'." % self.job.task_type_parameters[0] ) # Whatever happened, we conclude. evaluation["success"] = success evaluation["outcome"] = str(outcome) if outcome is not None else None evaluation["text"] = text delete_sandbox(sandbox) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) # Prepare the execution executable_filename = job.executables.keys()[0] language = get_language(job.language) commands = language.get_evaluation_commands( executable_filename, main="grader" if self._uses_grader() else executable_filename) executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None files_allowing_write = [] if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename else: files_allowing_write.append(output_filename) files_to_get = { input_filename: job.input } # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step( sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = None # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), output_filename] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox.create_file_from_storage( "res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox.file_exists(input_filename) sandbox.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox.max_processes = 1000 success, _ = evaluation_step( sandbox, [["./%s" % manager_filename, input_filename, "res.txt", output_filename]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2])
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return self.job.evaluations[test_number] = { 'sandboxes': [sandbox.path], 'plus': {} } evaluation = self.job.evaluations[test_number] outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.job.files: evaluation['success'] = True evaluation['outcome'] = "0.0" evaluation['text'] = "File not submitted." return True # First and only one step: diffing (manual or with manager). output_digest = self.job.files["output_%03d.txt" % test_number].digest # Put the files into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) sandbox.create_file_from_storage("output.txt", output_digest) if self.job.task_type_parameters[0] == "diff": # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") elif self.job.task_type_parameters[0] == "comparator": # Manager present: wonderful, he'll do all the job. manager_filename = "checker" if not manager_filename in self.job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')") success = False else: sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) input_digest = self.job.testcases[test_number].input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step(sandbox, [ "./%s" % manager_filename, "input.txt", "res.txt", "output.txt" ]) if success: outcome, text = extract_outcome_and_text(sandbox) else: raise ValueError("Unrecognized first parameter " "`%s' for OutputOnly tasktype. " "Should be `diff' or `comparator'." % self.job.task_type_parameters[0]) # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="manager_evaluate") sandbox_user = [ create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="user_evaluate") for i in indices] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: prepare the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = fifo_dir for filename, digest in iteritems(manager_executables_to_get): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in iteritems(manager_files_to_get): sandbox_mgr.create_file_from_storage(filename, digest) # Second step: load the executables for the user processes # (done before launching the manager so that it does not # impact its wall clock time). assert len(job.executables) == 1 executable_filename = next(iterkeys(job.executables)) executables_to_get = { executable_filename: job.executables[executable_filename].digest } for i in indices: for filename, digest in iteritems(executables_to_get): sandbox_user[i].create_file_from_storage( filename, digest, executable=True) # Third step: start the manager. manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Fourth step: start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands( executable_filename, main="stub", args=args) user_allow_dirs = [fifo_dir[i]] # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, [] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.submission.language header = HEADERS_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.submission.files) != 2: return self.finish_compilation( True, False, "Invalid files in submission", to_log="Submission contains %d files, expecting 2" % len(self.submission.files)) # First and only one compilation. sandbox = create_sandbox(self) files_to_get = {} # User's submissions and headers. source_filenames = [] for filename, _file in self.submission.files.iteritems(): source_filename = filename.replace("%l", language) source_filenames.append(source_filename) files_to_get[source_filename] = _file.digest # Headers. header_filename = filename.replace("%l", header) source_filenames.append(header_filename) files_to_get[header_filename] = \ self.submission.task.managers[header_filename].digest # Manager. manager_filename = "manager.%s" % language source_filenames.append(manager_filename) files_to_get[manager_filename] = \ self.submission.task.managers[manager_filename].digest # Manager's header. manager_filename = "manager.%s" % header source_filenames.append(manager_filename) files_to_get[manager_filename] = \ self.submission.task.managers[manager_filename].digest # Get compilation command and compile. executable_filename = "manager" command = get_compilation_command(language, source_filenames, executable_filename) operation_success, compilation_success, text, _ = \ self.compilation_step( sandbox, command, files_to_get, {executable_filename: "Executable %s for submission %s" % (executable_filename, self.submission.id)}) delete_sandbox(sandbox) # We had only one compilation, hence we pipe directly its # result to the finalization. return self.finish_compilation(operation_success, compilation_success, text)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) os.chmod(fifo_dir, 0o755) os.chmod(fifo_in, 0o666) os.chmod(fifo_out, 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = [fifo_dir] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: job.executables[executable_filename].digest } user_allow_dirs = [fifo_dir] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage(filename, digest, executable=True) process = evaluation_step_before_run(sandbox_user, command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) job.sandboxes = [sandbox_user.path, sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) if not config.keep_sandbox: rmtree(fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) # Prepare the execution assert len(job.executables) == 1 executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) commands = language.get_evaluation_commands( executable_filename, main="grader" if self._uses_grader() else executable_filename) executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None files_allowing_write = [] if len(input_filename) == 0: input_filename = "input.txt" stdin_redirect = input_filename if len(output_filename) == 0: output_filename = "output.txt" stdout_redirect = output_filename else: files_allowing_write.append(output_filename) files_to_get = {input_filename: job.input} # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = [] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox.file_exists(input_filename) sandbox.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox.max_processes = 1000 success, _ = evaluation_step( sandbox, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create the sandbox sandbox = create_sandbox(self) # Prepare the execution executable_filename = self.job.executables.keys()[0] command = [os.path.join(".", executable_filename)] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } input_filename, output_filename = self.job.task_type_parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = {input_filename: self.job.testcases[test_number].input} # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, command, self.job.time_limit, self.job.memory_limit, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) self.job.evaluations[test_number] = { 'sandboxes': [sandbox.path], 'plus': plus } outcome = None text = None evaluation = self.job.evaluations[test_number] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = "Execution didn't produce file %s" % \ (output_filename) if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = sandbox.get_file_to_storage( output_filename, "Output file for testcase %d in job %s" % (test_number, self.job.info), trunc_len=100 * 1024) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) # Check the solution with white_diff if self.job.task_type_parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.job.task_type_parameters[2] == "comparator": manager_filename = "checker" if not manager_filename in self.job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named 'checker')") success = False else: sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) success, _ = evaluation_step( sandbox, [ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s" % (e.message, )) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.job.task_type_parameters[2])
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} source_filenames = [] # Stub. stub_filename = "stub%s" % source_ext source_filenames.append(stub_filename) files_to_get[stub_filename] = job.managers[stub_filename].digest # User's submission. for filename, fileinfo in job.files.iteritems(): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = fileinfo.digest # Also copy all managers that might be useful during compilation. for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(source) for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(obj) for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = \ "_".join(pattern.replace(".%l", "") for pattern in job.files.keys()) commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension header_ext = language.header_extension if not check_files_number(job, 2): return files_to_get = {} source_filenames = [] # Manager. manager_filename = "manager%s" % source_ext if not check_manager_present(job, manager_filename): return source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # Manager's header. if header_ext is not None: manager_filename = "manager%s" % header_ext if not check_manager_present(job, manager_filename): return source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # User's submissions and headers. for filename, file_ in iteritems(job.files): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Headers (fixing compile error again here). if header_ext is not None: header_filename = filename.replace(".%l", header_ext) if not check_manager_present(job, header_filename): return source_filenames.append(header_filename) files_to_get[header_filename] = \ job.managers[header_filename].digest # Get compilation command. executable_filename = "manager" commands = language.get_compilation_commands( source_filenames, executable_filename) # Create the sandbox and put the required files in it. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.path) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] job.plus = {} logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[0]] = \ job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. compile_command = [] if self._uses_grader(): files_to_get["grader%s" % source_ext] = \ job.managers["grader%s" % source_ext].digest # For solutions using C or C++, # we first compile the grader source # file and then delete it from sandbox, # to prevent the user's solution # files from including it. try: compile_command = language.get_compilation_no_link_command( ["grader%s" % source_ext]) compile_command += [["/bin/rm", "grader%s" % source_ext]] except NotImplementedError: compile_command = [] source_filenames.insert(0, "grader%s" % source_ext) else: source_filenames.insert(0, "grader%s" % language.object_extension) # Also copy all managers that might be useful during compilation. for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in language.header_extensions): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(source) for source in language.source_extensions): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(obj) for obj in language.object_extensions): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = compile_command + language.get_compilation_commands( source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest first_sandbox = create_sandbox(file_cacher, name="first_evaluate") second_sandbox = create_sandbox(file_cacher, name="second_evaluate") job.sandboxes.append(first_sandbox.path) job.sandboxes.append(second_sandbox.path) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_command = ["./%s" % executable_filename, "0", fifo] first_executables_to_get = {executable_filename: executable_digest} first_files_to_get = { TwoSteps.INPUT_FILENAME: job.input } first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect=TwoSteps.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Second step: we start the second manager. second_command = ["./%s" % executable_filename, "1", fifo] second_executables_to_get = {executable_filename: executable_digest} second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect=TwoSteps.OUTPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Consume output. wait_without_std([second, first]) box_success_first, evaluation_success_first, first_stats = \ evaluation_step_after_run(first_sandbox) box_success_second, evaluation_success_second, second_stats = \ evaluation_step_after_run(second_sandbox) box_success = box_success_first and box_success_second evaluation_success = \ evaluation_success_first and evaluation_success_second stats = merge_execution_stats(first_stats, second_stats) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), TwoSteps.OUTPUT_FILENAME] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( TwoSteps.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=second_sandbox.relative_path( TwoSteps.OUTPUT_FILENAME)) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.submission.executables[first_filename].digest } first_files_to_get = { "input.txt": self.submission.task.testcases[test_number].input } first_allow_path = ["input.txt", fifo] first = self.evaluation_step_before_run( first_sandbox, first_command, first_executables_to_get, first_files_to_get, self.submission.task.time_limit, self.submission.task.memory_limit, first_allow_path, stdin_redirect="input.txt") # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.submission.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] second = self.evaluation_step_before_run( second_sandbox, second_command, second_executables_to_get, second_files_to_get, self.submission.task.time_limit, self.submission.task.memory_limit, second_allow_path) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, outcome_first, text_first, _ = \ self.evaluation_step_after_run(first_sandbox, final=False) success_second, outcome_second, text_second, plus = \ self.evaluation_step_after_run(second_sandbox, final=True) # If at least one evaluation had problems, we report the # problems. if not success_first: success, outcome, text = False, outcome_first, text_first elif not success_second: success, outcome, text = False, outcome_second, text_second # Otherwise, compare the second output to the results # (there is no comparator since the manager is expected to turn the # output in diffable form) else: success, outcome, text = self.white_diff_step( second_sandbox, "output.txt", "res.txt", {"res.txt": self.submission.task.testcases[test_number].output}) delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(job.executables) != 1: raise ValueError("Unexpected number of executables (%s)" % len(job.executables)) # Create the sandbox sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands( executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } stdin_redirect = None stdout_redirect = None files_allowing_write = [] if len(self.input_filename) == 0: self.input_filename = Batch.DEFAULT_INPUT_FILENAME stdin_redirect = self.input_filename if len(self.output_filename) == 0: self.output_filename = Batch.DEFAULT_OUTPUT_FILENAME stdout_redirect = self.output_filename else: files_allowing_write.append(self.output_filename) files_to_get = { self.input_filename: job.input } # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step( sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = [] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self.output_filename): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), self.output_filename] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self.output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Create a brand-new sandbox just for checking. Only admin # code runs in it, so we allow multithreading and many # processes (still with a limit to avoid fork-bombs). checkbox = create_sandbox( file_cacher, multithreaded=True, name="check") checkbox.max_processes = 1000 checker_success, outcome, text = self._eval_output( checkbox, job, sandbox.get_root_path()) success = success and checker_success # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # f stand for first, s for second. first_sandbox = create_sandbox(self) second_sandbox = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: self.job.executables[first_filename].digest } first_files_to_get = { "input.txt": self.job.testcases[test_number].input } first_allow_path = ["input.txt", fifo] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run(first_sandbox, first_command, self.job.time_limit, self.job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: self.job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo, "output.txt"] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run(second_sandbox, second_command, self.job.time_limit, self.job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) self.job.evaluations[test_number] = { 'sandboxes': [first_sandbox.path, second_sandbox.path], 'plus': second_plus } outcome = None text = None evaluation = self.job.evaluations[test_number] success = True # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = "Execution didn't produce file output.txt" if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = second_sandbox.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) outcome, text = white_diff_step(second_sandbox, "output.txt", "res.txt") # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = outcome evaluation['text'] = text delete_sandbox(first_sandbox) delete_sandbox(second_sandbox) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands(executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } files_to_get = {self._actual_input: job.input} # Check which redirect we need to perform, and in case we don't # manage the output via redirect, the submission needs to be able # to write on it. files_allowing_write = [] stdin_redirect = None stdout_redirect = None if len(self.input_filename) == 0: stdin_redirect = self._actual_input if len(self.output_filename) == 0: stdout_redirect = self._actual_output else: files_allowing_write.append(self._actual_output) # Create the sandbox sandbox = create_sandbox(file_cacher, name="evaluate") job.sandboxes.append(sandbox.path) # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution box_success, evaluation_success, stats = evaluation_step( sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, multiprocess=job.multithreaded_sandbox) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self._actual_output): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), self._actual_output ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self._actual_output, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, Batch.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=sandbox.relative_path( self._actual_output), user_output_filename=self.output_filename) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) # First step: execute the contestant program. This is also the # final step if we have a grader, otherwise we need to run also # a white_diff or a comparator. executable_filename = self.submission.executables.keys()[0] command = ["./%s" % executable_filename] executables_to_get = { executable_filename: self.submission.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = { input_filename: self.submission.task.testcases[test_number].input } allow_path = [input_filename, output_filename] success, outcome, text, plus = self.evaluation_step( sandbox, command, executables_to_get, files_to_get, self.submission.task.time_limit, self.submission.task.memory_limit, allow_path, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, final=False) # If an error occur (our or contestant's), return immediately. if not success or outcome is not None: delete_sandbox(sandbox) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus) # Second step: diffing (manual or with comparator). if self.parameters[2] == "diff": # We white_diff output.txt and res.txt. success, outcome, text = self.white_diff_step( sandbox, output_filename, "res.txt", {"res.txt": self.submission.task.testcases[test_number].output}) elif self.parameters[2] == "comparator": # Manager present: wonderful, it'll do all the job. manager_filename = self.submission.task.managers.keys()[0] success, outcome, text, _ = self.evaluation_step( sandbox, ["./%s" % manager_filename, input_filename, "res.txt", output_filename], {manager_filename: self.submission.task.managers[manager_filename].digest}, {"res.txt": self.submission.task.testcases[test_number].output}, allow_path=[input_filename, "res.txt", output_filename], final=True) else: raise ValueError("Unrecognized third parameter `%s' in for Batch " "tasktype." % self.parameters[2]) # Whatever happened, we conclude. delete_sandbox(sandbox) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.job.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, self.job.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage( filename, digest, executable=True) process = evaluation_step_before_run( sandbox_user, command, self.job.time_limit, self.job.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) self.job.evaluations[test_number] = \ {'sandboxes': [sandbox_user.path, sandbox_mgr.path], 'plus': plus_user} evaluation = self.job.evaluations[test_number] # If at least one evaluation had problems, we report the # problems. (TODO: shouldn't outcome and text better be None # and None?) if not success_user or not success_mgr: success, outcome, text = False, None, None # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if self.job.get_output: if sandbox_mgr.file_exists("output.txt"): evaluation['output'] = sandbox_mgr.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) else: evaluation['output'] = None # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return success
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%s.txt" % job.operation["testcase_codename"] \ not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return True # First and only one step: diffing (manual or with manager). output_digest = job.files["output_%s.txt" % job.operation["testcase_codename"]].digest # Put the files into the sandbox sandbox.create_file_from_storage( "res.txt", job.output) sandbox.create_file_from_storage( "output.txt", output_digest) if self.parameters[0] == "diff": # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step( sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": # Manager present: wonderful, he'll do all the job. manager_filename = "checker" if manager_filename not in job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) input_digest = job.input sandbox.create_file_from_storage( "input.txt", input_digest) success, _ = evaluation_step( sandbox, [["./%s" % manager_filename, "input.txt", "res.txt", "output.txt"]]) if success: outcome, text = extract_outcome_and_text(sandbox) else: raise ValueError("Unrecognized first parameter " "`%s' for OutputOnly tasktype. " "Should be `diff' or `comparator'." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension header_ext = language.header_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 2: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 2", len(job.files), extra={"operation": job.info}) return # First and only one compilation. sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="compile") job.sandboxes.append(sandbox.path) files_to_get = {} source_filenames = [] # Manager. manager_filename = "manager%s" % source_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # Manager's header. if header_ext is not None: manager_filename = "manager%s" % header_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # User's submissions and headers. for filename, file_ in iteritems(job.files): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Headers (fixing compile error again here). if header_ext is not None: header_filename = filename.replace(".%l", header_ext) source_filenames.append(header_filename) files_to_get[header_filename] = \ job.managers[header_filename].digest for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Get compilation command and compile. executable_filename = "manager" commands = language.get_compilation_commands(source_filenames, executable_filename) operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) os.chmod(fifo_dir, 0o755) os.chmod(fifo_in, 0o666) os.chmod(fifo_out, 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = [fifo_dir] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, job.time_limit, 0, allow_dirs=manager_allow_dirs, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: job.executables[executable_filename].digest } user_allow_dirs = [fifo_dir] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage( filename, digest, executable=True) process = evaluation_step_before_run( sandbox_user, command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) job.sandboxes = [sandbox_user.path, sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = str(outcome) if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) shutil.rmtree(fifo_dir)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # f stand for first, s for second. first_sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="first_evaluate") second_sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="second_evaluate") fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: job.executables[first_filename].digest } first_files_to_get = {"input.txt": job.input} first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run(first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run(second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) job.sandboxes = [first_sandbox.path, second_sandbox.path] job.plus = second_plus success = True outcome = None text = [] # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), "output.txt"] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", job.output) # If a checker is not provided, use white-diff if self.parameters[0] == "diff": outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": if TwoSteps.CHECKER_FILENAME not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: second_sandbox.create_file_from_storage( TwoSteps.CHECKER_FILENAME, job.managers[TwoSteps.CHECKER_FILENAME].digest, executable=True) # Rewrite input file, as in Batch.py try: second_sandbox.remove_file("input.txt") except OSError as e: assert not second_sandbox.file_exists( "input.txt") second_sandbox.create_file_from_storage( "input.txt", job.input) success, _ = evaluation_step( second_sandbox, [[ "./%s" % TwoSteps.CHECKER_FILENAME, "input.txt", "res.txt", "output.txt" ]]) if success: try: outcome, text = extract_outcome_and_text( second_sandbox) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e, extra={"operation": job.info}) success = False else: raise ValueError("Uncrecognized first parameter" " `%s' for TwoSteps tasktype." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = str(outcome) if outcome is not None else None job.text = text delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] # Stub. source_filenames.append("stub%s" % source_ext) files_to_get[source_filenames[-1]] = \ job.managers["stub%s" % source_ext].digest # User's submission. source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[-1]] = \ job.files[format_filename].digest # Also copy all managers that might be useful during compilation. for filename in job.managers.iterkeys(): if any( filename.endswith(header) for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest elif any( filename.endswith(source) for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest elif any( filename.endswith(obj) for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[0]] = \ job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. if self.parameters[0] == "grader": source_filenames.insert(0, "grader%s" % source_ext) files_to_get["grader%s" % source_ext] = \ job.managers["grader%s" % source_ext].digest # Also copy all *.h and *lib.pas graders for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] # Stub. source_filenames.append("stub%s" % source_ext) files_to_get[source_filenames[-1]] = job.managers["stub%s" % source_ext].digest # User's submission. source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[-1]] = job.files[format_filename].digest # Also copy all managers that might be useful during compilation. # We likely want to compile with .cpp or .o files, so add them to our # command line for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()): files_to_get[filename] = job.managers[filename].digest elif any(filename.endswith(source) for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()): files_to_get[filename] = job.managers[filename].digest if filename not in source_filenames: source_filenames.insert(1, filename) elif any(filename.endswith(obj) for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()): files_to_get[filename] = job.managers[filename].digest if filename not in source_filenames: source_filenames.insert(1, filename) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info) ) job.executables[executable_filename] = Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1" % len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[0]] = \ job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. if self.parameters[0] == "grader": source_filenames.insert(0, "grader%s" % source_ext) files_to_get["grader%s" % source_ext] = \ job.managers["grader%s" % source_ext].digest # Also copy all *.h and *lib.pas graders for filename in job.managers.iterkeys(): if filename.endswith('.h') or \ filename.endswith('lib.pas'): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher) # Prepare the execution executable_filename = job.executables.keys()[0] language = job.language commands = get_evaluation_commands(language, executable_filename) executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = {input_filename: job.input} # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = None # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox sandbox.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if not manager_filename in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) success, _ = evaluation_step( sandbox, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s" % (e.message, ), extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2])
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher) sandbox_user = [create_sandbox(file_cacher) for i in indices] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": job.input } manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage( filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: command = ["./%s" % executable_filename, fifo_out[i], fifo_in[i]] if num_processes != 1: command.append(str(i)) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage( filename, digest, executable=True) processes[i] = evaluation_step_before_run( sandbox_user[i], command, job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr) for s in sandbox_user: delete_sandbox(s) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return # Create the sandbox. sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="compile") job.sandboxes.append(sandbox.path) user_file_format = next(iterkeys(job.files)) user_source_filename = user_file_format.replace(".%l", source_ext) executable_filename = user_file_format.replace(".%l", "") # Copy required files in the sandbox (includes the grader if present). sandbox.create_file_from_storage(user_source_filename, job.files[user_file_format].digest) for filename in iterkeys(job.managers): if Batch._is_manager_for_compilation(filename): sandbox.create_file_from_storage(filename, job.managers[filename].digest) # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line. source_filenames = [user_source_filename] if self._uses_grader(): grader_source_filename = Batch.GRADER_BASENAME + source_ext source_filenames.insert(0, grader_source_filename) # Prepare the compilation command. commands = language.get_compilation_commands(source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox) sandbox_user = [ create_sandbox(file_cacher, job.multithreaded_sandbox) for i in indices ] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. language = get_language(job.language) executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="stub", args=args) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage(filename, digest, executable=True) # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if job.get_output: if sandbox_mgr.file_exists("output.txt"): job.user_output = sandbox_mgr.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) else: job.user_output = None # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.job.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.job.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run(sandbox_mgr, manager_command, self.job.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.job.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] for filename, digest in executables_to_get.iteritems(): sandbox_user.create_file_from_storage(filename, digest, executable=True) process = evaluation_step_before_run(sandbox_user, command, self.job.time_limit, self.job.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, plus_user = \ evaluation_step_after_run(sandbox_user) success_mgr, plus_mgr = \ evaluation_step_after_run(sandbox_mgr) self.job.evaluations[test_number] = \ {'sandboxes': [sandbox_user.path, sandbox_mgr.path], 'plus': plus_user} evaluation = self.job.evaluations[test_number] # If at least one evaluation had problems, we report the # problems. (TODO: shouldn't outcome and text better be None # and None?) if not success_user or not success_mgr: success, outcome, text = False, None, None # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) # Otherwise, we use the manager to obtain the outcome. else: success = True outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file, provided that it exists if self.job.get_output: if sandbox_mgr.file_exists("output.txt"): evaluation['output'] = sandbox_mgr.get_file_to_storage( "output.txt", "Output file for testcase %d in job %s" % (test_number, self.job.info)) else: evaluation['output'] = None # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return success
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return self.job.evaluations[test_number] = { 'sandboxes': [sandbox.path], 'plus': {} } evaluation = self.job.evaluations[test_number] outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.job.files: evaluation['success'] = True evaluation['outcome'] = 0.0 evaluation['text'] = "File not submitted." return True # First and only one step: diffing (manual or with manager). output_digest = self.job.files["output_%03d.txt" % test_number].digest # Put the files into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) sandbox.create_file_from_storage("output.txt", output_digest) # TODO: this should check self.parameters, not managers. if len(self.job.managers) == 0: # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") else: # Manager present: wonderful, he'll do all the job. manager_filename = self.job.managers.keys()[0] sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) input_digest = self.job.testcases[test_number].input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step(sandbox, [ "./%s" % manager_filename, "input.txt", "res.txt", "output.txt" ], allow_path=[ "input.txt", "output.txt", "res.txt" ]) if success: outcome, text = extract_outcome_and_text(sandbox) # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox) return success
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.job.language # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.job.files) != 1: self.job.success = True self.job.compilation_success = False self.job.text = "Invalid files in submission" logger.warning("Submission contains %d files, expecting 1" % len(self.job.files)) return True # Create the sandbox sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = self.job.files.keys()[0] source_filenames = [] source_filenames.append(format_filename.replace("%l", language)) files_to_get[source_filenames[0]] = \ self.job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. if self.job.task_type_parameters[0] == "grader": source_filenames.insert(0, "grader.%s" % language) files_to_get["grader.%s" % language] = \ self.job.managers["grader.%s" % language].digest # Also copy all *.h and *lib.pas graders for filename in self.job.managers.iterkeys(): if filename.endswith('.h') or \ filename.endswith('lib.pas'): files_to_get[filename] = \ self.job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") command = get_compilation_command(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, command) # Retrieve the compiled executables self.job.success = operation_success self.job.compilation_success = compilation_success self.job.plus = plus self.job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, self.job.info)) self.job.executables[executable_filename] = \ Executable(digest, executable_filename) # Cleanup delete_sandbox(sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%s.txt" % job.operation["testcase_codename"] \ not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return True # First and only one step: diffing (manual or with manager). output_digest = job.files["output_%s.txt" % job.operation["testcase_codename"]].digest # Put the files into the sandbox sandbox.create_file_from_storage("res.txt", job.output) sandbox.create_file_from_storage("output.txt", output_digest) if self.parameters[0] == "diff": # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": # Manager present: wonderful, it will do all the work. manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) input_digest = job.input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step(sandbox, [[ "./%s" % manager_filename, "input.txt", "res.txt", "output.txt" ]]) if success: outcome, text = extract_outcome_and_text(sandbox) else: raise ValueError("Unrecognized first parameter " "`%s' for OutputOnly tasktype. " "Should be `diff' or `comparator'." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox_mgr = create_sandbox(self) sandbox_user = create_sandbox(self) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo_in = os.path.join(fifo_dir, "in") fifo_out = os.path.join(fifo_dir, "out") os.mkfifo(fifo_in) os.mkfifo(fifo_out) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename, fifo_in, fifo_out] manager_executables_to_get = { manager_filename: self.submission.task.managers[manager_filename].digest } manager_files_to_get = { "input.txt": self.submission.task.testcases[test_number].input } manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out] manager = self.evaluation_step_before_run( sandbox_mgr, manager_command, manager_executables_to_get, manager_files_to_get, self.submission.task.time_limit, 0, manager_allow_path, stdin_redirect="input.txt") # Second step: we start the user submission compiled with the # stub. executable_filename = self.submission.executables.keys()[0] command = ["./%s" % executable_filename, fifo_out, fifo_in] executables_to_get = { executable_filename: self.submission.executables[executable_filename].digest } allow_path = [fifo_in, fifo_out] process = self.evaluation_step_before_run( sandbox_user, command, executables_to_get, {}, self.submission.task.time_limit, self.submission.task.memory_limit, allow_path) # Consume output. wait_without_std([process, manager]) # TODO: check exit codes with translate_box_exitcode. success_user, outcome_user, text_user, plus = \ self.evaluation_step_after_run(sandbox_user, final=False) success_mgr, outcome_mgr, text_mgr, _ = \ self.evaluation_step_after_run(sandbox_mgr, final=True) # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, outcome_user, text_user # If outcome_user is not None, it is 0.0 and it means that # there has been some errors in the user solution, and outcome # and text are meaningful, so we use them. elif outcome_user is not None: success, outcome, text = success_user, outcome_user, text_user # Otherwise, we use the manager to obtain the outcome. else: success, outcome, text = success_mgr, outcome_mgr, text_mgr delete_sandbox(sandbox_mgr) delete_sandbox(sandbox_user) return self.finish_evaluation_testcase( test_number, success, outcome, text, plus)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(job.executables) != 1: raise ValueError("Unexpected number of executables (%s)" % len(job.executables)) # Create the sandbox sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands(executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } stdin_redirect = None stdout_redirect = None files_allowing_write = [] if len(self.input_filename) == 0: self.input_filename = Batch.DEFAULT_INPUT_FILENAME stdin_redirect = self.input_filename if len(self.output_filename) == 0: self.output_filename = Batch.DEFAULT_OUTPUT_FILENAME stdout_redirect = self.output_filename else: files_allowing_write.append(self.output_filename) files_to_get = {self.input_filename: job.input} # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = [] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self.output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), self.output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self.output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Create a brand-new sandbox just for checking. Only admin # code runs in it, so we allow multithreading and many # processes (still with a limit to avoid fork-bombs). checkbox = create_sandbox(file_cacher, multithreaded=True, name="check") checkbox.max_processes = 1000 checker_success, outcome, text = self._eval_output( checkbox, job, sandbox.get_root_path()) success = success and checker_success # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher) # Prepare the execution executable_filename = job.executables.keys()[0] command = [os.path.join(".", executable_filename)] executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = { input_filename: job.input } # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step( sandbox, command, job.time_limit, job.memory_limit, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = None # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), output_filename] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox sandbox.create_file_from_storage( "res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if not manager_filename in job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) success, _ = evaluation_step( sandbox, ["./%s" % manager_filename, input_filename, "res.txt", output_filename]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s" % (e.message,), extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2])
def compile(self): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = self.job.language # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(self.job.files) != 1: self.job.success = True self.job.compilation_success = False self.job.text = "Invalid files in submission" logger.warning("Submission contains %d files, expecting 1" % len(self.job.files)) return True # Create the sandbox sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = self.job.files.keys()[0] source_filenames = [] # Stub. source_filenames.append("stub.%s" % language) files_to_get[source_filenames[1]] = \ self.job.managers["stub.%s" % language].digest # User's submission. source_filenames.append(format_filename.replace("%l", language)) files_to_get[source_filenames[0]] = \ self.job.files[format_filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") command = get_compilation_command(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, command) # Retrieve the compiled executables self.job.success = operation_success self.job.compilation_success = compilation_success self.job.plus = plus self.job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, self.job.info)) self.job.executables[executable_filename] = \ Executable(digest, executable_filename) # Cleanup delete_sandbox(sandbox)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, Communication.MANAGER_FILENAME): return manager_digest = job.managers[Communication.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # Create the manager sandbox and copy manager and input. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.path) sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME, job.input) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.path for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) if Communication.STUB_PRELOAD_FILENAME in job.managers: digest = job.managers[ Communication.STUB_PRELOAD_FILENAME].digest sandbox_user[i].create_file_from_storage( Communication.STUB_PRELOAD_FILENAME, digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % Communication.MANAGER_FILENAME] for i in indices: manager_command += [fifo_in[i], fifo_out[i]] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, allow_dirs=fifo_dir, writable_files=[Communication.OUTPUT_FILENAME], stdin_redirect=Communication.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) # Start the user submissions compiled with the stub. language = get_language(job.language) processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if self.num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="stub", args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) last_cmd = commands[-1] # Inject preload program if needed if Communication.STUB_PRELOAD_FILENAME in job.managers: last_cmd = [ "./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i], fifo_in[i] ] + commands[-1] processes[i] = evaluation_step_before_run( sandbox_user[i], last_cmd, job.time_limit, job.memory_limit, allow_dirs=[fifo_dir[i]], multiprocess=job.multithreaded_sandbox) # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None pass # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user, job.feedback_level) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( Communication.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" # Create the sandbox sandbox = create_sandbox(self) # Prepare the execution executable_filename = self.job.executables.keys()[0] command = [sandbox.relative_path(executable_filename)] executables_to_get = { executable_filename: self.job.executables[executable_filename].digest } input_filename, output_filename = self.job.task_type_parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = { input_filename: self.job.testcases[test_number].input } allow_path = [input_filename, output_filename] # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step( sandbox, command, self.job.time_limit, self.job.memory_limit, allow_path, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) self.job.evaluations[test_number] = {'sandboxes': [sandbox.path], 'plus': plus} outcome = None text = None evaluation = self.job.evaluations[test_number] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if self.job.get_output: evaluation['output'] = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = "Execution didn't produce file %s" % \ (output_filename) if self.job.get_output: evaluation['output'] = None else: # If asked so, put the output file into the storage if self.job.get_output: evaluation['output'] = sandbox.get_file_to_storage( output_filename, "Output file for testcase %d in job %s" % (test_number, self.job.info), trunc_len=100 * 1024) # If not asked otherwise, evaluate the output file if not self.job.only_execution: # Put the reference solution into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) # Check the solution with white_diff if self.job.task_type_parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.job.task_type_parameters[2] == "comparator": manager_filename = "checker" if not manager_filename in self.job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named 'checker')") success = False else: sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) success, _ = evaluation_step( sandbox, ["./%s" % manager_filename, input_filename, "res.txt", output_filename], allow_path=[input_filename, "res.txt", output_filename]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s" % (e.message,)) success = False # Unknown evaluationg parameter! else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.job.task_type_parameters[2])
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # f stand for first, s for second. first_sandbox = create_sandbox(file_cacher) second_sandbox = create_sandbox(file_cacher) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: job.executables[first_filename].digest } first_files_to_get = { "input.txt": job.input } first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in first_executables_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.iteritems(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in second_executables_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.iteritems(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) job.sandboxes = [first_sandbox.path, second_sandbox.path] job.plus = second_plus success = True outcome = None text = None # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), "output.txt"] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", job.output) # If a checker is not provided, use white-diff if self.parameters[0] == "diff": outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": if TwoSteps.CHECKER_FILENAME not in job.managers: logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: second_sandbox.create_file_from_storage( TwoSteps.CHECKER_FILENAME, job.managers[TwoSteps.CHECKER_FILENAME].digest, executable=True) # Rewrite input file, as in Batch.py try: second_sandbox.remove_file("input.txt") except OSError as e: assert not second_sandbox.file_exists( "input.txt") second_sandbox.create_file_from_storage( "input.txt", job.input) success, _ = evaluation_step( second_sandbox, [["./%s" % TwoSteps.CHECKER_FILENAME, "input.txt", "res.txt", "output.txt"]]) if success: try: outcome, text = extract_outcome_and_text( second_sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Uncrecognized first parameter" " `%s' for TwoSteps tasktype." % self.parameters[0])