def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" user_output_filename = self._get_user_output_filename(job) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if user_output_filename not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] job.plus = {} return # First and only step: eval the user output. box_success, outcome, text = eval_output( file_cacher, job, OutputOnly.CHECKER_CODENAME if self._uses_checker() else None, user_output_digest=job.files[user_output_filename].digest) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text # There is no actual evaluation, so no statistics. job.plus = {} if box_success else None
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands(executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } files_to_get = {self._actual_input: job.input} # Check which redirect we need to perform, and in case we don't # manage the output via redirect, the submission needs to be able # to write on it. files_allowing_write = [] stdin_redirect = None stdout_redirect = None if len(self.input_filename) == 0: stdin_redirect = self._actual_input if len(self.output_filename) == 0: stdout_redirect = self._actual_output else: files_allowing_write.append(self._actual_output) # Create the sandbox sandbox = create_sandbox(file_cacher, name="evaluate") job.sandboxes.append(sandbox.path) # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution box_success, evaluation_success, stats = evaluation_step( sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, multiprocess=job.multithreaded_sandbox) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self._actual_output): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), self._actual_output ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self._actual_output, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, Batch.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=sandbox.relative_path( self._actual_output), user_output_filename=self.output_filename) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iterkeys(job.executables)) executable_digest = job.executables[executable_filename].digest first_sandbox = create_sandbox(file_cacher, name="first_evaluate") second_sandbox = create_sandbox(file_cacher, name="second_evaluate") job.sandboxes.append(first_sandbox.path) job.sandboxes.append(second_sandbox.path) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_command = ["./%s" % executable_filename, "0", fifo] first_executables_to_get = {executable_filename: executable_digest} first_files_to_get = { TwoSteps.INPUT_FILENAME: job.input } first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect=TwoSteps.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Second step: we start the second manager. second_command = ["./%s" % executable_filename, "1", fifo] second_executables_to_get = {executable_filename: executable_digest} second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect=TwoSteps.OUTPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Consume output. wait_without_std([second, first]) box_success_first, evaluation_success_first, first_stats = \ evaluation_step_after_run(first_sandbox) box_success_second, evaluation_success_second, second_stats = \ evaluation_step_after_run(second_sandbox) box_success = box_success_first and box_success_second evaluation_success = \ evaluation_success_first and evaluation_success_second stats = merge_execution_stats(first_stats, second_stats) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), TwoSteps.OUTPUT_FILENAME] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( TwoSteps.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=second_sandbox.relative_path( TwoSteps.OUTPUT_FILENAME)) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success)