def setUp(self): super().setUp() self.p = ParameterTypeCollection("name", "shortname", "description", [ ParameterTypeInt("name0", "shortname0", "desc0"), ParameterTypeString("name1", "shortname1", "desc1"), ParameterTypeChoice("name2", "shortname2", "desc2", { "c1": "First choice", "c2": "Second choice", })])
class TwoSteps(TaskType): """Task type class for tasks where the user must submit two files with a function each; the first function compute some data, that get passed to the second function that must recover some data. The admins must provide a manager source file (for each language), called manager.%l, that get compiled with both two user sources, get the input as stdin, and get two parameters: 0 if it is the first instance, 1 if it is the second instance, and the name of the pipe. Admins must provide also header files, named "foo{.h|lib.pas}" for the three sources (manager and user provided). Parameters are given by a singleton list of strings (for possible expansions in the future), which may be 'diff' or 'comparator', specifying whether the evaluation is done using white diff or a comparator. """ # Codename of the checker, if it is used. CHECKER_CODENAME = "checker" # Filename of the input and of the contestant's solution. INPUT_FILENAME = "input.txt" OUTPUT_FILENAME = "output.txt" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" ALLOW_PARTIAL_SUBMISSION = False _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Two steps" def __init__(self, parameters): super().__init__(parameters) self.output_eval = self.parameters[0] def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" res = dict() for language in LANGUAGES: source_ext = language.source_extension header_ext = language.header_extension source_filenames = [] # Manager manager_source_filename = "manager%s" % source_ext source_filenames.append(manager_source_filename) # Manager's header. if header_ext is not None: manager_header_filename = "manager%s" % header_ext source_filenames.append(manager_header_filename) for filename in submission_format: source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) # Headers if header_ext is not None: header_filename = filename.replace(".%l", header_ext) source_filenames.append(header_filename) # Get compilation command and compile. executable_filename = "manager" commands = language.get_compilation_commands( source_filenames, executable_filename) res[language.name] = commands return res def get_user_managers(self): """See TaskType.get_user_managers.""" return ["manager.%l"] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_checker(self): return self.output_eval == TwoSteps.OUTPUT_EVAL_CHECKER def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension header_ext = language.header_extension if not check_files_number(job, 2): return files_to_get = {} source_filenames = [] # Manager. manager_filename = "manager%s" % source_ext if not check_manager_present(job, manager_filename): return source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # Manager's header. if header_ext is not None: manager_filename = "manager%s" % header_ext if not check_manager_present(job, manager_filename): return source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # User's submissions and headers. for filename, file_ in job.files.items(): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Headers (fixing compile error again here). if header_ext is not None: header_filename = filename.replace(".%l", header_ext) if not check_manager_present(job, header_filename): return source_filenames.append(header_filename) files_to_get[header_filename] = \ job.managers[header_filename].digest # Get compilation command. executable_filename = "manager" commands = language.get_compilation_commands(source_filenames, executable_filename) # Create the sandbox and put the required files in it. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.get_root_path()) for filename, digest in files_to_get.items(): sandbox.create_file_from_storage(filename, digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success, job.keep_sandbox) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iter(job.executables.keys())) executable_digest = job.executables[executable_filename].digest first_sandbox = create_sandbox(file_cacher, name="first_evaluate") second_sandbox = create_sandbox(file_cacher, name="second_evaluate") job.sandboxes.append(first_sandbox.get_root_path()) job.sandboxes.append(second_sandbox.get_root_path()) fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"] first_executables_to_get = {executable_filename: executable_digest} first_files_to_get = {TwoSteps.INPUT_FILENAME: job.input} # Put the required files into the sandbox for filename, digest in first_executables_to_get.items(): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in first_files_to_get.items(): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run( first_sandbox, first_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdin_redirect=TwoSteps.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Second step: we start the second manager. second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"] second_executables_to_get = {executable_filename: executable_digest} second_files_to_get = {} # Put the required files into the second sandbox for filename, digest in second_executables_to_get.items(): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in second_files_to_get.items(): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run( second_sandbox, second_command, job.time_limit, job.memory_limit, dirs_map={fifo_dir: ("/fifo", "rw")}, stdout_redirect=TwoSteps.OUTPUT_FILENAME, multiprocess=job.multithreaded_sandbox, wait=False) # Consume output. wait_without_std([second, first]) box_success_first, evaluation_success_first, first_stats = \ evaluation_step_after_run(first_sandbox) box_success_second, evaluation_success_second, second_stats = \ evaluation_step_after_run(second_sandbox) box_success = box_success_first and box_success_second evaluation_success = \ evaluation_success_first and evaluation_success_second stats = merge_execution_stats(first_stats, second_stats) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), TwoSteps.OUTPUT_FILENAME ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( TwoSteps.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=second_sandbox.relative_path( TwoSteps.OUTPUT_FILENAME)) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(first_sandbox, job.success, job.keep_sandbox) delete_sandbox(second_sandbox, job.success, job.keep_sandbox)
class Batch(TaskType): """Task type class for a unique standalone submission source, with comparator (or not). Parameters needs to be a list of three elements. The first element is 'grader' or 'alone': in the first case, the source file is to be compiled with a provided piece of software ('grader'); in the other by itself. The second element is a 2-tuple of the input file name and output file name. The input file may be '' to denote stdin, and similarly the output filename may be '' to denote stdout. The third element is 'diff' or 'comparator' and says whether the output is compared with a simple diff algorithm or using a comparator. Note: the first element is used only in the compilation step; the others only in the evaluation step. A comparator can read argv[1], argv[2], argv[3] (respectively, input, correct output and user output) and should write the outcome to stdout and the text to stderr. """ ALLOW_PARTIAL_SUBMISSION = False _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", { "alone": "Submissions are self-sufficient", "grader": "Submissions are compiled with a grader" }) _USE_FILE = ParameterTypeCollection( "I/O (blank for stdin/stdout)", "io", "", [ ParameterTypeString("Input file", "inputfile", ""), ParameterTypeString("Output file", "outputfile", ""), ]) _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a grader/comparator is used, etc... return "Batch" def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" source_filenames = [] # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. if self._uses_grader(): source_filenames.append("grader.%l") source_filenames.append(submission_format[0]) executable_filename = submission_format[0].replace(".%l", "") res = dict() for language in LANGUAGES: res[language.name] = language.get_compilation_commands([ source.replace(".%l", language.source_extension) for source in source_filenames ], executable_filename) return res def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_grader(self): return self.parameters[0] == "grader" def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = next(iterkeys(job.files)) source_filenames = [] source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[0]] = \ job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. if self._uses_grader(): source_filenames.insert(0, "grader%s" % source_ext) files_to_get["grader%s" % source_ext] = \ job.managers["grader%s" % source_ext].digest # Also copy all managers that might be useful during compilation. for filename in iterkeys(job.managers): if any(filename.endswith(header) for header in HEADER_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(source) for source in SOURCE_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(obj) for obj in OBJECT_EXTS): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = language.get_compilation_commands(source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) # Prepare the execution assert len(job.executables) == 1 executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) commands = language.get_evaluation_commands( executable_filename, main="grader" if self._uses_grader() else executable_filename) executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None files_allowing_write = [] if len(input_filename) == 0: input_filename = "input.txt" stdin_redirect = input_filename if len(output_filename) == 0: output_filename = "output.txt" stdout_redirect = output_filename else: files_allowing_write.append(output_filename) files_to_get = {input_filename: job.input} # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = [] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox.file_exists(input_filename) sandbox.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox.max_processes = 1000 success, _ = evaluation_step( sandbox, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
class Batch(TaskType): """Task type class for a unique standalone submission source, with comparator (or not). Parameters needs to be a list of three elements. The first element is 'grader' or 'alone': in the first case, the source file is to be compiled with a provided piece of software ('grader'); in the other by itself. The second element is a 2-tuple of the input file name and output file name. The input file may be '' to denote stdin, and similarly the output filename may be '' to denote stdout. The third element is 'diff' or 'comparator' and says whether the output is compared with a simple diff algorithm or using a comparator. Note: the first element is used only in the compilation step; the others only in the evaluation step. A comparator can read argv[1], argv[2], argv[3] (respectively, input, correct output and user output) and should write the outcome to stdout and the text to stderr. """ # Codename of the checker, if it is used. CHECKER_CODENAME = "checker" # Basename of the grader, used in the manager filename and as the main # class in languages that require us to specify it. GRADER_BASENAME = "grader" # Default input and output filenames when not provided as parameters. DEFAULT_INPUT_FILENAME = "input.txt" DEFAULT_OUTPUT_FILENAME = "output.txt" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" COMPILATION_ALONE = "alone" COMPILATION_GRADER = "grader" # Other constants to specify the task type behaviour and parameters. ALLOW_PARTIAL_SUBMISSION = False _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", { COMPILATION_ALONE: "Submissions are self-sufficient", COMPILATION_GRADER: "Submissions are compiled with a grader" }) _USE_FILE = ParameterTypeCollection( "I/O (blank for stdin/stdout)", "io", "", [ ParameterTypeString("Input file", "inputfile", ""), ParameterTypeString("Output file", "outputfile", ""), ]) _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a grader/comparator is used, etc... return "Batch" def __init__(self, parameters): super(Batch, self).__init__(parameters) # Data in the parameters. self.compilation = self.parameters[0] self.input_filename, self.output_filename = self.parameters[1] self.output_eval = self.parameters[2] # Actual input and output are the files used to store input and # where the output is checked, regardless of using redirects or not. self._actual_input = self.input_filename self._actual_output = self.output_filename if len(self.input_filename) == 0: self._actual_input = Batch.DEFAULT_INPUT_FILENAME if len(self.output_filename) == 0: self._actual_output = Batch.DEFAULT_OUTPUT_FILENAME def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" source_filenames = [] # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. if self._uses_grader(): source_filenames.append(Batch.GRADER_BASENAME + ".%l") source_filenames.append(submission_format[0]) executable_filename = submission_format[0].replace(".%l", "") res = dict() for language in LANGUAGES: res[language.name] = language.get_compilation_commands([ source.replace(".%l", language.source_extension) for source in source_filenames ], executable_filename) return res def get_user_managers(self): """See TaskType.get_user_managers.""" # In case the task uses a grader, we let the user provide their own # grader (which is usually a simplified grader provided by the admins). if self._uses_grader(): return ["grader.%l"] else: return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_grader(self): return self.compilation == Batch.COMPILATION_GRADER def _uses_checker(self): return self.output_eval == Batch.OUTPUT_EVAL_CHECKER def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension if not check_files_number(job, 1): return user_file_format = next(iterkeys(job.files)) user_source_filename = user_file_format.replace(".%l", source_ext) executable_filename = user_file_format.replace(".%l", "") # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line, and # we check that it exists. source_filenames = [user_source_filename] if self._uses_grader(): grader_source_filename = Batch.GRADER_BASENAME + source_ext if not check_manager_present(job, grader_source_filename): return source_filenames.insert(0, grader_source_filename) # Prepare the compilation command. commands = language.get_compilation_commands(source_filenames, executable_filename) # Create the sandbox. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.path) # Copy required files in the sandbox (includes the grader if present). sandbox.create_file_from_storage(user_source_filename, job.files[user_file_format].digest) for filename, manager in iteritems(job.managers): if is_manager_for_compilation(filename, language): sandbox.create_file_from_storage(filename, manager.digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables. job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup. delete_sandbox(sandbox, job.success) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands(executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } files_to_get = {self._actual_input: job.input} # Check which redirect we need to perform, and in case we don't # manage the output via redirect, the submission needs to be able # to write on it. files_allowing_write = [] stdin_redirect = None stdout_redirect = None if len(self.input_filename) == 0: stdin_redirect = self._actual_input if len(self.output_filename) == 0: stdout_redirect = self._actual_output else: files_allowing_write.append(self._actual_output) # Create the sandbox sandbox = create_sandbox(file_cacher, name="evaluate") job.sandboxes.append(sandbox.path) # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution box_success, evaluation_success, stats = evaluation_step( sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, multiprocess=job.multithreaded_sandbox) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self._actual_output): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), self._actual_output ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self._actual_output, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, Batch.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=sandbox.relative_path( self._actual_output), user_output_filename=self.output_filename) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(sandbox, job.success)
def setUp(self): super(TestParameterTypeChoice, self).setUp() self.p = ParameterTypeChoice("name", "shortname", "description", { "c1": "First choice", "c2": "Second choice", })
class TestParameterTypeChoice(unittest.TestCase): """Test the class ParameterTypeChoice.""" def setUp(self): super(TestParameterTypeChoice, self).setUp() self.p = ParameterTypeChoice("name", "shortname", "description", { "c1": "First choice", "c2": "Second choice", }) def test_validate_success(self): self.p.validate("c1") self.p.validate("c2") def test_validate_failure_wrong_type(self): with self.assertRaises(ValueError): self.p.validate("c3") with self.assertRaises(ValueError): self.p.validate(["c1"]) def test_parse_string(self): self.assertEqual(self.p.parse_string("c1"), "c1") with self.assertRaises(ValueError): self.p.parse_string("c3") def test_parse_handler(self): h = FakeHandler({ "ok_shortname": "c2", "fail_shortname": "c3", }) self.assertEqual(self.p.parse_handler(h, "ok_"), "c2") with self.assertRaises(ValueError): self.p.parse_handler(h, "fail_") with self.assertRaises(MissingArgumentError): self.p.parse_handler(h, "missing_")
class Communication2017Base(TaskType): """Task type class for tasks that requires: - a *manager* that reads the input file, work out the perfect solution on its own, and communicate the input (maybe with some modifications) on its standard output; it then reads the response of the user's solution from the standard input and write the outcome; - a *stub* that compiles with the user's source, reads from standard input what the manager says, and write back the user's solution to stdout. """ ALLOW_PARTIAL_SUBMISSION = False SUBMISSION_PAGE_MESSAGE = "" name = "Communication" _NUM_PROCESSES = ParameterTypeInt("Number of Processes", "num_processes", "") _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_NUM_PROCESSES, _EVALUATION] def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" res = dict() for language in LANGUAGES: source_ext = language.source_extension source_filenames = [] source_filenames.append("grader%s" % source_ext) executable_filename = \ "_".join(pattern.replace(".%l", "") for pattern in submission_format) for filename in submission_format: source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) commands = language.get_compilation_commands( source_filenames, executable_filename) res[language.name] = commands return res def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return None def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # Create the sandbox sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} source_filenames = [] # Stub. stub_filename = "grader%s" % source_ext source_filenames.append(stub_filename) files_to_get[stub_filename] = job.managers[stub_filename].digest # User's submission. for filename, fileinfo in job.files.iteritems(): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = fileinfo.digest # Also copy all managers that might be useful during compilation. for filename in job.managers.iterkeys(): if any(filename.endswith(header) for header in HEADER_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(source) for source in SOURCE_EXTS): files_to_get[filename] = \ job.managers[filename].digest elif any(filename.endswith(obj) for obj in OBJECT_EXTS): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = \ "_".join(pattern.replace(".%l", "") for pattern in job.files.keys()) commands = language.get_compilation_commands(source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(self.parameters) <= 0: num_processes = 1 else: num_processes = self.parameters[0] indices = range(num_processes) # Create sandboxes and FIFOs sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox) sandbox_user = [ create_sandbox(file_cacher, job.multithreaded_sandbox) for i in indices ] fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices] fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices] for i in indices: os.mkfifo(fifo_in[i]) os.mkfifo(fifo_out[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_in[i], 0o666) os.chmod(fifo_out[i], 0o666) # First step: we start the manager. manager_filename = "manager" manager_command = ["./%s" % manager_filename] for i in indices: manager_command.append(fifo_in[i]) manager_command.append(fifo_out[i]) manager_executables_to_get = { manager_filename: job.managers[manager_filename].digest } manager_files_to_get = {"input.txt": job.input} manager_allow_dirs = fifo_dir for filename, digest in manager_executables_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest, executable=True) for filename, digest in manager_files_to_get.iteritems(): sandbox_mgr.create_file_from_storage(filename, digest) manager = evaluation_step_before_run( sandbox_mgr, manager_command, num_processes * job.time_limit, 0, allow_dirs=manager_allow_dirs, writable_files=["output.txt"], stdin_redirect="input.txt", stdout_redirect="output.txt", ) # Second step: we start the user submission compiled with the # stub. language = get_language(job.language) executable_filename = job.executables.keys()[0] executables_to_get = { executable_filename: job.executables[executable_filename].digest } processes = [None for i in indices] for i in indices: args = [fifo_out[i], fifo_in[i]] if num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main="grader", args=args) user_allow_dirs = [fifo_dir[i]] for filename, digest in executables_to_get.iteritems(): sandbox_user[i].create_file_from_storage(filename, digest, executable=True) # Assumes that the actual execution of the user solution # is the last command in commands, and that the previous # are "setup" that doesn't need tight control. if len(commands) > 1: evaluation_step(sandbox_user[i], commands[:-1], 10, 256) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, allow_dirs=user_allow_dirs) # Consume output. wait_without_std(processes + [manager]) # TODO: check exit codes with translate_box_exitcode. user_results = [evaluation_step_after_run(s) for s in sandbox_user] success_user = all(r[0] for r in user_results) plus_user = reduce(merge_evaluation_results, [r[1] for r in user_results]) success_mgr, unused_plus_mgr = \ evaluation_step_after_run(sandbox_mgr) if plus_user['exit_status'] == Sandbox.EXIT_OK and \ plus_user["execution_time"] >= job.time_limit: plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT # Merge results. job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path] job.plus = plus_user # If at least one evaluation had problems, we report the # problems. if not success_user or not success_mgr: success, outcome, text = False, None, None # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not is_evaluation_passed(plus_user): success = True outcome, text = 0.0, human_evaluation_message(plus_user) if job.get_output: job.user_output = None # Otherwise, we use the manager to obtain the outcome. else: success = True outcome = None text = None input_filename = "input.txt" output_filename = "output.txt" # Check that the output file was created if not sandbox_mgr.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox_mgr.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=1024 * 1024 * 10) # If just asked to execute, fill text and set dummy # outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Put the reference solution into the sandbox sandbox_mgr.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[1] == "diff": outcome, text = white_diff_step( sandbox_mgr, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[1] == "comparator": manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox_mgr.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) # Rewrite input file. The untrusted # contestant program should not be able to # modify it; however, the grader may # destroy the input file to prevent the # contestant's program from directly # accessing it. Since we cannot create # files already existing in the sandbox, # we try removing the file first. try: sandbox_mgr.remove_file(input_filename) except OSError as e: # Let us be extra sure that the file # was actually removed and we did not # mess up with permissions. assert not sandbox_mgr.file_exists( input_filename) sandbox_mgr.create_file_from_storage( input_filename, job.input) # Allow using any number of processes (because e.g. # one may want to write a bash checker who calls # other processes). Set to a high number because # to avoid fork-bombing the worker. sandbox_mgr.max_processes = 1000 success, _ = evaluation_step( sandbox_mgr, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox_mgr) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e.message, extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized second parameter" " `%s' for Communication tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox_mgr, job.success) for s in sandbox_user: delete_sandbox(s, job.success) if not config.keep_sandbox: for d in fifo_dir: rmtree(d)
class Batch(TaskType): """Task type class for a unique standalone submission source, with comparator (or not). Parameters needs to be a list of three elements. The first element is 'grader' or 'alone': in the first case, the source file is to be compiled with a provided piece of software ('grader'); in the other by itself. The second element is a 2-tuple of the input file name and output file name. The input file may be '' to denote stdin, and similarly the output filename may be '' to denote stdout. The third element is 'diff' or 'comparator' and says whether the output is compared with a simple diff algorithm or using a comparator. Note: the first element is used only in the compilation step; the others only in the evaluation step. A comparator can read argv[1], argv[2], argv[3] (respectively, input, correct output and user output) and should write the outcome to stdout and the text to stderr. """ # Codename of the checker, if it is used. CHECKER_CODENAME = "checker" # Codename of the manager, when an interactive task is to be # evaluated in a single sandbox. MANAGER_CODENAME = "batchmanager" # Basename of the grader, used in the manager filename and as the main # class in languages that require us to specify it. GRADER_BASENAME = "grader" # Default input and output filenames when not provided as parameters. DEFAULT_INPUT_FILENAME = "input.txt" DEFAULT_OUTPUT_FILENAME = "output.txt" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" COMPILATION_ALONE = "alone" COMPILATION_GRADER = "grader" # Other constants to specify the task type behaviour and parameters. ALLOW_PARTIAL_SUBMISSION = False _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", {COMPILATION_ALONE: "Submissions are self-sufficient", COMPILATION_GRADER: "Submissions are compiled with a grader"}) _USE_FILE = ParameterTypeCollection( "I/O (blank for stdin/stdout)", "io", "", [ ParameterTypeString("Input file", "inputfile", ""), ParameterTypeString("Output file", "outputfile", ""), ]) _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", {OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator"}) ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a grader/comparator is used, etc... return "Batch" def __init__(self, parameters): super(Batch, self).__init__(parameters) # Data in the parameters. self.compilation = self.parameters[0] self.input_filename, self.output_filename = self.parameters[1] self.output_eval = self.parameters[2] # Actual input and output are the files used to store input and # where the output is checked, regardless of using redirects or not. self._actual_input = self.input_filename self._actual_output = self.output_filename if len(self.input_filename) == 0: self._actual_input = self.DEFAULT_INPUT_FILENAME if len(self.output_filename) == 0: self._actual_output = self.DEFAULT_OUTPUT_FILENAME def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" codenames_to_compile = [] if self._uses_grader(): codenames_to_compile.append(self.GRADER_BASENAME + ".%l") codenames_to_compile.extend(submission_format) executable_filename = self._executable_filename(submission_format) res = dict() for language in LANGUAGES: source_ext = language.source_extension res[language.name] = language.get_compilation_commands( [codename.replace(".%l", source_ext) for codename in codenames_to_compile], executable_filename) return res def get_user_managers(self): """See TaskType.get_user_managers.""" # In case the task uses a grader, we let the user provide their own # grader (which is usually a simplified grader provided by the admins). if self._uses_grader(): return [self.GRADER_BASENAME + ".%l"] else: return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_grader(self): return self.compilation == self.COMPILATION_GRADER def _uses_checker(self): return self.output_eval == self.OUTPUT_EVAL_CHECKER @staticmethod def _executable_filename(codenames): """Return the chosen executable name computed from the codenames. codenames ([str]): submission format or codename of submitted files, may contain %l. return (str): a deterministic executable name. """ return "_".join(sorted(codename.replace(".%l", "") for codename in codenames)) def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension if not check_files_number(job, 1, or_more=True): return # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line, and # we check that it exists. filenames_to_compile = [] filenames_and_digests_to_get = {} # The grader, that must have been provided (copy and add to # compilation). if self._uses_grader(): grader_filename = self.GRADER_BASENAME + source_ext if not check_manager_present(job, grader_filename): return filenames_to_compile.append(grader_filename) filenames_and_digests_to_get[grader_filename] = \ job.managers[grader_filename].digest # User's submitted file(s) (copy and add to compilation). for codename, file_ in iteritems(job.files): filename = codename.replace(".%l", source_ext) filenames_to_compile.append(filename) filenames_and_digests_to_get[filename] = file_.digest # Any other useful manager (just copy). for filename, manager in iteritems(job.managers): if is_manager_for_compilation(filename, language): filenames_and_digests_to_get[filename] = manager.digest # Prepare the compilation command. executable_filename = self._executable_filename(iterkeys(job.files)) commands = language.get_compilation_commands( filenames_to_compile, executable_filename) # Create the sandbox. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.get_root_path()) # Copy required files in the sandbox (includes the grader if present). for filename, digest in iteritems(filenames_and_digests_to_get): sandbox.create_file_from_storage(filename, digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables. job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup. delete_sandbox(sandbox, job) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = self.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands( executable_filename, main=main) # HACK for NECKLACE: one-time hack to support a task with very low memory limit if job.memory_limit == 3 and job.language == "Java / JDK": jvm_args = ["-Deval=true", "-Xmx4224k", "-Xss256k", "-XX:MaxMetaspaceSize=8704k"] commands = language.get_evaluation_commands( executable_filename, main=main, jvm_args=jvm_args) executables_to_get = { executable_filename: job.executables[executable_filename].digest } files_to_get = { self._actual_input: job.input } # Check which redirect we need to perform, and in case we don't # manage the output via redirect, the submission needs to be able # to write on it. files_allowing_write = [] stdin_redirect = None stdout_redirect = None if len(self.input_filename) == 0: stdin_redirect = self._actual_input if len(self.output_filename) == 0: stdout_redirect = self._actual_output else: files_allowing_write.append(self._actual_output) # Create the sandbox sandbox = create_sandbox(file_cacher, name="evaluate") job.sandboxes.append(sandbox.get_root_path()) # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Special handling: if there's a batchmanager, then this is really an # interactive task to be evaluated in a single sandbox. # Do NOT use check_manager_present() here, as it will raise an error # for normal tasks with no batchmanager. if Batch.MANAGER_CODENAME in job.managers: sandbox.create_file_from_storage(Batch.MANAGER_CODENAME, job.managers[Batch.MANAGER_CODENAME].digest, executable=True) # If there is a batchmanagermanager, run the last command with it. commands[-1][:0] = ["./%s" % Batch.MANAGER_CODENAME, self.input_filename, self.output_filename] # Actually performs the execution # HACK for NECKLACE: one-time hack to support a task with very low memory limit if job.memory_limit == 3 and job.language == "Java / JDK": memory_limit = 20 elif job.memory_limit == 3 and job.language == "Python 3 / CPython": memory_limit = 8 elif job.memory_limit == 3 and job.language == "C++11 / g++": memory_limit = 4 elif job.memory_limit == 3 and job.language == "C11 / gcc": memory_limit = 4 else: memory_limit = job.memory_limit box_success, evaluation_success, stats = evaluation_step( sandbox, commands, job.effective_time_limit(), memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, multiprocess=job.multithreaded_sandbox) outcome = None text = None # Error in the sandbox: nothing to do! if not box_success: pass # Contestant's error: the marks won't be good elif not evaluation_success: outcome = 0.0 text = human_evaluation_message(stats) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self._actual_output): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), self._actual_output] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self._actual_output, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: box_success, outcome, text = eval_output( file_cacher, job, self.CHECKER_CODENAME if self._uses_checker() else None, user_output_path=sandbox.relative_path( self._actual_output), user_output_filename=self.output_filename) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats delete_sandbox(sandbox, job)
def setUp(self): super().setUp() self.p = ParameterTypeChoice("name", "shortname", "description", { "c1": "First choice", "c2": "Second choice", })
class OutputOnly(TaskType): """Task type class for output only tasks, with submission composed of testcase_number text files, to be evaluated diffing or using a comparator. Parameters are a list of string with one element (for future possible expansions), which maybe 'diff' or 'comparator', meaning that the evaluation is done via white diff or via a comparator. """ ALLOW_PARTIAL_SUBMISSION = True SUBMISSION_PAGE_MESSAGE = """<span style='color: red'> You may submit any subset of outputs in a single submission. </span>""" _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Output only" testable = False def get_compilation_commands(self, unused_submission_format): """See TaskType.get_compilation_commands.""" return None def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def compile(self, job, file_cacher): """See TaskType.compile.""" # No compilation needed. job.success = True job.compilation_success = True job.text = [N_("No compilation needed")] job.plus = {} def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox) job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%s.txt" % job.operation["testcase_codename"] \ not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return True # First and only one step: diffing (manual or with manager). output_digest = job.files["output_%s.txt" % job.operation["testcase_codename"]].digest # Put the files into the sandbox sandbox.create_file_from_storage("res.txt", job.output) sandbox.create_file_from_storage("output.txt", output_digest) if self.parameters[0] == "diff": # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": # Manager present: wonderful, it will do all the work. manager_filename = "checker" if manager_filename not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) input_digest = job.input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step(sandbox, [[ "./%s" % manager_filename, "input.txt", "res.txt", "output.txt" ]]) if success: outcome, text = extract_outcome_and_text(sandbox) else: raise ValueError("Unrecognized first parameter " "`%s' for OutputOnly tasktype. " "Should be `diff' or `comparator'." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success)
class TestParameterTypeChoice(unittest.TestCase): """Test the class ParameterTypeChoice.""" def setUp(self): super().setUp() self.p = ParameterTypeChoice("name", "shortname", "description", { "c1": "First choice", "c2": "Second choice", }) def test_validate_success(self): self.p.validate("c1") self.p.validate("c2") def test_validate_failure_wrong_type(self): with self.assertRaises(ValueError): self.p.validate("c3") with self.assertRaises(ValueError): self.p.validate(["c1"]) def test_parse_string(self): self.assertEqual(self.p.parse_string("c1"), "c1") with self.assertRaises(ValueError): self.p.parse_string("c3") def test_parse_handler(self): h = FakeHandler({ "ok_shortname": "c2", "fail_shortname": "c3", }) self.assertEqual(self.p.parse_handler(h, "ok_"), "c2") with self.assertRaises(ValueError): self.p.parse_handler(h, "fail_") with self.assertRaises(MissingArgumentError): self.p.parse_handler(h, "missing_")
class Batch(TaskType): """Task type class for a unique standalone submission source, with comparator (or not). Parameters needs to be a list of three elements. The first element is 'grader' or 'alone': in the first case, the source file is to be compiled with a provided piece of software ('grader'); in the other by itself. The second element is a 2-tuple of the input file name and output file name. The input file may be '' to denote stdin, and similarly the output filename may be '' to denote stdout. The third element is 'diff' or 'comparator' and says whether the output is compared with a simple diff algorithm or using a comparator. Note: the first element is used only in the compilation step; the others only in the evaluation step. A comparator can read argv[1], argv[2], argv[3] (respectively, input, correct output and user output) and should write the outcome to stdout and the text to stderr. """ ALLOW_PARTIAL_SUBMISSION = False _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", { "alone": "Submissions are self-sufficient", "grader": "Submissions are compiled with a grader" }) _USE_FILE = ParameterTypeCollection( "I/O (blank for stdin/stdout)", "io", "", [ ParameterTypeString("Input file", "inputfile", ""), ParameterTypeString("Output file", "outputfile", ""), ]) _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a grader/comparator is used, etc... return "Batch" def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" res = dict() for language in LANGUAGES: format_filename = submission_format[0] source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] source_filenames = [] # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. if self.parameters[0] == "grader": source_filenames.append("grader%s" % source_ext) source_filenames.append(format_filename.replace(".%l", source_ext)) executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) res[language] = commands return res def get_user_managers(self, submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return None def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = job.language source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language] # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1" % len(job.files), extra={"operation": job.info}) return True # Create the sandbox sandbox = create_sandbox(file_cacher) job.sandboxes.append(sandbox.path) # Prepare the source files in the sandbox files_to_get = {} format_filename = job.files.keys()[0] source_filenames = [] source_filenames.append(format_filename.replace(".%l", source_ext)) files_to_get[source_filenames[0]] = \ job.files[format_filename].digest # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. The grader must # be the first file in source_filenames. if self.parameters[0] == "grader": source_filenames.insert(0, "grader%s" % source_ext) files_to_get["grader%s" % source_ext] = \ job.managers["grader%s" % source_ext].digest # Also copy all *.h and *lib.pas graders for filename in job.managers.iterkeys(): if filename.endswith('.h') or \ filename.endswith('lib.pas'): files_to_get[filename] = \ job.managers[filename].digest for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Prepare the compilation command executable_filename = format_filename.replace(".%l", "") commands = get_compilation_commands(language, source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # Create the sandbox sandbox = create_sandbox(file_cacher) # Prepare the execution executable_filename = job.executables.keys()[0] language = job.language commands = get_evaluation_commands(language, executable_filename) executables_to_get = { executable_filename: job.executables[executable_filename].digest } input_filename, output_filename = self.parameters[1] stdin_redirect = None stdout_redirect = None if input_filename == "": input_filename = "input.txt" stdin_redirect = input_filename if output_filename == "": output_filename = "output.txt" stdout_redirect = output_filename files_to_get = {input_filename: job.input} # Put the required files into the sandbox for filename, digest in executables_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in files_to_get.iteritems(): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = None # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = sandbox.get_file_to_storage( output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox sandbox.create_file_from_storage("res.txt", job.output) # Check the solution with white_diff if self.parameters[2] == "diff": outcome, text = white_diff_step( sandbox, output_filename, "res.txt") # Check the solution with a comparator elif self.parameters[2] == "comparator": manager_filename = "checker" if not manager_filename in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named 'checker')", extra={"operation": job.info}) success = False else: sandbox.create_file_from_storage( manager_filename, job.managers[manager_filename].digest, executable=True) success, _ = evaluation_step( sandbox, [[ "./%s" % manager_filename, input_filename, "res.txt", output_filename ]]) if success: try: outcome, text = \ extract_outcome_and_text(sandbox) except ValueError, e: logger.error("Invalid output from " "comparator: %s" % (e.message, ), extra={"operation": job.info}) success = False else: raise ValueError("Unrecognized third parameter" " `%s' for Batch tasktype." % self.parameters[2]) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox)
class Communication(TaskType): """Task type class for tasks with a fully admin-controlled process. The task type will run *manager*, an admin-provided executable, and one or more instances of the user solution, optionally compiled together with a language-specific stub. During the evaluation, the manager and each of the user processes communicate via FIFOs. The manager will read the input, send it (possibly with some modifications) to the user process(es). The user processes, either via functions provided by the stub or by themselves, will communicate with the manager. Finally, the manager will decide outcome and text, and print them on stdout and stderr. The manager reads the input from stdin and writes to stdout and stderr the standard manager output (that is, the outcome on stdout and the text on stderr, see trusted.py for more information). It receives as argument the names of the fifos: first from and to the first user process, then from and to the second user process, and so on. It can also print some information to a file named "output.txt"; the content of this file will be shown to users submitting a user test. The user process receives as argument the fifos (from and to the manager) and, if there are more than one user processes, the 0-based index of the process. The pipes can also be set up to be redirected to stdin/stdout: in that case the names of the pipes are not passed as arguments. """ # Filename of the manager (the stand-alone, admin-provided program). MANAGER_FILENAME = "manager" # Basename of the stub, used in the stub filename and as the main class in # languages that require us to specify it. STUB_BASENAME = "stub" # Filename of the input in the manager sandbox. The content will be # redirected to stdin, and managers should read from there. INPUT_FILENAME = "input.txt" # Filename where the manager can write additional output to show to users # in case of a user test. OUTPUT_FILENAME = "output.txt" # Constants used in the parameter definition. COMPILATION_ALONE = "alone" COMPILATION_STUB = "stub" USER_IO_STD = "std_io" USER_IO_FIFOS = "fifo_io" ALLOW_PARTIAL_SUBMISSION = False _NUM_PROCESSES = ParameterTypeInt("Number of Processes", "num_processes", "") _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", { COMPILATION_ALONE: "Submissions are self-sufficient", COMPILATION_STUB: "Submissions are compiled with a stub" }) _USER_IO = ParameterTypeChoice( "User I/O", "user_io", "", { USER_IO_STD: "User processes read from stdin and write to stdout", USER_IO_FIFOS: "User processes read from and write to fifos, " "whose paths are given as arguments" }) ACCEPTED_PARAMETERS = [_NUM_PROCESSES, _COMPILATION, _USER_IO] @property def name(self): """See TaskType.name.""" return "Communication" def __init__(self, parameters): super().__init__(parameters) self.num_processes = self.parameters[0] self.compilation = self.parameters[1] self.io = self.parameters[2] def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" codenames_to_compile = [] if self._uses_stub(): codenames_to_compile.append(self.STUB_BASENAME + ".%l") codenames_to_compile.extend(submission_format) executable_filename = self._executable_filename(submission_format) res = dict() for language in LANGUAGES: source_ext = language.source_extension res[language.name] = language.get_compilation_commands([ codename.replace(".%l", source_ext) for codename in codenames_to_compile ], executable_filename) return res def get_user_managers(self): """See TaskType.get_user_managers.""" if self._uses_stub(): return [self.STUB_BASENAME + ".%l"] else: return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [self.MANAGER_FILENAME] def _uses_stub(self): return self.compilation == self.COMPILATION_STUB def _uses_fifos(self): return self.io == self.USER_IO_FIFOS @staticmethod def _executable_filename(codenames): """Return the chosen executable name computed from the codenames. codenames ([str]): submission format or codename of submitted files, may contain %l. return (str): a deterministic executable name. """ return "_".join( sorted(codename.replace(".%l", "") for codename in codenames)) def compile(self, job, file_cacher): """See TaskType.compile.""" language = get_language(job.language) source_ext = language.source_extension if not check_files_number(job, 1, or_more=True): return # Prepare the files to copy in the sandbox and to add to the # compilation command. filenames_to_compile = [] filenames_and_digests_to_get = {} # The stub, that must have been provided (copy and add to compilation). if self._uses_stub(): stub_filename = self.STUB_BASENAME + source_ext if not check_manager_present(job, stub_filename): return filenames_to_compile.append(stub_filename) filenames_and_digests_to_get[stub_filename] = \ job.managers[stub_filename].digest # User's submitted file(s) (copy and add to compilation). for codename, file_ in job.files.items(): filename = codename.replace(".%l", source_ext) filenames_to_compile.append(filename) filenames_and_digests_to_get[filename] = file_.digest # Any other useful manager (just copy). for filename, manager in job.managers.items(): if is_manager_for_compilation(filename, language): filenames_and_digests_to_get[filename] = manager.digest # Prepare the compilation command executable_filename = self._executable_filename(job.files.keys()) commands = language.get_compilation_commands(filenames_to_compile, executable_filename) # Create the sandbox. sandbox = create_sandbox(file_cacher, name="compile") job.sandboxes.append(sandbox.get_root_path()) # Copy all required files in the sandbox. for filename, digest in filenames_and_digests_to_get.items(): sandbox.create_file_from_storage(filename, digest) # Run the compilation. box_success, compilation_success, text, stats = \ compilation_step(sandbox, commands) # Retrieve the compiled executables. job.success = box_success job.compilation_success = compilation_success job.text = text job.plus = stats if box_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup. delete_sandbox(sandbox, job.success, job.keep_sandbox) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if not check_executables_number(job, 1): return executable_filename = next(iter(job.executables.keys())) executable_digest = job.executables[executable_filename].digest # Make sure the required manager is among the job managers. if not check_manager_present(job, self.MANAGER_FILENAME): return manager_digest = job.managers[self.MANAGER_FILENAME].digest # Indices for the objects related to each user process. indices = range(self.num_processes) # Create FIFOs. fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] fifo_user_to_manager = [ os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices ] fifo_manager_to_user = [ os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices ] for i in indices: os.mkfifo(fifo_user_to_manager[i]) os.mkfifo(fifo_manager_to_user[i]) os.chmod(fifo_dir[i], 0o755) os.chmod(fifo_user_to_manager[i], 0o666) os.chmod(fifo_manager_to_user[i], 0o666) # Names of the fifos after being mapped inside the sandboxes. sandbox_fifo_dir = ["/fifo%d" % i for i in indices] sandbox_fifo_user_to_manager = [ os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices ] sandbox_fifo_manager_to_user = [ os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices ] # Create the manager sandbox and copy manager and input. sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate") job.sandboxes.append(sandbox_mgr.get_root_path()) sandbox_mgr.create_file_from_storage(self.MANAGER_FILENAME, manager_digest, executable=True) sandbox_mgr.create_file_from_storage(self.INPUT_FILENAME, job.input) # Create the user sandbox(es) and copy the executable. sandbox_user = [ create_sandbox(file_cacher, name="user_evaluate") for i in indices ] job.sandboxes.extend(s.get_root_path() for s in sandbox_user) for i in indices: sandbox_user[i].create_file_from_storage(executable_filename, executable_digest, executable=True) # Start the manager. Redirecting to stdin is unnecessary, but for # historical reasons the manager can choose to read from there # instead than from INPUT_FILENAME. manager_command = ["./%s" % self.MANAGER_FILENAME] for i in indices: manager_command += [ sandbox_fifo_user_to_manager[i], sandbox_fifo_manager_to_user[i] ] # We could use trusted_step for the manager, since it's fully # admin-controlled. But trusted_step is only synchronous at the moment. # Thus we use evaluation_step, and we set a time limit generous enough # to prevent user programs from sending the manager in timeout. # This means that: # - the manager wall clock timeout must be greater than the sum of all # wall clock timeouts of the user programs; # - with the assumption that the work the manager performs is not # greater than the work performed by the user programs, the manager # user timeout must be greater than the maximum allowed total time # of the user programs; in theory, this is the task's time limit, # but in practice is num_processes times that because the # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), config.trusted_sandbox_max_time_s) manager = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, config.trusted_sandbox_max_memory_kib // 1024, dirs_map=dict( (fifo_dir[i], (sandbox_fifo_dir[i], "rw")) for i in indices), writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, multiprocess=job.multithreaded_sandbox) # Start the user submissions compiled with the stub. language = get_language(job.language) main = self.STUB_BASENAME if self._uses_stub() else executable_filename processes = [None for i in indices] for i in indices: args = [] stdin_redirect = None stdout_redirect = None if self._uses_fifos(): args.extend([ sandbox_fifo_manager_to_user[i], sandbox_fifo_user_to_manager[i] ]) else: stdin_redirect = sandbox_fifo_manager_to_user[i] stdout_redirect = sandbox_fifo_user_to_manager[i] if self.num_processes != 1: args.append(str(i)) commands = language.get_evaluation_commands(executable_filename, main=main, args=args) # Assumes that the actual execution of the user solution is the # last command in commands, and that the previous are "setup" # that don't need tight control. if len(commands) > 1: trusted_step(sandbox_user[i], commands[:-1]) processes[i] = evaluation_step_before_run( sandbox_user[i], commands[-1], job.time_limit, job.memory_limit, dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")}, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect, multiprocess=job.multithreaded_sandbox) # Wait for the processes to conclude, without blocking them on I/O. wait_without_std(processes + [manager]) # Get the results of the manager sandbox. box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \ evaluation_step_after_run(sandbox_mgr) # Coalesce the results of the user sandboxes. user_results = [evaluation_step_after_run(s) for s in sandbox_user] box_success_user = all(r[0] for r in user_results) evaluation_success_user = all(r[1] for r in user_results) stats_user = reduce(merge_execution_stats, [r[2] for r in user_results]) # The actual running time is the sum of every user process, but each # sandbox can only check its own; if the sum is greater than the time # limit we adjust the result. if box_success_user and evaluation_success_user and \ stats_user["execution_time"] >= job.time_limit: evaluation_success_user = False stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT success = box_success_user \ and box_success_mgr and evaluation_success_mgr outcome = None text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). if not success: stats_user = None # If just asked to execute, fill text and set dummy outcome. elif job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # If the user sandbox detected some problem (timeout, ...), # the outcome is 0.0 and the text describes that problem. elif not evaluation_success_user: outcome = 0.0 text = human_evaluation_message(stats_user) # Otherwise, we use the manager to obtain the outcome. else: outcome, text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. if job.get_output: if sandbox_mgr.file_exists(self.OUTPUT_FILENAME): job.user_output = sandbox_mgr.get_file_to_storage( self.OUTPUT_FILENAME, "Output file in job %s" % job.info, trunc_len=100 * 1024) else: job.user_output = None # Fill in the job with the results. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox) for s in sandbox_user: delete_sandbox(s, job.success, job.keep_sandbox) if job.success and not config.keep_sandbox and not job.keep_sandbox: for d in fifo_dir: rmtree(d)
class TwoSteps(TaskType): """Task type class for tasks where the user must submit two files with a function each; the first function compute some data, that get passed to the second function that must recover some data. The admins must provide a manager source file (for each language), called manager.%l, that get compiled with both two user sources, get the input as stdin, and get two parameters: 0 if it is the first instance, 1 if it is the second instance, and the name of the pipe. Admins must provide also header files, named "foo{.h|lib.pas}" for the three sources (manager and user provided). Parameters are given by a singleton list of strings (for possible expansions in the future), which may be 'diff' or 'comparator', specifying whether the evaluation is done using white diff or a comparator. """ ALLOW_PARTIAL_SUBMISSION = False _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_EVALUATION] CHECKER_FILENAME = "checker" @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Two steps" def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" res = dict() for language in LANGUAGES: source_ext = language.source_extension header_ext = language.header_extension source_filenames = [] # Manager manager_source_filename = "manager%s" % source_ext source_filenames.append(manager_source_filename) # Manager's header. if header_ext is not None: manager_header_filename = "manager%s" % header_ext source_filenames.append(manager_header_filename) for filename in submission_format: source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) # Headers if header_ext is not None: header_filename = filename.replace(".%l", header_ext) source_filenames.append(header_filename) # Get compilation command and compile. executable_filename = "manager" commands = language.get_compilation_commands( source_filenames, executable_filename) res[language.name] = commands return res def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension header_ext = language.header_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 2: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 2", len(job.files), extra={"operation": job.info}) return # First and only one compilation. sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="compile") job.sandboxes.append(sandbox.path) files_to_get = {} source_filenames = [] # Manager. manager_filename = "manager%s" % source_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # Manager's header. if header_ext is not None: manager_filename = "manager%s" % header_ext source_filenames.append(manager_filename) files_to_get[manager_filename] = \ job.managers[manager_filename].digest # User's submissions and headers. for filename, file_ in iteritems(job.files): source_filename = filename.replace(".%l", source_ext) source_filenames.append(source_filename) files_to_get[source_filename] = file_.digest # Headers (fixing compile error again here). if header_ext is not None: header_filename = filename.replace(".%l", header_ext) source_filenames.append(header_filename) files_to_get[header_filename] = \ job.managers[header_filename].digest for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Get compilation command and compile. executable_filename = "manager" commands = language.get_compilation_commands(source_filenames, executable_filename) operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" # f stand for first, s for second. first_sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="first_evaluate") second_sandbox = create_sandbox( file_cacher, multithreaded=job.multithreaded_sandbox, name="second_evaluate") fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) os.chmod(fifo, 0o666) # First step: we start the first manager. first_filename = "manager" first_command = ["./%s" % first_filename, "0", fifo] first_executables_to_get = { first_filename: job.executables[first_filename].digest } first_files_to_get = {"input.txt": job.input} first_allow_path = [fifo_dir] # Put the required files into the sandbox for filename, digest in iteritems(first_executables_to_get): first_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(first_files_to_get): first_sandbox.create_file_from_storage(filename, digest) first = evaluation_step_before_run(first_sandbox, first_command, job.time_limit, job.memory_limit, first_allow_path, stdin_redirect="input.txt", wait=False) # Second step: we start the second manager. second_filename = "manager" second_command = ["./%s" % second_filename, "1", fifo] second_executables_to_get = { second_filename: job.executables[second_filename].digest } second_files_to_get = {} second_allow_path = [fifo_dir] # Put the required files into the second sandbox for filename, digest in iteritems(second_executables_to_get): second_sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(second_files_to_get): second_sandbox.create_file_from_storage(filename, digest) second = evaluation_step_before_run(second_sandbox, second_command, job.time_limit, job.memory_limit, second_allow_path, stdout_redirect="output.txt", wait=False) # Consume output. wait_without_std([second, first]) # TODO: check exit codes with translate_box_exitcode. success_first, first_plus = \ evaluation_step_after_run(first_sandbox) success_second, second_plus = \ evaluation_step_after_run(second_sandbox) job.sandboxes = [first_sandbox.path, second_sandbox.path] job.plus = second_plus success = True outcome = None text = [] # Error in the sandbox: report failure! if not success_first or not success_second: success = False # Contestant's error: the marks won't be good elif not is_evaluation_passed(first_plus) or \ not is_evaluation_passed(second_plus): outcome = 0.0 if not is_evaluation_passed(first_plus): text = human_evaluation_message(first_plus) else: text = human_evaluation_message(second_plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not second_sandbox.file_exists('output.txt'): outcome = 0.0 text = [N_("Evaluation didn't produce file %s"), "output.txt"] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage if job.get_output: job.user_output = second_sandbox.get_file_to_storage( "output.txt", "Output file in job %s" % job.info) # If not asked otherwise, evaluate the output file if not job.only_execution: # Put the reference solution into the sandbox second_sandbox.create_file_from_storage( "res.txt", job.output) # If a checker is not provided, use white-diff if self.parameters[0] == "diff": outcome, text = white_diff_step( second_sandbox, "output.txt", "res.txt") elif self.parameters[0] == "comparator": if TwoSteps.CHECKER_FILENAME not in job.managers: logger.error( "Configuration error: missing or " "invalid comparator (it must be " "named `checker')", extra={"operation": job.info}) success = False else: second_sandbox.create_file_from_storage( TwoSteps.CHECKER_FILENAME, job.managers[TwoSteps.CHECKER_FILENAME].digest, executable=True) # Rewrite input file, as in Batch.py try: second_sandbox.remove_file("input.txt") except OSError as e: assert not second_sandbox.file_exists( "input.txt") second_sandbox.create_file_from_storage( "input.txt", job.input) success, _ = evaluation_step( second_sandbox, [[ "./%s" % TwoSteps.CHECKER_FILENAME, "input.txt", "res.txt", "output.txt" ]]) if success: try: outcome, text = extract_outcome_and_text( second_sandbox) except ValueError as e: logger.error( "Invalid output from " "comparator: %s", e, extra={"operation": job.info}) success = False else: raise ValueError("Uncrecognized first parameter" " `%s' for TwoSteps tasktype." % self.parameters[0]) # Whatever happened, we conclude. job.success = success job.outcome = str(outcome) if outcome is not None else None job.text = text delete_sandbox(first_sandbox, job.success) delete_sandbox(second_sandbox, job.success) def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return ["manager.%l"]
class OutputOnly(TaskType): """Task type class for output only tasks, with submission composed of testcase_number text files, to be evaluated diffing or using a comparator. Parameters are a list of string with one element (for future possible expansions), which maybe 'diff' or 'comparator', meaning that the evaluation is done via white diff or via a comparator. """ ALLOW_PARTIAL_SUBMISSION = True _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { "diff": "Outputs compared with white diff", "comparator": "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Output only" testable = False def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" return None def get_user_managers(self, submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def compile(self): """See TaskType.compile.""" # No compilation needed. self.job.success = True self.job.compilation_success = True self.job.text = "No compilation needed." def evaluate_testcase(self, test_number): """See TaskType.evaluate_testcase.""" sandbox = create_sandbox(self) self.job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return self.job.evaluations[test_number] = { 'sandboxes': [sandbox.path], 'plus': {} } evaluation = self.job.evaluations[test_number] outcome = None text = None # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if "output_%03d.txt" % test_number not in self.job.files: evaluation['success'] = True evaluation['outcome'] = 0.0 evaluation['text'] = "File not submitted." return True # First and only one step: diffing (manual or with manager). output_digest = self.job.files["output_%03d.txt" % test_number].digest # Put the files into the sandbox sandbox.create_file_from_storage( "res.txt", self.job.testcases[test_number].output) sandbox.create_file_from_storage("output.txt", output_digest) # TODO: this should check self.parameters, not managers. if len(self.job.managers) == 0: # No manager: I'll do a white_diff between the submission # file and the correct output res.txt. success = True outcome, text = white_diff_step(sandbox, "output.txt", "res.txt") else: # Manager present: wonderful, he'll do all the job. manager_filename = self.job.managers.keys()[0] sandbox.create_file_from_storage( manager_filename, self.job.managers[manager_filename].digest, executable=True) input_digest = self.job.testcases[test_number].input sandbox.create_file_from_storage("input.txt", input_digest) success, _ = evaluation_step(sandbox, [ "./%s" % manager_filename, "input.txt", "res.txt", "output.txt" ], allow_path=[ "input.txt", "output.txt", "res.txt" ]) if success: outcome, text = extract_outcome_and_text(sandbox) # Whatever happened, we conclude. evaluation['success'] = success evaluation['outcome'] = str(outcome) if outcome is not None else None evaluation['text'] = text delete_sandbox(sandbox) return success
class OutputOnly(TaskType): """Task type class for output only tasks, with submission composed of testcase_number text files, to be evaluated diffing or using a comparator. Parameters are a list of string with one element (for future possible expansions), which maybe 'diff' or 'comparator', meaning that the evaluation is done via white diff or via a comparator. """ # Codename of the checker, if it is used. CHECKER_CODENAME = "checker" # Template for the filename of the output files provided by the user; %s # represent the testcase codename. USER_OUTPUT_FILENAME_TEMPLATE = "%s.out" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" # Other constants to specify the task type behaviour and parameters. ALLOW_PARTIAL_SUBMISSION = True _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", {OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator"}) ACCEPTED_PARAMETERS = [_EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Output only" testable = False def __init__(self, parameters): super().__init__(parameters) self.output_eval = self.parameters[0] def get_compilation_commands(self, unused_submission_format): """See TaskType.get_compilation_commands.""" return None def get_user_managers(self): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_checker(self): return self.output_eval == OutputOnly.OUTPUT_EVAL_CHECKER @staticmethod def _get_user_output_filename(job): return OutputOnly.USER_OUTPUT_FILENAME_TEMPLATE % \ job.operation.testcase_codename def compile(self, job, file_cacher): """See TaskType.compile.""" # No compilation needed. job.success = True job.compilation_success = True job.text = [N_("No compilation needed")] job.plus = {} def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" user_output_filename = self._get_user_output_filename(job) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if user_output_filename not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] job.plus = {} return # First and only step: eval the user output. box_success, outcome, text = eval_output( file_cacher, job, OutputOnly.CHECKER_CODENAME if self._uses_checker() else None, user_output_digest=job.files[user_output_filename].digest) # Fill in the job with the results. job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text # There is no actual evaluation, so no statistics. job.plus = {} if box_success else None
class Batch(TaskType): """Task type class for a unique standalone submission source, with comparator (or not). Parameters needs to be a list of three elements. The first element is 'grader' or 'alone': in the first case, the source file is to be compiled with a provided piece of software ('grader'); in the other by itself. The second element is a 2-tuple of the input file name and output file name. The input file may be '' to denote stdin, and similarly the output filename may be '' to denote stdout. The third element is 'diff' or 'comparator' and says whether the output is compared with a simple diff algorithm or using a comparator. Note: the first element is used only in the compilation step; the others only in the evaluation step. A comparator can read argv[1], argv[2], argv[3] (respectively, input, correct output and user output) and should write the outcome to stdout and the text to stderr. """ # Filename of the reference solution in the sandbox evaluating the output. CORRECT_OUTPUT_FILENAME = "res.txt" # Filename of the admin-provided comparator. CHECKER_FILENAME = "checker" # Basename of the grader, used in the manager filename and as the main # class in languages that require us to specify it. GRADER_BASENAME = "grader" # Default input and output filenames when not provided as parameters. DEFAULT_INPUT_FILENAME = "input.txt" DEFAULT_OUTPUT_FILENAME = "output.txt" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" COMPILATION_ALONE = "alone" COMPILATION_GRADER = "grader" # Other constants to specify the task type behaviour and parameters. ALLOW_PARTIAL_SUBMISSION = False _COMPILATION = ParameterTypeChoice( "Compilation", "compilation", "", { COMPILATION_ALONE: "Submissions are self-sufficient", COMPILATION_GRADER: "Submissions are compiled with a grader" }) _USE_FILE = ParameterTypeCollection( "I/O (blank for stdin/stdout)", "io", "", [ ParameterTypeString("Input file", "inputfile", ""), ParameterTypeString("Output file", "outputfile", ""), ]) _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a grader/comparator is used, etc... return "Batch" def __init__(self, parameters): super(Batch, self).__init__(parameters) self.compilation = self.parameters[0] self.input_filename, self.output_filename = self.parameters[1] self.output_eval = self.parameters[2] def get_compilation_commands(self, submission_format): """See TaskType.get_compilation_commands.""" source_filenames = [] # If a grader is specified, we add to the command line (and to # the files to get) the corresponding manager. if self._uses_grader(): source_filenames.append(Batch.GRADER_BASENAME + ".%l") source_filenames.append(submission_format[0]) executable_filename = submission_format[0].replace(".%l", "") res = dict() for language in LANGUAGES: res[language.name] = language.get_compilation_commands([ source.replace(".%l", language.source_extension) for source in source_filenames ], executable_filename) return res def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_grader(self): return self.compilation == Batch.COMPILATION_GRADER def _uses_checker(self): return self.output_eval == Batch.OUTPUT_EVAL_CHECKER @staticmethod def _is_manager_for_compilation(filename): """Return if a manager should be copied in the compilation sandbox""" return any(filename.endswith(header) for header in HEADER_EXTS) or \ any(filename.endswith(source) for source in SOURCE_EXTS) or \ any(filename.endswith(obj) for obj in OBJECT_EXTS) def compile(self, job, file_cacher): """See TaskType.compile.""" # Detect the submission's language. The checks about the # formal correctedness of the submission are done in CWS, # before accepting it. language = get_language(job.language) source_ext = language.source_extension # TODO: here we are sure that submission.files are the same as # task.submission_format. The following check shouldn't be # here, but in the definition of the task, since this actually # checks that task's task type and submission format agree. if len(job.files) != 1: job.success = True job.compilation_success = False job.text = [N_("Invalid files in submission")] logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info}) return # Create the sandbox. sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="compile") job.sandboxes.append(sandbox.path) user_file_format = next(iterkeys(job.files)) user_source_filename = user_file_format.replace(".%l", source_ext) executable_filename = user_file_format.replace(".%l", "") # Copy required files in the sandbox (includes the grader if present). sandbox.create_file_from_storage(user_source_filename, job.files[user_file_format].digest) for filename in iterkeys(job.managers): if Batch._is_manager_for_compilation(filename): sandbox.create_file_from_storage(filename, job.managers[filename].digest) # Create the list of filenames to be passed to the compiler. If we use # a grader, it needs to be in first position in the command line. source_filenames = [user_source_filename] if self._uses_grader(): grader_source_filename = Batch.GRADER_BASENAME + source_ext source_filenames.insert(0, grader_source_filename) # Prepare the compilation command. commands = language.get_compilation_commands(source_filenames, executable_filename) # Run the compilation operation_success, compilation_success, text, plus = \ compilation_step(sandbox, commands) # Retrieve the compiled executables job.success = operation_success job.compilation_success = compilation_success job.plus = plus job.text = text if operation_success and compilation_success: digest = sandbox.get_file_to_storage( executable_filename, "Executable %s for %s" % (executable_filename, job.info)) job.executables[executable_filename] = \ Executable(executable_filename, digest) # Cleanup delete_sandbox(sandbox, job.success) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" if len(job.executables) != 1: raise ValueError("Unexpected number of executables (%s)" % len(job.executables)) # Create the sandbox sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") # Prepare the execution executable_filename = next(iterkeys(job.executables)) language = get_language(job.language) main = Batch.GRADER_BASENAME \ if self._uses_grader() else executable_filename commands = language.get_evaluation_commands(executable_filename, main=main) executables_to_get = { executable_filename: job.executables[executable_filename].digest } stdin_redirect = None stdout_redirect = None files_allowing_write = [] if len(self.input_filename) == 0: self.input_filename = Batch.DEFAULT_INPUT_FILENAME stdin_redirect = self.input_filename if len(self.output_filename) == 0: self.output_filename = Batch.DEFAULT_OUTPUT_FILENAME stdout_redirect = self.output_filename else: files_allowing_write.append(self.output_filename) files_to_get = {self.input_filename: job.input} # Put the required files into the sandbox for filename, digest in iteritems(executables_to_get): sandbox.create_file_from_storage(filename, digest, executable=True) for filename, digest in iteritems(files_to_get): sandbox.create_file_from_storage(filename, digest) # Actually performs the execution success, plus = evaluation_step(sandbox, commands, job.time_limit, job.memory_limit, writable_files=files_allowing_write, stdin_redirect=stdin_redirect, stdout_redirect=stdout_redirect) job.sandboxes = [sandbox.path] job.plus = plus outcome = None text = [] # Error in the sandbox: nothing to do! if not success: pass # Contestant's error: the marks won't be good elif not is_evaluation_passed(plus): outcome = 0.0 text = human_evaluation_message(plus) if job.get_output: job.user_output = None # Otherwise, advance to checking the solution else: # Check that the output file was created if not sandbox.file_exists(self.output_filename): outcome = 0.0 text = [ N_("Evaluation didn't produce file %s"), self.output_filename ] if job.get_output: job.user_output = None else: # If asked so, put the output file into the storage. if job.get_output: job.user_output = sandbox.get_file_to_storage( self.output_filename, "Output file in job %s" % job.info, trunc_len=100 * 1024) # If just asked to execute, fill text and set dummy outcome. if job.only_execution: outcome = 0.0 text = [N_("Execution completed successfully")] # Otherwise evaluate the output file. else: # Create a brand-new sandbox just for checking. Only admin # code runs in it, so we allow multithreading and many # processes (still with a limit to avoid fork-bombs). checkbox = create_sandbox(file_cacher, multithreaded=True, name="check") checkbox.max_processes = 1000 checker_success, outcome, text = self._eval_output( checkbox, job, sandbox.get_root_path()) success = success and checker_success # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success) def _eval_output(self, sandbox, job, eval_sandbox_path): """Evaluate ("check") the output using a white diff or a checker. sandbox (Sandbox): the sandbox to use to eval the output. job (Job): the job triggering this checker run. return (bool, float|None, [str]): success (true if the checker was able to check the solution successfully), outcome and text. """ # Put the reference solution and input into the checkbox. sandbox.create_file_from_storage(Batch.CORRECT_OUTPUT_FILENAME, job.output) sandbox.create_file_from_storage(self.input_filename, job.input) # Put the user-produced output file into the checkbox output_src = os.path.join(eval_sandbox_path, self.output_filename) output_dst = os.path.join(sandbox.get_root_path(), self.output_filename) try: if os.path.islink(output_src): raise FileNotFoundError shutil.copyfile(output_src, output_dst) except FileNotFoundError: pass if self._uses_checker(): success, outcome, text = self._run_checker(sandbox, job) else: success = True outcome, text = white_diff_step(sandbox, self.output_filename, Batch.CORRECT_OUTPUT_FILENAME) delete_sandbox(sandbox, success) return success, outcome, text def _run_checker(self, sandbox, job): """Run the explicit checker given by the admins sandbox (Sandbox): the sandbox to run the checker in; should already contain input, correct output, and user output. job (Job): the job triggering this checker run. return (bool, float|None, [str]): success (true if the checker was able to check the solution successfully), outcome and text. """ # Copy the checker in the sandbox, after making sure it was provided. if Batch.CHECKER_FILENAME not in job.managers: logger.error( "Configuration error: missing or invalid comparator " "(it must be named '%s')", Batch.CHECKER_FILENAME, extra={"operation": job.info}) return False, None, [] sandbox.create_file_from_storage( Batch.CHECKER_FILENAME, job.managers[Batch.CHECKER_FILENAME].digest, executable=True) command = [ "./%s" % Batch.CHECKER_FILENAME, self.input_filename, Batch.CORRECT_OUTPUT_FILENAME, self.output_filename ] success, _ = evaluation_step(sandbox, [command]) if not success: return False, None, [] try: outcome, text = extract_outcome_and_text(sandbox) except ValueError as e: logger.error("Invalid output from comparator: %s", e, extra={"operation": job.info}) return False, None, [] return True, outcome, text
class OutputOnly(TaskType): """Task type class for output only tasks, with submission composed of testcase_number text files, to be evaluated diffing or using a comparator. Parameters are a list of string with one element (for future possible expansions), which maybe 'diff' or 'comparator', meaning that the evaluation is done via white diff or via a comparator. """ # Filename of the reference solution in the sandbox evaluating the output. CORRECT_OUTPUT_FILENAME = "res.txt" # Filename of the admin-provided comparator. CHECKER_FILENAME = "checker" # Name of input and user output files. INPUT_FILENAME = "input.txt" OUTPUT_FILENAME = "output.txt" # Template for the filename of the output files provided by the user; %s # represent the testcase codename. USER_OUTPUT_FILENAME_TEMPLATE = "output_%s.txt" # Constants used in the parameter definition. OUTPUT_EVAL_DIFF = "diff" OUTPUT_EVAL_CHECKER = "comparator" # Other constants to specify the task type behaviour and parameters. ALLOW_PARTIAL_SUBMISSION = True _EVALUATION = ParameterTypeChoice( "Output evaluation", "output_eval", "", { OUTPUT_EVAL_DIFF: "Outputs compared with white diff", OUTPUT_EVAL_CHECKER: "Outputs are compared by a comparator" }) ACCEPTED_PARAMETERS = [_EVALUATION] @property def name(self): """See TaskType.name.""" # TODO add some details if a comparator is used, etc... return "Output only" testable = False def __init__(self, parameters): super(OutputOnly, self).__init__(parameters) self.output_eval = self.parameters[0] def get_compilation_commands(self, unused_submission_format): """See TaskType.get_compilation_commands.""" return None def get_user_managers(self, unused_submission_format): """See TaskType.get_user_managers.""" return [] def get_auto_managers(self): """See TaskType.get_auto_managers.""" return [] def _uses_checker(self): return self.output_eval == OutputOnly.OUTPUT_EVAL_CHECKER @staticmethod def _get_user_output_filename(job): return OutputOnly.USER_OUTPUT_FILENAME_TEMPLATE % \ job.operation["testcase_codename"] def compile(self, job, file_cacher): """See TaskType.compile.""" # No compilation needed. job.success = True job.compilation_success = True job.text = [N_("No compilation needed")] job.plus = {} def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" sandbox = create_sandbox(file_cacher, multithreaded=job.multithreaded_sandbox, name="evaluate") job.sandboxes.append(sandbox.path) # Immediately prepare the skeleton to return job.sandboxes = [sandbox.path] job.plus = {} outcome = None text = [] user_output_filename = self._get_user_output_filename(job) # Since we allow partial submission, if the file is not # present we report that the outcome is 0. if user_output_filename not in job.files: job.success = True job.outcome = "0.0" job.text = [N_("File not submitted")] return # First and only one step: diffing (manual or with manager). # Put user output and reference solution into the sandbox. sandbox.create_file_from_storage( OutputOnly.OUTPUT_FILENAME, job.files[user_output_filename].digest) sandbox.create_file_from_storage(OutputOnly.CORRECT_OUTPUT_FILENAME, job.output) if self._uses_checker(): # Checker also requires the input file. sandbox.create_file_from_storage(OutputOnly.INPUT_FILENAME, job.input) success, outcome, text = OutputOnly._run_checker(sandbox, job) else: success = True outcome, text = white_diff_step(sandbox, OutputOnly.OUTPUT_FILENAME, OutputOnly.CORRECT_OUTPUT_FILENAME) # Whatever happened, we conclude. job.success = success job.outcome = "%s" % outcome if outcome is not None else None job.text = text delete_sandbox(sandbox, job.success) @staticmethod def _run_checker(sandbox, job): """Run the explicit checker given by the admins sandbox (Sandbox): the sandbox to run the checker in; should already contain input, correct output, and user output. job (Job): the job triggering this checker run. return (bool, float|None, [str]): success (true if the checker was able to check the solution successfully), outcome and text. """ # Copy the checker in the sandbox, after making sure it was provided. if OutputOnly.CHECKER_FILENAME not in job.managers: logger.error( "Configuration error: missing or invalid comparator " "(it must be named `%s')", OutputOnly.CHECKER_FILENAME, extra={"operation": job.info}) return False, None, [] sandbox.create_file_from_storage( OutputOnly.CHECKER_FILENAME, job.managers[OutputOnly.CHECKER_FILENAME].digest, executable=True) command = [ "./%s" % OutputOnly.CHECKER_FILENAME, OutputOnly.INPUT_FILENAME, OutputOnly.CORRECT_OUTPUT_FILENAME, OutputOnly.OUTPUT_FILENAME ] success, _ = evaluation_step(sandbox, [command]) if not success: return False, None, [] try: outcome, text = extract_outcome_and_text(sandbox) except ValueError as e: logger.error("Invalid output from comparator: %s", e, extra={"operation": job.info}) return False, None, [] return True, outcome, text