def test_correct_output_already_in_sandbox(self): self.sandbox.fake_file(trusted.CHECKER_CORRECT_OUTPUT_FILENAME, b"something") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") self.assertEqual(ret, (False, None, None)) self.assertLoggedError()
def test_missing_checker_text(self): self.mock_trusted_step.return_value = (True, True, {}) self.set_checker_output(b"0.123\n", None) ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") self.assertEqual(ret, (False, None, None)) self.assertLoggedError()
def test_invalid_checker_outcome(self): self.mock_trusted_step.return_value = (True, True, {}) self.set_checker_output(b"A0.123\n", b"Text.\n") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") self.assertEqual(ret, (False, None, None)) self.assertLoggedError()
def test_checker_failure(self): self.mock_trusted_step.return_value = (True, False, {}) # Output files are ignored. self.set_checker_output(b"0.123\n", b"Text.\n") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") self.assertEqual(ret, (False, None, None)) self.assertLoggedError()
def test_success(self): self.mock_trusted_step.return_value = (True, True, {}) self.set_checker_output(b"0.123\n", b"Text.\n") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") self.assertEqual(ret, (True, 0.123, ["Text."])) self.file_cacher.get_file_to_fobj.assert_has_calls([ call("c_dig", ANY), call("i_dig", ANY), call("co_dig", ANY), ], any_order=True) self.mock_trusted_step.assert_called_once_with( self.sandbox, [["./checker", trusted.CHECKER_INPUT_FILENAME, trusted.CHECKER_CORRECT_OUTPUT_FILENAME, "o"]]) self.assertLoggedError(False)
def eval_output(file_cacher, job, checker_codename, user_output_path=None, user_output_digest=None, user_output_filename=""): """Evaluate ("check") a user output using a white diff or a checker. file_cacher (FileCacher): file cacher to use to get files. job (Job): the job triggering this checker run. checker_codename (str|None): codename of the checker amongst the manager, or None to use white diff. user_output_path (str|None): full path of the user output file, None if using the digest (exactly one must be non-None). user_output_digest (str|None): digest of the user output file, None if using the path (exactly one must be non-None). user_output_filename (str): the filename the user was expected to write to, or empty if stdout (used to return an error to the user). return (bool, float|None, [str]|None): success (true if the checker was able to check the solution successfully), outcome and text (both None if success is False). """ if (user_output_path is None) == (user_output_digest is None): raise ValueError( "Exactly one of user_output_{path,digest} should be None.") if user_output_path is not None: # If a path was passed, it might not exist. First, check it does. We # also assume links are potential attacks, and therefore treat them # as if the file did not exist. if not os.path.exists(user_output_path) \ or os.path.islink(user_output_path): return True, 0.0, [EVALUATION_MESSAGES.get("nooutput").message, user_output_filename] if checker_codename is not None: if not check_manager_present(job, checker_codename): return False, None, None # Create a brand-new sandbox just for checking. sandbox = create_sandbox(file_cacher, name="check") job.sandboxes.append(sandbox.get_root_path()) # Put user output in the sandbox. if user_output_path is not None: shutil.copyfile(user_output_path, sandbox.relative_path(EVAL_USER_OUTPUT_FILENAME)) else: sandbox.create_file_from_storage(EVAL_USER_OUTPUT_FILENAME, user_output_digest) checker_digest = job.managers[checker_codename].digest \ if checker_codename in job.managers else None success, outcome, text = checker_step( sandbox, checker_digest, job.input, job.output, EVAL_USER_OUTPUT_FILENAME) delete_sandbox(sandbox, job, success) return success, outcome, text else: if user_output_path is not None: user_output_fobj = io.open(user_output_path, "rb") else: user_output_fobj = file_cacher.get_file(user_output_digest) with user_output_fobj: with file_cacher.get_file(job.output) as correct_output_fobj: outcome, text = white_diff_fobj_step( user_output_fobj, correct_output_fobj) return True, outcome, text
def test_missing_checker(self): ret = checker_step(self.sandbox, None, "i_dig", "co_dig", "o") self.mock_trusted_step.assert_not_called() self.assertEqual(ret, (False, None, None)) self.assertLoggedError()
def eval_output(file_cacher, job, checker_codename, user_output_path=None, user_output_digest=None, user_output_filename=""): """Evaluate ("check") a user output using a white diff or a checker. file_cacher (FileCacher): file cacher to use to get files. job (Job): the job triggering this checker run. checker_codename (str|None): codename of the checker amongst the manager, or None to use white diff. user_output_path (str|None): full path of the user output file, None if using the digest (exactly one must be non-None). user_output_digest (str|None): digest of the user output file, None if using the path (exactly one must be non-None). user_output_filename (str): the filename the user was expected to write to, or empty if stdout (used to return an error to the user). return (bool, float|None, [str]|None): success (true if the checker was able to check the solution successfully), outcome and text (both None if success is False). """ if (user_output_path is None) == (user_output_digest is None): raise ValueError( "Exactly one of user_output_{path,digest} should be None.") if user_output_path is not None: # If a path was passed, it might not exist. First, check it does. We # also assume links are potential attacks, and therefore treat them # as if the file did not exist. if not os.path.exists(user_output_path) \ or os.path.islink(user_output_path): return True, 0.0, [EVALUATION_MESSAGES.get("nooutput").message, user_output_filename] if checker_codename is not None: if not check_manager_present(job, checker_codename): return False, None, None # Create a brand-new sandbox just for checking. sandbox = create_sandbox(file_cacher, name="check") job.sandboxes.append(sandbox.get_root_path()) # Put user output in the sandbox. if user_output_path is not None: shutil.copyfile(user_output_path, sandbox.relative_path(EVAL_USER_OUTPUT_FILENAME)) else: sandbox.create_file_from_storage(EVAL_USER_OUTPUT_FILENAME, user_output_digest) checker_digest = job.managers[checker_codename].digest \ if checker_codename in job.managers else None success, outcome, text = checker_step( sandbox, checker_digest, job.input, job.output, EVAL_USER_OUTPUT_FILENAME) delete_sandbox(sandbox, success) return success, outcome, text else: if user_output_path is not None: user_output_fobj = io.open(user_output_path, "rb") else: user_output_fobj = file_cacher.get_file(user_output_digest) with user_output_fobj: with file_cacher.get_file(job.output) as correct_output_fobj: outcome, text = white_diff_fobj_step( user_output_fobj, correct_output_fobj) return True, outcome, text