def test_correct_output_already_in_sandbox(self):
self.sandbox.fake_file(trusted.CHECKER_CORRECT_OUTPUT_FILENAME,
b"something")
ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o")
self.assertEqual(ret, (False, None, None))
self.assertLoggedError()
def test_missing_checker_text(self):
self.mock_trusted_step.return_value = (True, True, {})
self.set_checker_output(b"0.123\n", None)
ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o")
self.assertEqual(ret, (False, None, None))
self.assertLoggedError()
def test_invalid_checker_outcome(self):
self.mock_trusted_step.return_value = (True, True, {})
self.set_checker_output(b"A0.123\n", b"Text.\n")
ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o")
self.assertEqual(ret, (False, None, None))
self.assertLoggedError()
def test_checker_failure(self):
self.mock_trusted_step.return_value = (True, False, {})
# Output files are ignored.
self.set_checker_output(b"0.123\n", b"Text.\n")
ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o")
self.assertEqual(ret, (False, None, None))
self.assertLoggedError()
def test_success(self):
self.mock_trusted_step.return_value = (True, True, {})
self.set_checker_output(b"0.123\n", b"Text.\n")
ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o")
self.assertEqual(ret, (True, 0.123, ["Text."]))
self.file_cacher.get_file_to_fobj.assert_has_calls([
call("c_dig", ANY),
call("i_dig", ANY),
call("co_dig", ANY),
], any_order=True)
self.mock_trusted_step.assert_called_once_with(
self.sandbox, [["./checker", trusted.CHECKER_INPUT_FILENAME,
trusted.CHECKER_CORRECT_OUTPUT_FILENAME, "o"]])
self.assertLoggedError(False)
def eval_output(file_cacher, job, checker_codename,
user_output_path=None, user_output_digest=None,
user_output_filename=""):
"""Evaluate ("check") a user output using a white diff or a checker.
file_cacher (FileCacher): file cacher to use to get files.
job (Job): the job triggering this checker run.
checker_codename (str|None): codename of the checker amongst the manager,
or None to use white diff.
user_output_path (str|None): full path of the user output file, None if
using the digest (exactly one must be non-None).
user_output_digest (str|None): digest of the user output file, None if
using the path (exactly one must be non-None).
user_output_filename (str): the filename the user was expected to write to,
or empty if stdout (used to return an error to the user).
return (bool, float|None, [str]|None): success (true if the checker was
able to check the solution successfully), outcome and text (both None
if success is False).
"""
if (user_output_path is None) == (user_output_digest is None):
raise ValueError(
"Exactly one of user_output_{path,digest} should be None.")
if user_output_path is not None:
# If a path was passed, it might not exist. First, check it does. We
# also assume links are potential attacks, and therefore treat them
# as if the file did not exist.
if not os.path.exists(user_output_path) \
or os.path.islink(user_output_path):
return True, 0.0, [EVALUATION_MESSAGES.get("nooutput").message,
user_output_filename]
if checker_codename is not None:
if not check_manager_present(job, checker_codename):
return False, None, None
# Create a brand-new sandbox just for checking.
sandbox = create_sandbox(file_cacher, name="check")
job.sandboxes.append(sandbox.get_root_path())
# Put user output in the sandbox.
if user_output_path is not None:
shutil.copyfile(user_output_path,
sandbox.relative_path(EVAL_USER_OUTPUT_FILENAME))
else:
sandbox.create_file_from_storage(EVAL_USER_OUTPUT_FILENAME,
user_output_digest)
checker_digest = job.managers[checker_codename].digest \
if checker_codename in job.managers else None
success, outcome, text = checker_step(
sandbox, checker_digest, job.input, job.output,
EVAL_USER_OUTPUT_FILENAME)
delete_sandbox(sandbox, job, success)
return success, outcome, text
else:
if user_output_path is not None:
user_output_fobj = io.open(user_output_path, "rb")
else:
user_output_fobj = file_cacher.get_file(user_output_digest)
with user_output_fobj:
with file_cacher.get_file(job.output) as correct_output_fobj:
outcome, text = white_diff_fobj_step(
user_output_fobj, correct_output_fobj)
return True, outcome, text
def test_missing_checker(self):
ret = checker_step(self.sandbox, None, "i_dig", "co_dig", "o")
self.mock_trusted_step.assert_not_called()
self.assertEqual(ret, (False, None, None))
self.assertLoggedError()
def eval_output(file_cacher, job, checker_codename,
user_output_path=None, user_output_digest=None,
user_output_filename=""):
"""Evaluate ("check") a user output using a white diff or a checker.
file_cacher (FileCacher): file cacher to use to get files.
job (Job): the job triggering this checker run.
checker_codename (str|None): codename of the checker amongst the manager,
or None to use white diff.
user_output_path (str|None): full path of the user output file, None if
using the digest (exactly one must be non-None).
user_output_digest (str|None): digest of the user output file, None if
using the path (exactly one must be non-None).
user_output_filename (str): the filename the user was expected to write to,
or empty if stdout (used to return an error to the user).
return (bool, float|None, [str]|None): success (true if the checker was
able to check the solution successfully), outcome and text (both None
if success is False).
"""
if (user_output_path is None) == (user_output_digest is None):
raise ValueError(
"Exactly one of user_output_{path,digest} should be None.")
if user_output_path is not None:
# If a path was passed, it might not exist. First, check it does. We
# also assume links are potential attacks, and therefore treat them
# as if the file did not exist.
if not os.path.exists(user_output_path) \
or os.path.islink(user_output_path):
return True, 0.0, [EVALUATION_MESSAGES.get("nooutput").message,
user_output_filename]
if checker_codename is not None:
if not check_manager_present(job, checker_codename):
return False, None, None
# Create a brand-new sandbox just for checking.
sandbox = create_sandbox(file_cacher, name="check")
job.sandboxes.append(sandbox.get_root_path())
# Put user output in the sandbox.
if user_output_path is not None:
shutil.copyfile(user_output_path,
sandbox.relative_path(EVAL_USER_OUTPUT_FILENAME))
else:
sandbox.create_file_from_storage(EVAL_USER_OUTPUT_FILENAME,
user_output_digest)
checker_digest = job.managers[checker_codename].digest \
if checker_codename in job.managers else None
success, outcome, text = checker_step(
sandbox, checker_digest, job.input, job.output,
EVAL_USER_OUTPUT_FILENAME)
delete_sandbox(sandbox, success)
return success, outcome, text
else:
if user_output_path is not None:
user_output_fobj = io.open(user_output_path, "rb")
else:
user_output_fobj = file_cacher.get_file(user_output_digest)
with user_output_fobj:
with file_cacher.get_file(job.output) as correct_output_fobj:
outcome, text = white_diff_fobj_step(
user_output_fobj, correct_output_fobj)
return True, outcome, text
