def test_alone_two_files(self):
tt = Batch(["alone", ["", ""], "diff"])
cc = tt.get_compilation_commands(["foo.%l", "bar.%l"])
self.assertEqual(cc, {
"L1": fake_compilation_commands(
COMPILATION_COMMAND_1, ["foo.l1", "bar.l1"], "bar_foo"),
"L2": fake_compilation_commands(
COMPILATION_COMMAND_2, ["foo.l2", "bar.l2"], "bar_foo"),
})
def test_grader(self):
tt = Batch(["grader", ["", ""], "diff"])
cc = tt.get_compilation_commands(["foo.%l"])
self.assertEqual(cc, {
"L1": fake_compilation_commands(
COMPILATION_COMMAND_1, ["foo.l1", "grader.l1"], "foo"),
"L2": fake_compilation_commands(
COMPILATION_COMMAND_2, ["foo.l2", "grader.l2"], "foo"),
})
def prepare(self, parameters, files=None, managers=None,
compilation_step_return_value=(True, True, TEXT, STATS_OK)):
tt = Batch(parameters)
job = self.job(files, managers)
if compilation_step_return_value is not None:
self.compilation_step.return_value = compilation_step_return_value
return tt, job
def prepare(self, parameters, executables):
tt = Batch(parameters)
job = self.job(executables)
self.evaluation_step.return_value = (True, True, STATS_OK)
self.eval_output.return_value = (True, OUTCOME, TEXT)
return tt, job
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# f stand for first, s for second.
if "user test" in job.info:
return Batch(
parameters=["grader", ("", ""), self.parameters[0]]).evaluate(
job, file_cacher)
first_sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
second_sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
language = get_language(job.language)
# First step: we start the first manager.
first_filename = "manager"
first_command = language.get_evaluation_commands(first_filename,
main="grader",
args=["0", fifo])
if len(first_command) > 1:
job.success = False
logger.error(
"Language contains %d commands for "
"evaluation, expecting 1",
len(first_command),
extra={"operation": job.info})
return
else:
first_command = first_command[0]
first_executables_to_get = {
first_filename: job.executables[first_filename].digest
}
first_files_to_get = {"input.txt": job.input}
first_allow_path = [fifo_dir]
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.iteritems():
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in first_files_to_get.iteritems():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
first_allow_path,
stdin_redirect="input.txt",
wait=False)
# Second step: we start the second manager.
second_filename = "manager"
second_command = language.get_evaluation_commands(second_filename,
main="grader",
args=["1", fifo])
if len(second_command) > 1:
job.success = False
logger.error(
"Language contains %d commands for "
"evaluation, expecting 1",
len(second_command),
extra={"operation": job.info})
return
second_command = second_command[0]
second_executables_to_get = {
second_filename: job.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo_dir]
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.iteritems():
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in second_files_to_get.iteritems():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
second_allow_path,
stdout_redirect="output.txt",
wait=False)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, first_plus = \
evaluation_step_after_run(first_sandbox)
success_second, second_plus = \
evaluation_step_after_run(second_sandbox)
job.sandboxes = [first_sandbox.path, second_sandbox.path]
job.plus = second_plus
success = True
outcome = None
text = None
# Error in the sandbox: report failure!
if not success_first or not success_second:
success = False
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(first_plus) or \
not is_evaluation_passed(second_plus):
outcome = 0.0
if not is_evaluation_passed(first_plus):
text = human_evaluation_message(first_plus)
else:
text = human_evaluation_message(second_plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists('output.txt'):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"), "output.txt"]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
# If not asked otherwise, evaluate the output file
if not job.only_execution:
# Put the reference solution into the sandbox
second_sandbox.create_file_from_storage(
"res.txt", job.output)
# If a checker is not provided, use white-diff
if self.parameters[0] == "diff":
outcome, text = white_diff_step(
second_sandbox, "output.txt", "res.txt")
elif self.parameters[0] == "comparator":
if TwoSteps2017.CHECKER_FILENAME not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named `checker')",
extra={"operation": job.info})
success = False
else:
second_sandbox.create_file_from_storage(
TwoSteps2017.CHECKER_FILENAME,
job.managers[
TwoSteps2017.CHECKER_FILENAME].digest,
executable=True)
# Rewrite input file, as in Batch.py
try:
second_sandbox.remove_file("input.txt")
except OSError as e:
assert not second_sandbox.file_exists(
"input.txt")
second_sandbox.create_file_from_storage(
"input.txt", job.input)
success, _ = evaluation_step(
second_sandbox, [[
"./%s" % TwoSteps2017.CHECKER_FILENAME,
"input.txt", "res.txt", "output.txt"
]],
allow_dirs=second_allow_path)
if success:
try:
outcome, text = extract_outcome_and_text(
second_sandbox)
except ValueError, e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Uncrecognized first parameter"
" `%s' for TwoSteps tasktype." %
self.parameters[0])
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
if "user test" in job.info:
return Batch(
parameters=["grader", ("", ""), self.parameters[0]]).compile(
job, file_cacher)
language = get_language(job.language)
source_ext = language.source_extension
header_ext = language.header_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.plus = {}
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 2",
len(job.files),
extra={"operation": job.info})
return True
# First and only one compilation.
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
files_to_get = {}
source_filenames = []
# Manager.
manager_filename = "grader%s" % source_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# User's submissions and headers.
for filename, file_ in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Headers (fixing compile error again here).
if header_ext is not None:
header_filename = filename.replace(".%l", header_ext)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
job.managers[header_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
commands = language.get_compilation_commands(source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)