def _eval_output(self, sandbox, job, eval_sandbox_path):
"""Evaluate ("check") the output using a white diff or a checker.
sandbox (Sandbox): the sandbox to use to eval the output.
job (Job): the job triggering this checker run.
return (bool, float|None, [str]): success (true if the checker was able
to check the solution successfully), outcome and text.
"""
# Put the reference solution and input into the checkbox.
sandbox.create_file_from_storage(Batch.CORRECT_OUTPUT_FILENAME,
job.output)
sandbox.create_file_from_storage(self.input_filename, job.input)
# Put the user-produced output file into the checkbox
output_src = os.path.join(eval_sandbox_path, self.output_filename)
output_dst = os.path.join(sandbox.get_root_path(),
self.output_filename)
try:
if os.path.islink(output_src):
raise FileNotFoundError
shutil.copyfile(output_src, output_dst)
except FileNotFoundError:
pass
if self._uses_checker():
success, outcome, text = self._run_checker(sandbox, job)
else:
success = True
outcome, text = white_diff_step(sandbox, self.output_filename,
Batch.CORRECT_OUTPUT_FILENAME)
delete_sandbox(sandbox, success)
return success, outcome, text
def _eval_output(self, sandbox, job, eval_sandbox_path):
"""Evaluate ("check") the output using a white diff or a checker.
sandbox (Sandbox): the sandbox to use to eval the output.
job (Job): the job triggering this checker run.
return (bool, float|None, [str]): success (true if the checker was able
to check the solution successfully), outcome and text.
"""
# Put the reference solution and input into the checkbox.
sandbox.create_file_from_storage(
Batch.CORRECT_OUTPUT_FILENAME, job.output)
sandbox.create_file_from_storage(self.input_filename, job.input)
# Put the user-produced output file into the checkbox
output_src = os.path.join(eval_sandbox_path, self.output_filename)
output_dst = os.path.join(
sandbox.get_root_path(), self.output_filename)
try:
if os.path.islink(output_src):
raise FileNotFoundError
shutil.copyfile(output_src, output_dst)
except FileNotFoundError:
pass
if self._uses_checker():
success, outcome, text = self._run_checker(sandbox, job)
else:
success = True
outcome, text = white_diff_step(
sandbox, self.output_filename, Batch.CORRECT_OUTPUT_FILENAME)
delete_sandbox(sandbox, success)
return success, outcome, text
def compile(self, job, file_cacher):
"""See TaskType.compile."""
language = get_language(job.language)
source_ext = language.source_extension
# Prepare the files to copy in the sandbox and to add to the
# compilation command.
files_to_get = {}
source_filenames = []
# The stub, that must have been provided (copy and add to compilation).
stub_filename = "stub%s" % source_ext
if not check_manager_present(job, stub_filename):
return
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submitted file(s) (copy and add to compilation).
for codename, file_ in iteritems(job.files):
source_filename = codename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Any other useful manager (just copy).
for filename, manager in iteritems(job.managers):
if is_manager_for_compilation(filename, language):
files_to_get[filename] = manager.digest
# Prepare the compilation command
executable_filename = self._executable_filename(iterkeys(job.files))
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Create the sandbox.
sandbox = create_sandbox(file_cacher, name="compile")
job.sandboxes.append(sandbox.path)
# Copy all required files in the sandbox.
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Run the compilation.
box_success, compilation_success, text, stats = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables.
job.success = box_success
job.compilation_success = compilation_success
job.text = text
job.plus = stats
if box_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup.
delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
language = get_language(job.language)
source_ext = language.source_extension
if not check_files_number(job, 1):
return
user_file_format = next(iterkeys(job.files))
user_source_filename = user_file_format.replace(".%l", source_ext)
executable_filename = user_file_format.replace(".%l", "")
# Create the list of filenames to be passed to the compiler. If we use
# a grader, it needs to be in first position in the command line, and
# we check that it exists.
source_filenames = [user_source_filename]
if self._uses_grader():
grader_source_filename = Batch.GRADER_BASENAME + source_ext
if not check_manager_present(job, grader_source_filename):
return
source_filenames.insert(0, grader_source_filename)
# Prepare the compilation command.
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Create the sandbox.
sandbox = create_sandbox(file_cacher, name="compile")
job.sandboxes.append(sandbox.path)
# Copy required files in the sandbox (includes the grader if present).
sandbox.create_file_from_storage(user_source_filename,
job.files[user_file_format].digest)
for filename, manager in iteritems(job.managers):
if is_manager_for_compilation(filename, language):
sandbox.create_file_from_storage(filename, manager.digest)
# Run the compilation.
box_success, compilation_success, text, stats = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables.
job.success = box_success
job.compilation_success = compilation_success
job.text = text
job.plus = stats
if box_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup.
delete_sandbox(sandbox, job.success)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.job.language
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.job.files) != 1:
self.job.success = True
self.job.compilation_success = False
self.job.text = "Invalid files in submission"
logger.error("Submission contains %d files, expecting 1" % len(self.job.files))
return True
# Create the sandbox
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = self.job.files.keys()[0]
source_filenames = []
# Stub.
source_filenames.append("stub.%s" % language)
files_to_get[source_filenames[-1]] = self.job.managers["stub.%s" % language].digest
# User's submission.
source_filenames.append(format_filename.replace("%l", language))
files_to_get[source_filenames[-1]] = self.job.files[format_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
command = get_compilation_command(language, source_filenames, executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = compilation_step(sandbox, command)
# Retrieve the compiled executables
self.job.success = operation_success
self.job.compilation_success = compilation_success
self.job.plus = plus
self.job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename, "Executable %s for %s" % (executable_filename, self.job.info)
)
self.job.executables[executable_filename] = Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
sandbox = create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="evaluate")
job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
job.sandboxes = [sandbox.path]
job.plus = {}
outcome = None
text = []
user_output_filename = self._get_user_output_filename(job)
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if user_output_filename not in job.files:
job.success = True
job.outcome = "0.0"
job.text = [N_("File not submitted")]
return
# First and only one step: diffing (manual or with manager).
# Put user output and reference solution into the sandbox.
sandbox.create_file_from_storage(
OutputOnly.OUTPUT_FILENAME, job.files[user_output_filename].digest)
sandbox.create_file_from_storage(
OutputOnly.CORRECT_OUTPUT_FILENAME, job.output)
if self._uses_checker():
# Checker also requires the input file.
sandbox.create_file_from_storage(
OutputOnly.INPUT_FILENAME, job.input)
success, outcome, text = OutputOnly._run_checker(sandbox, job)
else:
success = True
outcome, text = white_diff_step(
sandbox,
OutputOnly.OUTPUT_FILENAME,
OutputOnly.CORRECT_OUTPUT_FILENAME)
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
sandbox = create_sandbox(file_cacher,
multithreaded=job.multithreaded_sandbox,
name="evaluate")
job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
job.sandboxes = [sandbox.path]
job.plus = {}
outcome = None
text = []
user_output_filename = self._get_user_output_filename(job)
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if user_output_filename not in job.files:
job.success = True
job.outcome = "0.0"
job.text = [N_("File not submitted")]
return
# First and only one step: diffing (manual or with manager).
# Put user output and reference solution into the sandbox.
sandbox.create_file_from_storage(
OutputOnly.OUTPUT_FILENAME, job.files[user_output_filename].digest)
sandbox.create_file_from_storage(OutputOnly.CORRECT_OUTPUT_FILENAME,
job.output)
if self._uses_checker():
# Checker also requires the input file.
sandbox.create_file_from_storage(OutputOnly.INPUT_FILENAME,
job.input)
success, outcome, text = OutputOnly._run_checker(sandbox, job)
else:
success = True
outcome, text = white_diff_step(sandbox,
OutputOnly.OUTPUT_FILENAME,
OutputOnly.CORRECT_OUTPUT_FILENAME)
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.submission.language
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.submission.files) != 1:
return self.finish_compilation(
True, False, "Invalid files in submission",
to_log="Submission contains %d files, expecting 1" %
len(self.submission.files))
# First and only one compilation.
sandbox = create_sandbox(self)
files_to_get = {}
format_filename = self.submission.files.keys()[0]
source_filenames = [format_filename.replace("%l", language)]
files_to_get[source_filenames[0]] = \
self.submission.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager.
if self.parameters[0] == "grader":
source_filenames.append("grader.%s" % language)
files_to_get[source_filenames[1]] = \
self.submission.task.managers["grader.%s" % language].digest
executable_filename = format_filename.replace(".%l", "")
command = get_compilation_command(language,
source_filenames,
executable_filename)
operation_success, compilation_success, text, _ = \
self.compilation_step(
sandbox,
command,
files_to_get,
{executable_filename: "Executable %s for submission %s" %
(executable_filename, self.submission.id)})
delete_sandbox(sandbox)
# We had only one compilation, hence we pipe directly its
# result to the finalization.
return self.finish_compilation(operation_success, compilation_success,
text)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox = create_sandbox(self)
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%03d.txt" % test_number not in self.submission.files:
return self.finish_evaluation_testcase(
test_number,
True, 0.0, "File not submitted.")
# First and only one step: diffing (manual or with manager).
output_digest = self.submission.files["output_%03d.txt" %
test_number].digest
# TODO: this should check self.parameters, not managers.
if len(self.submission.task.managers) == 0:
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success, outcome, text = self.white_diff_step(
sandbox,
"output.txt", "res.txt",
{"res.txt":
self.submission.task.testcases[test_number].output,
"output.txt": output_digest})
else:
# Manager present: wonderful, he'll do all the job.
manager_filename = self.submission.task.managers.keys()[0]
success, outcome, text = self.evaluation_step(
sandbox,
["./%s" % manager_filename,
"input.txt", "res.txt", "output.txt"],
{manager_filename:
self.submission.task.managers[manager_filename].digest},
{"output.txt": output_digest,
"res.txt": self.submission.task.testcases[test_number].output,
"input.txt":
self.submission.task.testcases[test_number].input},
allow_path=["input.txt", "output.txt", "res.txt"],
final=True)
# Whatever happened, we conclude.
delete_sandbox(sandbox)
return self.finish_evaluation_testcase(test_number,
success, outcome, text)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# Create the sandbox
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
# Prepare the execution
assert len(job.executables) == 1
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
commands = language.get_evaluation_commands(
executable_filename,
main="grader" if self._uses_grader() else executable_filename)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
input_filename, output_filename = self.parameters[1]
stdin_redirect = None
stdout_redirect = None
files_allowing_write = []
if len(input_filename) == 0:
input_filename = "input.txt"
stdin_redirect = input_filename
if len(output_filename) == 0:
output_filename = "output.txt"
stdout_redirect = output_filename
else:
files_allowing_write.append(output_filename)
files_to_get = {input_filename: job.input}
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
success, plus = evaluation_step(sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect)
job.sandboxes = [sandbox.path]
job.plus = plus
outcome = None
text = []
# Error in the sandbox: nothing to do!
if not success:
pass
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(plus):
outcome = 0.0
text = human_evaluation_message(plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(output_filename):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"), output_filename
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
output_filename,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy
# outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
# Put the reference solution into the sandbox
sandbox.create_file_from_storage("res.txt", job.output)
# Check the solution with white_diff
if self.parameters[2] == "diff":
outcome, text = white_diff_step(
sandbox, output_filename, "res.txt")
# Check the solution with a comparator
elif self.parameters[2] == "comparator":
manager_filename = "checker"
if manager_filename not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named 'checker')",
extra={"operation": job.info})
success = False
else:
sandbox.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
# Rewrite input file. The untrusted
# contestant program should not be able to
# modify it; however, the grader may
# destroy the input file to prevent the
# contestant's program from directly
# accessing it. Since we cannot create
# files already existing in the sandbox,
# we try removing the file first.
try:
sandbox.remove_file(input_filename)
except OSError as e:
# Let us be extra sure that the file
# was actually removed and we did not
# mess up with permissions.
assert not sandbox.file_exists(input_filename)
sandbox.create_file_from_storage(
input_filename, job.input)
# Allow using any number of processes (because e.g.
# one may want to write a bash checker who calls
# other processes). Set to a high number because
# to avoid fork-bombing the worker.
sandbox.max_processes = 1000
success, _ = evaluation_step(
sandbox, [[
"./%s" % manager_filename, input_filename,
"res.txt", output_filename
]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError as e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
sandbox_user = [
create_sandbox(file_cacher, job.multithreaded_sandbox)
for i in indices
]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename: job.managers[manager_filename].digest
}
manager_files_to_get = {"input.txt": job.input}
manager_allow_dirs = fifo_dir
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt",
stdout_redirect="output.txt",
)
# Second step: we start the user submission compiled with the
# stub.
language = get_language(job.language)
executable_filename = job.executables.keys()[0]
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="grader",
args=args)
user_allow_dirs = [fifo_dir[i]]
for filename, digest in executables_to_get.iteritems():
sandbox_user[i].create_file_from_storage(filename,
digest,
executable=True)
# Assumes that the actual execution of the user solution
# is the last command in commands, and that the previous
# are "setup" that doesn't need tight control.
if len(commands) > 1:
evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
if job.get_output:
job.user_output = None
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome = None
text = None
input_filename = "input.txt"
output_filename = "output.txt"
# Check that the output file was created
if not sandbox_mgr.file_exists(output_filename):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"), output_filename
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = sandbox_mgr.get_file_to_storage(
output_filename,
"Output file in job %s" % job.info,
trunc_len=1024 * 1024 * 10)
# If just asked to execute, fill text and set dummy
# outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
# Put the reference solution into the sandbox
sandbox_mgr.create_file_from_storage("res.txt", job.output)
# Check the solution with white_diff
if self.parameters[1] == "diff":
outcome, text = white_diff_step(
sandbox_mgr, output_filename, "res.txt")
# Check the solution with a comparator
elif self.parameters[1] == "comparator":
manager_filename = "checker"
if manager_filename not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named 'checker')",
extra={"operation": job.info})
success = False
else:
sandbox_mgr.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
# Rewrite input file. The untrusted
# contestant program should not be able to
# modify it; however, the grader may
# destroy the input file to prevent the
# contestant's program from directly
# accessing it. Since we cannot create
# files already existing in the sandbox,
# we try removing the file first.
try:
sandbox_mgr.remove_file(input_filename)
except OSError as e:
# Let us be extra sure that the file
# was actually removed and we did not
# mess up with permissions.
assert not sandbox_mgr.file_exists(
input_filename)
sandbox_mgr.create_file_from_storage(
input_filename, job.input)
# Allow using any number of processes (because e.g.
# one may want to write a bash checker who calls
# other processes). Set to a high number because
# to avoid fork-bombing the worker.
sandbox_mgr.max_processes = 1000
success, _ = evaluation_step(
sandbox_mgr, [[
"./%s" % manager_filename, input_filename,
"res.txt", output_filename
]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox_mgr)
except ValueError as e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized second parameter"
" `%s' for Communication tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher)
sandbox_user = create_sandbox(file_cacher)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_in = os.path.join(fifo_dir, "in")
fifo_out = os.path.join(fifo_dir, "out")
os.mkfifo(fifo_in)
os.mkfifo(fifo_out)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo_in, 0o666)
os.chmod(fifo_out, 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
manager_executables_to_get = {
manager_filename: job.managers[manager_filename].digest
}
manager_files_to_get = {"input.txt": job.input}
manager_allow_dirs = [fifo_dir]
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(sandbox_mgr,
manager_command,
job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = job.executables.keys()[0]
command = ["./%s" % executable_filename, fifo_out, fifo_in]
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
user_allow_dirs = [fifo_dir]
for filename, digest in executables_to_get.iteritems():
sandbox_user.create_file_from_storage(filename,
digest,
executable=True)
process = evaluation_step_before_run(sandbox_user,
command,
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std([process, manager])
# TODO: check exit codes with translate_box_exitcode.
success_user, plus_user = \
evaluation_step_after_run(sandbox_user)
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
job.sandboxes = [sandbox_user.path, sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr)
delete_sandbox(sandbox_user)
if not config.keep_sandbox:
rmtree(fifo_dir)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox = create_sandbox(self)
# First step: execute the contestant program. This is also the
# final step if we have a grader, otherwise we need to run also
# a white_diff or a comparator.
executable_filename = self.submission.executables.keys()[0]
command = ["./%s" % executable_filename]
executables_to_get = {
executable_filename:
self.submission.executables[executable_filename].digest
}
input_filename, output_filename = self.parameters[1]
stdin_redirect = None
stdout_redirect = None
if input_filename == "":
input_filename = "input.txt"
stdin_redirect = input_filename
if output_filename == "":
output_filename = "output.txt"
stdout_redirect = output_filename
files_to_get = {
input_filename: self.submission.task.testcases[test_number].input
}
allow_path = [input_filename, output_filename]
success, outcome, text, plus = self.evaluation_step(
sandbox,
command,
executables_to_get,
files_to_get,
self.submission.task.time_limit,
self.submission.task.memory_limit,
allow_path,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
final=False)
# If an error occur (our or contestant's), return immediately.
if not success or outcome is not None:
delete_sandbox(sandbox)
return self.finish_evaluation_testcase(
test_number, success, outcome, text, plus)
# Second step: diffing (manual or with comparator).
if self.parameters[2] == "diff":
# We white_diff output.txt and res.txt.
success, outcome, text = self.white_diff_step(
sandbox,
output_filename, "res.txt",
{"res.txt":
self.submission.task.testcases[test_number].output})
elif self.parameters[2] == "comparator":
# Manager present: wonderful, it'll do all the job.
manager_filename = self.submission.task.managers.keys()[0]
success, outcome, text, _ = self.evaluation_step(
sandbox,
["./%s" % manager_filename,
input_filename, "res.txt", output_filename],
{manager_filename:
self.submission.task.managers[manager_filename].digest},
{"res.txt":
self.submission.task.testcases[test_number].output},
allow_path=[input_filename, "res.txt", output_filename],
final=True)
else:
raise ValueError("Unrecognized third parameter `%s' in for Batch "
"tasktype." % self.parameters[2])
# Whatever happened, we conclude.
delete_sandbox(sandbox)
return self.finish_evaluation_testcase(
test_number, success, outcome, text, plus)
sandbox.create_file_from_storage(
input_filename, job.input)
success, _ = evaluation_step(
sandbox, [[
"./%s" % manager_filename, input_filename,
"res.txt", output_filename
]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError, e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
self.job.evaluations[test_number] = {'sandboxes': [sandbox.path],
'plus': {}}
evaluation = self.job.evaluations[test_number]
outcome = None
text = None
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%03d.txt" % test_number not in self.job.files:
evaluation['success'] = True
evaluation['outcome'] = 0.0
evaluation['text'] = "File not submitted."
return True
# First and only one step: diffing (manual or with manager).
output_digest = self.job.files["output_%03d.txt" %
test_number].digest
# Put the files into the sandbox
sandbox.create_file_from_storage(
"res.txt",
self.job.testcases[test_number].output)
sandbox.create_file_from_storage(
"output.txt",
output_digest)
# TODO: this should check self.parameters, not managers.
if len(self.job.managers) == 0:
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success = True
outcome, text = white_diff_step(
sandbox, "output.txt", "res.txt")
else:
# Manager present: wonderful, he'll do all the job.
manager_filename = self.job.managers.keys()[0]
sandbox.create_file_from_storage(
manager_filename,
self.job.managers[manager_filename].digest,
executable=True)
input_digest = self.job.testcases[test_number].input
sandbox.create_file_from_storage(
"input.txt",
input_digest)
success, _ = evaluation_step(
sandbox,
["./%s" % manager_filename,
"input.txt", "res.txt", "output.txt"],
allow_path=["input.txt", "output.txt", "res.txt"])
if success:
outcome, text = extract_outcome_and_text(sandbox)
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = str(outcome) if outcome is not None else None
evaluation['text'] = text
delete_sandbox(sandbox)
return success
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
# f stand for first, s for second.
first_sandbox = create_sandbox(self)
second_sandbox = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
# First step: we start the first manager.
first_filename = "manager"
first_command = ["./%s" % first_filename, "0", fifo]
first_executables_to_get = {
first_filename: self.job.executables[first_filename].digest
}
first_files_to_get = {
"input.txt": self.job.testcases[test_number].input
}
first_allow_path = ["input.txt", fifo]
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.iteritems():
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in first_files_to_get.iteritems():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(first_sandbox,
first_command,
self.job.time_limit,
self.job.memory_limit,
first_allow_path,
stdin_redirect="input.txt",
wait=False)
# Second step: we start the second manager.
second_filename = "manager"
second_command = ["./%s" % second_filename, "1", fifo]
second_executables_to_get = {
second_filename: self.job.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo, "output.txt"]
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.iteritems():
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in second_files_to_get.iteritems():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(second_sandbox,
second_command,
self.job.time_limit,
self.job.memory_limit,
second_allow_path,
stdout_redirect="output.txt",
wait=False)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, first_plus = \
evaluation_step_after_run(first_sandbox)
success_second, second_plus = \
evaluation_step_after_run(second_sandbox)
self.job.evaluations[test_number] = {
'sandboxes': [first_sandbox.path, second_sandbox.path],
'plus': second_plus
}
outcome = None
text = None
evaluation = self.job.evaluations[test_number]
success = True
# Error in the sandbox: report failure!
if not success_first or not success_second:
success = False
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(first_plus) or \
not is_evaluation_passed(second_plus):
outcome = 0.0
if not is_evaluation_passed(first_plus):
text = human_evaluation_message(first_plus)
else:
text = human_evaluation_message(second_plus)
if self.job.get_output:
evaluation['output'] = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists('output.txt'):
outcome = 0.0
text = "Execution didn't produce file output.txt"
if self.job.get_output:
evaluation['output'] = None
else:
# If asked so, put the output file into the storage
if self.job.get_output:
evaluation['output'] = second_sandbox.get_file_to_storage(
"output.txt", "Output file for testcase %d in job %s" %
(test_number, self.job.info))
# If not asked otherwise, evaluate the output file
if not self.job.only_execution:
# Put the reference solution into the sandbox
second_sandbox.create_file_from_storage(
"res.txt", self.job.testcases[test_number].output)
outcome, text = white_diff_step(second_sandbox,
"output.txt", "res.txt")
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = outcome
evaluation['text'] = text
delete_sandbox(first_sandbox)
delete_sandbox(second_sandbox)
return success
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(self)
sandbox_user = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_in = os.path.join(fifo_dir, "in")
fifo_out = os.path.join(fifo_dir, "out")
os.mkfifo(fifo_in)
os.mkfifo(fifo_out)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
manager_executables_to_get = {
manager_filename:
self.job.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": self.job.testcases[test_number].input
}
manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(
filename, digest, executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
self.job.time_limit,
0,
manager_allow_path,
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = self.job.executables.keys()[0]
command = ["./%s" % executable_filename, fifo_out, fifo_in]
executables_to_get = {
executable_filename:
self.job.executables[executable_filename].digest
}
allow_path = [fifo_in, fifo_out]
for filename, digest in executables_to_get.iteritems():
sandbox_user.create_file_from_storage(
filename, digest, executable=True)
process = evaluation_step_before_run(
sandbox_user,
command,
self.job.time_limit,
self.job.memory_limit,
allow_path)
# Consume output.
wait_without_std([process, manager])
# TODO: check exit codes with translate_box_exitcode.
success_user, plus_user = \
evaluation_step_after_run(sandbox_user)
success_mgr, plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
self.job.evaluations[test_number] = \
{'sandboxes': [sandbox_user.path,
sandbox_mgr.path],
'plus': plus_user}
evaluation = self.job.evaluations[test_number]
# If at least one evaluation had problems, we report the
# problems. (TODO: shouldn't outcome and text better be None
# and None?)
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If outcome_user is not None, it is 0.0 and it means that
# there has been some errors in the user solution, and outcome
# and text are meaningful, so we use them.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if self.job.get_output:
if sandbox_mgr.file_exists("output.txt"):
evaluation['output'] = sandbox_mgr.get_file_to_storage(
"output.txt",
"Output file for testcase %d in job %s" %
(test_number, self.job.info))
else:
evaluation['output'] = None
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = str(outcome) if outcome is not None else None
evaluation['text'] = text
delete_sandbox(sandbox_mgr)
delete_sandbox(sandbox_user)
return success
class TwoSteps2017(TaskType):
"""Task type class for tasks where the user must submit two files
with a function each; the first function compute some data, that
get passed to the second function that must recover some data.
The admins must provide a manager source file (for each language),
called manager.%l, that get compiled with both two user sources,
get the input as stdin, and get two parameters: 0 if it is the
first instance, 1 if it is the second instance, and the name of
the pipe.
Admins must provide also header files, named "foo{.h|lib.pas}" for
the three sources (manager and user provided).
Parameters are given by a singleton list of strings (for possible
expansions in the future), which may be 'diff' or 'comparator',
specifying whether the evaluation is done using white diff or a
comparator.
"""
ALLOW_PARTIAL_SUBMISSION = False
SUBMISSION_PAGE_MESSAGE = ""
_EVALUATION = ParameterTypeChoice(
"Output evaluation", "output_eval", "", {
"diff": "Outputs compared with white diff",
"comparator": "Outputs are compared by a comparator"
})
ACCEPTED_PARAMETERS = [_EVALUATION]
CHECKER_FILENAME = "checker"
@property
def name(self):
"""See TaskType.name."""
# TODO add some details if a comparator is used, etc...
return "Two steps"
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
res = dict()
for language in LANGUAGES:
source_ext = language.source_extension
header_ext = language.header_extension
source_filenames = []
# Manager
manager_source_filename = "grader%s" % source_ext
source_filenames.append(manager_source_filename)
# Manager's header.
for filename in submission_format:
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
# Headers
if header_ext is not None:
header_filename = filename.replace(".%l", header_ext)
source_filenames.append(header_filename)
# Get compilation command and compile.
executable_filename = "manager"
commands = language.get_compilation_commands(
source_filenames, executable_filename)
res[language.name] = commands
return res
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
if "user test" in job.info:
return Batch(
parameters=["grader", ("", ""), self.parameters[0]]).compile(
job, file_cacher)
language = get_language(job.language)
source_ext = language.source_extension
header_ext = language.header_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.plus = {}
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 2",
len(job.files),
extra={"operation": job.info})
return True
# First and only one compilation.
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
files_to_get = {}
source_filenames = []
# Manager.
manager_filename = "grader%s" % source_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# User's submissions and headers.
for filename, file_ in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Headers (fixing compile error again here).
if header_ext is not None:
header_filename = filename.replace(".%l", header_ext)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
job.managers[header_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
commands = language.get_compilation_commands(source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# f stand for first, s for second.
if "user test" in job.info:
return Batch(
parameters=["grader", ("", ""), self.parameters[0]]).evaluate(
job, file_cacher)
first_sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
second_sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
language = get_language(job.language)
# First step: we start the first manager.
first_filename = "manager"
first_command = language.get_evaluation_commands(first_filename,
main="grader",
args=["0", fifo])
if len(first_command) > 1:
job.success = False
logger.error(
"Language contains %d commands for "
"evaluation, expecting 1",
len(first_command),
extra={"operation": job.info})
return
else:
first_command = first_command[0]
first_executables_to_get = {
first_filename: job.executables[first_filename].digest
}
first_files_to_get = {"input.txt": job.input}
first_allow_path = [fifo_dir]
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.iteritems():
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in first_files_to_get.iteritems():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
first_allow_path,
stdin_redirect="input.txt",
wait=False)
# Second step: we start the second manager.
second_filename = "manager"
second_command = language.get_evaluation_commands(second_filename,
main="grader",
args=["1", fifo])
if len(second_command) > 1:
job.success = False
logger.error(
"Language contains %d commands for "
"evaluation, expecting 1",
len(second_command),
extra={"operation": job.info})
return
second_command = second_command[0]
second_executables_to_get = {
second_filename: job.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo_dir]
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.iteritems():
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in second_files_to_get.iteritems():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
second_allow_path,
stdout_redirect="output.txt",
wait=False)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, first_plus = \
evaluation_step_after_run(first_sandbox)
success_second, second_plus = \
evaluation_step_after_run(second_sandbox)
job.sandboxes = [first_sandbox.path, second_sandbox.path]
job.plus = second_plus
success = True
outcome = None
text = None
# Error in the sandbox: report failure!
if not success_first or not success_second:
success = False
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(first_plus) or \
not is_evaluation_passed(second_plus):
outcome = 0.0
if not is_evaluation_passed(first_plus):
text = human_evaluation_message(first_plus)
else:
text = human_evaluation_message(second_plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists('output.txt'):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"), "output.txt"]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
# If not asked otherwise, evaluate the output file
if not job.only_execution:
# Put the reference solution into the sandbox
second_sandbox.create_file_from_storage(
"res.txt", job.output)
# If a checker is not provided, use white-diff
if self.parameters[0] == "diff":
outcome, text = white_diff_step(
second_sandbox, "output.txt", "res.txt")
elif self.parameters[0] == "comparator":
if TwoSteps2017.CHECKER_FILENAME not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named `checker')",
extra={"operation": job.info})
success = False
else:
second_sandbox.create_file_from_storage(
TwoSteps2017.CHECKER_FILENAME,
job.managers[
TwoSteps2017.CHECKER_FILENAME].digest,
executable=True)
# Rewrite input file, as in Batch.py
try:
second_sandbox.remove_file("input.txt")
except OSError as e:
assert not second_sandbox.file_exists(
"input.txt")
second_sandbox.create_file_from_storage(
"input.txt", job.input)
success, _ = evaluation_step(
second_sandbox, [[
"./%s" % TwoSteps2017.CHECKER_FILENAME,
"input.txt", "res.txt", "output.txt"
]],
allow_dirs=second_allow_path)
if success:
try:
outcome, text = extract_outcome_and_text(
second_sandbox)
except ValueError, e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Uncrecognized first parameter"
" `%s' for TwoSteps tasktype." %
self.parameters[0])
# Whatever happened, we conclude.
job.success = success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
delete_sandbox(first_sandbox, job.success)
delete_sandbox(second_sandbox, job.success)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox_mgr = create_sandbox(self)
sandbox_user = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_in = os.path.join(fifo_dir, "in")
fifo_out = os.path.join(fifo_dir, "out")
os.mkfifo(fifo_in)
os.mkfifo(fifo_out)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
manager_executables_to_get = {
manager_filename:
self.submission.task.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": self.submission.task.testcases[test_number].input
}
manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
manager = self.evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_executables_to_get,
manager_files_to_get,
self.submission.task.time_limit,
0,
manager_allow_path,
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = self.submission.executables.keys()[0]
command = ["./%s" % executable_filename, fifo_out, fifo_in]
executables_to_get = {
executable_filename:
self.submission.executables[executable_filename].digest
}
allow_path = [fifo_in, fifo_out]
process = self.evaluation_step_before_run(
sandbox_user,
command,
executables_to_get,
{},
self.submission.task.time_limit,
self.submission.task.memory_limit,
allow_path)
# Consume output.
wait_without_std([process, manager])
# TODO: check exit codes with translate_box_exitcode.
success_user, outcome_user, text_user, plus = \
self.evaluation_step_after_run(sandbox_user, final=False)
success_mgr, outcome_mgr, text_mgr, _ = \
self.evaluation_step_after_run(sandbox_mgr, final=True)
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, outcome_user, text_user
# If outcome_user is not None, it is 0.0 and it means that
# there has been some errors in the user solution, and outcome
# and text are meaningful, so we use them.
elif outcome_user is not None:
success, outcome, text = success_user, outcome_user, text_user
# Otherwise, we use the manager to obtain the outcome.
else:
success, outcome, text = success_mgr, outcome_mgr, text_mgr
delete_sandbox(sandbox_mgr)
delete_sandbox(sandbox_user)
return self.finish_evaluation_testcase(
test_number, success, outcome, text, plus)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = Batch.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(executable_filename,
main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {self._actual_input: job.input}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.path)
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
self._actual_output
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher,
job,
Batch.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
job.sandboxes.append(first_sandbox.path)
job.sandboxes.append(second_sandbox.path)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_command = ["./%s" % executable_filename, "0", fifo]
first_executables_to_get = {executable_filename: executable_digest}
first_files_to_get = {
TwoSteps.INPUT_FILENAME: job.input
}
first_allow_path = [fifo_dir]
# Put the required files into the sandbox
for filename, digest in iteritems(first_executables_to_get):
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(first_files_to_get):
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
first_allow_path,
stdin_redirect=TwoSteps.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Second step: we start the second manager.
second_command = ["./%s" % executable_filename, "1", fifo]
second_executables_to_get = {executable_filename: executable_digest}
second_files_to_get = {}
second_allow_path = [fifo_dir]
# Put the required files into the second sandbox
for filename, digest in iteritems(second_executables_to_get):
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(second_files_to_get):
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
second_allow_path,
stdout_redirect=TwoSteps.OUTPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Consume output.
wait_without_std([second, first])
box_success_first, evaluation_success_first, first_stats = \
evaluation_step_after_run(first_sandbox)
box_success_second, evaluation_success_second, second_stats = \
evaluation_step_after_run(second_sandbox)
box_success = box_success_first and box_success_second
evaluation_success = \
evaluation_success_first and evaluation_success_second
stats = merge_execution_stats(first_stats, second_stats)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
TwoSteps.OUTPUT_FILENAME]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
TwoSteps.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
TwoSteps.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=second_sandbox.relative_path(
TwoSteps.OUTPUT_FILENAME))
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(first_sandbox, job.success)
delete_sandbox(second_sandbox, job.success)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
language = get_language(job.language)
source_ext = language.source_extension
header_ext = language.header_extension
if not check_files_number(job, 2):
return
files_to_get = {}
source_filenames = []
# Manager.
manager_filename = "manager%s" % source_ext
if not check_manager_present(job, manager_filename):
return
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# Manager's header.
if header_ext is not None:
manager_filename = "manager%s" % header_ext
if not check_manager_present(job, manager_filename):
return
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# User's submissions and headers.
for filename, file_ in iteritems(job.files):
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Headers (fixing compile error again here).
if header_ext is not None:
header_filename = filename.replace(".%l", header_ext)
if not check_manager_present(job, header_filename):
return
source_filenames.append(header_filename)
files_to_get[header_filename] = \
job.managers[header_filename].digest
# Get compilation command.
executable_filename = "manager"
commands = language.get_compilation_commands(
source_filenames, executable_filename)
# Create the sandbox and put the required files in it.
sandbox = create_sandbox(file_cacher, name="compile")
job.sandboxes.append(sandbox.path)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Run the compilation.
box_success, compilation_success, text, stats = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = box_success
job.compilation_success = compilation_success
job.text = text
job.plus = stats
if box_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
header_ext = LANGUAGE_TO_HEADER_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 2:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 2",
len(job.files), extra={"operation": job.info})
return True
# First and only one compilation.
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
files_to_get = {}
# User's submissions and headers.
source_filenames = []
for filename, file_ in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Headers.
header_filename = filename.replace(".%l", header_ext)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
job.managers[header_filename].digest
# Manager.
manager_filename = "manager%s" % source_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# Manager's header.
manager_filename = "manager%s" % header_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
commands = get_compilation_commands(language,
source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
self.job.evaluations[test_number] = {"sandboxes": [sandbox.path], "plus": {}}
evaluation = self.job.evaluations[test_number]
outcome = None
text = None
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%03d.txt" % test_number not in self.job.files:
evaluation["success"] = True
evaluation["outcome"] = 0.0
evaluation["text"] = "File not submitted."
return True
# First and only one step: diffing (manual or with manager).
output_digest = self.job.files["output_%03d.txt" % test_number].digest
# Put the files into the sandbox
sandbox.create_file_from_storage("res.txt", self.job.testcases[test_number].output)
sandbox.create_file_from_storage("output.txt", output_digest)
if self.job.task_type_parameters[0] == "diff":
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success = True
outcome, text = white_diff_step(sandbox, "output.txt", "res.txt")
elif self.job.task_type_parameters[0] == "comparator":
# Manager present: wonderful, he'll do all the job.
manager_filename = "checker"
if not manager_filename in self.job.managers:
logger.error("Configuration error: missing or " "invalid comparator (it must be " "named `checker')")
success = False
else:
sandbox.create_file_from_storage(
manager_filename, self.job.managers[manager_filename].digest, executable=True
)
input_digest = self.job.testcases[test_number].input
sandbox.create_file_from_storage("input.txt", input_digest)
success, _ = evaluation_step(
sandbox,
["./%s" % manager_filename, "input.txt", "res.txt", "output.txt"],
allow_path=["input.txt", "output.txt", "res.txt"],
)
if success:
outcome, text = extract_outcome_and_text(sandbox)
else:
raise ValueError(
"Unrecognized first parameter "
"`%s' for OutputOnly tasktype. "
"Should be `diff' or `comparator'." % self.job.task_type_parameters[0]
)
# Whatever happened, we conclude.
evaluation["success"] = success
evaluation["outcome"] = str(outcome) if outcome is not None else None
evaluation["text"] = text
delete_sandbox(sandbox)
return success
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
header_ext = language.header_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 2:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 2",
len(job.files),
extra={"operation": job.info})
return
# First and only one compilation.
sandbox = create_sandbox(file_cacher,
multithreaded=job.multithreaded_sandbox,
name="compile")
job.sandboxes.append(sandbox.path)
files_to_get = {}
source_filenames = []
# Manager.
manager_filename = "manager%s" % source_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# Manager's header.
if header_ext is not None:
manager_filename = "manager%s" % header_ext
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
job.managers[manager_filename].digest
# User's submissions and headers.
for filename, file_ in iteritems(job.files):
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Headers (fixing compile error again here).
if header_ext is not None:
header_filename = filename.replace(".%l", header_ext)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
job.managers[header_filename].digest
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
commands = language.get_compilation_commands(source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
job.plus = {}
logger.error("Submission contains %d files, expecting 1",
len(job.files), extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[0]] = \
job.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager. The grader must
# be the first file in source_filenames.
compile_command = []
if self._uses_grader():
files_to_get["grader%s" % source_ext] = \
job.managers["grader%s" % source_ext].digest
# For solutions using C or C++,
# we first compile the grader source
# file and then delete it from sandbox,
# to prevent the user's solution
# files from including it.
try:
compile_command = language.get_compilation_no_link_command(
["grader%s" % source_ext])
compile_command += [["/bin/rm", "grader%s" % source_ext]]
except NotImplementedError:
compile_command = []
source_filenames.insert(0, "grader%s" % source_ext)
else:
source_filenames.insert(0, "grader%s" %
language.object_extension)
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(filename.endswith(header) for header in
language.header_extensions):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(source) for source in
language.source_extensions):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(obj) for obj in
language.object_extensions):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = compile_command + language.get_compilation_commands(
source_filenames, executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# f stand for first, s for second.
first_sandbox = create_sandbox(file_cacher,
multithreaded=job.multithreaded_sandbox,
name="first_evaluate")
second_sandbox = create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="second_evaluate")
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_filename = "manager"
first_command = ["./%s" % first_filename, "0", fifo]
first_executables_to_get = {
first_filename: job.executables[first_filename].digest
}
first_files_to_get = {"input.txt": job.input}
first_allow_path = [fifo_dir]
# Put the required files into the sandbox
for filename, digest in iteritems(first_executables_to_get):
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(first_files_to_get):
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
first_allow_path,
stdin_redirect="input.txt",
wait=False)
# Second step: we start the second manager.
second_filename = "manager"
second_command = ["./%s" % second_filename, "1", fifo]
second_executables_to_get = {
second_filename: job.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo_dir]
# Put the required files into the second sandbox
for filename, digest in iteritems(second_executables_to_get):
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(second_files_to_get):
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
second_allow_path,
stdout_redirect="output.txt",
wait=False)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, first_plus = \
evaluation_step_after_run(first_sandbox)
success_second, second_plus = \
evaluation_step_after_run(second_sandbox)
job.sandboxes = [first_sandbox.path, second_sandbox.path]
job.plus = second_plus
success = True
outcome = None
text = []
# Error in the sandbox: report failure!
if not success_first or not success_second:
success = False
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(first_plus) or \
not is_evaluation_passed(second_plus):
outcome = 0.0
if not is_evaluation_passed(first_plus):
text = human_evaluation_message(first_plus)
else:
text = human_evaluation_message(second_plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists('output.txt'):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"), "output.txt"]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
# If not asked otherwise, evaluate the output file
if not job.only_execution:
# Put the reference solution into the sandbox
second_sandbox.create_file_from_storage(
"res.txt", job.output)
# If a checker is not provided, use white-diff
if self.parameters[0] == "diff":
outcome, text = white_diff_step(
second_sandbox, "output.txt", "res.txt")
elif self.parameters[0] == "comparator":
if TwoSteps.CHECKER_FILENAME not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named `checker')",
extra={"operation": job.info})
success = False
else:
second_sandbox.create_file_from_storage(
TwoSteps.CHECKER_FILENAME,
job.managers[TwoSteps.CHECKER_FILENAME].digest,
executable=True)
# Rewrite input file, as in Batch.py
try:
second_sandbox.remove_file("input.txt")
except OSError as e:
assert not second_sandbox.file_exists(
"input.txt")
second_sandbox.create_file_from_storage(
"input.txt", job.input)
success, _ = evaluation_step(
second_sandbox, [[
"./%s" % TwoSteps.CHECKER_FILENAME,
"input.txt", "res.txt", "output.txt"
]])
if success:
try:
outcome, text = extract_outcome_and_text(
second_sandbox)
except ValueError as e:
logger.error(
"Invalid output from "
"comparator: %s",
e,
extra={"operation": job.info})
success = False
else:
raise ValueError("Uncrecognized first parameter"
" `%s' for TwoSteps tasktype." %
self.parameters[0])
# Whatever happened, we conclude.
job.success = success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
delete_sandbox(first_sandbox, job.success)
delete_sandbox(second_sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="manager_evaluate")
sandbox_user = [
create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="user_evaluate")
for i in indices]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: prepare the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename:
job.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": job.input
}
manager_allow_dirs = fifo_dir
for filename, digest in iteritems(manager_executables_to_get):
sandbox_mgr.create_file_from_storage(
filename, digest, executable=True)
for filename, digest in iteritems(manager_files_to_get):
sandbox_mgr.create_file_from_storage(filename, digest)
# Second step: load the executables for the user processes
# (done before launching the manager so that it does not
# impact its wall clock time).
assert len(job.executables) == 1
executable_filename = next(iterkeys(job.executables))
executables_to_get = {
executable_filename:
job.executables[executable_filename].digest
}
for i in indices:
for filename, digest in iteritems(executables_to_get):
sandbox_user[i].create_file_from_storage(
filename, digest, executable=True)
# Third step: start the manager.
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt")
# Fourth step: start the user submissions compiled with the stub.
language = get_language(job.language)
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(
executable_filename,
main="stub",
args=args)
user_allow_dirs = [fifo_dir[i]]
# Assumes that the actual execution of the user solution
# is the last command in commands, and that the previous
# are "setup" that doesn't need tight control.
if len(commands) > 1:
evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, []
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt",
"Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
else:
sandbox.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
success, _ = evaluation_step(
sandbox,
[["./%s" % manager_filename,
input_filename, "res.txt", output_filename]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError, e:
logger.error("Invalid output from "
"comparator: %s", e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.job.language
header = HEADERS_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.job.files) != 2:
self.job.success = True
self.job.compilation_success = False
self.job.text = "Invalid files in submission"
logger.warning("Submission contains %d files, expecting 2" %
len(self.job.files))
return True
# First and only one compilation.
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
files_to_get = {}
# User's submissions and headers.
source_filenames = []
for filename, _file in self.job.files.iteritems():
source_filename = filename.replace("%l", language)
source_filenames.append(source_filename)
files_to_get[source_filename] = _file.digest
# Headers.
header_filename = filename.replace("%l", header)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
self.job.managers[header_filename].digest
# Manager.
manager_filename = "manager.%s" % language
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.job.managers[manager_filename].digest
# Manager's header.
manager_filename = "manager.%s" % header
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.job.managers[manager_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
command = get_compilation_command(language, source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, command)
# Retrieve the compiled executables
self.job.success = operation_success
self.job.compilation_success = compilation_success
self.job.plus = plus
self.job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, self.job.info))
self.job.executables[executable_filename] = \
Executable(digest, executable_filename)
# Cleanup
delete_sandbox(sandbox)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1",
len(job.files), extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[0]] = \
job.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager. The grader must
# be the first file in source_filenames.
if self.parameters[0] == "grader":
source_filenames.insert(0, "grader%s" % source_ext)
files_to_get["grader%s" % source_ext] = \
job.managers["grader%s" % source_ext].digest
# Also copy all *.h and *lib.pas graders
for filename in job.managers.iterkeys():
if any(filename.endswith(header)
for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language,
source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
# f stand for first, s for second.
first_sandbox = create_sandbox(self)
second_sandbox = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
# First step: we start the first manager.
first_filename = "manager"
first_command = ["./%s" % first_filename, "0", fifo]
first_executables_to_get = {
first_filename:
self.submission.executables[first_filename].digest
}
first_files_to_get = {
"input.txt": self.submission.task.testcases[test_number].input
}
first_allow_path = ["input.txt", fifo]
first = self.evaluation_step_before_run(
first_sandbox,
first_command,
first_executables_to_get,
first_files_to_get,
self.submission.task.time_limit,
self.submission.task.memory_limit,
first_allow_path,
stdin_redirect="input.txt")
# Second step: we start the second manager.
second_filename = "manager"
second_command = ["./%s" % second_filename, "1", fifo]
second_executables_to_get = {
second_filename:
self.submission.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo, "output.txt"]
second = self.evaluation_step_before_run(
second_sandbox,
second_command,
second_executables_to_get,
second_files_to_get,
self.submission.task.time_limit,
self.submission.task.memory_limit,
second_allow_path)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, outcome_first, text_first, _ = \
self.evaluation_step_after_run(first_sandbox, final=False)
success_second, outcome_second, text_second, plus = \
self.evaluation_step_after_run(second_sandbox, final=True)
# If at least one evaluation had problems, we report the
# problems.
if not success_first:
success, outcome, text = False, outcome_first, text_first
elif not success_second:
success, outcome, text = False, outcome_second, text_second
# Otherwise, compare the second output to the results
# (there is no comparator since the manager is expected to turn the
# output in diffable form)
else:
success, outcome, text = self.white_diff_step(
second_sandbox,
"output.txt", "res.txt",
{"res.txt":
self.submission.task.testcases[test_number].output})
delete_sandbox(first_sandbox)
delete_sandbox(second_sandbox)
return self.finish_evaluation_testcase(
test_number, success, outcome, text, plus)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1", len(job.files), extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
# Stub.
source_filenames.append("stub%s" % source_ext)
files_to_get[source_filenames[-1]] = job.managers["stub%s" % source_ext].digest
# User's submission.
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[-1]] = job.files[format_filename].digest
# Also copy all managers that might be useful during compilation.
# We likely want to compile with .cpp or .o files, so add them to our
# command line
for filename in job.managers.iterkeys():
if any(filename.endswith(header) for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()):
files_to_get[filename] = job.managers[filename].digest
elif any(filename.endswith(source) for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()):
files_to_get[filename] = job.managers[filename].digest
if filename not in source_filenames:
source_filenames.insert(1, filename)
elif any(filename.endswith(obj) for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()):
files_to_get[filename] = job.managers[filename].digest
if filename not in source_filenames:
source_filenames.insert(1, filename)
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language, source_filenames, executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename, "Executable %s for %s" % (executable_filename, job.info)
)
job.executables[executable_filename] = Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
job.sandboxes = [sandbox.path]
job.plus = {}
outcome = None
text = None
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%s.txt" % job.operation["testcase_codename"] \
not in job.files:
job.success = True
job.outcome = "0.0"
job.text = [N_("File not submitted")]
return True
# First and only one step: diffing (manual or with manager).
output_digest = job.files["output_%s.txt" %
job.operation["testcase_codename"]].digest
# Put the files into the sandbox
sandbox.create_file_from_storage(
"res.txt",
job.output)
sandbox.create_file_from_storage(
"output.txt",
output_digest)
if self.parameters[0] == "diff":
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success = True
outcome, text = white_diff_step(
sandbox, "output.txt", "res.txt")
elif self.parameters[0] == "comparator":
# Manager present: wonderful, he'll do all the job.
manager_filename = "checker"
if manager_filename not in job.managers:
logger.error("Configuration error: missing or "
"invalid comparator (it must be "
"named `checker')", extra={"operation": job.info})
success = False
else:
sandbox.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
input_digest = job.input
sandbox.create_file_from_storage(
"input.txt",
input_digest)
success, _ = evaluation_step(
sandbox,
[["./%s" % manager_filename,
"input.txt", "res.txt", "output.txt"]])
if success:
outcome, text = extract_outcome_and_text(sandbox)
else:
raise ValueError("Unrecognized first parameter "
"`%s' for OutputOnly tasktype. "
"Should be `diff' or `comparator'." %
self.parameters[0])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
# f stand for first, s for second.
first_sandbox = create_sandbox(self)
second_sandbox = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
# First step: we start the first manager.
first_filename = "manager"
first_command = ["./%s" % first_filename, "0", fifo]
first_executables_to_get = {
first_filename:
self.job.executables[first_filename].digest
}
first_files_to_get = {
"input.txt": self.job.testcases[test_number].input
}
first_allow_path = ["input.txt", fifo]
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.iteritems():
first_sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in first_files_to_get.iteritems():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
self.job.time_limit,
self.job.memory_limit,
first_allow_path,
stdin_redirect="input.txt",
wait=False)
# Second step: we start the second manager.
second_filename = "manager"
second_command = ["./%s" % second_filename, "1", fifo]
second_executables_to_get = {
second_filename:
self.job.executables[second_filename].digest
}
second_files_to_get = {}
second_allow_path = [fifo, "output.txt"]
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.iteritems():
second_sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in second_files_to_get.iteritems():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
self.job.time_limit,
self.job.memory_limit,
second_allow_path,
stdout_redirect="output.txt",
wait=False)
# Consume output.
wait_without_std([second, first])
# TODO: check exit codes with translate_box_exitcode.
success_first, first_plus = \
evaluation_step_after_run(first_sandbox)
success_second, second_plus = \
evaluation_step_after_run(second_sandbox)
self.job.evaluations[test_number] = {'sandboxes': [first_sandbox.path,
second_sandbox.path],
'plus': second_plus}
outcome = None
text = None
evaluation = self.job.evaluations[test_number]
success = True
# Error in the sandbox: report failure!
if not success_first or not success_second:
success = False
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(first_plus) or \
not is_evaluation_passed(second_plus):
outcome = 0.0
if not is_evaluation_passed(first_plus):
text = human_evaluation_message(first_plus)
else:
text = human_evaluation_message(second_plus)
if self.job.get_output:
evaluation['output'] = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists('output.txt'):
outcome = 0.0
text = "Execution didn't produce file output.txt"
if self.job.get_output:
evaluation['output'] = None
else:
# If asked so, put the output file into the storage
if self.job.get_output:
evaluation['output'] = second_sandbox.get_file_to_storage(
"output.txt",
"Output file for testcase %d in job %s" %
(test_number, self.job.info))
# If not asked otherwise, evaluate the output file
if not self.job.only_execution:
# Put the reference solution into the sandbox
second_sandbox.create_file_from_storage(
"res.txt",
self.job.testcases[test_number].output)
outcome, text = white_diff_step(
second_sandbox, "output.txt", "res.txt")
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = outcome
evaluation['text'] = text
delete_sandbox(first_sandbox)
delete_sandbox(second_sandbox)
return success
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher)
sandbox_user = create_sandbox(file_cacher)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_in = os.path.join(fifo_dir, "in")
fifo_out = os.path.join(fifo_dir, "out")
os.mkfifo(fifo_in)
os.mkfifo(fifo_out)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo_in, 0o666)
os.chmod(fifo_out, 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
manager_executables_to_get = {
manager_filename:
job.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": job.input
}
manager_allow_dirs = [fifo_dir]
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(
filename, digest, executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
job.time_limit,
0,
allow_dirs=manager_allow_dirs,
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = job.executables.keys()[0]
command = ["./%s" % executable_filename, fifo_out, fifo_in]
executables_to_get = {
executable_filename:
job.executables[executable_filename].digest
}
user_allow_dirs = [fifo_dir]
for filename, digest in executables_to_get.iteritems():
sandbox_user.create_file_from_storage(
filename, digest, executable=True)
process = evaluation_step_before_run(
sandbox_user,
command,
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std([process, manager])
# TODO: check exit codes with translate_box_exitcode.
success_user, plus_user = \
evaluation_step_after_run(sandbox_user)
success_mgr, plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
job.sandboxes = [sandbox_user.path,
sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt",
"Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr)
delete_sandbox(sandbox_user)
shutil.rmtree(fifo_dir)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
source_filenames = []
# Stub.
stub_filename = "stub%s" % source_ext
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submission.
for filename, fileinfo in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = fileinfo.digest
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(filename.endswith(header)
for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(source)
for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(obj)
for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in job.files.keys())
commands = get_compilation_commands(language,
source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1",
len(job.files),
extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
# Stub.
source_filenames.append("stub%s" % source_ext)
files_to_get[source_filenames[-1]] = \
job.managers["stub%s" % source_ext].digest
# User's submission.
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[-1]] = \
job.files[format_filename].digest
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(
filename.endswith(header)
for header in LANGUAGE_TO_HEADER_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
elif any(
filename.endswith(source)
for source in LANGUAGE_TO_SOURCE_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
elif any(
filename.endswith(obj)
for obj in LANGUAGE_TO_OBJ_EXT_MAP.itervalues()):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language, source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# Create the sandbox
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
source_filenames = []
# Stub.
stub_filename = "stub%s" % source_ext
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submission.
for filename, fileinfo in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = fileinfo.digest
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(filename.endswith(header) for header in HEADER_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(source) for source in SOURCE_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(obj) for obj in OBJECT_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in job.files.keys())
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1" %
len(job.files),
extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[0]] = \
job.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager. The grader must
# be the first file in source_filenames.
if self.parameters[0] == "grader":
source_filenames.insert(0, "grader%s" % source_ext)
files_to_get["grader%s" % source_ext] = \
job.managers["grader%s" % source_ext].digest
# Also copy all *.h and *lib.pas graders
for filename in job.managers.iterkeys():
if filename.endswith('.h') or \
filename.endswith('lib.pas'):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language, source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
# one may want to write a bash checker who calls
# other processes). Set to a high number because
# to avoid fork-bombing the worker.
sandbox.max_processes = 1000
success, _ = evaluation_step(
sandbox,
[["./%s" % manager_filename,
input_filename, "res.txt", output_filename]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError, e:
logger.error("Invalid output from "
"comparator: %s", e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.job.language
header = HEADERS_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.job.files) != 2:
self.job.success = True
self.job.compilation_success = False
self.job.text = "Invalid files in submission"
logger.warning("Submission contains %d files, expecting 2" %
len(self.job.files))
return True
# First and only one compilation.
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
files_to_get = {}
# User's submissions and headers.
source_filenames = []
for filename, _file in self.job.files.iteritems():
source_filename = filename.replace("%l", language)
source_filenames.append(source_filename)
files_to_get[source_filename] = _file.digest
# Headers.
header_filename = filename.replace("%l", header)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
self.job.managers[header_filename].digest
# Manager.
manager_filename = "manager.%s" % language
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.job.managers[manager_filename].digest
# Manager's header.
manager_filename = "manager.%s" % header
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.job.managers[manager_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Get compilation command and compile.
executable_filename = "manager"
command = get_compilation_command(language,
source_filenames,
executable_filename)
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, command)
# Retrieve the compiled executables
self.job.success = operation_success
self.job.compilation_success = compilation_success
self.job.plus = plus
self.job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, self.job.info))
self.job.executables[executable_filename] = \
Executable(digest, executable_filename)
# Cleanup
delete_sandbox(sandbox)
class Batch(TaskType):
"""Task type class for a unique standalone submission source, with
comparator (or not).
Parameters needs to be a list of three elements.
The first element is 'grader' or 'alone': in the first
case, the source file is to be compiled with a provided piece of
software ('grader'); in the other by itself.
The second element is a 2-tuple of the input file name and output file
name. The input file may be '' to denote stdin, and similarly the
output filename may be '' to denote stdout.
The third element is 'diff' or 'comparator' and says whether the
output is compared with a simple diff algorithm or using a
comparator.
Note: the first element is used only in the compilation step; the
others only in the evaluation step.
A comparator can read argv[1], argv[2], argv[3] (respectively,
input, correct output and user output) and should write the
outcome to stdout and the text to stderr.
"""
ALLOW_PARTIAL_SUBMISSION = False
_COMPILATION = ParameterTypeChoice(
"Compilation", "compilation", "", {
"alone": "Submissions are self-sufficient",
"grader": "Submissions are compiled with a grader"
})
_USE_FILE = ParameterTypeCollection(
"I/O (blank for stdin/stdout)", "io", "", [
ParameterTypeString("Input file", "inputfile", ""),
ParameterTypeString("Output file", "outputfile", ""),
])
_EVALUATION = ParameterTypeChoice(
"Output evaluation", "output_eval", "", {
"diff": "Outputs compared with white diff",
"comparator": "Outputs are compared by a comparator"
})
ACCEPTED_PARAMETERS = [_COMPILATION, _USE_FILE, _EVALUATION]
@property
def name(self):
"""See TaskType.name."""
# TODO add some details if a grader/comparator is used, etc...
return "Batch"
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
res = dict()
for language in LANGUAGES:
format_filename = submission_format[0]
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
source_filenames = []
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager.
if self.parameters[0] == "grader":
source_filenames.append("grader%s" % source_ext)
source_filenames.append(format_filename.replace(".%l", source_ext))
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language, source_filenames,
executable_filename)
res[language] = commands
return res
def get_user_managers(self, submission_format):
"""See TaskType.get_user_managers."""
return []
def get_auto_managers(self):
"""See TaskType.get_auto_managers."""
return None
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = job.language
source_ext = LANGUAGE_TO_SOURCE_EXT_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1" %
len(job.files),
extra={"operation": job.info})
return True
# Create the sandbox
sandbox = create_sandbox(file_cacher)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = job.files.keys()[0]
source_filenames = []
source_filenames.append(format_filename.replace(".%l", source_ext))
files_to_get[source_filenames[0]] = \
job.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager. The grader must
# be the first file in source_filenames.
if self.parameters[0] == "grader":
source_filenames.insert(0, "grader%s" % source_ext)
files_to_get["grader%s" % source_ext] = \
job.managers["grader%s" % source_ext].digest
# Also copy all *.h and *lib.pas graders
for filename in job.managers.iterkeys():
if filename.endswith('.h') or \
filename.endswith('lib.pas'):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
commands = get_compilation_commands(language, source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
# Create the sandbox
sandbox = create_sandbox(file_cacher)
# Prepare the execution
executable_filename = job.executables.keys()[0]
language = job.language
commands = get_evaluation_commands(language, executable_filename)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
input_filename, output_filename = self.parameters[1]
stdin_redirect = None
stdout_redirect = None
if input_filename == "":
input_filename = "input.txt"
stdin_redirect = input_filename
if output_filename == "":
output_filename = "output.txt"
stdout_redirect = output_filename
files_to_get = {input_filename: job.input}
# Put the required files into the sandbox
for filename, digest in executables_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
success, plus = evaluation_step(sandbox,
commands,
job.time_limit,
job.memory_limit,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect)
job.sandboxes = [sandbox.path]
job.plus = plus
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not success:
pass
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(plus):
outcome = 0.0
text = human_evaluation_message(plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(output_filename):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"), output_filename
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
output_filename,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If not asked otherwise, evaluate the output file
if not job.only_execution:
# Put the reference solution into the sandbox
sandbox.create_file_from_storage("res.txt", job.output)
# Check the solution with white_diff
if self.parameters[2] == "diff":
outcome, text = white_diff_step(
sandbox, output_filename, "res.txt")
# Check the solution with a comparator
elif self.parameters[2] == "comparator":
manager_filename = "checker"
if not manager_filename in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named 'checker')",
extra={"operation": job.info})
success = False
else:
sandbox.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
success, _ = evaluation_step(
sandbox, [[
"./%s" % manager_filename, input_filename,
"res.txt", output_filename
]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError, e:
logger.error("Invalid output from "
"comparator: %s" % (e.message, ),
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher)
sandbox_user = [create_sandbox(file_cacher) for i in indices]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename:
job.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": job.input
}
manager_allow_dirs = fifo_dir
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(
filename, digest, executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = job.executables.keys()[0]
executables_to_get = {
executable_filename:
job.executables[executable_filename].digest
}
processes = [None for i in indices]
for i in indices:
command = ["./%s" % executable_filename, fifo_out[i], fifo_in[i]]
if num_processes != 1:
command.append(str(i))
user_allow_dirs = [fifo_dir[i]]
for filename, digest in executables_to_get.iteritems():
sandbox_user[i].create_file_from_storage(
filename, digest, executable=True)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
command,
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt",
"Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr)
for s in sandbox_user:
delete_sandbox(s)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1",
len(job.files),
extra={"operation": job.info})
return
# Create the sandbox.
sandbox = create_sandbox(file_cacher,
multithreaded=job.multithreaded_sandbox,
name="compile")
job.sandboxes.append(sandbox.path)
user_file_format = next(iterkeys(job.files))
user_source_filename = user_file_format.replace(".%l", source_ext)
executable_filename = user_file_format.replace(".%l", "")
# Copy required files in the sandbox (includes the grader if present).
sandbox.create_file_from_storage(user_source_filename,
job.files[user_file_format].digest)
for filename in iterkeys(job.managers):
if Batch._is_manager_for_compilation(filename):
sandbox.create_file_from_storage(filename,
job.managers[filename].digest)
# Create the list of filenames to be passed to the compiler. If we use
# a grader, it needs to be in first position in the command line.
source_filenames = [user_source_filename]
if self._uses_grader():
grader_source_filename = Batch.GRADER_BASENAME + source_ext
source_filenames.insert(0, grader_source_filename)
# Prepare the compilation command.
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
sandbox_user = [
create_sandbox(file_cacher, job.multithreaded_sandbox)
for i in indices
]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename: job.managers[manager_filename].digest
}
manager_files_to_get = {"input.txt": job.input}
manager_allow_dirs = fifo_dir
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
language = get_language(job.language)
executable_filename = job.executables.keys()[0]
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="stub",
args=args)
user_allow_dirs = [fifo_dir[i]]
for filename, digest in executables_to_get.iteritems():
sandbox_user[i].create_file_from_storage(filename,
digest,
executable=True)
# Assumes that the actual execution of the user solution
# is the last command in commands, and that the previous
# are "setup" that doesn't need tight control.
if len(commands) > 1:
evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.job.language
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.job.files) != 1:
self.job.success = True
self.job.compilation_success = False
self.job.text = "Invalid files in submission"
logger.warning("Submission contains %d files, expecting 1" %
len(self.job.files))
return True
# Create the sandbox
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = self.job.files.keys()[0]
source_filenames = []
# Stub.
source_filenames.append("stub.%s" % language)
files_to_get[source_filenames[1]] = \
self.job.managers["stub.%s" % language].digest
# User's submission.
source_filenames.append(format_filename.replace("%l", language))
files_to_get[source_filenames[0]] = \
self.job.files[format_filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
command = get_compilation_command(language, source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, command)
# Retrieve the compiled executables
self.job.success = operation_success
self.job.compilation_success = compilation_success
self.job.plus = plus
self.job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, self.job.info))
self.job.executables[executable_filename] = \
Executable(digest, executable_filename)
# Cleanup
delete_sandbox(sandbox)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
self.job.evaluations[test_number] = {
'sandboxes': [sandbox.path],
'plus': {}
}
evaluation = self.job.evaluations[test_number]
outcome = None
text = None
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%03d.txt" % test_number not in self.job.files:
evaluation['success'] = True
evaluation['outcome'] = 0.0
evaluation['text'] = "File not submitted."
return True
# First and only one step: diffing (manual or with manager).
output_digest = self.job.files["output_%03d.txt" % test_number].digest
# Put the files into the sandbox
sandbox.create_file_from_storage(
"res.txt", self.job.testcases[test_number].output)
sandbox.create_file_from_storage("output.txt", output_digest)
# TODO: this should check self.parameters, not managers.
if len(self.job.managers) == 0:
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success = True
outcome, text = white_diff_step(sandbox, "output.txt", "res.txt")
else:
# Manager present: wonderful, he'll do all the job.
manager_filename = self.job.managers.keys()[0]
sandbox.create_file_from_storage(
manager_filename,
self.job.managers[manager_filename].digest,
executable=True)
input_digest = self.job.testcases[test_number].input
sandbox.create_file_from_storage("input.txt", input_digest)
success, _ = evaluation_step(sandbox, [
"./%s" % manager_filename, "input.txt", "res.txt", "output.txt"
],
allow_path=[
"input.txt", "output.txt",
"res.txt"
])
if success:
outcome, text = extract_outcome_and_text(sandbox)
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = str(outcome) if outcome is not None else None
evaluation['text'] = text
delete_sandbox(sandbox)
return success
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(job.executables) != 1:
raise ValueError("Unexpected number of executables (%s)" %
len(job.executables))
# Create the sandbox
sandbox = create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="evaluate")
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = Batch.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main)
executables_to_get = {
executable_filename:
job.executables[executable_filename].digest
}
stdin_redirect = None
stdout_redirect = None
files_allowing_write = []
if len(self.input_filename) == 0:
self.input_filename = Batch.DEFAULT_INPUT_FILENAME
stdin_redirect = self.input_filename
if len(self.output_filename) == 0:
self.output_filename = Batch.DEFAULT_OUTPUT_FILENAME
stdout_redirect = self.output_filename
else:
files_allowing_write.append(self.output_filename)
files_to_get = {
self.input_filename: job.input
}
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
success, plus = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect)
job.sandboxes = [sandbox.path]
job.plus = plus
outcome = None
text = []
# Error in the sandbox: nothing to do!
if not success:
pass
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(plus):
outcome = 0.0
text = human_evaluation_message(plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self.output_filename):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
self.output_filename]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self.output_filename,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
# Create a brand-new sandbox just for checking. Only admin
# code runs in it, so we allow multithreading and many
# processes (still with a limit to avoid fork-bombs).
checkbox = create_sandbox(
file_cacher,
multithreaded=True,
name="check")
checkbox.max_processes = 1000
checker_success, outcome, text = self._eval_output(
checkbox, job, sandbox.get_root_path())
success = success and checker_success
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
# one may want to write a bash checker who calls
# other processes). Set to a high number because
# to avoid fork-bombing the worker.
sandbox.max_processes = 1000
success, _ = evaluation_step(
sandbox,
[["./%s" % manager_filename,
input_filename, "res.txt", output_filename]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox)
except ValueError, e:
logger.error("Invalid output from "
"comparator: %s", e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized third parameter"
" `%s' for Batch tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.submission.language
header = HEADERS_MAP[language]
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.submission.files) != 2:
return self.finish_compilation(
True, False, "Invalid files in submission",
to_log="Submission contains %d files, expecting 2" %
len(self.submission.files))
# First and only one compilation.
sandbox = create_sandbox(self)
files_to_get = {}
# User's submissions and headers.
source_filenames = []
for filename, _file in self.submission.files.iteritems():
source_filename = filename.replace("%l", language)
source_filenames.append(source_filename)
files_to_get[source_filename] = _file.digest
# Headers.
header_filename = filename.replace("%l", header)
source_filenames.append(header_filename)
files_to_get[header_filename] = \
self.submission.task.managers[header_filename].digest
# Manager.
manager_filename = "manager.%s" % language
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.submission.task.managers[manager_filename].digest
# Manager's header.
manager_filename = "manager.%s" % header
source_filenames.append(manager_filename)
files_to_get[manager_filename] = \
self.submission.task.managers[manager_filename].digest
# Get compilation command and compile.
executable_filename = "manager"
command = get_compilation_command(language,
source_filenames,
executable_filename)
operation_success, compilation_success, text, _ = \
self.compilation_step(
sandbox,
command,
files_to_get,
{executable_filename: "Executable %s for submission %s" %
(executable_filename, self.submission.id)})
delete_sandbox(sandbox)
# We had only one compilation, hence we pipe directly its
# result to the finalization.
return self.finish_compilation(operation_success, compilation_success,
text)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(job.executables) != 1:
raise ValueError("Unexpected number of executables (%s)" %
len(job.executables))
# Create the sandbox
sandbox = create_sandbox(file_cacher,
multithreaded=job.multithreaded_sandbox,
name="evaluate")
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = Batch.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(executable_filename,
main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
stdin_redirect = None
stdout_redirect = None
files_allowing_write = []
if len(self.input_filename) == 0:
self.input_filename = Batch.DEFAULT_INPUT_FILENAME
stdin_redirect = self.input_filename
if len(self.output_filename) == 0:
self.output_filename = Batch.DEFAULT_OUTPUT_FILENAME
stdout_redirect = self.output_filename
else:
files_allowing_write.append(self.output_filename)
files_to_get = {self.input_filename: job.input}
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
success, plus = evaluation_step(sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect)
job.sandboxes = [sandbox.path]
job.plus = plus
outcome = None
text = []
# Error in the sandbox: nothing to do!
if not success:
pass
# Contestant's error: the marks won't be good
elif not is_evaluation_passed(plus):
outcome = 0.0
text = human_evaluation_message(plus)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self.output_filename):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
self.output_filename
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self.output_filename,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
# Create a brand-new sandbox just for checking. Only admin
# code runs in it, so we allow multithreading and many
# processes (still with a limit to avoid fork-bombs).
checkbox = create_sandbox(file_cacher,
multithreaded=True,
name="check")
checkbox.max_processes = 1000
checker_success, outcome, text = self._eval_output(
checkbox, job, sandbox.get_root_path())
success = success and checker_success
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Immediately prepare the skeleton to return
job.sandboxes = [sandbox.path]
job.plus = {}
outcome = None
text = None
# Since we allow partial submission, if the file is not
# present we report that the outcome is 0.
if "output_%s.txt" % job.operation["testcase_codename"] \
not in job.files:
job.success = True
job.outcome = "0.0"
job.text = [N_("File not submitted")]
return True
# First and only one step: diffing (manual or with manager).
output_digest = job.files["output_%s.txt" %
job.operation["testcase_codename"]].digest
# Put the files into the sandbox
sandbox.create_file_from_storage("res.txt", job.output)
sandbox.create_file_from_storage("output.txt", output_digest)
if self.parameters[0] == "diff":
# No manager: I'll do a white_diff between the submission
# file and the correct output res.txt.
success = True
outcome, text = white_diff_step(sandbox, "output.txt", "res.txt")
elif self.parameters[0] == "comparator":
# Manager present: wonderful, it will do all the work.
manager_filename = "checker"
if manager_filename not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named `checker')",
extra={"operation": job.info})
success = False
else:
sandbox.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
input_digest = job.input
sandbox.create_file_from_storage("input.txt", input_digest)
success, _ = evaluation_step(sandbox, [[
"./%s" % manager_filename, "input.txt", "res.txt",
"output.txt"
]])
if success:
outcome, text = extract_outcome_and_text(sandbox)
else:
raise ValueError("Unrecognized first parameter "
"`%s' for OutputOnly tasktype. "
"Should be `diff' or `comparator'." %
self.parameters[0])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox, job.success)
def evaluate_testcase(self, test_number):
"""See TaskType.evaluate_testcase."""
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(self)
sandbox_user = create_sandbox(self)
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_in = os.path.join(fifo_dir, "in")
fifo_out = os.path.join(fifo_dir, "out")
os.mkfifo(fifo_in)
os.mkfifo(fifo_out)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename, fifo_in, fifo_out]
manager_executables_to_get = {
manager_filename: self.job.managers[manager_filename].digest
}
manager_files_to_get = {
"input.txt": self.job.testcases[test_number].input
}
manager_allow_path = ["input.txt", "output.txt", fifo_in, fifo_out]
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(sandbox_mgr,
manager_command,
self.job.time_limit,
0,
manager_allow_path,
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
executable_filename = self.job.executables.keys()[0]
command = ["./%s" % executable_filename, fifo_out, fifo_in]
executables_to_get = {
executable_filename:
self.job.executables[executable_filename].digest
}
allow_path = [fifo_in, fifo_out]
for filename, digest in executables_to_get.iteritems():
sandbox_user.create_file_from_storage(filename,
digest,
executable=True)
process = evaluation_step_before_run(sandbox_user, command,
self.job.time_limit,
self.job.memory_limit, allow_path)
# Consume output.
wait_without_std([process, manager])
# TODO: check exit codes with translate_box_exitcode.
success_user, plus_user = \
evaluation_step_after_run(sandbox_user)
success_mgr, plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
self.job.evaluations[test_number] = \
{'sandboxes': [sandbox_user.path,
sandbox_mgr.path],
'plus': plus_user}
evaluation = self.job.evaluations[test_number]
# If at least one evaluation had problems, we report the
# problems. (TODO: shouldn't outcome and text better be None
# and None?)
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If outcome_user is not None, it is 0.0 and it means that
# there has been some errors in the user solution, and outcome
# and text are meaningful, so we use them.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if self.job.get_output:
if sandbox_mgr.file_exists("output.txt"):
evaluation['output'] = sandbox_mgr.get_file_to_storage(
"output.txt", "Output file for testcase %d in job %s" %
(test_number, self.job.info))
else:
evaluation['output'] = None
# Whatever happened, we conclude.
evaluation['success'] = success
evaluation['outcome'] = str(outcome) if outcome is not None else None
evaluation['text'] = text
delete_sandbox(sandbox_mgr)
delete_sandbox(sandbox_user)
return success
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, Communication.MANAGER_FILENAME):
return
manager_digest = job.managers[Communication.MANAGER_FILENAME].digest
# Indices for the objects related to each user process.
indices = range(self.num_processes)
# Create FIFOs.
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# Create the manager sandbox and copy manager and input.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.path)
sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME,
manager_digest,
executable=True)
sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME,
job.input)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [
create_sandbox(file_cacher, name="user_evaluate") for i in indices
]
job.sandboxes.extend(s.path for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(executable_filename,
executable_digest,
executable=True)
if Communication.STUB_PRELOAD_FILENAME in job.managers:
digest = job.managers[
Communication.STUB_PRELOAD_FILENAME].digest
sandbox_user[i].create_file_from_storage(
Communication.STUB_PRELOAD_FILENAME,
digest,
executable=True)
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % Communication.MANAGER_FILENAME]
for i in indices:
manager_command += [fifo_in[i], fifo_out[i]]
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib // 1024,
allow_dirs=fifo_dir,
writable_files=[Communication.OUTPUT_FILENAME],
stdin_redirect=Communication.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox)
# Start the user submissions compiled with the stub.
language = get_language(job.language)
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if self.num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="stub",
args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
last_cmd = commands[-1]
# Inject preload program if needed
if Communication.STUB_PRELOAD_FILENAME in job.managers:
last_cmd = [
"./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i],
fifo_in[i]
] + commands[-1]
processes[i] = evaluation_step_before_run(
sandbox_user[i],
last_cmd,
job.time_limit,
job.memory_limit,
allow_dirs=[fifo_dir[i]],
multiprocess=job.multithreaded_sandbox)
# Wait for the processes to conclude, without blocking them on I/O.
wait_without_std(processes + [manager])
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
pass
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user, job.feedback_level)
# Otherwise, we use the manager to obtain the outcome.
else:
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
Communication.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def compile(self):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = self.job.language
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(self.job.files) != 1:
self.job.success = True
self.job.compilation_success = False
self.job.text = "Invalid files in submission"
logger.warning("Submission contains %d files, expecting 1" %
len(self.job.files))
return True
# Create the sandbox
sandbox = create_sandbox(self)
self.job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
format_filename = self.job.files.keys()[0]
source_filenames = []
source_filenames.append(format_filename.replace("%l", language))
files_to_get[source_filenames[0]] = \
self.job.files[format_filename].digest
# If a grader is specified, we add to the command line (and to
# the files to get) the corresponding manager. The grader must
# be the first file in source_filenames.
if self.job.task_type_parameters[0] == "grader":
source_filenames.insert(0, "grader.%s" % language)
files_to_get["grader.%s" % language] = \
self.job.managers["grader.%s" % language].digest
# Also copy all *.h and *lib.pas graders
for filename in self.job.managers.iterkeys():
if filename.endswith('.h') or \
filename.endswith('lib.pas'):
files_to_get[filename] = \
self.job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = format_filename.replace(".%l", "")
command = get_compilation_command(language,
source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, command)
# Retrieve the compiled executables
self.job.success = operation_success
self.job.compilation_success = compilation_success
self.job.plus = plus
self.job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, self.job.info))
self.job.executables[executable_filename] = \
Executable(digest, executable_filename)
# Cleanup
delete_sandbox(sandbox)
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# TODO: here we are sure that submission.files are the same as
# task.submission_format. The following check shouldn't be
# here, but in the definition of the task, since this actually
# checks that task's task type and submission format agree.
if len(job.files) != 1:
job.success = True
job.compilation_success = False
job.text = [N_("Invalid files in submission")]
logger.error("Submission contains %d files, expecting 1",
len(job.files), extra={"operation": job.info})
return
# Create the sandbox.
sandbox = create_sandbox(
file_cacher,
multithreaded=job.multithreaded_sandbox,
name="compile")
job.sandboxes.append(sandbox.path)
user_file_format = next(iterkeys(job.files))
user_source_filename = user_file_format.replace(".%l", source_ext)
executable_filename = user_file_format.replace(".%l", "")
# Copy required files in the sandbox (includes the grader if present).
sandbox.create_file_from_storage(
user_source_filename, job.files[user_file_format].digest)
for filename in iterkeys(job.managers):
if Batch._is_manager_for_compilation(filename):
sandbox.create_file_from_storage(
filename, job.managers[filename].digest)
# Create the list of filenames to be passed to the compiler. If we use
# a grader, it needs to be in first position in the command line.
source_filenames = [user_source_filename]
if self._uses_grader():
grader_source_filename = Batch.GRADER_BASENAME + source_ext
source_filenames.insert(0, grader_source_filename)
# Prepare the compilation command.
commands = language.get_compilation_commands(
source_filenames, executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" %
(executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
