def setUp(self):
super().setUp()
self.p = ParameterTypeCollection("name", "shortname", "description", [
ParameterTypeInt("name0", "shortname0", "desc0"),
ParameterTypeString("name1", "shortname1", "desc1"),
ParameterTypeChoice("name2", "shortname2", "desc2", {
"c1": "First choice",
"c2": "Second choice",
})])
class Communication(TaskType):
"""Task type class for tasks that requires:
- a *manager* that reads the input file, work out the perfect
solution on its own, and communicate the input (maybe with some
modifications) on its standard output; it then reads the
response of the user's solution from the standard input and
write the outcome;
- a *stub* that compiles with the user's source, reads from
standard input what the manager says, and write back the user's
solution to stdout.
"""
ALLOW_PARTIAL_SUBMISSION = False
name = "Communication"
_NUM_PROCESSES = ParameterTypeInt("Number of Processes", "num_processes",
"")
ACCEPTED_PARAMETERS = [_NUM_PROCESSES]
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
res = dict()
for language in LANGUAGES:
source_ext = language.source_extension
source_filenames = []
source_filenames.append("stub%s" % source_ext)
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in submission_format)
for filename in submission_format:
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
commands = language.get_compilation_commands(
source_filenames, executable_filename)
res[language.name] = commands
return res
def get_user_managers(self, unused_submission_format):
"""See TaskType.get_user_managers."""
return ["stub.%l"]
def get_auto_managers(self):
"""See TaskType.get_auto_managers."""
return ["manager"]
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# Create the sandbox
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
source_filenames = []
# Stub.
stub_filename = "stub%s" % source_ext
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submission.
for filename, fileinfo in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = fileinfo.digest
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(filename.endswith(header) for header in HEADER_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(source) for source in SOURCE_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(obj) for obj in OBJECT_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in job.files.keys())
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
sandbox_user = [
create_sandbox(file_cacher, job.multithreaded_sandbox)
for i in indices
]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename: job.managers[manager_filename].digest
}
manager_files_to_get = {"input.txt": job.input}
manager_allow_dirs = fifo_dir
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt")
# Second step: we start the user submission compiled with the
# stub.
language = get_language(job.language)
executable_filename = job.executables.keys()[0]
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="stub",
args=args)
user_allow_dirs = [fifo_dir[i]]
for filename, digest in executables_to_get.iteritems():
sandbox_user[i].create_file_from_storage(filename,
digest,
executable=True)
# Assumes that the actual execution of the user solution
# is the last command in commands, and that the previous
# are "setup" that doesn't need tight control.
if len(commands) > 1:
evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file, provided that it exists
if job.get_output:
if sandbox_mgr.file_exists("output.txt"):
job.user_output = sandbox_mgr.get_file_to_storage(
"output.txt", "Output file in job %s" % job.info)
else:
job.user_output = None
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def setUp(self):
super(TestParameterTypeInt, self).setUp()
self.p = ParameterTypeInt("name", "shortname", "description")
class TestParameterTypeInt(unittest.TestCase):
"""Test the class ParameterTypeInt."""
def setUp(self):
super(TestParameterTypeInt, self).setUp()
self.p = ParameterTypeInt("name", "shortname", "description")
def test_validate_success(self):
self.p.validate(1)
self.p.validate(-1)
self.p.validate(0)
self.p.validate(2**34)
def test_validate_failure_wrong_type(self):
with self.assertRaises(ValueError):
self.p.validate("1")
with self.assertRaises(ValueError):
self.p.validate(1.0)
with self.assertRaises(ValueError):
self.p.validate(1j)
with self.assertRaises(ValueError):
self.p.validate([1])
def test_parse_string(self):
self.assertEqual(self.p.parse_string("123"), 123)
self.assertEqual(self.p.parse_string("-123"), -123)
def test_parse_handler(self):
h = FakeHandler({
"ok_shortname": "-45",
"fail_shortname": "not an int",
})
self.assertEqual(self.p.parse_handler(h, "ok_"), -45)
with self.assertRaises(ValueError):
self.p.parse_handler(h, "fail_")
with self.assertRaises(MissingArgumentError):
self.p.parse_handler(h, "missing_")
class Communication2017Base(TaskType):
"""Task type class for tasks that requires:
- a *manager* that reads the input file, work out the perfect
solution on its own, and communicate the input (maybe with some
modifications) on its standard output; it then reads the
response of the user's solution from the standard input and
write the outcome;
- a *stub* that compiles with the user's source, reads from
standard input what the manager says, and write back the user's
solution to stdout.
"""
ALLOW_PARTIAL_SUBMISSION = False
SUBMISSION_PAGE_MESSAGE = ""
name = "Communication"
_NUM_PROCESSES = ParameterTypeInt("Number of Processes", "num_processes",
"")
_EVALUATION = ParameterTypeChoice(
"Output evaluation", "output_eval", "", {
"diff": "Outputs compared with white diff",
"comparator": "Outputs are compared by a comparator"
})
ACCEPTED_PARAMETERS = [_NUM_PROCESSES, _EVALUATION]
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
res = dict()
for language in LANGUAGES:
source_ext = language.source_extension
source_filenames = []
source_filenames.append("grader%s" % source_ext)
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in submission_format)
for filename in submission_format:
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
commands = language.get_compilation_commands(
source_filenames, executable_filename)
res[language.name] = commands
return res
def get_user_managers(self, unused_submission_format):
"""See TaskType.get_user_managers."""
return []
def get_auto_managers(self):
"""See TaskType.get_auto_managers."""
return None
def compile(self, job, file_cacher):
"""See TaskType.compile."""
# Detect the submission's language. The checks about the
# formal correctedness of the submission are done in CWS,
# before accepting it.
language = get_language(job.language)
source_ext = language.source_extension
# Create the sandbox
sandbox = create_sandbox(file_cacher, job.multithreaded_sandbox)
job.sandboxes.append(sandbox.path)
# Prepare the source files in the sandbox
files_to_get = {}
source_filenames = []
# Stub.
stub_filename = "grader%s" % source_ext
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submission.
for filename, fileinfo in job.files.iteritems():
source_filename = filename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = fileinfo.digest
# Also copy all managers that might be useful during compilation.
for filename in job.managers.iterkeys():
if any(filename.endswith(header) for header in HEADER_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(source) for source in SOURCE_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
elif any(filename.endswith(obj) for obj in OBJECT_EXTS):
files_to_get[filename] = \
job.managers[filename].digest
for filename, digest in files_to_get.iteritems():
sandbox.create_file_from_storage(filename, digest)
# Prepare the compilation command
executable_filename = \
"_".join(pattern.replace(".%l", "")
for pattern in job.files.keys())
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Run the compilation
operation_success, compilation_success, text, plus = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables
job.success = operation_success
job.compilation_success = compilation_success
job.plus = plus
job.text = text
if operation_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if len(self.parameters) <= 0:
num_processes = 1
else:
num_processes = self.parameters[0]
indices = range(num_processes)
# Create sandboxes and FIFOs
sandbox_mgr = create_sandbox(file_cacher, job.multithreaded_sandbox)
sandbox_user = [
create_sandbox(file_cacher, job.multithreaded_sandbox)
for i in indices
]
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# First step: we start the manager.
manager_filename = "manager"
manager_command = ["./%s" % manager_filename]
for i in indices:
manager_command.append(fifo_in[i])
manager_command.append(fifo_out[i])
manager_executables_to_get = {
manager_filename: job.managers[manager_filename].digest
}
manager_files_to_get = {"input.txt": job.input}
manager_allow_dirs = fifo_dir
for filename, digest in manager_executables_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in manager_files_to_get.iteritems():
sandbox_mgr.create_file_from_storage(filename, digest)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
num_processes * job.time_limit,
0,
allow_dirs=manager_allow_dirs,
writable_files=["output.txt"],
stdin_redirect="input.txt",
stdout_redirect="output.txt",
)
# Second step: we start the user submission compiled with the
# stub.
language = get_language(job.language)
executable_filename = job.executables.keys()[0]
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="grader",
args=args)
user_allow_dirs = [fifo_dir[i]]
for filename, digest in executables_to_get.iteritems():
sandbox_user[i].create_file_from_storage(filename,
digest,
executable=True)
# Assumes that the actual execution of the user solution
# is the last command in commands, and that the previous
# are "setup" that doesn't need tight control.
if len(commands) > 1:
evaluation_step(sandbox_user[i], commands[:-1], 10, 256)
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
allow_dirs=user_allow_dirs)
# Consume output.
wait_without_std(processes + [manager])
# TODO: check exit codes with translate_box_exitcode.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
success_user = all(r[0] for r in user_results)
plus_user = reduce(merge_evaluation_results,
[r[1] for r in user_results])
success_mgr, unused_plus_mgr = \
evaluation_step_after_run(sandbox_mgr)
if plus_user['exit_status'] == Sandbox.EXIT_OK and \
plus_user["execution_time"] >= job.time_limit:
plus_user['exit_status'] = Sandbox.EXIT_TIMEOUT
# Merge results.
job.sandboxes = [s.path for s in sandbox_user] + [sandbox_mgr.path]
job.plus = plus_user
# If at least one evaluation had problems, we report the
# problems.
if not success_user or not success_mgr:
success, outcome, text = False, None, None
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not is_evaluation_passed(plus_user):
success = True
outcome, text = 0.0, human_evaluation_message(plus_user)
if job.get_output:
job.user_output = None
# Otherwise, we use the manager to obtain the outcome.
else:
success = True
outcome = None
text = None
input_filename = "input.txt"
output_filename = "output.txt"
# Check that the output file was created
if not sandbox_mgr.file_exists(output_filename):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"), output_filename
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = sandbox_mgr.get_file_to_storage(
output_filename,
"Output file in job %s" % job.info,
trunc_len=1024 * 1024 * 10)
# If just asked to execute, fill text and set dummy
# outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
# Put the reference solution into the sandbox
sandbox_mgr.create_file_from_storage("res.txt", job.output)
# Check the solution with white_diff
if self.parameters[1] == "diff":
outcome, text = white_diff_step(
sandbox_mgr, output_filename, "res.txt")
# Check the solution with a comparator
elif self.parameters[1] == "comparator":
manager_filename = "checker"
if manager_filename not in job.managers:
logger.error(
"Configuration error: missing or "
"invalid comparator (it must be "
"named 'checker')",
extra={"operation": job.info})
success = False
else:
sandbox_mgr.create_file_from_storage(
manager_filename,
job.managers[manager_filename].digest,
executable=True)
# Rewrite input file. The untrusted
# contestant program should not be able to
# modify it; however, the grader may
# destroy the input file to prevent the
# contestant's program from directly
# accessing it. Since we cannot create
# files already existing in the sandbox,
# we try removing the file first.
try:
sandbox_mgr.remove_file(input_filename)
except OSError as e:
# Let us be extra sure that the file
# was actually removed and we did not
# mess up with permissions.
assert not sandbox_mgr.file_exists(
input_filename)
sandbox_mgr.create_file_from_storage(
input_filename, job.input)
# Allow using any number of processes (because e.g.
# one may want to write a bash checker who calls
# other processes). Set to a high number because
# to avoid fork-bombing the worker.
sandbox_mgr.max_processes = 1000
success, _ = evaluation_step(
sandbox_mgr, [[
"./%s" % manager_filename, input_filename,
"res.txt", output_filename
]])
if success:
try:
outcome, text = \
extract_outcome_and_text(sandbox_mgr)
except ValueError as e:
logger.error(
"Invalid output from "
"comparator: %s",
e.message,
extra={"operation": job.info})
success = False
else:
raise ValueError("Unrecognized second parameter"
" `%s' for Communication tasktype." %
self.parameters[2])
# Whatever happened, we conclude.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
class Communication(TaskType):
"""Task type class for tasks with a fully admin-controlled process.
The task type will run *manager*, an admin-provided executable, and one or
more instances of the user solution, compiled together with a
language-specific stub.
During the evaluation, the manager and each of the user processes
communicate via FIFOs. The manager will read the input, send it (possibly
with some modifications) to the user process(es). The user processes, via
functions provided by the stub, will communicate with the manager. Finally,
the manager will decide outcome and text, and print them on stdout and
stderr.
The manager reads the input from stdin and writes to stdout and stderr the
standard manager output (that is, the outcome on stdout and the text on
stderr, see trusted.py for more information). It receives as argument the
names of the fifos: first from and to the first user process, then from and
to the second user process, and so on. It can also print some information
to a file named "output.txt"; the content of this file will be shown to
users submitting a user test.
The stub receives as argument the fifos (from and to the manager) and if
there are more than one user processes, the 0-based index of the process.
"""
# Filename of the manager (the stand-alone, admin-provided program).
MANAGER_FILENAME = "manager"
# Filename of the input in the manager sandbox. The content will be
# redirected to stdin, and managers should read from there.
INPUT_FILENAME = "input.txt"
# Filename where the manager can write additional output to show to users
# in case of a user test.
OUTPUT_FILENAME = "output.txt"
STUB_PRELOAD_FILENAME = "stub_preload"
ALLOW_PARTIAL_SUBMISSION = False
_NUM_PROCESSES = ParameterTypeInt("Number of Processes", "num_processes",
"")
ACCEPTED_PARAMETERS = [_NUM_PROCESSES]
@property
def name(self):
"""See TaskType.name."""
return "Communication"
def __init__(self, parameters):
super(Communication, self).__init__(parameters)
self.num_processes = 1
if len(self.parameters) > 0:
self.num_processes = self.parameters[0]
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
res = dict()
for language in LANGUAGES:
# Collect source filenames.
source_ext = language.source_extension
source_filenames = ["stub%s" % source_ext]
for codename in submission_format:
source_filenames.append(codename.replace(".%l", source_ext))
# Compute executable name.
executable_filename = self._executable_filename(submission_format)
# Build the compilation commands.
commands = language.get_compilation_commands(
source_filenames, executable_filename)
res[language.name] = commands
return res
def get_user_managers(self):
"""See TaskType.get_user_managers."""
return ["stub.%l"]
def get_auto_managers(self):
"""See TaskType.get_auto_managers."""
return ["manager"]
@staticmethod
def _executable_filename(codenames):
"""Return the chosen executable name computed from the codenames.
codenames ([str]): submission format or codename of submitted files,
may contain %l.
return (str): a deterministic executable name.
"""
return "_".join(
sorted(codename.replace(".%l", "") for codename in codenames))
def compile(self, job, file_cacher):
"""See TaskType.compile."""
language = get_language(job.language)
source_ext = language.source_extension
# Prepare the files to copy in the sandbox and to add to the
# compilation command.
files_to_get = {}
source_filenames = []
# The stub, that must have been provided (copy and add to compilation).
stub_filename = "stub%s" % source_ext
if not check_manager_present(job, stub_filename):
return
source_filenames.append(stub_filename)
files_to_get[stub_filename] = job.managers[stub_filename].digest
# User's submitted file(s) (copy and add to compilation).
for codename, file_ in iteritems(job.files):
source_filename = codename.replace(".%l", source_ext)
source_filenames.append(source_filename)
files_to_get[source_filename] = file_.digest
# Any other useful manager (just copy).
for filename, manager in iteritems(job.managers):
if is_manager_for_compilation(filename, language):
files_to_get[filename] = manager.digest
# Prepare the compilation command
executable_filename = self._executable_filename(iterkeys(job.files))
commands = language.get_compilation_commands(source_filenames,
executable_filename)
# Create the sandbox.
sandbox = create_sandbox(file_cacher, name="compile")
job.sandboxes.append(sandbox.path)
# Copy all required files in the sandbox.
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Run the compilation.
box_success, compilation_success, text, stats = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables.
job.success = box_success
job.compilation_success = compilation_success
job.text = text
job.plus = stats
if box_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup.
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, Communication.MANAGER_FILENAME):
return
manager_digest = job.managers[Communication.MANAGER_FILENAME].digest
# Indices for the objects related to each user process.
indices = range(self.num_processes)
# Create FIFOs.
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# Create the manager sandbox and copy manager and input.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.path)
sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME,
manager_digest,
executable=True)
sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME,
job.input)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [
create_sandbox(file_cacher, name="user_evaluate") for i in indices
]
job.sandboxes.extend(s.path for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(executable_filename,
executable_digest,
executable=True)
if Communication.STUB_PRELOAD_FILENAME in job.managers:
digest = job.managers[
Communication.STUB_PRELOAD_FILENAME].digest
sandbox_user[i].create_file_from_storage(
Communication.STUB_PRELOAD_FILENAME,
digest,
executable=True)
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % Communication.MANAGER_FILENAME]
for i in indices:
manager_command += [fifo_in[i], fifo_out[i]]
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib // 1024,
allow_dirs=fifo_dir,
writable_files=[Communication.OUTPUT_FILENAME],
stdin_redirect=Communication.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox)
# Start the user submissions compiled with the stub.
language = get_language(job.language)
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if self.num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="stub",
args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
last_cmd = commands[-1]
# Inject preload program if needed
if Communication.STUB_PRELOAD_FILENAME in job.managers:
last_cmd = [
"./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i],
fifo_in[i]
] + commands[-1]
processes[i] = evaluation_step_before_run(
sandbox_user[i],
last_cmd,
job.time_limit,
job.memory_limit,
allow_dirs=[fifo_dir[i]],
multiprocess=job.multithreaded_sandbox)
# Wait for the processes to conclude, without blocking them on I/O.
wait_without_std(processes + [manager])
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
pass
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user, job.feedback_level)
# Otherwise, we use the manager to obtain the outcome.
else:
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
Communication.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def setUp(self):
super().setUp()
self.p = ParameterTypeInt("name", "shortname", "description")
class TestParameterTypeInt(unittest.TestCase):
"""Test the class ParameterTypeInt."""
def setUp(self):
super().setUp()
self.p = ParameterTypeInt("name", "shortname", "description")
def test_validate_success(self):
self.p.validate(1)
self.p.validate(-1)
self.p.validate(0)
self.p.validate(2**34)
def test_validate_failure_wrong_type(self):
with self.assertRaises(ValueError):
self.p.validate("1")
with self.assertRaises(ValueError):
self.p.validate(1.0)
with self.assertRaises(ValueError):
self.p.validate(1j)
with self.assertRaises(ValueError):
self.p.validate([1])
def test_parse_string(self):
self.assertEqual(self.p.parse_string("123"), 123)
self.assertEqual(self.p.parse_string("-123"), -123)
def test_parse_handler(self):
h = FakeHandler({
"ok_shortname": "-45",
"fail_shortname": "not an int",
})
self.assertEqual(self.p.parse_handler(h, "ok_"), -45)
with self.assertRaises(ValueError):
self.p.parse_handler(h, "fail_")
with self.assertRaises(MissingArgumentError):
self.p.parse_handler(h, "missing_")
class Communication(TaskType):
"""Task type class for tasks with a fully admin-controlled process.
The task type will run *manager*, an admin-provided executable, and one or
more instances of the user solution, optionally compiled together with a
language-specific stub.
During the evaluation, the manager and each of the user processes
communicate via FIFOs. The manager will read the input, send it (possibly
with some modifications) to the user process(es). The user processes, either
via functions provided by the stub or by themselves, will communicate with
the manager. Finally, the manager will decide outcome and text, and print
them on stdout and stderr.
The manager reads the input from stdin and writes to stdout and stderr the
standard manager output (that is, the outcome on stdout and the text on
stderr, see trusted.py for more information). It receives as argument the
names of the fifos: first from and to the first user process, then from and
to the second user process, and so on. It can also print some information
to a file named "output.txt"; the content of this file will be shown to
users submitting a user test.
The user process receives as argument the fifos (from and to the manager)
and, if there are more than one user processes, the 0-based index of the
process. The pipes can also be set up to be redirected to stdin/stdout: in
that case the names of the pipes are not passed as arguments.
"""
# Filename of the manager (the stand-alone, admin-provided program).
MANAGER_FILENAME = "manager"
# Filename of the checker.
CHECKER_FILENAME = "checker"
# Basename of the stub, used in the stub filename and as the main class in
# languages that require us to specify it.
STUB_BASENAME = "stub"
# Filename of the input in the manager sandbox. The content will be
# redirected to stdin, and managers should read from there.
INPUT_FILENAME = "input.txt"
# Filename where the manager can write additional output to show to users
# in case of a user test.
OUTPUT_FILENAME = "output.txt"
# Constants used in the parameter definition.
COMPILATION_ALONE = "alone"
COMPILATION_STUB = "stub"
USER_IO_STD = "std_io"
USER_IO_FIFOS = "fifo_io"
OUTPUT_EVAL_MANAGER = "eval_manager"
OUTPUT_EVAL_DIFF = "eval_diff"
OUTPUT_EVAL_CHECKER = "eval_checker"
ALLOW_PARTIAL_SUBMISSION = False
_NUM_PROCESSES = ParameterTypeInt(
"Number of Processes",
"num_processes",
"")
_COMPILATION = ParameterTypeChoice(
"Compilation",
"compilation",
"",
{COMPILATION_ALONE: "Submissions are self-sufficient",
COMPILATION_STUB: "Submissions are compiled with a stub"})
_USER_IO = ParameterTypeChoice(
"User I/O",
"user_io",
"",
{USER_IO_STD: "User processes read from stdin and write to stdout",
USER_IO_FIFOS: "User processes read from and write to fifos, "
"whose paths are given as arguments"})
_EVALUATION = ParameterTypeChoice(
"Output evaluation",
"output_eval",
"",
{OUTPUT_EVAL_MANAGER: "Use manager output as evaluation result",
OUTPUT_EVAL_DIFF: "Use white diff to evaluate manager output",
OUTPUT_EVAL_CHECKER: "Use checker to evaluate manager output"})
ACCEPTED_PARAMETERS = [_NUM_PROCESSES, _COMPILATION, _USER_IO, _EVALUATION]
@property
def name(self):
"""See TaskType.name."""
return "Communication"
def __init__(self, parameters):
super().__init__(parameters)
self.num_processes = self.parameters[0]
self.compilation = self.parameters[1]
self.io = self.parameters[2]
self.eval = self.parameters[3]
def get_compilation_commands(self, submission_format):
"""See TaskType.get_compilation_commands."""
codenames_to_compile = []
if self._uses_stub():
codenames_to_compile.append(self.STUB_BASENAME + ".%l")
codenames_to_compile.extend(submission_format)
executable_filename = self._executable_filename(submission_format)
res = dict()
for language in LANGUAGES:
source_ext = language.source_extension
res[language.name] = language.get_compilation_commands(
[codename.replace(".%l", source_ext)
for codename in codenames_to_compile],
executable_filename)
return res
def get_user_managers(self):
"""See TaskType.get_user_managers."""
if self._uses_stub():
return [self.STUB_BASENAME + ".%l"]
else:
return []
def get_auto_managers(self):
"""See TaskType.get_auto_managers."""
return [self.MANAGER_FILENAME]
def _uses_stub(self):
return self.compilation == self.COMPILATION_STUB
def _uses_fifos(self):
return self.io == self.USER_IO_FIFOS
def _uses_white_diff(self):
return self.eval == self.OUTPUT_EVAL_DIFF
def _uses_checker(self):
return self.eval == self.OUTPUT_EVAL_CHECKER
@staticmethod
def _executable_filename(codenames):
"""Return the chosen executable name computed from the codenames.
codenames ([str]): submission format or codename of submitted files,
may contain %l.
return (str): a deterministic executable name.
"""
return "_".join(sorted(codename.replace(".%l", "")
for codename in codenames))
def compile(self, job, file_cacher):
"""See TaskType.compile."""
language = get_language(job.language)
source_ext = language.source_extension
if not check_files_number(job, 1, or_more=True):
return
# Prepare the files to copy in the sandbox and to add to the
# compilation command.
filenames_to_compile = []
filenames_and_digests_to_get = {}
# The stub, that must have been provided (copy and add to compilation).
if self._uses_stub():
stub_filename = self.STUB_BASENAME + source_ext
if not check_manager_present(job, stub_filename):
return
filenames_to_compile.append(stub_filename)
filenames_and_digests_to_get[stub_filename] = \
job.managers[stub_filename].digest
# User's submitted file(s) (copy and add to compilation).
for codename, file_ in job.files.items():
filename = codename.replace(".%l", source_ext)
filenames_to_compile.append(filename)
filenames_and_digests_to_get[filename] = file_.digest
# Any other useful manager (just copy).
for filename, manager in job.managers.items():
if is_manager_for_compilation(filename, language):
filenames_and_digests_to_get[filename] = manager.digest
# Prepare the compilation command
executable_filename = self._executable_filename(job.files.keys())
commands = language.get_compilation_commands(
filenames_to_compile, executable_filename)
# Create the sandbox.
sandbox = create_sandbox(file_cacher, name="compile")
job.sandboxes.append(sandbox.get_root_path())
# Copy all required files in the sandbox.
for filename, digest in filenames_and_digests_to_get.items():
sandbox.create_file_from_storage(filename, digest)
# Run the compilation.
box_success, compilation_success, text, stats = \
compilation_step(sandbox, commands)
# Retrieve the compiled executables.
job.success = box_success
job.compilation_success = compilation_success
job.text = text
job.plus = stats
if box_success and compilation_success:
digest = sandbox.get_file_to_storage(
executable_filename,
"Executable %s for %s" % (executable_filename, job.info))
job.executables[executable_filename] = \
Executable(executable_filename, digest)
# Cleanup.
delete_sandbox(sandbox, job.success, job.keep_sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iter(job.executables.keys()))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, self.MANAGER_FILENAME):
return
manager_digest = job.managers[self.MANAGER_FILENAME].digest
# Indices for the objects related to each user process.
indices = range(self.num_processes)
# Create FIFOs.
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_user_to_manager = [
os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices]
fifo_manager_to_user = [
os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_user_to_manager[i])
os.mkfifo(fifo_manager_to_user[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_user_to_manager[i], 0o666)
os.chmod(fifo_manager_to_user[i], 0o666)
# Names of the fifos after being mapped inside the sandboxes.
sandbox_fifo_dir = ["/fifo%d" % i for i in indices]
sandbox_fifo_user_to_manager = [
os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices]
sandbox_fifo_manager_to_user = [
os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices]
# Create the manager sandbox and copy manager and input.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.get_root_path())
sandbox_mgr.create_file_from_storage(
self.MANAGER_FILENAME, manager_digest, executable=True)
sandbox_mgr.create_file_from_storage(
self.INPUT_FILENAME, job.input)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [create_sandbox(file_cacher, name="user_evaluate")
for i in indices]
job.sandboxes.extend(s.get_root_path() for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(
executable_filename, executable_digest, executable=True)
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % self.MANAGER_FILENAME]
for i in indices:
manager_command += [sandbox_fifo_user_to_manager[i],
sandbox_fifo_manager_to_user[i]]
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib * 1024,
dirs_map=dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw"))
for i in indices),
writable_files=[self.OUTPUT_FILENAME],
stdin_redirect=self.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox)
# Start the user submissions compiled with the stub.
language = get_language(job.language)
main = self.STUB_BASENAME if self._uses_stub() else executable_filename
processes = [None for i in indices]
for i in indices:
args = []
stdin_redirect = None
stdout_redirect = None
if self._uses_fifos():
args.extend([sandbox_fifo_manager_to_user[i],
sandbox_fifo_user_to_manager[i]])
else:
stdin_redirect = sandbox_fifo_manager_to_user[i]
stdout_redirect = sandbox_fifo_user_to_manager[i]
if self.num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(
executable_filename,
main=main,
args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")},
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
# Wait for the processes to conclude, without blocking them on I/O.
wait_without_std(processes + [manager])
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user)
# Otherwise, we use the manager to obtain the outcome.
else:
# If the manager output is used as evaluation result
if not self._uses_white_diff() and not self._uses_checker():
outcome, text = extract_outcome_and_text(sandbox_mgr)
# Otherwise evaluate manager output to get evaluation result
else:
if not sandbox_mgr.file_exists(sandbox_mgr.stdout_file):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
sandbox_mgr.stdout_file]
else:
success, outcome, text = eval_output(
file_cacher, job,
self.CHECKER_FILENAME if self._uses_checker() else None,
user_output_path=sandbox_mgr.relative_path(
sandbox_mgr.stdout_file))
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(self.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
self.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox)
for s in sandbox_user:
delete_sandbox(s, job.success, job.keep_sandbox)
if job.success and not config.keep_sandbox and not job.keep_sandbox:
for d in fifo_dir:
rmtree(d)