def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iter(job.executables.keys()))
executable_digest = job.executables[executable_filename].digest
first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
job.sandboxes.append(first_sandbox.get_root_path())
job.sandboxes.append(second_sandbox.get_root_path())
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
first_executables_to_get = {executable_filename: executable_digest}
first_files_to_get = {TwoSteps.INPUT_FILENAME: job.input}
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.items():
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in first_files_to_get.items():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdin_redirect=TwoSteps.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Second step: we start the second manager.
second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
second_executables_to_get = {executable_filename: executable_digest}
second_files_to_get = {}
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.items():
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in second_files_to_get.items():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdout_redirect=TwoSteps.OUTPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Consume output.
wait_without_std([second, first])
box_success_first, evaluation_success_first, first_stats = \
evaluation_step_after_run(first_sandbox)
box_success_second, evaluation_success_second, second_stats = \
evaluation_step_after_run(second_sandbox)
box_success = box_success_first and box_success_second
evaluation_success = \
evaluation_success_first and evaluation_success_second
stats = merge_execution_stats(first_stats, second_stats)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
TwoSteps.OUTPUT_FILENAME
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
TwoSteps.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher,
job,
TwoSteps.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=second_sandbox.relative_path(
TwoSteps.OUTPUT_FILENAME))
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(first_sandbox, job.success, job.keep_sandbox)
delete_sandbox(second_sandbox, job.success, job.keep_sandbox)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = Batch.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(executable_filename,
main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {self._actual_input: job.input}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.path)
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
self._actual_output
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher,
job,
Batch.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
job.sandboxes.append(first_sandbox.get_root_path())
job.sandboxes.append(second_sandbox.get_root_path())
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
first_executables_to_get = {executable_filename: executable_digest}
first_files_to_get = {
TwoSteps.INPUT_FILENAME: job.input
}
# Put the required files into the sandbox
for filename, digest in iteritems(first_executables_to_get):
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(first_files_to_get):
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdin_redirect=TwoSteps.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Second step: we start the second manager.
second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
second_executables_to_get = {executable_filename: executable_digest}
second_files_to_get = {}
# Put the required files into the second sandbox
for filename, digest in iteritems(second_executables_to_get):
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(second_files_to_get):
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdout_redirect=TwoSteps.OUTPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Consume output.
wait_without_std([second, first])
box_success_first, evaluation_success_first, first_stats = \
evaluation_step_after_run(first_sandbox)
box_success_second, evaluation_success_second, second_stats = \
evaluation_step_after_run(second_sandbox)
box_success = box_success_first and box_success_second
evaluation_success = \
evaluation_success_first and evaluation_success_second
stats = merge_execution_stats(first_stats, second_stats)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
TwoSteps.OUTPUT_FILENAME]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
TwoSteps.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
TwoSteps.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=second_sandbox.relative_path(
TwoSteps.OUTPUT_FILENAME))
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(first_sandbox, job.success)
delete_sandbox(second_sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, Communication.MANAGER_FILENAME):
return
manager_digest = job.managers[Communication.MANAGER_FILENAME].digest
# Indices for the objects related to each user process.
indices = range(self.num_processes)
# Create FIFOs.
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
fifo_in = [os.path.join(fifo_dir[i], "in%d" % i) for i in indices]
fifo_out = [os.path.join(fifo_dir[i], "out%d" % i) for i in indices]
for i in indices:
os.mkfifo(fifo_in[i])
os.mkfifo(fifo_out[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_in[i], 0o666)
os.chmod(fifo_out[i], 0o666)
# Create the manager sandbox and copy manager and input.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.path)
sandbox_mgr.create_file_from_storage(Communication.MANAGER_FILENAME,
manager_digest,
executable=True)
sandbox_mgr.create_file_from_storage(Communication.INPUT_FILENAME,
job.input)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [
create_sandbox(file_cacher, name="user_evaluate") for i in indices
]
job.sandboxes.extend(s.path for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(executable_filename,
executable_digest,
executable=True)
if Communication.STUB_PRELOAD_FILENAME in job.managers:
digest = job.managers[
Communication.STUB_PRELOAD_FILENAME].digest
sandbox_user[i].create_file_from_storage(
Communication.STUB_PRELOAD_FILENAME,
digest,
executable=True)
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % Communication.MANAGER_FILENAME]
for i in indices:
manager_command += [fifo_in[i], fifo_out[i]]
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib // 1024,
allow_dirs=fifo_dir,
writable_files=[Communication.OUTPUT_FILENAME],
stdin_redirect=Communication.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox)
# Start the user submissions compiled with the stub.
language = get_language(job.language)
processes = [None for i in indices]
for i in indices:
args = [fifo_out[i], fifo_in[i]]
if self.num_processes != 1:
args.append(str(i))
commands = language.get_evaluation_commands(executable_filename,
main="stub",
args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
last_cmd = commands[-1]
# Inject preload program if needed
if Communication.STUB_PRELOAD_FILENAME in job.managers:
last_cmd = [
"./%s" % Communication.STUB_PRELOAD_FILENAME, fifo_out[i],
fifo_in[i]
] + commands[-1]
processes[i] = evaluation_step_before_run(
sandbox_user[i],
last_cmd,
job.time_limit,
job.memory_limit,
allow_dirs=[fifo_dir[i]],
multiprocess=job.multithreaded_sandbox)
# Wait for the processes to conclude, without blocking them on I/O.
wait_without_std(processes + [manager])
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
pass
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user, job.feedback_level)
# Otherwise, we use the manager to obtain the outcome.
else:
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(Communication.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
Communication.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
def evaluate(self, job, file_cacher):
if not check_executables_number(job, 1):
return
executable_filename = next(iter(job.executables.keys()))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, self.MANAGER_FILENAME):
return
manager_digest = job.managers[self.MANAGER_FILENAME].digest
# Create FIFO dirs and first batch of FIFos
fifo_dir_base = tempfile.mkdtemp(dir=config.temp_dir)
def fifo_dir(i, j):
p = os.path.join(fifo_dir_base, f"fifo{i}_{j}")
if not os.path.exists(p):
os.mkdir(p)
return p
abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq")
fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq")
os.mkfifo(fifo_solution_quitter)
os.mkfifo(fifo_manager_quitter)
os.chmod(abortion_control_fifo_dir, 0o755)
os.chmod(fifo_solution_quitter, 0o666)
os.chmod(fifo_manager_quitter, 0o666)
sandbox_abortion_control_fifo_dir = "/abort"
sandbox_fifo_solution_quitter = \
os.path.join(sandbox_abortion_control_fifo_dir, "sq")
sandbox_fifo_manager_quitter = \
os.path.join(sandbox_abortion_control_fifo_dir, "mq")
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % self.MANAGER_FILENAME]
manager_command += [
sandbox_fifo_solution_quitter, sandbox_fifo_manager_quitter
]
# Create the manager sandbox and copy manager and input and
# reference output.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.get_root_path())
sandbox_mgr.create_file_from_storage(self.MANAGER_FILENAME,
manager_digest,
executable=True)
sandbox_mgr.create_file_from_storage(self.INPUT_FILENAME, job.input)
sandbox_mgr.create_file_from_storage(self.OK_FILENAME, job.output)
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
sandbox_fifo_dir_base = "/fifo"
def sandbox_fifo_dir(i, j):
return f"{sandbox_fifo_dir_base}/fifo{i}_{j}"
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager_dirs_map = {
abortion_control_fifo_dir:
(sandbox_abortion_control_fifo_dir, "rw")
}
# TODO: can we avoid creating all these directories?
MAX_NUM_INSTANCES = 42
list_of_fifo_dirs = []
for pr in range(0, self.num_processes):
for i in range(0, MAX_NUM_INSTANCES):
d = fifo_dir(i, pr)
list_of_fifo_dirs.append(d)
manager_dirs_map[d] = (sandbox_fifo_dir(i, pr), "rw")
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib * 1024,
dirs_map=manager_dirs_map,
writable_files=[self.OUTPUT_FILENAME],
stdin_redirect=self.INPUT_FILENAME,
multiprocess=True)
solution_quitter = open(fifo_solution_quitter, "r")
manager_quitter = open(fifo_manager_quitter, "w")
def finish_run():
wait_without_std(processes)
L = [finish_run_single(i) for i in indices]
return all(L)
def finish_run_single(i):
nonlocal wall_clock_acc
nonlocal num_runs
user_results.append(evaluation_step_after_run(sandbox_user[i]))
wall_clock_acc += user_results[-1][2]["execution_wall_clock_time"]
num_runs += 1
runtimes[i].append(user_results[-1][2]["execution_time"])
# Convert tuple to list for write access to entries
L = list(user_results[-1])
L[2]["execution_time"] = runtimes[i][-1] / max_num_runs
if (L[2]["execution_time"] >= job.time_limit):
L[2]["exit_status"] = Sandbox.EXIT_TIMEOUT
user_results[-1] = tuple(L)
if not self._uses_stub():
# It can happen that the submission runs out of memory and then
# gets killed by the manager while it is being shut down, in
# which case isolate does not report a signal as the exit
# status. To catch this, we look for cg-oom-killed in the logs
sandbox_user[i].get_log()
if user_results[-1][1] and \
"cg-oom-killed" in sandbox_user[i].log:
# Convert tuple to list for write access to entries
r = list(user_results[-1])
r[1] = False
r[2]["status"] = ["SG"]
r[2]["exit_status"] = "signal"
r[2]["signal"] = -41 # sit by a lake
r[2]["message"] = ["out of memory"]
user_results[-1] = tuple(r)
return user_results[-1][0] and user_results[-1][1]
def respond(okay=True):
manager_quitter.write("O" if okay else "X")
manager_quitter.flush()
def read_int_from_manager():
L = []
while True:
c = solution_quitter.read(1)
if c == 'B':
break
else:
L.append(c)
return int("".join(L))
quit = False
for pr in range(0, self.num_processes):
if quit:
break
wall_clock_acc = 0
num_runs = 0
# Startup message to sync pipes
manager_quitter.write('S')
manager_quitter.flush()
# Ask the manager for the number of processes
num_instances = read_int_from_manager()
indices = range(0, num_instances)
max_num_runs = read_int_from_manager()
# Create remaining FIFOs
fifo_user_to_manager = [
os.path.join(fifo_dir(i, pr), f"u{pr}_{i}_to_m")
for i in indices
]
fifo_manager_to_user = [
os.path.join(fifo_dir(i, pr), f"m_to_u{pr}_{i}")
for i in indices
]
for i in indices:
os.mkfifo(fifo_user_to_manager[i])
os.mkfifo(fifo_manager_to_user[i])
os.chmod(fifo_dir(i, pr), 0o755)
os.chmod(fifo_user_to_manager[i], 0o666)
os.chmod(fifo_manager_to_user[i], 0o666)
# Names of the fifos after being mapped inside the sandboxes.
sandbox_fifo_user_to_manager = \
[os.path.join(sandbox_fifo_dir(i, pr),
f"u{pr}_{i}_to_m") for i in indices]
sandbox_fifo_manager_to_user = \
[os.path.join(sandbox_fifo_dir(i, pr),
f"m_to_u{pr}_{i}") for i in indices]
for i in indices:
print(sandbox_fifo_user_to_manager[i],
file=manager_quitter,
flush=True)
print(sandbox_fifo_manager_to_user[i],
file=manager_quitter,
flush=True)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [
create_sandbox(file_cacher, name="user_evaluate")
for i in indices
]
job.sandboxes.extend(s.get_root_path() for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(executable_filename,
executable_digest,
executable=True)
# Prepare the user submissions
language = get_language(job.language)
main = self.STUB_BASENAME if self._uses_stub() \
else os.path.splitext(executable_filename)[0]
processes = [None for i in indices]
user_results = []
args = []
stdin_redirect = None
stdout_redirect = None
if self._uses_fifos():
args.extend([
sandbox_fifo_manager_to_user[i],
sandbox_fifo_user_to_manager[i]
])
if self.num_processes != 1:
args.append(str(i))
if self._uses_stub():
main = self.STUB_BASENAME
else:
main = executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main, args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
processes = [None for _ in indices]
runtimes = [[] for _ in indices]
while True:
for i in indices:
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit * max_num_runs * num_instances,
job.memory_limit,
dirs_map={
fifo_dir(i, pr): (sandbox_fifo_dir(i, pr), "rw")
},
stdin_redirect=sandbox_fifo_manager_to_user[i],
stdout_redirect=sandbox_fifo_user_to_manager[i],
multiprocess=job.multithreaded_sandbox)
response = solution_quitter.read(1)
if response == "C": # continue
if not finish_run():
# this run was not successful, time to call it quits
quit = True
respond(okay=False)
break
respond()
elif response == "N": # next process
if not finish_run():
# this run was not successful, time to call it quits
quit = True
respond(okay=False)
break
respond()
break
elif response == "Q":
if not self._uses_stub():
time.sleep(.01)
processes[i].send_signal(signal.SIGINT)
finish_run()
respond()
quit = True
break
else:
raise RuntimeError("Received '{}' ".format(response) +
"through solution_quitter.")
# Wait for the manager to conclude, without blocking them on I/O.
wait_without_std([manager])
solution_quitter.close()
manager_quitter.close()
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(my_merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user)
# Otherwise, we use the manager to obtain the outcome.
else:
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(self.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
self.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox)
for s in sandbox_user:
delete_sandbox(s, job.success, job.keep_sandbox)
if job.success and not config.keep_sandbox and not job.keep_sandbox:
rmtree(fifo_dir_base)
rmtree(abortion_control_fifo_dir)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
# Make sure the required manager is among the job managers.
if not check_manager_present(job, self.MANAGER_FILENAME):
return
manager_digest = job.managers[self.MANAGER_FILENAME].digest
# Indices for the objects related to each user process.
indices = range(self.num_processes)
# Create FIFOs.
fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices]
if not self._uses_grader():
abortion_control_fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo_user_to_manager = [
os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices]
fifo_manager_to_user = [
os.path.join(fifo_dir[i], "m_to_u%d" % i) for i in indices]
if not self._uses_grader():
fifo_solution_quitter = os.path.join(abortion_control_fifo_dir, "sq")
fifo_manager_quitter = os.path.join(abortion_control_fifo_dir, "mq")
for i in indices:
os.mkfifo(fifo_user_to_manager[i])
os.mkfifo(fifo_manager_to_user[i])
os.chmod(fifo_dir[i], 0o755)
os.chmod(fifo_user_to_manager[i], 0o666)
os.chmod(fifo_manager_to_user[i], 0o666)
if not self._uses_grader():
os.mkfifo(fifo_solution_quitter)
os.mkfifo(fifo_manager_quitter)
os.chmod(abortion_control_fifo_dir, 0o755)
os.chmod(fifo_solution_quitter, 0o666)
os.chmod(fifo_manager_quitter, 0o666)
# Names of the fifos after being mapped inside the sandboxes.
sandbox_fifo_dir = ["/fifo%d" % i for i in indices]
sandbox_fifo_user_to_manager = [
os.path.join(sandbox_fifo_dir[i], "u%d_to_m" % i) for i in indices]
sandbox_fifo_manager_to_user = [
os.path.join(sandbox_fifo_dir[i], "m_to_u%d" % i) for i in indices]
if not self._uses_grader():
sandbox_abortion_control_fifo_dir = "/abort"
sandbox_fifo_solution_quitter = \
os.path.join(sandbox_abortion_control_fifo_dir, "sq")
sandbox_fifo_manager_quitter = \
os.path.join(sandbox_abortion_control_fifo_dir, "mq")
# Create the manager sandbox and copy manager and input and
# reference output.
sandbox_mgr = create_sandbox(file_cacher, name="manager_evaluate")
job.sandboxes.append(sandbox_mgr.get_root_path())
sandbox_mgr.create_file_from_storage(
self.MANAGER_FILENAME, manager_digest, executable=True)
sandbox_mgr.create_file_from_storage(
self.INPUT_FILENAME, job.input)
sandbox_mgr.create_file_from_storage(
self.OK_FILENAME, job.output)
# Create the user sandbox(es) and copy the executable.
sandbox_user = [create_sandbox(file_cacher, name="user_evaluate")
for i in indices]
job.sandboxes.extend(s.get_root_path() for s in sandbox_user)
for i in indices:
sandbox_user[i].create_file_from_storage(
executable_filename, executable_digest, executable=True)
# Start the manager. Redirecting to stdin is unnecessary, but for
# historical reasons the manager can choose to read from there
# instead than from INPUT_FILENAME.
manager_command = ["./%s" % self.MANAGER_FILENAME]
for i in indices:
manager_command += [sandbox_fifo_user_to_manager[i],
sandbox_fifo_manager_to_user[i]]
if not self._uses_grader():
manager_command += [sandbox_fifo_solution_quitter,
sandbox_fifo_manager_quitter]
# We could use trusted_step for the manager, since it's fully
# admin-controlled. But trusted_step is only synchronous at the moment.
# Thus we use evaluation_step, and we set a time limit generous enough
# to prevent user programs from sending the manager in timeout.
# This means that:
# - the manager wall clock timeout must be greater than the sum of all
# wall clock timeouts of the user programs;
# - with the assumption that the work the manager performs is not
# greater than the work performed by the user programs, the manager
# user timeout must be greater than the maximum allowed total time
# of the user programs; in theory, this is the task's time limit,
# but in practice is num_processes times that because the
# constraint on the total time can only be enforced after all user
# programs terminated.
manager_time_limit = max(self.num_processes * (job.time_limit + 1.0),
config.trusted_sandbox_max_time_s)
manager_dirs_map = dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw"))
for i in indices)
if not self._uses_grader():
manager_dirs_map[abortion_control_fifo_dir] = \
(sandbox_abortion_control_fifo_dir, "rw")
manager = evaluation_step_before_run(
sandbox_mgr,
manager_command,
manager_time_limit,
config.trusted_sandbox_max_memory_kib // 1024,
dirs_map=manager_dirs_map,
writable_files=[self.OUTPUT_FILENAME],
stdin_redirect=self.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox)
if not self._uses_grader():
solution_quitter = open(fifo_solution_quitter, "r")
manager_quitter = open(fifo_manager_quitter, "w")
manager_quitter_open = True
# Start the user submissions compiled with the stub.
language = get_language(job.language)
processes = [None for i in indices]
for i in indices:
args = [sandbox_fifo_manager_to_user[i],
sandbox_fifo_user_to_manager[i]]
if self.num_processes != 1:
args.append(str(i))
if self._uses_grader():
main = self.STUB_BASENAME
else:
main = executable_filename
commands = language.get_evaluation_commands(
executable_filename,
main=main,
args=args)
# Assumes that the actual execution of the user solution is the
# last command in commands, and that the previous are "setup"
# that don't need tight control.
if len(commands) > 1:
trusted_step(sandbox_user[i], commands[:-1])
processes[i] = evaluation_step_before_run(
sandbox_user[i],
commands[-1],
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir[i]: (sandbox_fifo_dir[i], "rw")},
stdin_redirect=sandbox_fifo_manager_to_user[i],
stdout_redirect=sandbox_fifo_user_to_manager[i],
multiprocess=job.multithreaded_sandbox)
if not self._uses_grader():
# Manager still running but wants to quit
if solution_quitter.read() == "<3":
for i in indices:
processes[i].send_signal(signal.SIGINT) # Kill user
wait_without_std(processes)
manager_quitter.close()
manager_quitter_open = False
# Wait for the processes to conclude, without blocking them on I/O.
wait_without_std(processes + [manager])
if not self._uses_grader():
solution_quitter.close()
if manager_quitter_open:
manager_quitter.close()
# Get the results of the manager sandbox.
box_success_mgr, evaluation_success_mgr, unused_stats_mgr = \
evaluation_step_after_run(sandbox_mgr)
# Coalesce the results of the user sandboxes.
user_results = [evaluation_step_after_run(s) for s in sandbox_user]
box_success_user = all(r[0] for r in user_results)
evaluation_success_user = all(r[1] for r in user_results)
stats_user = reduce(merge_execution_stats,
[r[2] for r in user_results])
# The actual running time is the sum of every user process, but each
# sandbox can only check its own; if the sum is greater than the time
# limit we adjust the result.
if box_success_user and evaluation_success_user and \
stats_user["execution_time"] >= job.time_limit:
evaluation_success_user = False
stats_user['exit_status'] = Sandbox.EXIT_TIMEOUT
success = box_success_user \
and box_success_mgr and evaluation_success_mgr
outcome = None
text = None
# If at least one sandbox had problems, or the manager did not
# terminate correctly, we report an error (and no need for user stats).
if not success:
stats_user = None
pass
# If just asked to execute, fill text and set dummy outcome.
elif job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# If the user sandbox detected some problem (timeout, ...),
# the outcome is 0.0 and the text describes that problem.
elif not evaluation_success_user:
outcome = 0.0
text = human_evaluation_message(stats_user)
# Otherwise, we use the manager to obtain the outcome.
else:
outcome, text = extract_outcome_and_text(sandbox_mgr)
# If asked so, save the output file with additional information,
# provided that it exists.
if job.get_output:
if sandbox_mgr.file_exists(self.OUTPUT_FILENAME):
job.user_output = sandbox_mgr.get_file_to_storage(
self.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
else:
job.user_output = None
# Fill in the job with the results.
job.success = success
job.outcome = "%s" % outcome if outcome is not None else None
job.text = text
job.plus = stats_user
delete_sandbox(sandbox_mgr, job.success)
for s in sandbox_user:
delete_sandbox(s, job.success)
if not config.keep_sandbox:
for d in fifo_dir:
rmtree(d)
if not self._uses_grader():
rmtree(abortion_control_fifo_dir)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = self.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main)
# HACK for NECKLACE: one-time hack to support a task with very low memory limit
if job.memory_limit == 3 and job.language == "Java / JDK":
jvm_args = ["-Deval=true", "-Xmx4224k", "-Xss256k", "-XX:MaxMetaspaceSize=8704k"]
commands = language.get_evaluation_commands(
executable_filename, main=main, jvm_args=jvm_args)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {
self._actual_input: job.input
}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.get_root_path())
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Special handling: if there's a batchmanager, then this is really an
# interactive task to be evaluated in a single sandbox.
# Do NOT use check_manager_present() here, as it will raise an error
# for normal tasks with no batchmanager.
if Batch.MANAGER_CODENAME in job.managers:
sandbox.create_file_from_storage(Batch.MANAGER_CODENAME,
job.managers[Batch.MANAGER_CODENAME].digest, executable=True)
# If there is a batchmanagermanager, run the last command with it.
commands[-1][:0] = ["./%s" % Batch.MANAGER_CODENAME,
self.input_filename, self.output_filename]
# Actually performs the execution
# HACK for NECKLACE: one-time hack to support a task with very low memory limit
if job.memory_limit == 3 and job.language == "Java / JDK":
memory_limit = 20
elif job.memory_limit == 3 and job.language == "Python 3 / CPython":
memory_limit = 8
elif job.memory_limit == 3 and job.language == "C++11 / g++":
memory_limit = 4
elif job.memory_limit == 3 and job.language == "C11 / gcc":
memory_limit = 4
else:
memory_limit = job.memory_limit
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.effective_time_limit(),
memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
self._actual_output]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
self.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = self.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {
self._actual_input: job.input
}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.get_root_path())
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
self._actual_output]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
self.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job.success)
