def test_failure_second_none(self):
with self.assertRaises(ValueError):
merge_execution_stats(None, None)
r0 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK)
with self.assertRaises(ValueError):
merge_execution_stats(r0, None)
def test_failure_second_none(self):
with self.assertRaises(ValueError):
merge_execution_stats(None, None)
r0 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK)
with self.assertRaises(ValueError):
merge_execution_stats(r0, None)
def test_success_first_status_ok(self):
self.assertStats(
merge_execution_stats(get_stats(0, 0, 0, Sandbox.EXIT_OK),
get_stats(0, 0, 0, Sandbox.EXIT_TIMEOUT)),
get_stats(0, 0, 0, Sandbox.EXIT_TIMEOUT))
self.assertStats(
merge_execution_stats(
get_stats(0, 0, 0, Sandbox.EXIT_OK),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11)),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11))
self.assertStats(
merge_execution_stats(
get_stats(0, 0, 0, Sandbox.EXIT_OK),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11)),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11))
def test_success_output_joined(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="e1")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK,
stdout="o1\n===\no2", stderr="e1\n===\ne2"))
def test_success_first_status_not_ok(self):
self.assertStats(
merge_execution_stats(
get_stats(0, 0, 0, Sandbox.EXIT_TIMEOUT),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11)),
get_stats(0, 0, 0, Sandbox.EXIT_TIMEOUT))
self.assertStats(
merge_execution_stats(
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=9),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=11)),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=9))
self.assertStats(
merge_execution_stats(
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=9),
get_stats(0, 0, 0, Sandbox.EXIT_OK)),
get_stats(0, 0, 0, Sandbox.EXIT_SIGNAL, signal=9))
def test_many_processes_merged_timeout(self):
# Solution was ok, but considering all runtimes, it hit timeout.
tt, job = self.prepare(
[2, "stub", "fifo_io"],
{"foo": EXE_FOO}, {"manager": MANAGER})
job.time_limit = 2.5
stats0 = dict(STATS_OK)
stats0["execution_time"] = 1.0
stats1 = dict(STATS_OK)
stats1["execution_time"] = 2.0
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
self._set_evaluation_step_return_values({
sandbox_mgr: (True, True, STATS_OK),
sandbox_usr0: (True, True, stats0),
sandbox_usr1: (True, True, stats1),
})
tt.evaluate(job, self.file_cacher)
# The stats are the merge of the two, but the status is changed to
# timeout since the sum of the cpu times is over the time limit.
stats = merge_execution_stats(stats0, stats1)
stats["exit_status"] = "timeout"
self.assertResultsInJob(
job, True, str(0.0), self.human_evaluation_message.return_value,
stats)
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def test_many_processes_merged_timeout(self):
# Solution was ok, but considering all runtimes, it hit timeout.
tt, job = self.prepare([2, "stub", "fifo_io", "eval_manager"],
{"foo": EXE_FOO}, {"manager": MANAGER})
job.time_limit = 2.5
stats0 = dict(STATS_OK)
stats0["execution_time"] = 1.0
stats1 = dict(STATS_OK)
stats1["execution_time"] = 2.0
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
self._set_evaluation_step_return_values({
sandbox_mgr: (True, True, STATS_OK),
sandbox_usr0: (True, True, stats0),
sandbox_usr1: (True, True, stats1),
})
tt.evaluate(job, self.file_cacher)
# The stats are the merge of the two, but the status is changed to
# timeout since the sum of the cpu times is over the time limit.
stats = merge_execution_stats(stats0, stats1)
stats["exit_status"] = "timeout"
self.assertResultsInJob(job, True, str(0.0),
self.human_evaluation_message.return_value,
stats)
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def test_empty_outputs_are_preserved(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK,
stdout="o1\n===\n", stderr="\n===\ne2"))
def test_success_sequential(self):
# In non-concurrent mode memory is max'd and wall clock is added.
self.assertStats(
merge_execution_stats(get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK),
get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK),
concurrent=False),
get_stats(1.1, 2.2, 300.0, Sandbox.EXIT_OK))
def test_success_sequential(self):
# In non-concurrent mode memory is max'd and wall clock is added.
self.assertStats(
merge_execution_stats(
get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK),
get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK),
concurrent=False),
get_stats(1.1, 2.2, 300.0, Sandbox.EXIT_OK))
def test_success_stats_are_not_modified(self):
r0 = get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK)
r1 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11)
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(1.1, 2.0, 300.3, Sandbox.EXIT_SIGNAL, signal=11))
self.assertStats(r0, get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK))
self.assertStats(
r1, get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11))
def test_success_stats_are_not_modified(self):
r0 = get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK)
r1 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11)
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(1.1, 2.0, 300.3, Sandbox.EXIT_SIGNAL, signal=11))
self.assertStats(
r0, get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK))
self.assertStats(
r1, get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11))
def test_success_output_joined(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="e1")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m,
get_stats(0,
0,
0,
Sandbox.EXIT_OK,
stdout="o1\n===\no2",
stderr="e1\n===\ne2"))
def test_empty_outputs_are_preserved(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m,
get_stats(0,
0,
0,
Sandbox.EXIT_OK,
stdout="o1\n===\n",
stderr="\n===\ne2"))
def test_many_processes_last_user_failure(self):
# One of the user programs had problems, it's the user's fault.
tt, job = self.prepare([2], {"foo": EXE_FOO}, {"manager": MANAGER})
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
self._set_evaluation_step_return_values({
sandbox_mgr: (True, True, STATS_OK),
sandbox_usr0: (True, True, STATS_OK),
sandbox_usr1: (True, False, STATS_RE),
})
tt.evaluate(job, self.file_cacher)
self.assertResultsInJob(job, True, str(0.0),
self.human_evaluation_message.return_value,
merge_execution_stats(STATS_OK, STATS_RE))
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def test_many_processes_last_user_failure(self):
# One of the user programs had problems, it's the user's fault.
tt, job = self.prepare(
["grader", 2], {"foo": EXE_FOO}, {"manager": MANAGER})
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
self._set_evaluation_step_return_values({
sandbox_mgr: (True, True, STATS_OK),
sandbox_usr0: (True, True, STATS_OK),
sandbox_usr1: (True, False, STATS_RE),
})
tt.evaluate(job, self.file_cacher)
self.assertResultsInJob(
job, True, str(0.0), self.human_evaluation_message.return_value,
merge_execution_stats(STATS_OK, STATS_RE))
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def test_success_output_missing_one(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="e2"))
def test_success_first_none(self):
r1 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11)
m = merge_execution_stats(None, r1)
self.assertStats(m, r1)
self.assertIsNot(r1, m)
def test_many_processes_success(self):
tt, job = self.prepare([2, "stub", "fifo_io", "eval_manager"],
{"foo": EXE_FOO}, {"manager": MANAGER})
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
tt.evaluate(job, self.file_cacher)
# Sandboxes created with the correct file cacher and names.
self.Sandbox.assert_has_calls([
call(self.file_cacher, name="manager_evaluate"),
call(self.file_cacher, name="user_evaluate"),
call(self.file_cacher, name="user_evaluate"),
],
any_order=False)
self.assertEqual(self.Sandbox.call_count, 3)
# We need input (with the default filename for redirection) and
# executable copied in the sandbox.
sandbox_mgr.create_file_from_storage.assert_has_calls([
call("manager", "digest of manager", executable=True),
call("input.txt", "digest of input"),
],
any_order=True)
self.assertEqual(sandbox_mgr.create_file_from_storage.call_count, 2)
# Same content in both user sandboxes.
for s in [sandbox_usr0, sandbox_usr1]:
s.create_file_from_storage.assert_has_calls([
call("foo", "digest of foo", executable=True),
],
any_order=True)
self.assertEqual(s.create_file_from_storage.call_count, 1)
# Evaluation step called with the right arguments, in particular
# redirects, and no (other) writable files. For the user's command,
# see fake_evaluation_commands in the mixin.
cmdline_mgr = [
"./manager", "/fifo0/u0_to_m", "/fifo0/m_to_u0", "/fifo1/u1_to_m",
"/fifo1/m_to_u1"
]
cmdline_usr0 = [
"run1", "foo", "stub", "/fifo0/m_to_u0", "/fifo0/u0_to_m", "0"
]
cmdline_usr1 = [
"run1", "foo", "stub", "/fifo1/m_to_u1", "/fifo1/u1_to_m", "1"
]
self.evaluation_step_before_run.assert_has_calls([
call(sandbox_mgr,
cmdline_mgr,
4321,
1234 * 1024 * 1024,
dirs_map={
os.path.join(self.base_dir, "0"): ("/fifo0", "rw"),
os.path.join(self.base_dir, "1"): ("/fifo1", "rw"),
},
writable_files=["output.txt"],
stdin_redirect="input.txt",
multiprocess=True),
call(sandbox_usr0,
cmdline_usr0,
2.5,
123 * 1024 * 1024,
dirs_map={os.path.join(self.base_dir, "0"): ("/fifo0", "rw")},
stdin_redirect=None,
stdout_redirect=None,
multiprocess=True),
call(sandbox_usr1,
cmdline_usr1,
2.5,
123 * 1024 * 1024,
dirs_map={os.path.join(self.base_dir, "1"): ("/fifo1", "rw")},
stdin_redirect=None,
stdout_redirect=None,
multiprocess=True),
],
any_order=True)
self.assertEqual(self.evaluation_step_before_run.call_count, 3)
self.assertEqual(self.evaluation_step_after_run.call_count, 3)
# Results put in job and sandbox deleted.
self.assertResultsInJob(job, True, str(OUTCOME), TEXT,
merge_execution_stats(STATS_OK, STATS_OK))
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iterkeys(job.executables))
executable_digest = job.executables[executable_filename].digest
first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
job.sandboxes.append(first_sandbox.get_root_path())
job.sandboxes.append(second_sandbox.get_root_path())
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
first_executables_to_get = {executable_filename: executable_digest}
first_files_to_get = {
TwoSteps.INPUT_FILENAME: job.input
}
# Put the required files into the sandbox
for filename, digest in iteritems(first_executables_to_get):
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(first_files_to_get):
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdin_redirect=TwoSteps.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Second step: we start the second manager.
second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
second_executables_to_get = {executable_filename: executable_digest}
second_files_to_get = {}
# Put the required files into the second sandbox
for filename, digest in iteritems(second_executables_to_get):
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in iteritems(second_files_to_get):
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdout_redirect=TwoSteps.OUTPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Consume output.
wait_without_std([second, first])
box_success_first, evaluation_success_first, first_stats = \
evaluation_step_after_run(first_sandbox)
box_success_second, evaluation_success_second, second_stats = \
evaluation_step_after_run(second_sandbox)
box_success = box_success_first and box_success_second
evaluation_success = \
evaluation_success_first and evaluation_success_second
stats = merge_execution_stats(first_stats, second_stats)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
TwoSteps.OUTPUT_FILENAME]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
TwoSteps.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
TwoSteps.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=second_sandbox.relative_path(
TwoSteps.OUTPUT_FILENAME))
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(first_sandbox, job.success)
delete_sandbox(second_sandbox, job.success)
def test_many_processes_success(self):
tt, job = self.prepare(
[2, "stub", "fifo_io"],
{"foo": EXE_FOO}, {"manager": MANAGER})
sandbox_mgr = self.expect_sandbox()
sandbox_usr0 = self.expect_sandbox()
sandbox_usr1 = self.expect_sandbox()
tt.evaluate(job, self.file_cacher)
# Sandboxes created with the correct file cacher and names.
self.Sandbox.assert_has_calls([
call(self.file_cacher, name="manager_evaluate"),
call(self.file_cacher, name="user_evaluate"),
call(self.file_cacher, name="user_evaluate"),
], any_order=False)
self.assertEqual(self.Sandbox.call_count, 3)
# We need input (with the default filename for redirection) and
# executable copied in the sandbox.
sandbox_mgr.create_file_from_storage.assert_has_calls([
call("manager", "digest of manager", executable=True),
call("input.txt", "digest of input"),
], any_order=True)
self.assertEqual(sandbox_mgr.create_file_from_storage.call_count, 2)
# Same content in both user sandboxes.
for s in [sandbox_usr0, sandbox_usr1]:
s.create_file_from_storage.assert_has_calls([
call("foo", "digest of foo", executable=True),
], any_order=True)
self.assertEqual(s.create_file_from_storage.call_count, 1)
# Evaluation step called with the right arguments, in particular
# redirects, and no (other) writable files. For the user's command,
# see fake_evaluation_commands in the mixin.
cmdline_mgr = ["./manager",
"/fifo0/u0_to_m", "/fifo0/m_to_u0",
"/fifo1/u1_to_m", "/fifo1/m_to_u1"]
cmdline_usr0 = ["run1", "foo", "stub",
"/fifo0/m_to_u0", "/fifo0/u0_to_m", "0"]
cmdline_usr1 = ["run1", "foo", "stub",
"/fifo1/m_to_u1", "/fifo1/u1_to_m", "1"]
self.evaluation_step_before_run.assert_has_calls([
call(sandbox_mgr, cmdline_mgr, 4321, 1234 * 1024 * 1024,
dirs_map={
os.path.join(self.base_dir, "0"): ("/fifo0", "rw"),
os.path.join(self.base_dir, "1"): ("/fifo1", "rw"),
},
writable_files=["output.txt"],
stdin_redirect="input.txt", multiprocess=True),
call(sandbox_usr0, cmdline_usr0, 2.5, 123 * 1024 * 1024,
dirs_map={os.path.join(self.base_dir, "0"): ("/fifo0", "rw")},
stdin_redirect=None,
stdout_redirect=None,
multiprocess=True),
call(sandbox_usr1, cmdline_usr1, 2.5, 123 * 1024 * 1024,
dirs_map={os.path.join(self.base_dir, "1"): ("/fifo1", "rw")},
stdin_redirect=None,
stdout_redirect=None,
multiprocess=True),
], any_order=True)
self.assertEqual(self.evaluation_step_before_run.call_count, 3)
self.assertEqual(self.evaluation_step_after_run.call_count, 3)
# Results put in job and sandbox deleted.
self.assertResultsInJob(job, True, str(OUTCOME), TEXT,
merge_execution_stats(STATS_OK, STATS_OK))
sandbox_mgr.cleanup.assert_called_once_with(delete=True)
sandbox_usr0.cleanup.assert_called_once_with(delete=True)
sandbox_usr1.cleanup.assert_called_once_with(delete=True)
def test_missing_outputs_are_not_preserved(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK)
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2"))
def test_missing_outputs_are_not_preserved(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK)
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o2", stderr="e2"))
def test_success_first_none(self):
r1 = get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_SIGNAL, signal=11)
m = merge_execution_stats(None, r1)
self.assertStats(m, r1)
self.assertIsNot(r1, m)
def test_success_status_ok(self):
self.assertStats(
merge_execution_stats(get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK),
get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK)),
get_stats(1.1, 2.0, 300.3, Sandbox.EXIT_OK))
def test_success_output_missing_one(self):
r0 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1")
r1 = get_stats(0, 0, 0, Sandbox.EXIT_OK, stderr="e2")
m = merge_execution_stats(r0, r1)
self.assertStats(
m, get_stats(0, 0, 0, Sandbox.EXIT_OK, stdout="o1", stderr="e2"))
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
executable_filename = next(iter(job.executables.keys()))
executable_digest = job.executables[executable_filename].digest
first_sandbox = create_sandbox(file_cacher, name="first_evaluate")
second_sandbox = create_sandbox(file_cacher, name="second_evaluate")
job.sandboxes.append(first_sandbox.get_root_path())
job.sandboxes.append(second_sandbox.get_root_path())
fifo_dir = tempfile.mkdtemp(dir=config.temp_dir)
fifo = os.path.join(fifo_dir, "fifo")
os.mkfifo(fifo)
os.chmod(fifo_dir, 0o755)
os.chmod(fifo, 0o666)
# First step: we start the first manager.
first_command = ["./%s" % executable_filename, "0", "/fifo/fifo"]
first_executables_to_get = {executable_filename: executable_digest}
first_files_to_get = {TwoSteps.INPUT_FILENAME: job.input}
# Put the required files into the sandbox
for filename, digest in first_executables_to_get.items():
first_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in first_files_to_get.items():
first_sandbox.create_file_from_storage(filename, digest)
first = evaluation_step_before_run(
first_sandbox,
first_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdin_redirect=TwoSteps.INPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Second step: we start the second manager.
second_command = ["./%s" % executable_filename, "1", "/fifo/fifo"]
second_executables_to_get = {executable_filename: executable_digest}
second_files_to_get = {}
# Put the required files into the second sandbox
for filename, digest in second_executables_to_get.items():
second_sandbox.create_file_from_storage(filename,
digest,
executable=True)
for filename, digest in second_files_to_get.items():
second_sandbox.create_file_from_storage(filename, digest)
second = evaluation_step_before_run(
second_sandbox,
second_command,
job.time_limit,
job.memory_limit,
dirs_map={fifo_dir: ("/fifo", "rw")},
stdout_redirect=TwoSteps.OUTPUT_FILENAME,
multiprocess=job.multithreaded_sandbox,
wait=False)
# Consume output.
wait_without_std([second, first])
box_success_first, evaluation_success_first, first_stats = \
evaluation_step_after_run(first_sandbox)
box_success_second, evaluation_success_second, second_stats = \
evaluation_step_after_run(second_sandbox)
box_success = box_success_first and box_success_second
evaluation_success = \
evaluation_success_first and evaluation_success_second
stats = merge_execution_stats(first_stats, second_stats)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not second_sandbox.file_exists(TwoSteps.OUTPUT_FILENAME):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
TwoSteps.OUTPUT_FILENAME
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage
if job.get_output:
job.user_output = second_sandbox.get_file_to_storage(
TwoSteps.OUTPUT_FILENAME,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher,
job,
TwoSteps.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=second_sandbox.relative_path(
TwoSteps.OUTPUT_FILENAME))
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(first_sandbox, job.success, job.keep_sandbox)
delete_sandbox(second_sandbox, job.success, job.keep_sandbox)
def test_success_status_ok(self):
self.assertStats(
merge_execution_stats(
get_stats(1.0, 2.0, 300, Sandbox.EXIT_OK),
get_stats(0.1, 0.2, 0.3, Sandbox.EXIT_OK)),
get_stats(1.1, 2.0, 300.3, Sandbox.EXIT_OK))
