def test_out_of_memory(self):
run_config = RunConfig(time_limit=3.0, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemOOM.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.ACCEPTED, TestStatus.MEMORY_LIMIT, TestStatus.ACCEPTED]
)
def test_unsuccessful_execution_cpp(self):
run_config = RunConfig(time_limit=0.2, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemWA1.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.ACCEPTED, TestStatus.WRONG_ANSWER, TestStatus.ACCEPTED]
)
def test_segfaults_4M_elements(self):
run_config = RunConfig(time_limit=0.5, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemSeg3.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.ACCEPTED, TestStatus.ACCEPTED, TestStatus.RUNTIME_ERROR]
)
def test_counting_total_time_of_all_processes(self):
run_config = RunConfig(time_limit=0.5, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemTime.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.ACCEPTED, TestStatus.TIME_LIMIT, TestStatus.ACCEPTED]
)
def test_exceeding_output_limit(self):
run_config = RunConfig(time_limit=3.0, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemRE2.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.ACCEPTED, TestStatus.RUNTIME_ERROR, TestStatus.ACCEPTED]
)
def test_floating_point_comparison(self):
run_config = RunConfig(time_limit=0.2, memory_limit=67108864, executable_path="fake/path", compare_floats=False)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblem.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
expected_results=[TestStatus.WRONG_ANSWER, TestStatus.WRONG_ANSWER, TestStatus.WRONG_ANSWER]
)
def test_exec_from_program(self):
run_config = RunConfig(time_limit=0.5, memory_limit=67108864, executable_path="fake/path", compare_floats=True)
self.exec_helper(
path_source=os.path.join(self.PATH_FIXTURES, "ABProblem/Solutions/ABProblemExec.cpp"),
tests=self.tests_abproblem,
run_config=run_config,
# Currently there is no limitation to exec with system() and possibly others =(
expected_results=[TestStatus.ACCEPTED, TestStatus.ACCEPTED, TestStatus.ACCEPTED]
)
def process_problem(self):
start_time = perf_counter()
completed_successfully = True
run_config = RunConfig(time_limit=self.time_limit,
memory_limit=self.memory_limit,
executable_path=self.path_executable,
checker_path=self.path_checker_executable,
tester_path=self.path_tester_executable,
compare_floats=self.floats)
test_futures = []
result_id = 0
for test in self.tests:
future = common.job_pool.submit(execute_problem, self.updater,
self.id, result_id, test,
run_config)
test_futures.append((test, future))
result_id += 1
for test, future in test_futures:
try:
future.result() # Wait for the test to be executed
except Exception as ex:
completed_successfully = False
logger.error(
"Submit {id} | Exception on test {test_name} ({test_hash}): {ex}"
.format(id=self.id,
test_name=test.inpFile,
test_hash=test.inpHash,
ex=str(ex)))
logger.info(
"Submit {id} | >> executed {cnt} tests in {time:.3f}s.".format(
id=self.id,
cnt=len(self.tests),
time=perf_counter() - start_time))
return completed_successfully
def process_game(self):
start_time = perf_counter()
completed_successfully = True
run_config = RunConfig(time_limit=self.time_limit,
memory_limit=self.memory_limit,
executable_path=self.path_executable,
tester_path=self.path_tester_executable)
result_id = 0
for match in self.matches:
logger.info(
"Submit {id} | -- running game {player1} vs {player2}...".
format(id=self.id,
player1=match["player_one_name"],
player2=match["player_two_name"]))
# Get and compile the opponent's solution
opponent_language = match["language"]
opponent_path_source = os.path.join(
self.path_sandbox, config.OPPONENT_SOURCE_NAME +
common.get_source_extension(opponent_language))
opponent_path_executable = os.path.join(
self.path_sandbox, config.OPPONENT_EXECUTABLE_NAME +
common.get_executable_extension(opponent_language))
logger.info(
"Submit {id} | ++ writing opponent's source...".format(
id=self.id))
if not self.write_source(match["source"], opponent_path_source):
logger.error(
"Submit {id} | Could not write opponent's source!".format(
id=self.id))
continue
logger.info(
"Submit {id} | ++ compiling opponent's source...".format(
id=self.id))
if self.compile(opponent_language, opponent_path_source,
opponent_path_executable) != "":
logger.error(
"Submit {id} | Could not compile opponent's source!".
format(id=self.id))
continue
# Run all of the game's tests for this pair of solutions
test_futures = []
for test in self.tests:
# Play forward game
future = common.job_pool.submit(
execute_game, self.updater, self.id, result_id, test,
run_config, match["player_one_id"],
match["player_one_name"], self.path_executable,
match["player_two_id"], match["player_two_name"],
opponent_path_executable)
test_futures.append([test, future])
result_id += 1
# Play also reversed game (first player as second) so it is fair
future = common.job_pool.submit(
execute_game, self.updater, self.id, result_id, test,
run_config, match["player_two_id"],
match["player_two_name"], opponent_path_executable,
match["player_one_id"], match["player_one_name"],
self.path_executable)
test_futures.append([test, future])
result_id += 1
for test, future in test_futures:
try:
future.result() # Wait for the test to be executed
except Exception as ex:
completed_successfully = False
logger.error(
"Submit {id} | Exception on test {test_name} ({test_hash}): {ex}"
.format(id=self.id,
test_name=test.inpFile,
test_hash=test.inpHash,
ex=str(ex)))
logger.info(
"Submit {id} | -- executed {cnt} matches in {time:.3f}s.".
format(id=self.id,
cnt=len(self.matches),
time=perf_counter() - start_time))
return completed_successfully
