def test_update_interval(self, send_request_mock):
updater = Updater("localhost", 1337, TESTS)
updater.add_info(status=TestStatus.COMPILING)
# An update should be sent shortly within the new info arrives
sleep(0.1)
self.assertTrue(send_request_mock.call_count == 1, "An update to the frontend is not being sent.")
updater.add_info(status=TestStatus.TESTING)
# But it shouldn't be called too often...
sleep(0.1)
self.assertTrue(send_request_mock.call_count == 1, "Updates to the frontend are sent too often.")
# ... instead, they should be called every UPDATE_INTERVAL seconds
sleep(config.UPDATE_INTERVAL)
self.assertTrue(send_request_mock.call_count == 2, "A second update was not sent after update INTERVAL.")
class Evaluator:
def __init__(self, data):
# Submit information
self.id = data["id"]
self.source = data["source"]
self.language = data["language"]
self.time_limit = data["timeLimit"]
self.memory_limit = data[
"memoryLimit"] * 1048576 # Given in MiB, convert to bytes
self.problem_type = data["problemType"]
# Front-end endpoint
self.update_url = data["updateEndpoint"]
# List of tests and endpoint where to download them from
self.tests = [
TestInfo(inpFile=test["inpFile"],
inpHash=test["inpHash"],
solFile=test["solFile"],
solHash=test["solHash"],
position=test["position"]) for test in data["tests"]
]
self.tests_url = data["testsEndpoint"]
# If a task with checker, there should also be an endpoint where to download it from
self.path_checker_source = None
self.path_checker_executable = None
if "checker" in data and data["checker"] != "":
self.path_checker_source = os.path.join(
config.PATH_CHECKERS,
data["checker"] + config.SOURCE_EXTENSION_CPP)
self.path_checker_executable = os.path.join(
config.PATH_CHECKERS,
data["checker"] + config.EXECUTABLE_EXTENSION_CPP)
self.checker_url = data[
"checkerEndpoint"] if "checkerEndpoint" in data else None
# If a task with tester, there should also be an endpoint where to download it from
self.path_tester_source = None
self.path_tester_executable = None
if "tester" in data and data["tester"] != "":
self.path_tester_source = os.path.join(
config.PATH_TESTERS,
data["tester"] + config.SOURCE_EXTENSION_CPP)
self.path_tester_executable = os.path.join(
config.PATH_TESTERS,
data["tester"] + config.EXECUTABLE_EXTENSION_CPP)
self.tester_url = data[
"testerEndpoint"] if "testerEndpoint" in data else None
# If a game, there should also be a list of matches (opponents' names and solutions)
self.matches = data["matches"] if "matches" in data else None
# Whether to use relative or absolute floating point comparison
self.floats = data["floats"]
# Path to sandbox and files inside
self.path_sandbox = os.path.join(config.PATH_SANDBOX, "submits",
"submit_{:06d}/".format(self.id))
self.path_source = os.path.join(
self.path_sandbox,
config.SOURCE_NAME + common.get_source_extension(self.language))
self.path_executable = os.path.join(
self.path_sandbox, config.EXECUTABLE_NAME +
common.get_executable_extension(self.language))
# Frontend server update logic
self.updater = Updater(self.update_url, self.id, self.tests)
def __del__(self):
self.cleanup()
def evaluate(self):
# Send an update that preparation has been started for executing this submission
logger.info("Submit {id} | Evaluating submit {submit_id}".format(
id=self.id, submit_id=self.id))
self.updater.add_info(status=TestStatus.PREPARING)
# Create sandbox directory
logger.info("Submit {id} | >> creating sandbox directory...".format(
id=self.id))
if not self.create_sandbox_dir():
self.updater.add_info(
message="Error while creating sandbox directory!",
status=TestStatus.INTERNAL_ERROR)
return
# Download the test files (if not downloaded already)
logger.info(
"Submit {id} | >> downloading test files...".format(id=self.id))
if not self.download_tests():
self.updater.add_info(
message="Error while downloading test files!",
status=TestStatus.INTERNAL_ERROR)
return
# Download and compile the checker (if not already available)
if self.path_checker_executable is not None:
if not os.path.exists(self.path_checker_executable):
logger.info(
"Submit {id} | >> downloading and compiling checker...".
format(id=self.id))
if not self.setup_utility_file(self.path_checker_source,
self.path_checker_executable,
self.checker_url):
self.updater.add_info(
message="Error while setting up checker!",
status=TestStatus.INTERNAL_ERROR)
return
# Download and compile the tester (if not already available)
if self.path_tester_executable is not None:
if not os.path.exists(self.path_tester_executable):
logger.info(
"Submit {id} | >> downloading and compiling tester...".
format(id=self.id))
if not self.setup_utility_file(self.path_tester_source,
self.path_tester_executable,
self.tester_url):
self.updater.add_info(
message="Error while setting up tester!",
status=TestStatus.INTERNAL_ERROR)
return
# Send an update that the compilation has been started for this submission
self.updater.add_info(status=TestStatus.COMPILING)
# Save the source to a file so we can compile it later
logger.info("Submit {id} | >> writing source code to file...".format(
id=self.id))
if not self.write_source(self.source, self.path_source):
self.updater.add_info(
message="Error while writing the source to a file!",
status=TestStatus.INTERNAL_ERROR)
return
# Compile
logger.info(
"Submit {id} | >> compiling solution...".format(id=self.id))
compilation_status = self.compile(self.language, self.path_source,
self.path_executable)
if compilation_status != "":
self.updater.add_info(message=compilation_status,
status=TestStatus.COMPILATION_ERROR)
return
# Run the solution on the problem's tests or with the provided tester
logger.info(
"Submit {id} | >> starting evaluation of solution...".format(
id=self.id))
if not self.run_solution():
self.updater.add_info(
message="Error while processing the solution!",
status=TestStatus.INTERNAL_ERROR)
return
# Finished with this submission
logger.info(
"Submit {id} | Completed evaluating submit {submit_id}.".format(
id=self.id, submit_id=self.id))
self.updater.add_info(message="DONE")
def create_sandbox_dir(self):
try:
# Delete if already present (maybe regrade?)
if os.path.exists(self.path_sandbox):
shutil.rmtree(self.path_sandbox)
# Create the submit testing directory
if not os.path.exists(self.path_sandbox):
os.makedirs(self.path_sandbox)
except OSError as ex:
logger.error(
"Submit {id} | Could not create sandbox directory. Exception was: {ex}"
.format(id=self.id, ex=str(ex)))
return False
return True
def download_test(self, file_name, file_hash, file_path):
# Download only if the file doesn't already exist
if not os.path.exists(file_path):
logger.info(
"Submit {id} | Downloading file {file_name} with hash {file_hash} from URL: {url}"
.format(id=self.id,
file_name=file_name,
file_hash=file_hash,
url=self.tests_url + file_name))
network.download_file(self.tests_url + file_name, file_path)
def download_tests(self):
# In case the directory for the tests does not exist, create it
if not os.path.exists(config.PATH_TESTS):
os.makedirs(config.PATH_TESTS)
try:
for test in self.tests:
self.download_test(test.inpFile, test.inpHash, test.inpPath)
self.download_test(test.solFile, test.solHash, test.solPath)
except Exception as ex:
logger.error(
"Submit {id} | Could not download tests. Exception was: {ex}".
format(id=self.id, ex=str(ex)))
return False
return True
def compile(self,
language,
path_source,
path_executable,
print_error=False):
try:
compile_status = Compiler.compile(language, path_source,
path_executable)
if compile_status != "":
error_log_func = logger.error if print_error else logger.info
error_log_func(
"Submit {id} | Could not compile file {file_path}!".format(
id=self.id, file_path=path_source))
return compile_status
except Exception as ex:
# If a non-compiler error occurred, log the message in addition to sending it to the user
logger.error(
"Submit {id} | Could not compile file {file_path}! Exception was: {ex}"
.format(id=self.id, file_path=path_source, ex=str(ex)))
return "Internal Error: " + str(ex)
def compile_utility_file(self, path_source, path_executable):
# If already compiled, return directly
if os.path.exists(path_executable):
return True
# Otherwise do the compilation
logger.info(
"Submit {id} | >> compiling utility file {file_name}...".format(
id=self.id, file_name=os.path.basename(path_source)))
return self.compile(config.LANGUAGE_CPP, path_source, path_executable,
True) == ""
def download_utility_file(self, download_url, destination):
# If already downloaded, return directly
if os.path.exists(destination):
return True
# Otherwise download it
logger.info(
"Submit {id} | >> downloading utility file {file_name}".format(
id=self.id, file_name=download_url.split('/')[-1]))
try:
network.download_file(download_url, destination)
except RuntimeError:
return False
return True
def setup_utility_file(self, path_source, path_executable, download_url):
if not self.download_utility_file(download_url, path_source):
return False
if not self.compile_utility_file(path_source, path_executable):
return False
return True
def write_source(self, source, destination):
try:
with open(destination, "w") as file:
file.write(source)
except OSError as ex:
logger.error(
"Submit {id} | Could not write source file. Exception was: {ex}"
.format(id=self.id, ex=str(ex)))
return False
return True
def process_problem(self):
start_time = perf_counter()
completed_successfully = True
run_config = RunConfig(time_limit=self.time_limit,
memory_limit=self.memory_limit,
executable_path=self.path_executable,
checker_path=self.path_checker_executable,
tester_path=self.path_tester_executable,
compare_floats=self.floats)
test_futures = []
result_id = 0
for test in self.tests:
future = common.job_pool.submit(execute_problem, self.updater,
self.id, result_id, test,
run_config)
test_futures.append((test, future))
result_id += 1
for test, future in test_futures:
try:
future.result() # Wait for the test to be executed
except Exception as ex:
completed_successfully = False
logger.error(
"Submit {id} | Exception on test {test_name} ({test_hash}): {ex}"
.format(id=self.id,
test_name=test.inpFile,
test_hash=test.inpHash,
ex=str(ex)))
logger.info(
"Submit {id} | >> executed {cnt} tests in {time:.3f}s.".format(
id=self.id,
cnt=len(self.tests),
time=perf_counter() - start_time))
return completed_successfully
def process_game(self):
start_time = perf_counter()
completed_successfully = True
run_config = RunConfig(time_limit=self.time_limit,
memory_limit=self.memory_limit,
executable_path=self.path_executable,
tester_path=self.path_tester_executable)
result_id = 0
for match in self.matches:
logger.info(
"Submit {id} | -- running game {player1} vs {player2}...".
format(id=self.id,
player1=match["player_one_name"],
player2=match["player_two_name"]))
# Get and compile the opponent's solution
opponent_language = match["language"]
opponent_path_source = os.path.join(
self.path_sandbox, config.OPPONENT_SOURCE_NAME +
common.get_source_extension(opponent_language))
opponent_path_executable = os.path.join(
self.path_sandbox, config.OPPONENT_EXECUTABLE_NAME +
common.get_executable_extension(opponent_language))
logger.info(
"Submit {id} | ++ writing opponent's source...".format(
id=self.id))
if not self.write_source(match["source"], opponent_path_source):
logger.error(
"Submit {id} | Could not write opponent's source!".format(
id=self.id))
continue
logger.info(
"Submit {id} | ++ compiling opponent's source...".format(
id=self.id))
if self.compile(opponent_language, opponent_path_source,
opponent_path_executable) != "":
logger.error(
"Submit {id} | Could not compile opponent's source!".
format(id=self.id))
continue
# Run all of the game's tests for this pair of solutions
test_futures = []
for test in self.tests:
# Play forward game
future = common.job_pool.submit(
execute_game, self.updater, self.id, result_id, test,
run_config, match["player_one_id"],
match["player_one_name"], self.path_executable,
match["player_two_id"], match["player_two_name"],
opponent_path_executable)
test_futures.append([test, future])
result_id += 1
# Play also reversed game (first player as second) so it is fair
future = common.job_pool.submit(
execute_game, self.updater, self.id, result_id, test,
run_config, match["player_two_id"],
match["player_two_name"], opponent_path_executable,
match["player_one_id"], match["player_one_name"],
self.path_executable)
test_futures.append([test, future])
result_id += 1
for test, future in test_futures:
try:
future.result() # Wait for the test to be executed
except Exception as ex:
completed_successfully = False
logger.error(
"Submit {id} | Exception on test {test_name} ({test_hash}): {ex}"
.format(id=self.id,
test_name=test.inpFile,
test_hash=test.inpHash,
ex=str(ex)))
logger.info(
"Submit {id} | -- executed {cnt} matches in {time:.3f}s.".
format(id=self.id,
cnt=len(self.matches),
time=perf_counter() - start_time))
return completed_successfully
def run_solution(self):
if self.problem_type == "interactive" or self.problem_type == "game":
if self.path_tester_executable is None:
logger.error(
"Submit {id} | Game or interactive problem without a tester!"
.format(id=self.id))
return False
if self.path_tester_executable is not None and self.problem_type != "interactive":
return self.process_game()
else:
return self.process_problem()
def cleanup(self):
# Clean up remaining files
if os.path.exists(self.path_sandbox):
shutil.rmtree(self.path_sandbox)
class Evaluator:
def __init__(self, data):
# Sleep for a very short while so werkzeug can print its log BEFORE we start printing from here
sleep(0.01)
# Submit information
self.id = data["id"]
self.source = data["source"]
self.language = data["language"]
self.time_limit = data["timeLimit"]
self.memory_limit = data["memoryLimit"] * 1048576 # Given in MiB, convert to bytes
# Front-end endpoint
self.update_url = data["updateEndpoint"]
# List of tests and endpoint where to download them from
self.tests = data["tests"]
self.tests_url = data["testsEndpoint"]
# If a task with checker, there should also be an endpoint where to download it from
self.checker = data["checker"] if ("checker" in data and data["checker"] != "") else None
self.checker_url = data["checkerEndpoint"] if "checkerEndpoint" in data else None
# If a game, there should also be a tester and a list of matches (opponents' names and solutions)
self.tester = data["tester"] if ("tester" in data and data["tester"] != "") else None
self.tester_url = data["testerEndpoint"] if "testerEndpoint" in data else None
self.matches = data["matches"] if "matches" in data else None
# Whether to use relative or absolute floating point comparison
self.floats = data["floats"]
# Path to sandbox and files inside
self.path_sandbox = config.PATH_SANDBOX + "submit_{:06d}/".format(self.id)
self.path_source = self.path_sandbox + config.SOURCE_NAME + common.get_source_extension(self.language)
self.path_executable = self.path_sandbox + config.EXECUTABLE_NAME + common.get_executable_extension(self.language)
# Frontend server update logic
self.updater = Updater(self.update_url, self.id, self.tests)
# Configure logger
self.logger = logging.getLogger("evltr")
def __del__(self):
# Clean up remaining files
self.cleanup()
def evaluate(self):
# Send an update that preparation has been started for executing this submission
self.logger.info("[Submission {}] Evaluating submission {}".format(self.id, self.id))
self.updater.add_info("", None, TestStatus.PREPARING)
# Create sandbox directory
self.logger.info("[Submission {}] >> creating sandbox directory...".format(self.id))
if not self.create_sandbox_dir():
self.updater.add_info("Error while creating sandbox directory!", None, TestStatus.INTERNAL_ERROR)
return
# Download the test files (if not downloaded already)
self.logger.info("[Submission {}] >> downloading test files...".format(self.id))
if not self.download_tests():
self.updater.add_info("Error while downloading test files!", None, TestStatus.INTERNAL_ERROR)
return
# Download and compile the checker (if not already available)
if self.checker is not None and not path.exists(config.PATH_CHECKERS + self.checker):
self.logger.info("[Submission {}] >> updating checker file...".format(self.id))
if not self.download_and_compile_utility_file(config.PATH_CHECKERS, self.checker, self.checker_url):
self.updater.add_info("Error while setting up checker!", None, TestStatus.INTERNAL_ERROR)
return
# Download and compile the tester (if not already available)
if self.tester is not None and not path.exists(config.PATH_TESTERS + self.tester):
self.logger.info("[Submission {}] >> updating tester file...".format(self.id))
if not self.download_and_compile_utility_file(config.PATH_TESTERS, self.tester, self.tester_url):
self.updater.add_info("Error while setting up tester!", None, TestStatus.INTERNAL_ERROR)
return
# Save the source to a file so we can compile it later
self.logger.info("[Submission {}] >> writing source code to file...".format(self.id))
if not self.write_source(self.source, self.path_source):
self.updater.add_info("Error while writing the source to a file!", None, TestStatus.INTERNAL_ERROR)
return
# Send an update that the compilation has been started for this submission
self.updater.add_info("", None, TestStatus.COMPILING)
# Compile
self.logger.info("[Submission {}] >> compiling solution...".format(self.id))
compilation_status = self.compile(self.language, self.path_source, self.path_executable)
if compilation_status != "":
self.logger.info("[Submission {}] Compilation error! Aborting...".format(self.id))
self.updater.add_info(compilation_status, None, TestStatus.COMPILATION_ERROR)
return
# If a standard task, just run the solution on the given tests
self.logger.info("[Submission {}] >> starting evaluation of solution...".format(self.id))
if not self.run_solution():
self.logger.info("[Submission {}] Error while processing the solution! Aborting...".format(self.id))
self.updater.add_info("Error while processing the solution!", None, TestStatus.INTERNAL_ERROR)
return
# Finished with this submission
self.logger.info("[Submission {}] >> done with {}!".format(self.id, self.id))
self.updater.add_info("DONE", None, None)
def create_sandbox_dir(self):
try:
# Delete if already present (maybe regrade?)
if path.exists(self.path_sandbox):
shutil.rmtree(self.path_sandbox)
# Create the submit testing directory
if not path.exists(self.path_sandbox):
makedirs(self.path_sandbox)
except OSError as ex:
self.logger.error("[Submission {}] Could not create sandbox directory. Error was: {}".format(self.id, str(ex)))
return False
return True
def download_test(self, test_name, test_hash):
test_path = config.PATH_TESTS + test_hash
# Download only if the file doesn't already exist
if not path.exists(test_path):
self.logger.info("[Submission {}] Downloading file {} with hash {} from URL: {}".format(
self.id, test_name, test_hash, self.tests_url + test_name))
common.download_file(self.tests_url + test_name, test_path)
def download_tests(self):
# In case the directory for the tests does not exist, create it
if not path.exists(config.PATH_DATA):
makedirs(config.PATH_DATA)
if not path.exists(config.PATH_TESTS):
makedirs(config.PATH_TESTS)
try:
for test in self.tests:
self.download_test(test["inpFile"], test["inpHash"])
self.download_test(test["solFile"], test["solHash"])
except Exception as ex:
self.logger.error("[Submission {}] Could not download tests properly. Error was: {}".format(self.id, str(ex)))
return False
return True
def compile(self, language, path_source, path_executable):
try:
return common.executor.submit(Compiler.compile, language, path_source, path_executable).result()
except ValueError as ex:
# If a non-compiler error occurred, log the message in addition to sending it to the user
self.logger.error("[Submission {}] Could not compile file {}! Error was: {}".format(
self.id, path_source, str(ex)))
return "Internal Error: " + str(ex)
def compile_utility_file(self, path_source, path_executable):
# Only compile if not already compiled
if not path.exists(path_executable):
self.logger.info("[Submission {}] >> compiling utility file {}...".format(
self.id, path.basename(path_source)))
return self.compile("C++", path_source, path_executable) == ""
return True
def download_utility_file(self, url, destination):
# Only download if not downloaded already
if not path.exists(destination):
self.logger.info("[Submission {}] >> downloading utility file {}".format(self.id, url.split('/')[-1]))
try:
common.download_file(url, destination)
except RuntimeError:
return False
return True
def download_and_compile_utility_file(self, directory, file_hash, url):
path_source = directory + file_hash + config.SOURCE_EXTENSION_CPP
path_executable = directory + file_hash + config.EXECUTABLE_EXTENSION_CPP
if not self.download_utility_file(url, path_source):
return False
if not self.compile_utility_file(path_source, path_executable):
return False
return True
def write_source(self, source, destination):
try:
with open(destination, "w") as file:
file.write(source)
except OSError as ex:
self.logger.error("[Submission {}] Could not write source file. Error: ".format(self.id, str(ex)))
return False
return True
def process_tests(self):
start_time = perf_counter()
runner = Runner(self)
errors = ""
test_futures = []
for result_id in range(len(self.tests)):
test_futures.append([self.tests[result_id],
common.executor.submit(runner.run, result_id, self.tests[result_id])])
for test, future in test_futures:
try:
# Wait for the test to be executed
future.result()
except Exception as ex:
errors += "Internal error on test " + test["inpFile"] + "(" + test["inpHash"] + "): " + str(ex)
self.logger.error("[Submission {}] Got exception: {}".format(self.id, str(ex)))
break
self.logger.info("[Submission {}] -- executed {} tests in {:.3f}s.".format(
self.id, len(self.tests), perf_counter() - start_time))
return errors
def process_games(self):
start_time = perf_counter()
runner = Runner(self)
errors = ""
result_id = 0
for match in self.matches:
self.logger.info("[Submission {}] -- running game {} vs {}...".format(
self.id, match["player_one_name"], match["player_two_name"]))
# Get and compile the opponent's solution
opponent_language = match["language"]
opponent_path_source = self.path_sandbox + config.OPPONENT_SOURCE_NAME +\
common.get_source_extension(opponent_language)
opponent_path_executable = self.path_sandbox + config.OPPONENT_EXECUTABLE_NAME +\
common.get_executable_extension(opponent_language)
self.logger.info("[Submission {}] ++ writing opponent's source...".format(self.id))
if not self.write_source(match["source"], opponent_path_source):
self.logger.error("[Submission {}] Could not write opponent's source!".format(self.id))
continue
self.logger.info("[Submission {}] ++ compiling opponent's source...".format(self.id))
if self.compile(opponent_language, opponent_path_source, opponent_path_executable) != "":
self.logger.error("[Submission {}] Could not compile opponent's source!".format(self.id))
continue
# Run all of the game's tests for this pair of solutions
test_futures = []
for test in self.tests:
# Play forward game
future = common.executor.submit(runner.play, result_id, test, self.tester,
match["player_one_id"], match["player_one_name"], self.path_executable,
match["player_two_id"], match["player_two_name"], opponent_path_executable)
test_futures.append([test, future])
result_id += 1
# Play also reversed game (first player as second) so it is fair
future = common.executor.submit(runner.play, result_id, test, self.tester,
match["player_two_id"], match["player_two_name"], opponent_path_executable,
match["player_one_id"], match["player_one_name"], self.path_executable)
test_futures.append([test, future])
result_id += 1
for test_future in test_futures:
test, future = test_future
try:
# Wait for the test to be executed
future.result()
except ValueError as ex:
errors += "Internal error on test " + test["inpFile"] + "(" + test["inpHash"] + "): " + str(ex)
self.logger.error("[Submission {}] {}".format(self.id, str(ex)))
break
except Exception as ex:
self.logger.error("[Submission {}] Got exception: {}".format(self.id, str(ex)))
self.logger.info("[Submission {}] -- executed {} matches in {:.3f}s.".format(
self.id, len(self.matches), perf_counter() - start_time))
return errors
def run_solution(self):
if self.tester is None:
run_status = self.process_tests()
if run_status != "":
self.logger.info("[Submission {}] Error while processing the tests: {}!".format(self.id, run_status))
return False
# If a game, set-up the runner and opponents' solutions, then simulate the game
else:
run_status = self.process_games()
if run_status != "":
self.logger.info("[Submission {}] Error while processing the games: {}!".format(self.id, run_status))
return False
return True
def cleanup(self):
self.logger.info("[Submission {}] Cleaning up sandbox...".format(self.id))
if path.exists(self.path_sandbox):
shutil.rmtree(self.path_sandbox)
def test_send_update(self, send_request_mock):
updater = Updater("localhost", 1337, TESTS)
updater.add_info(status=TestStatus.COMPILING)
# An update should be sent shortly within the new info arrives
sleep(0.1)
self.assertTrue(send_request_mock.call_count == 1, "An update to the frontend is not being sent.")
