def compile(self, submission_id=None, report_status=False):
def report(status=STATUS_RUNABLE):
if report_status:
self.post_id += 1
result = {
"post_id": self.post_id,
"submission_id": submission_id,
"status_id": status
}
self.cloud.post_result('api_compile_result', result)
if submission_id == None:
# compile in current directory
compiler.compile_anything()
else:
submission_dir = self.submission_dir(submission_id)
download_dir = self.download_dir(submission_id)
if os.path.exists(submission_dir):
log.info("Already compiled: %s" % submission_id)
if self.functional_test(submission_id):
report(STATUS_RUNABLE)
return True
else:
report(STATUS_TEST_ERROR)
return False
if not os.path.exists(download_dir):
if not self.download_submission(submission_id):
report(STATUS_DOWNLOAD_ERROR)
log.error("Download Error")
return False
if len(os.listdir(download_dir)) == 1:
if not self.unpack(submission_id):
report(STATUS_UNPACK_ERROR)
log.error("Unpack Error")
return False
log.info("Compiling %s " % submission_id)
detected_lang, errors = compiler.compile_anything(download_dir)
if not detected_lang:
shutil.rmtree(download_dir)
log.error('\n'.join(errors))
report(STATUS_COMPILE_ERROR)
log.error("Compile Error")
return False
else:
if not os.path.exists(os.path.split(submission_dir)[0]):
os.makedirs(os.path.split(submission_dir)[0])
os.rename(download_dir, submission_dir)
if self.functional_test(submission_id):
report(STATUS_RUNABLE)
return True
else:
log.error("Functional Test Error")
report(STATUS_TEST_ERROR)
return False
def testStarterPackages(self):
'''Archive the starter packages like the website would'''
SP_DIR = "starterpackages"
if os.path.isdir(SP_DIR):
shutil.rmtree(SP_DIR)
os.system("cd ../../website/; ./archiveStarterPackages.sh");
shutil.copytree("../../website/downloads/starterpackages/", SP_DIR)
for f in glob.glob(os.path.join(SP_DIR, "*.zip")):
zip_ref = zipfile.ZipFile(f, 'r')
zip_ref.extractall(SP_DIR)
zip_ref.close()
folderName = os.path.splitext(os.path.basename(f))[0]
folder = os.path.join(SP_DIR, os.path.splitext(os.path.basename(f))[0])
for filename in os.listdir(folder):
if os.path.splitext(filename)[0] == "RandomBot":
os.remove(os.path.join(folder, filename))
expectedLanguage = folderName.split("-")[1]
print("Expected Language: " + expectedLanguage)
language, errors = compiler.compile_anything(folder)
if errors is not None: print("Errors: " + "\n".join(errors))
print("Language: " + language)
assert language == expectedLanguage
assert errors == None
def testStarterPackages(self):
'''Archive the starter packages like the website would'''
SP_DIR = "starterpackages"
if os.path.isdir(SP_DIR):
shutil.rmtree(SP_DIR)
os.system("cd ../../website/; ./archiveStarterPackages.sh")
shutil.copytree("../../website/downloads/starterpackages/", SP_DIR)
for f in glob.glob(os.path.join(SP_DIR, "*.zip")):
zip_ref = zipfile.ZipFile(f, 'r')
zip_ref.extractall(SP_DIR)
zip_ref.close()
folderName = os.path.splitext(os.path.basename(f))[0]
folder = os.path.join(SP_DIR,
os.path.splitext(os.path.basename(f))[0])
for filename in os.listdir(folder):
if os.path.splitext(filename)[0] == "RandomBot":
os.remove(os.path.join(folder, filename))
expectedLanguage = folderName.split("-")[1]
print("Expected Language: " + expectedLanguage)
language, errors = compiler.compile_anything(folder)
if errors is not None: print("Errors: " + "\n".join(errors))
print("Language: " + language)
assert language == expectedLanguage
assert errors == None
def testLanguageOverride(self):
'''Use a LANGUAGE file to override the detected language'''
LANGUAGE_BOT_PATH = "languageBot"
bot_dir = os.path.join(OUR_PATH, LANGUAGE_BOT_PATH)
expectedLanguage = "TestLanguage"
language, errors = compiler.compile_anything(bot_dir)
if errors is not None: print("Errors: " + "\n".join(errors))
print("Language: " + language)
assert language == expectedLanguage
assert errors == None
def testLanguageOverride(self):
'''Use a LANGUAGE file to override the detected language'''
LANGUAGE_BOT_PATH = "languageBot"
bot_dir = os.path.join(OUR_PATH, LANGUAGE_BOT_PATH)
expectedLanguage = "TestLanguage"
language, errors = compiler.compile_anything(bot_dir)
if errors is not None: print("Errors: " + "\n".join(errors))
print("Language: " + language)
assert language == expectedLanguage
assert errors == None
def testStarterPackages(self):
'''Archive the starter packages like the website would'''
SP_DIR = "starterpackages"
if os.path.isdir(SP_DIR):
shutil.rmtree(SP_DIR)
os.system("cd ../../website/; ./archiveStarterPackages.sh");
shutil.copytree("../../website/downloads/starterpackages/", SP_DIR)
for f in glob.glob(os.path.join(SP_DIR, "*.zip")):
zip_ref = zipfile.ZipFile(f, 'r')
zip_ref.extractall(SP_DIR)
zip_ref.close()
folderName = os.path.splitext(os.path.basename(f))[0]
folder = os.path.join(SP_DIR, os.path.splitext(os.path.basename(f))[0])
expectedLanguage = folderName.split("-")[1]
language, errors = compiler.compile_anything(folder)
print(errors)
print(language)
assert language == expectedLanguage
assert errors == None
def setupParticipant(user_index, user, temp_dir):
"""
Download and set up the bot for a game participant.
"""
# Include username to deal with duplicate bots
bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"])
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
archive.unpack(backend.storeBotLocally(user["user_id"],
user["bot_id"], bot_dir))
if user.get("requires_compilation"):
compile_dir = bot_dir + '_compile'
try:
# Move to temp directory to avoid permission problems
# (can't chown files created by compile user back to us)
shutil.move(bot_dir, compile_dir)
# Give the compilation user access
os.chmod(compile_dir, 0o2755)
# User needs to be able to write to the directory
give_ownership(compile_dir, "bots", 0o2774)
language, errors = compiler.compile_anything(compile_dir)
didCompile = errors is None
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
if not didCompile:
# Abort and upload an error log
rm_as_user("bot_compilation", compile_dir)
raise OndemandCompileError(language, '\n'.join(errors))
# Move back to original directory
try:
shutil.copytree(compile_dir, bot_dir)
except shutil.Error as e:
logging.error("Could not compile bot ondemand", e)
rm_as_user("bot_compilation", compile_dir)
# Make the start script executable
os.chmod(os.path.join(bot_dir, RUNFILE), 0o755)
# Give the bot user ownership of their directory
# We should set up each user's default group as a group that the
# worker is also a part of. Then we always have access to their
# files, but not vice versa.
# https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group
bot_user = "******".format(user_index)
bot_group = "bots_{}".format(user_index)
bot_cgroup = "bot_{}".format(user_index)
# We want 775 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the
# group
give_ownership(bot_dir, bot_group, 0o2775)
bot_command = BOT_COMMAND.format(
cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_group=bot_group,
bot_user=bot_user,
runfile=RUNFILE,
)
bot_name = "{} v{}".format(user["username"], user["version_number"])
return bot_command, bot_name, bot_dir
def executeCompileTask(user_id, bot_id, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
logging.debug("Compiling a bot with userID %s\n" % str(user_id))
errors = []
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
try:
bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir,
is_compile=True)
archive.unpack(bot_path)
# Make sure things are in the top-level directory
while len([
name for name in os.listdir(temp_dir)
if os.path.isfile(os.path.join(temp_dir, name))
]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1:
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder:
singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0]
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), bufferFolder)
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), temp_dir)
# Context manager takes care of buffer folder
# Delete any symlinks
subprocess.call(["find", temp_dir, "-type", "l", "-delete"])
# Give the compilation user access
os.chmod(temp_dir, 0o755)
# User needs to be able to write to the directory and create files
give_ownership(temp_dir, "bots", 0o2770)
# Reset cwd before compilation, in case it was in a
# deleted temporary folder
os.chdir(os.path.dirname(os.path.realpath(sys.argv[0])))
language, more_errors = compiler.compile_anything(temp_dir)
didCompile = more_errors is None
if more_errors:
errors.extend(more_errors)
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
try:
if didCompile:
logging.debug("Bot did compile\n")
archive_path = os.path.join(temp_dir, str(user_id)+".zip")
archive.zipFolder(temp_dir, archive_path)
backend.storeBotRemotely(user_id, bot_id, archive_path)
else:
logging.debug("Bot did not compile\n")
logging.debug("Bot errors %s\n" % str(errors))
backend.compileResult(user_id, bot_id, didCompile, language,
errors=(None if didCompile else "\n".join(errors)))
except Exception as e:
logging.error("Bot did not upload", e)
errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc())
backend.compileResult(user_id, bot_id, False, language,
errors="\n".join(errors))
finally:
# Remove files as bot user (Python will clean up tempdir, but we don't
# necessarily have permissions to clean up files)
rm_as_user("bot_compilation", temp_dir)
def compile(self, submission_id=None, report_status=(False, False), run_test=True):
report_success, report_failure = report_status
def report(status, language="Unknown", errors=None):
# oooh, tricky, a terinary in an if
if report_success if type(errors) != list else report_failure:
self.post_id += 1
result = {"post_id": self.post_id,
"submission_id": submission_id,
"status_id": status,
"language": language }
if status != 40:
if type(errors) != list:
errors = [errors] # for valid json
result['errors'] = json.dumps(errors)
return self.cloud.post_result('api_compile_result', result)
else:
return True
if submission_id == None:
# compile in current directory
compiler.compile_anything(os.getcwd())
else:
submission_dir = self.submission_dir(submission_id)
if os.path.exists(submission_dir):
log.info("Already compiled: %s" % submission_id)
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
if report(STATUS_RUNABLE, compiler.get_run_lang(submission_dir)):
return True
else:
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
else:
report(STATUS_TEST_ERROR, compiler.get_run_lang(submission_dir), errors)
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
if (not submission_id in self.download_dirs or
len(os.listdir(self.download_dir(submission_id))) == 0):
if not self.download_submission(submission_id):
report(STATUS_DOWNLOAD_ERROR)
log.error("Download Error")
return False
download_dir = self.download_dir(submission_id)
if not os.path.exists(os.path.join(self.download_dir(submission_id),
'bot')):
if len(os.listdir(download_dir)) == 1:
if not self.unpack(submission_id):
report(STATUS_UNPACK_ERROR)
log.error("Unpack Error")
return False
log.info("Compiling %s " % submission_id)
bot_dir = os.path.join(download_dir, 'bot')
timelimit = 10 * 60 # 10 minute limit to compile submission
if not run_test:
# give it 50% more time if this isn't the initial compilation
# this is to try and prevent the situation where the initial
# compilation just makes it in the time limit and then a
# subsequent compilation fails when another worker goes to
# play a game with it
timelimit += timelimit * 0.5
detected_lang, errors = compiler.compile_anything(bot_dir,
timelimit)
if errors != None:
log.error(errors)
if not self.debug:
shutil.rmtree(download_dir)
log.error(detected_lang)
report(STATUS_COMPILE_ERROR, detected_lang, errors=errors);
log.error("Compile Error")
return False
else:
log.info("Detected language: {0}".format(detected_lang))
if not os.path.exists(os.path.split(submission_dir)[0]):
os.makedirs(os.path.split(submission_dir)[0])
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
os.rename(download_dir, submission_dir)
del self.download_dirs[submission_id]
if report(STATUS_RUNABLE, detected_lang):
return True
else:
# could not report back to server, cleanup compiled dir
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
else:
log.info("Functional Test Failure")
report(STATUS_TEST_ERROR, detected_lang, errors)
return False
def compile(self, submission_id=None, report_status=False, run_test=True):
def report(status, language="Unknown", errors=None):
if report_status:
self.post_id += 1
result = {"post_id": self.post_id,
"submission_id": submission_id,
"status_id": status,
"language": language }
if status != 40:
if type(errors) != list:
errors = [errors] # for valid json according to php
# get rid of any binary garbage by decoding to UTF-8
for i in range(len(errors)):
try:
errors[i] = errors[i].decode("UTF-8", "replace")
except AttributeError:
pass
result['errors'] = json.dumps(errors)
return self.cloud.post_result('api_compile_result', result)
else:
return True
if submission_id == None:
# compile in current directory
compiler.compile_anything(os.getcwd())
else:
submission_dir = self.submission_dir(submission_id)
if os.path.exists(submission_dir):
log.info("Already compiled: %s" % submission_id)
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
if report(STATUS_RUNABLE, compiler.get_run_lang(submission_dir)):
return True
else:
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
else:
report(STATUS_TEST_ERROR, compiler.get_run_lang(submission_dir), errors)
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
if (not submission_id in self.download_dirs or
len(os.listdir(self.download_dir(submission_id))) == 0):
if not self.download_submission(submission_id):
report(STATUS_DOWNLOAD_ERROR)
log.error("Download Error")
return False
download_dir = self.download_dir(submission_id)
if not os.path.exists(os.path.join(self.download_dir(submission_id),
'bot')):
if len(os.listdir(download_dir)) == 1:
if not self.unpack(submission_id):
report(STATUS_UNPACK_ERROR)
log.error("Unpack Error")
return False
log.info("Compiling %s " % submission_id)
bot_dir = os.path.join(download_dir, 'bot')
detected_lang, errors = compiler.compile_anything(bot_dir)
if errors != None:
log.info(errors)
if not self.debug:
shutil.rmtree(download_dir)
log.error(str(errors))
log.error(detected_lang)
report(STATUS_COMPILE_ERROR, detected_lang, errors=errors);
log.error("Compile Error")
return False
else:
log.info("Detected language: {0}".format(detected_lang))
if not os.path.exists(os.path.split(submission_dir)[0]):
os.makedirs(os.path.split(submission_dir)[0])
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
os.rename(download_dir, submission_dir)
del self.download_dirs[submission_id]
if report(STATUS_RUNABLE, detected_lang):
return True
else:
# could not report back to server, cleanup compiled dir
log.debug("Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
else:
log.info("Functional Test Failure")
report(STATUS_TEST_ERROR, detected_lang, errors)
return False
def handle_compile_task(bot_id):
"""Downloads and compiles a bot, then posts the compiled bot archive back through the API"""
errors = []
util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
try:
bot_path = backend.download_bot(bot_id,
temp_dir,
is_compiled=False)
archive.unpack(bot_path)
# Make sure things are in the top-level directory so people can zip their bot from inside or outside the directory
while len([
name for name in os.listdir(temp_dir)
if os.path.isfile(os.path.join(temp_dir, name))
]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1:
with tempfile.TemporaryDirectory(
dir=TEMP_DIR) as buffer_folder:
single_folder = glob.glob(os.path.join(temp_dir, "*"))[0]
for filename in os.listdir(single_folder):
shutil.move(os.path.join(single_folder, filename),
buffer_folder)
os.rmdir(single_folder)
for filename in os.listdir(buffer_folder):
shutil.move(os.path.join(buffer_folder, filename),
temp_dir)
# Delete any symlinks
subprocess.call(["find", temp_dir, "-type", "l", "-delete"])
# Replace DOS line endings with unix line endings in sh scripts
try:
logging.debug(
"Replacing DOS line endings with UNIX line endings...")
ps_find = subprocess.Popen([
'find', temp_dir, '-name', '*.sh', '-type', 'f', '-print0'
],
stdout=subprocess.PIPE)
dos2unix_output = subprocess.check_output(
['xargs', '-0', 'dos2unix'], stdin=ps_find.stdout)
ps_find.wait()
logging.debug(dos2unix_output)
except Exception as ex:
logging.error(
"An error occured while converting dos line endings in sh files",
ex)
# Give the compilation user access
os.chmod(temp_dir, 0o755)
# Compilation user needs to be able to write to the directory and create files
util.give_ownership(temp_dir, 0o2770, "bot_compilation")
# Reset cwd before compilation, in case it was in a deleted temporary folder
os.chdir(os.path.dirname(os.path.realpath(sys.argv[0])))
language, more_errors = compiler.compile_anything(temp_dir)
did_compile = more_errors is None
if more_errors:
errors.extend(more_errors)
except Exception as ex:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc(),
str(ex)] + errors
did_compile = False
try:
if did_compile:
logging.debug('Bot did compile')
# Make things group-readable
subprocess.call([
"sudo", "-H", "-u", "bot_compilation", "-s", "chmod", "-R",
"g+r", temp_dir
],
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL)
archive_path = os.path.join(temp_dir, str(bot_id) + ".zip")
archive.pack(temp_dir, archive_path)
backend.upload_bot(bot_id, archive_path)
else:
logging.debug("Bot did not compile")
logging.debug("Bot errors: %s" % "\n".join(errors))
backend.send_compilation_result(bot_id, did_compile, language,
"\n".join(errors))
except Exception as ex:
logging.error("Bot did not upload", ex)
errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc())
backend.send_compilation_result(bot_id, False, language,
"\n".join(errors))
finally:
# Remove files as bot user (Python will clean up tempdir, but we don't necessarily have permissions to clean up files)
util.chmod_recursive("bot_compilation", temp_dir, "755")
util.rm_as_user("bot_compilation", temp_dir)
util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
def compile(self,
submission_id=None,
report_status=(False, False),
run_test=True):
report_success, report_failure = report_status
def report(status, language="Unknown", errors=None):
# oooh, tricky, a terinary in an if
if report_success if type(errors) != list else report_failure:
self.post_id += 1
result = {
"post_id": self.post_id,
"submission_id": submission_id,
"status_id": status,
"language": language
}
if status != 40:
if type(errors) != list:
errors = [errors] # for valid json
result['errors'] = json.dumps(errors)
return self.cloud.post_result('api_compile_result', result)
else:
return True
if submission_id == None:
# compile in current directory
compiler.compile_anything(os.getcwd())
else:
submission_dir = self.submission_dir(submission_id)
if os.path.exists(submission_dir):
log.info("Already compiled: %s" % submission_id)
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
if report(STATUS_RUNABLE,
compiler.get_run_lang(submission_dir)):
return True
else:
log.debug("Cleanup of compiled dir: {0}".format(
submission_dir))
shutil.rmtree(submission_dir)
return False
else:
report(STATUS_TEST_ERROR,
compiler.get_run_lang(submission_dir), errors)
log.debug(
"Cleanup of compiled dir: {0}".format(submission_dir))
shutil.rmtree(submission_dir)
return False
if (not submission_id in self.download_dirs
or len(os.listdir(self.download_dir(submission_id))) == 0):
if not self.download_submission(submission_id):
report(STATUS_DOWNLOAD_ERROR)
log.error("Download Error")
return False
download_dir = self.download_dir(submission_id)
if not os.path.exists(
os.path.join(self.download_dir(submission_id), 'bot')):
if len(os.listdir(download_dir)) == 1:
if not self.unpack(submission_id):
report(STATUS_UNPACK_ERROR)
log.error("Unpack Error")
return False
log.info("Compiling %s " % submission_id)
bot_dir = os.path.join(download_dir, 'bot')
timelimit = 10 * 60 # 10 minute limit to compile submission
if not run_test:
# give it 50% more time if this isn't the initial compilation
# this is to try and prevent the situation where the initial
# compilation just makes it in the time limit and then a
# subsequent compilation fails when another worker goes to
# play a game with it
timelimit += timelimit * 0.5
detected_lang, errors = compiler.compile_anything(
bot_dir, timelimit)
if errors != None:
log.error(errors)
if not self.debug:
shutil.rmtree(download_dir)
log.error(detected_lang)
report(STATUS_COMPILE_ERROR, detected_lang, errors=errors)
log.error("Compile Error")
return False
else:
log.info("Detected language: {0}".format(detected_lang))
if not os.path.exists(os.path.split(submission_dir)[0]):
os.makedirs(os.path.split(submission_dir)[0])
if run_test:
errors = self.functional_test(submission_id)
else:
errors = None
if errors == None:
os.rename(download_dir, submission_dir)
del self.download_dirs[submission_id]
if report(STATUS_RUNABLE, detected_lang):
return True
else:
# could not report back to server, cleanup compiled dir
log.debug("Cleanup of compiled dir: {0}".format(
submission_dir))
shutil.rmtree(submission_dir)
return False
else:
log.info("Functional Test Failure")
report(STATUS_TEST_ERROR, detected_lang, errors)
return False
