def downloadUsers(users):
for user in users:
userDir = str(user["userID"])
if os.path.isdir(userDir):
shutil.rmtree(userDir)
os.mkdir(userDir)
archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def downloadUsers(users):
for user in users:
userDir = str(user["userID"])
if os.path.isdir(userDir):
shutil.rmtree(userDir)
os.mkdir(userDir)
archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s" % (user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath)
zip.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(name)]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(
os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(
os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = [
"Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.",
"For our reference, here is the trace of the error: " + str(e)]
didCompile = False
if didCompile:
print("Bot did compile")
zip.zipFolder(workingPath, os.path.join(
workingPath, user["userID"] + ".zip"))
backend.storeBotRemotely(
int(user["userID"]), os.path.join(workingPath, user["userID"] + ".zip"))
else:
print("Bot did not compile")
print(str(errors))
sendEmail("Halite Bot Compilation Error", "<h2>The bot that you recently submitted to the Halite competition would not compile on our servers.</h2> <p>Our autocompile script <b>thought that your bot was written in \"" + language +
".\"</b> If that is incorrect, please change your code's file extensions to <code>cpp</code> and <code>h</code> for C++11, <code>java</code> for Java 7, <code>py</code> for Python3, and <code>rs</code> for Rust 1.10. Make sure to include a <code>Cargo.toml</code> file if you are using Rust. Please make sure that your <b>main file is named MyBot</b> (not main, not BasicBot).</p> <b>Here is a description of the compilation error</b>:<br><pre><code>" + "<br>".join(errors) + "</code></pre>", user["email"])
backend.compileResult(int(user["userID"]), didCompile, language)
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s\n" % str(user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True)
archive.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
# Rm symlinks
os.system("find "+workingPath+" -type l -delete")
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()]
didCompile = False
if didCompile:
print("Bot did compile\n")
archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip"))
backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip"))
else:
print("Bot did not compile\n")
print("Bot errors %s\n" % str(errors))
backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors)))
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def executeCompileTask(user, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
print("Compiling a bot with userID %s\n" % str(user["userID"]))
try:
workingPath = "workingPath"
makePath(workingPath)
botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True)
archive.unpack(botPath)
while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1:
singleFolder = glob.glob(os.path.join(workingPath, "*"))[0]
bufferFolder = os.path.join(workingPath, SECRET_FOLDER)
os.mkdir(bufferFolder)
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename))
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename))
os.rmdir(bufferFolder)
# Rm symlinks
os.system("find "+workingPath+" -type l -delete")
language, errors = compile_anything(workingPath)
didCompile = True if errors == None else False
except Exception as e:
language = "Other"
errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()]
didCompile = False
if didCompile:
print("Bot did compile\n")
archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip"))
backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip"))
else:
print("Bot did not compile\n")
print("Bot errors %s\n" % str(errors))
backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors)))
if os.path.isdir(workingPath):
shutil.rmtree(workingPath)
def setupParticipant(user_index, user, temp_dir):
"""
Download and set up the bot for a game participant.
"""
# Include username to deal with duplicate bots
bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"])
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
archive.unpack(backend.storeBotLocally(user["user_id"],
user["bot_id"], bot_dir))
if user.get("requires_compilation"):
compile_dir = bot_dir + '_compile'
try:
# Move to temp directory to avoid permission problems
# (can't chown files created by compile user back to us)
shutil.move(bot_dir, compile_dir)
# Give the compilation user access
os.chmod(compile_dir, 0o2755)
# User needs to be able to write to the directory
give_ownership(compile_dir, "bots", 0o2774)
language, errors = compiler.compile_anything(compile_dir)
didCompile = errors is None
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
if not didCompile:
# Abort and upload an error log
rm_as_user("bot_compilation", compile_dir)
raise OndemandCompileError(language, '\n'.join(errors))
# Move back to original directory
try:
shutil.copytree(compile_dir, bot_dir)
except shutil.Error as e:
logging.error("Could not compile bot ondemand", e)
rm_as_user("bot_compilation", compile_dir)
# Make the start script executable
os.chmod(os.path.join(bot_dir, RUNFILE), 0o755)
# Give the bot user ownership of their directory
# We should set up each user's default group as a group that the
# worker is also a part of. Then we always have access to their
# files, but not vice versa.
# https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group
bot_user = "******".format(user_index)
bot_group = "bots_{}".format(user_index)
bot_cgroup = "bot_{}".format(user_index)
# We want 775 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the
# group
give_ownership(bot_dir, bot_group, 0o2775)
bot_command = BOT_COMMAND.format(
cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_group=bot_group,
bot_user=bot_user,
runfile=RUNFILE,
)
bot_name = "{} v{}".format(user["username"], user["version_number"])
return bot_command, bot_name, bot_dir
def executeCompileTask(user_id, bot_id, backend):
"""Downloads and compiles a bot. Posts the compiled bot files to the manager."""
logging.debug("Compiling a bot with userID %s\n" % str(user_id))
errors = []
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
try:
bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir,
is_compile=True)
archive.unpack(bot_path)
# Make sure things are in the top-level directory
while len([
name for name in os.listdir(temp_dir)
if os.path.isfile(os.path.join(temp_dir, name))
]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1:
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder:
singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0]
for filename in os.listdir(singleFolder):
shutil.move(os.path.join(singleFolder, filename), bufferFolder)
os.rmdir(singleFolder)
for filename in os.listdir(bufferFolder):
shutil.move(os.path.join(bufferFolder, filename), temp_dir)
# Context manager takes care of buffer folder
# Delete any symlinks
subprocess.call(["find", temp_dir, "-type", "l", "-delete"])
# Give the compilation user access
os.chmod(temp_dir, 0o755)
# User needs to be able to write to the directory and create files
give_ownership(temp_dir, "bots", 0o2770)
# Reset cwd before compilation, in case it was in a
# deleted temporary folder
os.chdir(os.path.dirname(os.path.realpath(sys.argv[0])))
language, more_errors = compiler.compile_anything(temp_dir)
didCompile = more_errors is None
if more_errors:
errors.extend(more_errors)
except Exception:
language = "Other"
errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors
didCompile = False
try:
if didCompile:
logging.debug("Bot did compile\n")
archive_path = os.path.join(temp_dir, str(user_id)+".zip")
archive.zipFolder(temp_dir, archive_path)
backend.storeBotRemotely(user_id, bot_id, archive_path)
else:
logging.debug("Bot did not compile\n")
logging.debug("Bot errors %s\n" % str(errors))
backend.compileResult(user_id, bot_id, didCompile, language,
errors=(None if didCompile else "\n".join(errors)))
except Exception as e:
logging.error("Bot did not upload", e)
errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc())
backend.compileResult(user_id, bot_id, False, language,
errors="\n".join(errors))
finally:
# Remove files as bot user (Python will clean up tempdir, but we don't
# necessarily have permissions to clean up files)
rm_as_user("bot_compilation", temp_dir)
def runGame(width, height, users):
with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir:
shutil.copy(ENVIRONMENT, os.path.join(temp_dir, ENVIRONMENT))
command = [
"./" + ENVIRONMENT,
"-d",
"{} {}".format(width, height),
"-q",
"-o",
]
# Make sure bots have access to the temp dir as a whole
# Otherwise, Python can't import modules from the bot dir
# Based on strace, Python lstat()s the full dir path to the dir it's
# in, and fails when it tries to lstat the temp dir, which this
# fixes
os.chmod(temp_dir, 0o755)
for user_index, user in enumerate(users):
bot_dir = "{}_{}".format(user["user_id"], user["bot_id"])
bot_dir = os.path.join(temp_dir, bot_dir)
os.mkdir(bot_dir)
archive.unpack(
backend.storeBotLocally(user["user_id"], user["bot_id"],
bot_dir))
# Make the start script executable
os.chmod(os.path.join(bot_dir, RUNFILE), 0o755)
# Give the bot user ownership of their directory
# We should set up each user's default group as a group that the
# worker is also a part of. Then we always have access to their
# files, but not vice versa.
# https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group
bot_user = "******".format(user_index)
bot_cgroup = "bot_{}".format(user_index)
# We want 775 so that the bot can create files still; leading 2
# is equivalent to g+s which forces new files to be owned by the
# group
give_ownership(bot_dir, "bots", 0o2775)
command.append(
BOT_COMMAND.format(
cgroup=bot_cgroup,
bot_dir=bot_dir,
bot_user=bot_user,
runfile=RUNFILE,
))
command.append("{} v{}".format(user["username"],
user["version_number"]))
logging.debug("Run game command %s\n" % command)
logging.debug("Waiting for game output...\n")
lines = subprocess.Popen(
command,
stdout=subprocess.PIPE).stdout.read().decode('utf-8').split('\n')
logging.debug("\n-----Here is game output: -----")
logging.debug("\n".join(lines))
logging.debug("--------------------------------\n")
# tempdir will automatically be cleaned up, but we need to do things
# manually because the bot might have made files it owns
for user_index, user in enumerate(users):
bot_user = "******".format(user_index)
rm_as_user(bot_user, temp_dir)
# The processes won't necessarily be automatically cleaned up, so
# let's do it ourselves
util.kill_processes_as(bot_user, "bash")
return lines
