def downloadUsers(users): for user in users: userDir = str(user["userID"]) if os.path.isdir(userDir): shutil.rmtree(userDir) os.mkdir(userDir) archive.unpack(backend.storeBotLocally(user["userID"], userDir))
def executeCompileTask(user, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" print("Compiling a bot with userID %s" % (user["userID"])) try: workingPath = "workingPath" makePath(workingPath) botPath = backend.storeBotLocally(int(user["userID"]), workingPath) zip.unpack(botPath) while len([name for name in os.listdir(workingPath) if os.path.isfile(name)]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1: singleFolder = glob.glob(os.path.join(workingPath, "*"))[0] bufferFolder = os.path.join(workingPath, SECRET_FOLDER) os.mkdir(bufferFolder) for filename in os.listdir(singleFolder): shutil.move( os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename)) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move( os.path.join(bufferFolder, filename), os.path.join(workingPath, filename)) os.rmdir(bufferFolder) language, errors = compile_anything(workingPath) didCompile = True if errors == None else False except Exception as e: language = "Other" errors = [ "Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + str(e)] didCompile = False if didCompile: print("Bot did compile") zip.zipFolder(workingPath, os.path.join( workingPath, user["userID"] + ".zip")) backend.storeBotRemotely( int(user["userID"]), os.path.join(workingPath, user["userID"] + ".zip")) else: print("Bot did not compile") print(str(errors)) sendEmail("Halite Bot Compilation Error", "<h2>The bot that you recently submitted to the Halite competition would not compile on our servers.</h2> <p>Our autocompile script <b>thought that your bot was written in \"" + language + ".\"</b> If that is incorrect, please change your code's file extensions to <code>cpp</code> and <code>h</code> for C++11, <code>java</code> for Java 7, <code>py</code> for Python3, and <code>rs</code> for Rust 1.10. Make sure to include a <code>Cargo.toml</code> file if you are using Rust. Please make sure that your <b>main file is named MyBot</b> (not main, not BasicBot).</p> <b>Here is a description of the compilation error</b>:<br><pre><code>" + "<br>".join(errors) + "</code></pre>", user["email"]) backend.compileResult(int(user["userID"]), didCompile, language) if os.path.isdir(workingPath): shutil.rmtree(workingPath)
def executeCompileTask(user, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" print("Compiling a bot with userID %s\n" % str(user["userID"])) try: workingPath = "workingPath" makePath(workingPath) botPath = backend.storeBotLocally(int(user["userID"]), workingPath, isCompile=True) archive.unpack(botPath) while len([name for name in os.listdir(workingPath) if os.path.isfile(os.path.join(workingPath, name))]) == 0 and len(glob.glob(os.path.join(workingPath, "*"))) == 1: singleFolder = glob.glob(os.path.join(workingPath, "*"))[0] bufferFolder = os.path.join(workingPath, SECRET_FOLDER) os.mkdir(bufferFolder) for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), os.path.join(bufferFolder, filename)) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), os.path.join(workingPath, filename)) os.rmdir(bufferFolder) # Rm symlinks os.system("find "+workingPath+" -type l -delete") language, errors = compile_anything(workingPath) didCompile = True if errors == None else False except Exception as e: language = "Other" errors = ["Your bot caused unexpected behavior in our servers. If you cannot figure out why this happened, please email us at [email protected]. We can help.", "For our reference, here is the trace of the error: " + traceback.format_exc()] didCompile = False if didCompile: print("Bot did compile\n") archive.zipFolder(workingPath, os.path.join(workingPath, user["userID"]+".zip")) backend.storeBotRemotely(int(user["userID"]), os.path.join(workingPath, user["userID"]+".zip")) else: print("Bot did not compile\n") print("Bot errors %s\n" % str(errors)) backend.compileResult(int(user["userID"]), didCompile, language, errors=(None if didCompile else "\n".join(errors))) if os.path.isdir(workingPath): shutil.rmtree(workingPath)
def setupParticipant(user_index, user, temp_dir): """ Download and set up the bot for a game participant. """ # Include username to deal with duplicate bots bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"]) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) archive.unpack(backend.storeBotLocally(user["user_id"], user["bot_id"], bot_dir)) if user.get("requires_compilation"): compile_dir = bot_dir + '_compile' try: # Move to temp directory to avoid permission problems # (can't chown files created by compile user back to us) shutil.move(bot_dir, compile_dir) # Give the compilation user access os.chmod(compile_dir, 0o2755) # User needs to be able to write to the directory give_ownership(compile_dir, "bots", 0o2774) language, errors = compiler.compile_anything(compile_dir) didCompile = errors is None except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False if not didCompile: # Abort and upload an error log rm_as_user("bot_compilation", compile_dir) raise OndemandCompileError(language, '\n'.join(errors)) # Move back to original directory try: shutil.copytree(compile_dir, bot_dir) except shutil.Error as e: logging.error("Could not compile bot ondemand", e) rm_as_user("bot_compilation", compile_dir) # Make the start script executable os.chmod(os.path.join(bot_dir, RUNFILE), 0o755) # Give the bot user ownership of their directory # We should set up each user's default group as a group that the # worker is also a part of. Then we always have access to their # files, but not vice versa. # https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group bot_user = "******".format(user_index) bot_group = "bots_{}".format(user_index) bot_cgroup = "bot_{}".format(user_index) # We want 775 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the # group give_ownership(bot_dir, bot_group, 0o2775) bot_command = BOT_COMMAND.format( cgroup=bot_cgroup, bot_dir=bot_dir, bot_group=bot_group, bot_user=bot_user, runfile=RUNFILE, ) bot_name = "{} v{}".format(user["username"], user["version_number"]) return bot_command, bot_name, bot_dir
def executeCompileTask(user_id, bot_id, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" logging.debug("Compiling a bot with userID %s\n" % str(user_id)) errors = [] with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: try: bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir, is_compile=True) archive.unpack(bot_path) # Make sure things are in the top-level directory while len([ name for name in os.listdir(temp_dir) if os.path.isfile(os.path.join(temp_dir, name)) ]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder: singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0] for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), bufferFolder) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), temp_dir) # Context manager takes care of buffer folder # Delete any symlinks subprocess.call(["find", temp_dir, "-type", "l", "-delete"]) # Give the compilation user access os.chmod(temp_dir, 0o755) # User needs to be able to write to the directory and create files give_ownership(temp_dir, "bots", 0o2770) # Reset cwd before compilation, in case it was in a # deleted temporary folder os.chdir(os.path.dirname(os.path.realpath(sys.argv[0]))) language, more_errors = compiler.compile_anything(temp_dir) didCompile = more_errors is None if more_errors: errors.extend(more_errors) except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False try: if didCompile: logging.debug("Bot did compile\n") archive_path = os.path.join(temp_dir, str(user_id)+".zip") archive.zipFolder(temp_dir, archive_path) backend.storeBotRemotely(user_id, bot_id, archive_path) else: logging.debug("Bot did not compile\n") logging.debug("Bot errors %s\n" % str(errors)) backend.compileResult(user_id, bot_id, didCompile, language, errors=(None if didCompile else "\n".join(errors))) except Exception as e: logging.error("Bot did not upload", e) errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc()) backend.compileResult(user_id, bot_id, False, language, errors="\n".join(errors)) finally: # Remove files as bot user (Python will clean up tempdir, but we don't # necessarily have permissions to clean up files) rm_as_user("bot_compilation", temp_dir)
def runGame(width, height, users): with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: shutil.copy(ENVIRONMENT, os.path.join(temp_dir, ENVIRONMENT)) command = [ "./" + ENVIRONMENT, "-d", "{} {}".format(width, height), "-q", "-o", ] # Make sure bots have access to the temp dir as a whole # Otherwise, Python can't import modules from the bot dir # Based on strace, Python lstat()s the full dir path to the dir it's # in, and fails when it tries to lstat the temp dir, which this # fixes os.chmod(temp_dir, 0o755) for user_index, user in enumerate(users): bot_dir = "{}_{}".format(user["user_id"], user["bot_id"]) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) archive.unpack( backend.storeBotLocally(user["user_id"], user["bot_id"], bot_dir)) # Make the start script executable os.chmod(os.path.join(bot_dir, RUNFILE), 0o755) # Give the bot user ownership of their directory # We should set up each user's default group as a group that the # worker is also a part of. Then we always have access to their # files, but not vice versa. # https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group bot_user = "******".format(user_index) bot_cgroup = "bot_{}".format(user_index) # We want 775 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the # group give_ownership(bot_dir, "bots", 0o2775) command.append( BOT_COMMAND.format( cgroup=bot_cgroup, bot_dir=bot_dir, bot_user=bot_user, runfile=RUNFILE, )) command.append("{} v{}".format(user["username"], user["version_number"])) logging.debug("Run game command %s\n" % command) logging.debug("Waiting for game output...\n") lines = subprocess.Popen( command, stdout=subprocess.PIPE).stdout.read().decode('utf-8').split('\n') logging.debug("\n-----Here is game output: -----") logging.debug("\n".join(lines)) logging.debug("--------------------------------\n") # tempdir will automatically be cleaned up, but we need to do things # manually because the bot might have made files it owns for user_index, user in enumerate(users): bot_user = "******".format(user_index) rm_as_user(bot_user, temp_dir) # The processes won't necessarily be automatically cleaned up, so # let's do it ourselves util.kill_processes_as(bot_user, "bash") return lines