def compile(self, submission_id=None, report_status=False): def report(status=STATUS_RUNABLE): if report_status: self.post_id += 1 result = { "post_id": self.post_id, "submission_id": submission_id, "status_id": status } self.cloud.post_result('api_compile_result', result) if submission_id == None: # compile in current directory compiler.compile_anything() else: submission_dir = self.submission_dir(submission_id) download_dir = self.download_dir(submission_id) if os.path.exists(submission_dir): log.info("Already compiled: %s" % submission_id) if self.functional_test(submission_id): report(STATUS_RUNABLE) return True else: report(STATUS_TEST_ERROR) return False if not os.path.exists(download_dir): if not self.download_submission(submission_id): report(STATUS_DOWNLOAD_ERROR) log.error("Download Error") return False if len(os.listdir(download_dir)) == 1: if not self.unpack(submission_id): report(STATUS_UNPACK_ERROR) log.error("Unpack Error") return False log.info("Compiling %s " % submission_id) detected_lang, errors = compiler.compile_anything(download_dir) if not detected_lang: shutil.rmtree(download_dir) log.error('\n'.join(errors)) report(STATUS_COMPILE_ERROR) log.error("Compile Error") return False else: if not os.path.exists(os.path.split(submission_dir)[0]): os.makedirs(os.path.split(submission_dir)[0]) os.rename(download_dir, submission_dir) if self.functional_test(submission_id): report(STATUS_RUNABLE) return True else: log.error("Functional Test Error") report(STATUS_TEST_ERROR) return False
def testStarterPackages(self): '''Archive the starter packages like the website would''' SP_DIR = "starterpackages" if os.path.isdir(SP_DIR): shutil.rmtree(SP_DIR) os.system("cd ../../website/; ./archiveStarterPackages.sh"); shutil.copytree("../../website/downloads/starterpackages/", SP_DIR) for f in glob.glob(os.path.join(SP_DIR, "*.zip")): zip_ref = zipfile.ZipFile(f, 'r') zip_ref.extractall(SP_DIR) zip_ref.close() folderName = os.path.splitext(os.path.basename(f))[0] folder = os.path.join(SP_DIR, os.path.splitext(os.path.basename(f))[0]) for filename in os.listdir(folder): if os.path.splitext(filename)[0] == "RandomBot": os.remove(os.path.join(folder, filename)) expectedLanguage = folderName.split("-")[1] print("Expected Language: " + expectedLanguage) language, errors = compiler.compile_anything(folder) if errors is not None: print("Errors: " + "\n".join(errors)) print("Language: " + language) assert language == expectedLanguage assert errors == None
def testStarterPackages(self): '''Archive the starter packages like the website would''' SP_DIR = "starterpackages" if os.path.isdir(SP_DIR): shutil.rmtree(SP_DIR) os.system("cd ../../website/; ./archiveStarterPackages.sh") shutil.copytree("../../website/downloads/starterpackages/", SP_DIR) for f in glob.glob(os.path.join(SP_DIR, "*.zip")): zip_ref = zipfile.ZipFile(f, 'r') zip_ref.extractall(SP_DIR) zip_ref.close() folderName = os.path.splitext(os.path.basename(f))[0] folder = os.path.join(SP_DIR, os.path.splitext(os.path.basename(f))[0]) for filename in os.listdir(folder): if os.path.splitext(filename)[0] == "RandomBot": os.remove(os.path.join(folder, filename)) expectedLanguage = folderName.split("-")[1] print("Expected Language: " + expectedLanguage) language, errors = compiler.compile_anything(folder) if errors is not None: print("Errors: " + "\n".join(errors)) print("Language: " + language) assert language == expectedLanguage assert errors == None
def testLanguageOverride(self): '''Use a LANGUAGE file to override the detected language''' LANGUAGE_BOT_PATH = "languageBot" bot_dir = os.path.join(OUR_PATH, LANGUAGE_BOT_PATH) expectedLanguage = "TestLanguage" language, errors = compiler.compile_anything(bot_dir) if errors is not None: print("Errors: " + "\n".join(errors)) print("Language: " + language) assert language == expectedLanguage assert errors == None
def testStarterPackages(self): '''Archive the starter packages like the website would''' SP_DIR = "starterpackages" if os.path.isdir(SP_DIR): shutil.rmtree(SP_DIR) os.system("cd ../../website/; ./archiveStarterPackages.sh"); shutil.copytree("../../website/downloads/starterpackages/", SP_DIR) for f in glob.glob(os.path.join(SP_DIR, "*.zip")): zip_ref = zipfile.ZipFile(f, 'r') zip_ref.extractall(SP_DIR) zip_ref.close() folderName = os.path.splitext(os.path.basename(f))[0] folder = os.path.join(SP_DIR, os.path.splitext(os.path.basename(f))[0]) expectedLanguage = folderName.split("-")[1] language, errors = compiler.compile_anything(folder) print(errors) print(language) assert language == expectedLanguage assert errors == None
def setupParticipant(user_index, user, temp_dir): """ Download and set up the bot for a game participant. """ # Include username to deal with duplicate bots bot_dir = "{}_{}_{}".format(user["user_id"], user["bot_id"], user["username"]) bot_dir = os.path.join(temp_dir, bot_dir) os.mkdir(bot_dir) archive.unpack(backend.storeBotLocally(user["user_id"], user["bot_id"], bot_dir)) if user.get("requires_compilation"): compile_dir = bot_dir + '_compile' try: # Move to temp directory to avoid permission problems # (can't chown files created by compile user back to us) shutil.move(bot_dir, compile_dir) # Give the compilation user access os.chmod(compile_dir, 0o2755) # User needs to be able to write to the directory give_ownership(compile_dir, "bots", 0o2774) language, errors = compiler.compile_anything(compile_dir) didCompile = errors is None except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False if not didCompile: # Abort and upload an error log rm_as_user("bot_compilation", compile_dir) raise OndemandCompileError(language, '\n'.join(errors)) # Move back to original directory try: shutil.copytree(compile_dir, bot_dir) except shutil.Error as e: logging.error("Could not compile bot ondemand", e) rm_as_user("bot_compilation", compile_dir) # Make the start script executable os.chmod(os.path.join(bot_dir, RUNFILE), 0o755) # Give the bot user ownership of their directory # We should set up each user's default group as a group that the # worker is also a part of. Then we always have access to their # files, but not vice versa. # https://superuser.com/questions/102253/how-to-make-files-created-in-a-directory-owned-by-directory-group bot_user = "******".format(user_index) bot_group = "bots_{}".format(user_index) bot_cgroup = "bot_{}".format(user_index) # We want 775 so that the bot can create files still; leading 2 # is equivalent to g+s which forces new files to be owned by the # group give_ownership(bot_dir, bot_group, 0o2775) bot_command = BOT_COMMAND.format( cgroup=bot_cgroup, bot_dir=bot_dir, bot_group=bot_group, bot_user=bot_user, runfile=RUNFILE, ) bot_name = "{} v{}".format(user["username"], user["version_number"]) return bot_command, bot_name, bot_dir
def executeCompileTask(user_id, bot_id, backend): """Downloads and compiles a bot. Posts the compiled bot files to the manager.""" logging.debug("Compiling a bot with userID %s\n" % str(user_id)) errors = [] with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: try: bot_path = backend.storeBotLocally(user_id, bot_id, temp_dir, is_compile=True) archive.unpack(bot_path) # Make sure things are in the top-level directory while len([ name for name in os.listdir(temp_dir) if os.path.isfile(os.path.join(temp_dir, name)) ]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1: with tempfile.TemporaryDirectory(dir=TEMP_DIR) as bufferFolder: singleFolder = glob.glob(os.path.join(temp_dir, "*"))[0] for filename in os.listdir(singleFolder): shutil.move(os.path.join(singleFolder, filename), bufferFolder) os.rmdir(singleFolder) for filename in os.listdir(bufferFolder): shutil.move(os.path.join(bufferFolder, filename), temp_dir) # Context manager takes care of buffer folder # Delete any symlinks subprocess.call(["find", temp_dir, "-type", "l", "-delete"]) # Give the compilation user access os.chmod(temp_dir, 0o755) # User needs to be able to write to the directory and create files give_ownership(temp_dir, "bots", 0o2770) # Reset cwd before compilation, in case it was in a # deleted temporary folder os.chdir(os.path.dirname(os.path.realpath(sys.argv[0]))) language, more_errors = compiler.compile_anything(temp_dir) didCompile = more_errors is None if more_errors: errors.extend(more_errors) except Exception: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc()] + errors didCompile = False try: if didCompile: logging.debug("Bot did compile\n") archive_path = os.path.join(temp_dir, str(user_id)+".zip") archive.zipFolder(temp_dir, archive_path) backend.storeBotRemotely(user_id, bot_id, archive_path) else: logging.debug("Bot did not compile\n") logging.debug("Bot errors %s\n" % str(errors)) backend.compileResult(user_id, bot_id, didCompile, language, errors=(None if didCompile else "\n".join(errors))) except Exception as e: logging.error("Bot did not upload", e) errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc()) backend.compileResult(user_id, bot_id, False, language, errors="\n".join(errors)) finally: # Remove files as bot user (Python will clean up tempdir, but we don't # necessarily have permissions to clean up files) rm_as_user("bot_compilation", temp_dir)
def compile(self, submission_id=None, report_status=(False, False), run_test=True): report_success, report_failure = report_status def report(status, language="Unknown", errors=None): # oooh, tricky, a terinary in an if if report_success if type(errors) != list else report_failure: self.post_id += 1 result = {"post_id": self.post_id, "submission_id": submission_id, "status_id": status, "language": language } if status != 40: if type(errors) != list: errors = [errors] # for valid json result['errors'] = json.dumps(errors) return self.cloud.post_result('api_compile_result', result) else: return True if submission_id == None: # compile in current directory compiler.compile_anything(os.getcwd()) else: submission_dir = self.submission_dir(submission_id) if os.path.exists(submission_dir): log.info("Already compiled: %s" % submission_id) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: if report(STATUS_RUNABLE, compiler.get_run_lang(submission_dir)): return True else: log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False else: report(STATUS_TEST_ERROR, compiler.get_run_lang(submission_dir), errors) log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False if (not submission_id in self.download_dirs or len(os.listdir(self.download_dir(submission_id))) == 0): if not self.download_submission(submission_id): report(STATUS_DOWNLOAD_ERROR) log.error("Download Error") return False download_dir = self.download_dir(submission_id) if not os.path.exists(os.path.join(self.download_dir(submission_id), 'bot')): if len(os.listdir(download_dir)) == 1: if not self.unpack(submission_id): report(STATUS_UNPACK_ERROR) log.error("Unpack Error") return False log.info("Compiling %s " % submission_id) bot_dir = os.path.join(download_dir, 'bot') timelimit = 10 * 60 # 10 minute limit to compile submission if not run_test: # give it 50% more time if this isn't the initial compilation # this is to try and prevent the situation where the initial # compilation just makes it in the time limit and then a # subsequent compilation fails when another worker goes to # play a game with it timelimit += timelimit * 0.5 detected_lang, errors = compiler.compile_anything(bot_dir, timelimit) if errors != None: log.error(errors) if not self.debug: shutil.rmtree(download_dir) log.error(detected_lang) report(STATUS_COMPILE_ERROR, detected_lang, errors=errors); log.error("Compile Error") return False else: log.info("Detected language: {0}".format(detected_lang)) if not os.path.exists(os.path.split(submission_dir)[0]): os.makedirs(os.path.split(submission_dir)[0]) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: os.rename(download_dir, submission_dir) del self.download_dirs[submission_id] if report(STATUS_RUNABLE, detected_lang): return True else: # could not report back to server, cleanup compiled dir log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False else: log.info("Functional Test Failure") report(STATUS_TEST_ERROR, detected_lang, errors) return False
def compile(self, submission_id=None, report_status=False, run_test=True): def report(status, language="Unknown", errors=None): if report_status: self.post_id += 1 result = {"post_id": self.post_id, "submission_id": submission_id, "status_id": status, "language": language } if status != 40: if type(errors) != list: errors = [errors] # for valid json according to php # get rid of any binary garbage by decoding to UTF-8 for i in range(len(errors)): try: errors[i] = errors[i].decode("UTF-8", "replace") except AttributeError: pass result['errors'] = json.dumps(errors) return self.cloud.post_result('api_compile_result', result) else: return True if submission_id == None: # compile in current directory compiler.compile_anything(os.getcwd()) else: submission_dir = self.submission_dir(submission_id) if os.path.exists(submission_dir): log.info("Already compiled: %s" % submission_id) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: if report(STATUS_RUNABLE, compiler.get_run_lang(submission_dir)): return True else: log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False else: report(STATUS_TEST_ERROR, compiler.get_run_lang(submission_dir), errors) log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False if (not submission_id in self.download_dirs or len(os.listdir(self.download_dir(submission_id))) == 0): if not self.download_submission(submission_id): report(STATUS_DOWNLOAD_ERROR) log.error("Download Error") return False download_dir = self.download_dir(submission_id) if not os.path.exists(os.path.join(self.download_dir(submission_id), 'bot')): if len(os.listdir(download_dir)) == 1: if not self.unpack(submission_id): report(STATUS_UNPACK_ERROR) log.error("Unpack Error") return False log.info("Compiling %s " % submission_id) bot_dir = os.path.join(download_dir, 'bot') detected_lang, errors = compiler.compile_anything(bot_dir) if errors != None: log.info(errors) if not self.debug: shutil.rmtree(download_dir) log.error(str(errors)) log.error(detected_lang) report(STATUS_COMPILE_ERROR, detected_lang, errors=errors); log.error("Compile Error") return False else: log.info("Detected language: {0}".format(detected_lang)) if not os.path.exists(os.path.split(submission_dir)[0]): os.makedirs(os.path.split(submission_dir)[0]) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: os.rename(download_dir, submission_dir) del self.download_dirs[submission_id] if report(STATUS_RUNABLE, detected_lang): return True else: # could not report back to server, cleanup compiled dir log.debug("Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False else: log.info("Functional Test Failure") report(STATUS_TEST_ERROR, detected_lang, errors) return False
def handle_compile_task(bot_id): """Downloads and compiles a bot, then posts the compiled bot archive back through the API""" errors = [] util.rm_everything_owned_in_tmp_and_home_by("bot_compilation") with tempfile.TemporaryDirectory(dir=TEMP_DIR) as temp_dir: try: bot_path = backend.download_bot(bot_id, temp_dir, is_compiled=False) archive.unpack(bot_path) # Make sure things are in the top-level directory so people can zip their bot from inside or outside the directory while len([ name for name in os.listdir(temp_dir) if os.path.isfile(os.path.join(temp_dir, name)) ]) == 0 and len(glob.glob(os.path.join(temp_dir, "*"))) == 1: with tempfile.TemporaryDirectory( dir=TEMP_DIR) as buffer_folder: single_folder = glob.glob(os.path.join(temp_dir, "*"))[0] for filename in os.listdir(single_folder): shutil.move(os.path.join(single_folder, filename), buffer_folder) os.rmdir(single_folder) for filename in os.listdir(buffer_folder): shutil.move(os.path.join(buffer_folder, filename), temp_dir) # Delete any symlinks subprocess.call(["find", temp_dir, "-type", "l", "-delete"]) # Replace DOS line endings with unix line endings in sh scripts try: logging.debug( "Replacing DOS line endings with UNIX line endings...") ps_find = subprocess.Popen([ 'find', temp_dir, '-name', '*.sh', '-type', 'f', '-print0' ], stdout=subprocess.PIPE) dos2unix_output = subprocess.check_output( ['xargs', '-0', 'dos2unix'], stdin=ps_find.stdout) ps_find.wait() logging.debug(dos2unix_output) except Exception as ex: logging.error( "An error occured while converting dos line endings in sh files", ex) # Give the compilation user access os.chmod(temp_dir, 0o755) # Compilation user needs to be able to write to the directory and create files util.give_ownership(temp_dir, 0o2770, "bot_compilation") # Reset cwd before compilation, in case it was in a deleted temporary folder os.chdir(os.path.dirname(os.path.realpath(sys.argv[0]))) language, more_errors = compiler.compile_anything(temp_dir) did_compile = more_errors is None if more_errors: errors.extend(more_errors) except Exception as ex: language = "Other" errors = [COMPILE_ERROR_MESSAGE + traceback.format_exc(), str(ex)] + errors did_compile = False try: if did_compile: logging.debug('Bot did compile') # Make things group-readable subprocess.call([ "sudo", "-H", "-u", "bot_compilation", "-s", "chmod", "-R", "g+r", temp_dir ], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) archive_path = os.path.join(temp_dir, str(bot_id) + ".zip") archive.pack(temp_dir, archive_path) backend.upload_bot(bot_id, archive_path) else: logging.debug("Bot did not compile") logging.debug("Bot errors: %s" % "\n".join(errors)) backend.send_compilation_result(bot_id, did_compile, language, "\n".join(errors)) except Exception as ex: logging.error("Bot did not upload", ex) errors.append(UPLOAD_ERROR_MESSAGE + traceback.format_exc()) backend.send_compilation_result(bot_id, False, language, "\n".join(errors)) finally: # Remove files as bot user (Python will clean up tempdir, but we don't necessarily have permissions to clean up files) util.chmod_recursive("bot_compilation", temp_dir, "755") util.rm_as_user("bot_compilation", temp_dir) util.rm_everything_owned_in_tmp_and_home_by("bot_compilation")
def compile(self, submission_id=None, report_status=(False, False), run_test=True): report_success, report_failure = report_status def report(status, language="Unknown", errors=None): # oooh, tricky, a terinary in an if if report_success if type(errors) != list else report_failure: self.post_id += 1 result = { "post_id": self.post_id, "submission_id": submission_id, "status_id": status, "language": language } if status != 40: if type(errors) != list: errors = [errors] # for valid json result['errors'] = json.dumps(errors) return self.cloud.post_result('api_compile_result', result) else: return True if submission_id == None: # compile in current directory compiler.compile_anything(os.getcwd()) else: submission_dir = self.submission_dir(submission_id) if os.path.exists(submission_dir): log.info("Already compiled: %s" % submission_id) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: if report(STATUS_RUNABLE, compiler.get_run_lang(submission_dir)): return True else: log.debug("Cleanup of compiled dir: {0}".format( submission_dir)) shutil.rmtree(submission_dir) return False else: report(STATUS_TEST_ERROR, compiler.get_run_lang(submission_dir), errors) log.debug( "Cleanup of compiled dir: {0}".format(submission_dir)) shutil.rmtree(submission_dir) return False if (not submission_id in self.download_dirs or len(os.listdir(self.download_dir(submission_id))) == 0): if not self.download_submission(submission_id): report(STATUS_DOWNLOAD_ERROR) log.error("Download Error") return False download_dir = self.download_dir(submission_id) if not os.path.exists( os.path.join(self.download_dir(submission_id), 'bot')): if len(os.listdir(download_dir)) == 1: if not self.unpack(submission_id): report(STATUS_UNPACK_ERROR) log.error("Unpack Error") return False log.info("Compiling %s " % submission_id) bot_dir = os.path.join(download_dir, 'bot') timelimit = 10 * 60 # 10 minute limit to compile submission if not run_test: # give it 50% more time if this isn't the initial compilation # this is to try and prevent the situation where the initial # compilation just makes it in the time limit and then a # subsequent compilation fails when another worker goes to # play a game with it timelimit += timelimit * 0.5 detected_lang, errors = compiler.compile_anything( bot_dir, timelimit) if errors != None: log.error(errors) if not self.debug: shutil.rmtree(download_dir) log.error(detected_lang) report(STATUS_COMPILE_ERROR, detected_lang, errors=errors) log.error("Compile Error") return False else: log.info("Detected language: {0}".format(detected_lang)) if not os.path.exists(os.path.split(submission_dir)[0]): os.makedirs(os.path.split(submission_dir)[0]) if run_test: errors = self.functional_test(submission_id) else: errors = None if errors == None: os.rename(download_dir, submission_dir) del self.download_dirs[submission_id] if report(STATUS_RUNABLE, detected_lang): return True else: # could not report back to server, cleanup compiled dir log.debug("Cleanup of compiled dir: {0}".format( submission_dir)) shutil.rmtree(submission_dir) return False else: log.info("Functional Test Failure") report(STATUS_TEST_ERROR, detected_lang, errors) return False