def compile_worker(submissionid): """ Performs a compilation job as specified in submissionid Message format: {submissionid} A single string containing the submissionid Filesystem structure: /box/ `-- source.zip `-- src/ | `-- <package name>/ | `-- all .java sources `-- build/ `-- classes/ `-- <package-name>/ | `-- all compiled .class files `-- player.zip """ client = storage.Client() bucket = client.get_bucket(GCLOUD_BUCKET_SUBMISSION) rootdir = os.path.join('/', 'box') sourcedir = os.path.join(rootdir, 'src') builddir = os.path.join(rootdir, 'build') classdir = os.path.join(builddir, 'classes') try: # Obtain compressed archive of the submission try: os.mkdir(sourcedir) with open(os.path.join(rootdir, 'source.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join( submissionid, 'source.zip')).download_to_file(file_obj) except: compile_log_error(submissionid, 'Could not retrieve source file from bucket') # Decompress submission archive result = util.monitor_command(['unzip', 'source.zip', '-d', sourcedir], cwd=rootdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: compile_log_error(submissionid, 'Could not decompress source file') # Update distribution util.pull_distribution( rootdir, lambda: compile_log_error(submissionid, 'Could not pull distribution')) # Execute compilation result = util.monitor_command( ['./gradlew', 'build', '-Psource={}'.format(sourcedir)], cwd=rootdir, timeout=TIMEOUT_COMPILE) # Only one package allowed per submission try: packages = os.listdir(classdir) except: # No classes were generated after compiling compile_report_result(submissionid, COMPILE_FAILED) else: if result[0] == 0 and len(packages) == 1: # Compress compiled classes result = util.monitor_command( ['zip', '-r', 'player.zip', packages[0]], cwd=classdir, timeout=TIMEOUT_COMPILE) # Send package to bucket for storage if result[0] == 0: try: with open(os.path.join(classdir, 'player.zip'), 'rb') as file_obj: bucket.blob( os.path.join( submissionid, 'player.zip')).upload_from_file(file_obj) except: compile_log_error( submissionid, 'Could not send executable to bucket') logging.info('Compilation succeeded') compile_report_result(submissionid, COMPILE_SUCCESS) else: compile_log_error(submissionid, 'Could not compress compiled classes') else: logging.info('Compilation failed') compile_report_result(submissionid, COMPILE_FAILED) finally: # Clean up working directory try: shutil.rmtree(sourcedir) os.remove(os.path.join(rootdir, 'source.zip')) shutil.rmtree(builddir) except: logging.warning('Could not clean up compilation directory')
def game_worker(gameinfo): """ Runs a game as specified by the message Message contains JSON data, with string parameters: gametype: string, either "scrimmage" or "tournament" gameid: string, id of the game player1: string, id of red player submission player2: string, id of blue player submission name1: string, team name of the red player name2: string, team name of the blue player replay: string, a unique identifier for the name of the replay Filesystem structure: /box/ `-- bots/ | `-- player1.zip | `-- player2.zip | `-- player1/ | | `-- <player1 package>/ | | `-- all .py files in player1.zip | `-- player2/ | `-- <player2 package>/ | `-- all .py files in player2.zip `-- replay.txt """ client = storage.Client() bucket = client.get_bucket(GCLOUD_BUCKET_SUBMISSION) try: gameinfo = json.loads(gameinfo) gametype = gameinfo['gametype'] gameid = gameinfo['gameid'] player1 = gameinfo['player1'] player2 = gameinfo['player2'] replay = gameinfo['replay'] name1 = gameinfo['name1'] name2 = gameinfo['name2'] # For reverse-compatibility except: game_log_error(gametype, gameid, 'Game information in incorrect format') rootdir = os.path.join('/', 'box') botdir = os.path.join(rootdir, 'bots') try: # Obtain player executables try: os.mkdir(botdir) with open(os.path.join(botdir, 'player1.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join( player1, 'source.zip')).download_to_file(file_obj) with open(os.path.join(botdir, 'player2.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join( player2, 'source.zip')).download_to_file(file_obj) except: game_log_error(gametype, gameid, 'Could not retrieve submissions from bucket') # Decompress zip archives of player classes try: # Unzip player 1 result = util.monitor_command( ['unzip', 'player1.zip', '-d', 'player1'], cwd=botdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError # Unzip player 2 result = util.monitor_command( ['unzip', 'player2.zip', '-d', 'player2'], cwd=botdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError except: game_log_error(gametype, gameid, 'Could not decompress player zips') # Determine player packages try: package1 = os.listdir(os.path.join(botdir, 'player1')) try: package1.remove('__MACOSX') except: print('cool it wasn\' a mac') assert (len(package1) == 1) assert (not package1[0] == 'bot.py') package1 = os.path.join(botdir, "player1", package1[0]) package2 = os.listdir(os.path.join(botdir, 'player2')) try: package2.remove('__MACOSX') except: print('cool it wasn\' a mac') assert (len(package2) == 1) assert (not package2[0] == 'bot.py') package2 = os.path.join(botdir, "player2", package2[0]) except: game_log_error(gametype, gameid, 'Could not determine player packages', ack=True) logging.info('Player files downloaded, unzipped, determined') # Update distribution util.pull_distribution( rootdir, lambda: game_log_error(gametype, gameid, 'Could not pull distribution')) logging.info( util.monitor_command(['pip3', 'show', 'battlehack20'], botdir)[1]) # Execute game logging.info('Engine updated, running game') result = util.monitor_command([ 'python', 'engine/run.py', package1, package2, '--raw-text', '--delay', '0', '--seed', str(random.randint(1, 1000000)) ], cwd=rootdir, timeout=TIMEOUT_GAME) logging.info('Game execution finished') # Try to parse result anyway, despite exit # if result[0] != 0: # game_log_error(gametype, gameid, 'Game execution had non-zero return code') # "make replay" with open(os.path.join(rootdir, 'replay.txt'), 'w') as file_obj: file_obj.write('Team 1: {}\n'.format(name1)) file_obj.write('Team 2: {}\n'.format(name2)) file_obj.write('Battlehack Version: ') file_obj.write( util.monitor_command(['pip3', 'show', 'battlehack20'], botdir)[1]) file_obj.write('\n') file_obj.write(result[1]) # Upload replay file bucket = client.get_bucket(GCLOUD_BUCKET_REPLAY) try: with open(os.path.join(rootdir, 'replay.txt'), 'rb') as file_obj: bucket.blob(os.path.join( 'replays', '{}.txt'.format(replay))).upload_from_file(file_obj) except: game_log_error(gametype, gameid, 'Could not send replay file to bucket') # Interpret game result server_output = result[1].split('\n') wins = [0, 0] try: # Read the winner of each game from the engine for line in server_output[-5:]: if re.fullmatch('Team.WHITE wins!', line): game_winner = 'A' wins[0] += 1 elif re.fullmatch('Team.BLACK wins!', line): game_winner = 'B' wins[1] += 1 assert (wins[0] + wins[1] == 1) if result[0] != 0: logging.info( 'Game Execution had non-zero return code. Game finished anyway' ) logging.info('Game ended. Result {}:{}'.format(wins[0], wins[1])) except: if result[0] != 0: game_log_error(gametype, gameid, 'Game execution had non-zero return code') game_log_error(gametype, gameid, 'Could not determine winner') else: if wins[0] > wins[1]: game_report_result(gametype, gameid, GAME_REDWON) elif wins[1] > wins[0]: game_report_result(gametype, gameid, GAME_BLUEWON) else: game_log_error(gametype, gameid, 'Ended in draw, which should not happen') finally: # Clean up working directory try: shutil.rmtree(botdir) # os.remove(os.path.join(rootdir, 'replay.bc20')) except: logging.warning('Could not clean up game execution directory')
def game_worker(gameinfo): """ Runs a game as specified by the message Message contains JSON data, with string parameters: gametype: string, either "scrimmage" or "tournament" gameid: string, id of the game player1: string, id of red player submission player2: string, id of blue player submission maps: string, comma separated list of maps replay: string, a unique identifier for the name of the replay Filesystem structure: /box/ `-- classes/ | `-- player1.zip | `-- player2.zip | `-- player1/ | | `-- <player1 package>/ | | `-- all compiled .class files in player1.zip | `-- player2/ | `-- <player2 package>/ | `-- all compiled .class files in player2.zip `-- build/ | `-- some nonsense that gradle creates `-- replay.bc20 """ client = storage.Client() bucket = client.get_bucket(GCLOUD_BUCKET_SUBMISSION) try: gameinfo = json.loads(gameinfo) gametype = gameinfo['gametype'] gameid = gameinfo['gameid'] player1 = gameinfo['player1'] player2 = gameinfo['player2'] maps = gameinfo['maps'] replay = gameinfo['replay'] except: game_log_error(gametype, gameid, 'Game information in incorrect format') rootdir = os.path.join('/', 'box') classdir = os.path.join(rootdir, 'classes') builddir = os.path.join(rootdir, 'build') try: # Obtain player executables try: os.mkdir(classdir) with open(os.path.join(classdir, 'player1.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join( player1, 'player.zip')).download_to_file(file_obj) with open(os.path.join(classdir, 'player2.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join( player2, 'player.zip')).download_to_file(file_obj) except: game_log_error(gametype, gameid, 'Could not retrieve submissions from bucket') # Decompress zip archives of player classes try: # Unzip player 1 result = util.monitor_command( ['unzip', 'player1.zip', '-d', 'player1'], cwd=classdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError # Unzip player 2 result = util.monitor_command( ['unzip', 'player2.zip', '-d', 'player2'], cwd=classdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError except: game_log_error(gametype, gameid, 'Could not decompress player zips') # Determine player packages try: package1 = os.listdir(os.path.join(classdir, 'player1')) assert (len(package1) == 1) package1 = package1[0] package2 = os.listdir(os.path.join(classdir, 'player2')) assert (len(package2) == 1) package2 = package2[0] except: game_log_error(gametype, gameid, 'Could not determine player packages') # Update distribution util.pull_distribution( rootdir, lambda: game_log_error(gametype, gameid, 'Could not pull distribution')) # Execute game result = util.monitor_command([ './gradlew', 'run', '-PteamA={}'.format(package1), '-PteamB={}'.format(package2), '-Pmaps={}'.format(maps), '-PclassLocationA={}'.format(os.path.join( classdir, 'player1')), '-PclassLocationB={}'.format( os.path.join(classdir, 'player2')), '-Preplay=replay.bc20' ], cwd=rootdir, timeout=TIMEOUT_GAME) if result[0] != 0: game_log_error(gametype, gameid, 'Game execution had non-zero return code') # Upload replay file bucket = client.get_bucket(GCLOUD_BUCKET_REPLAY) try: with open(os.path.join(rootdir, 'replay.bc20'), 'rb') as file_obj: bucket.blob(os.path.join( 'replays', '{}.bc20'.format(replay))).upload_from_file(file_obj) except: game_log_error(gametype, gameid, 'Could not send replay file to bucket') # Interpret game result server_output = result[1].decode().split('\n') wins = [0, 0] try: # Read the winner of each game from the engine for line in server_output: if re.fullmatch(GAME_WINNER, line): game_winner = line[line.rfind('wins') - 3] assert (game_winner == 'A' or game_winner == 'B') if game_winner == 'A': wins[0] += 1 elif game_winner == 'B': wins[1] += 1 # We should have as many game wins as games played assert (wins[0] + wins[1] == len(maps.split(','))) logging.info('Game ended. Result {}:{}'.format(wins[0], wins[1])) except: game_log_error(gametype, gameid, 'Could not determine winner') else: if wins[0] > wins[1]: game_report_result(gametype, gameid, GAME_REDWON) elif wins[1] > wins[0]: game_report_result(gametype, gameid, GAME_BLUEWON) else: game_log_error(gametype, gameid, 'Ended in draw, which should not happen') finally: # Clean up working directory try: shutil.rmtree(classdir) shutil.rmtree(builddir) os.remove(os.path.join(rootdir, 'replay.bc20')) except: logging.warning('Could not clean up game execution directory')
def game_worker(gameinfo): """ Runs a game as specified by the message Message contains JSON data, with string parameters: gametype: string, either "scrimmage" or "tournament" gameid: string, id of the game player1: string, id of red player submission player2: string, id of blue player submission name1: string, team name of the red player name2: string, team name of the blue player maps: string, comma separated list of maps replay: string, a unique identifier for the name of the replay tourmode: string, "True" to use an experimental tournament mode which alternates team sides for each match, and saves replays as a comma-separated list of replay ids for each game Filesystem structure: /box/ `-- classes/ | `-- player1.zip | `-- player2.zip | `-- player1/ | | `-- <player1 package>/ | | `-- all compiled .class files in player1.zip | `-- player2/ | `-- <player2 package>/ | `-- all compiled .class files in player2.zip `-- build/ | `-- some nonsense that gradle creates `-- replay.bc20 """ client = storage.Client() bucket = client.get_bucket(GCLOUD_BUCKET_SUBMISSION) try: gameinfo = json.loads(gameinfo) gametype = gameinfo['gametype'] gameid = gameinfo['gameid'] player1 = gameinfo['player1'] player2 = gameinfo['player2'] maps = gameinfo['maps'] replay = gameinfo['replay'] tourmode = False if 'tourmode' in gameinfo and gameinfo['tourmode'] == True: tourmode = True # For reverse-compatibility if 'name1' in gameinfo: teamname1 = gameinfo['name1'] else: teamname1 = player1 if 'name2' in gameinfo: teamname2 = gameinfo['name2'] else: teamname2 = player2 except Exception as ex: game_log_error(gametype, gameid, 'Game information in incorrect format') rootdir = os.path.join('/', 'box') classdir = os.path.join(rootdir, 'classes') builddir = os.path.join(rootdir, 'build') try: # Obtain player executables attempts = 10 for i in range(attempts): try: os.mkdir(classdir) with open(os.path.join(classdir, 'player1.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join(player1, 'player.zip')).download_to_file(file_obj) with open(os.path.join(classdir, 'player2.zip'), 'wb') as file_obj: bucket.get_blob(os.path.join(player2, 'player.zip')).download_to_file(file_obj) break # exit loop and continue on our merry way except: if i >= attempts-1: #Tried {attempts} times, give up game_log_error(gametype, gameid, 'Could not retrieve submissions from bucket') else: #Try again logging.warn('Could not retrieve submissions from bucket, retrying...') # Decompress zip archives of player classes try: # Unzip player 1 result = util.monitor_command( ['unzip', 'player1.zip', '-d', 'player1'], cwd=classdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError # Unzip player 2 result = util.monitor_command( ['unzip', 'player2.zip', '-d', 'player2'], cwd=classdir, timeout=TIMEOUT_UNZIP) if result[0] != 0: raise RuntimeError except: game_log_error(gametype, gameid, 'Could not decompress player zips') # Determine player packages try: package1 = os.listdir(os.path.join(classdir, 'player1')) assert (len(package1) == 1) package1 = package1[0] package2 = os.listdir(os.path.join(classdir, 'player2')) assert (len(package2) == 1) package2 = package2[0] except: game_log_error(gametype, gameid, 'Could not determine player packages') # Update distribution util.pull_distribution(rootdir, lambda: game_log_error(gametype, gameid, 'Could not pull distribution')) # Prep maps # We want the maps as a list, so we can iterate. # For tournament mode, we want to split up map list into separate parts so we can run on each map individually; # For regular mode, we want to pass the map list in as a string of comma-separated maps, # but we wrap it in a list so we can "iterate" still while retaining old behavior. if tourmode: maps = maps.split(',') else: maps = [maps] # Initialize win count, to count for each game wins_overall = [0, 0] # Initialize a list of all the replay ids, in case we have to use multiple replay links for the same match replay_ids = [] # For tour mode, game_number represents which game (of a match) we're in; # in regular mode, game_number only takes on a value of 0 and doesn't really mean much # (since all the maps get played in the the same engine run) for game_number in range (0, len(maps)): # Prep game arguments, making sure to switch teams each game. # If game_number is even, then team A in the engine is player1, etc. teamA_is_player1 = (game_number%2==0) player1_info = (teamname1, os.path.join(classdir, 'player1'), package1) player2_info = (teamname2, os.path.join(classdir, 'player2'), package2) (teamA_arg, classLocationA_arg, packageNameA_arg) = player1_info if teamA_is_player1 else player2_info (teamB_arg, classLocationB_arg, packageNameB_arg) = player2_info if teamA_is_player1 else player1_info maps_arg = maps[game_number] # Execute game result = util.monitor_command( ['./gradlew', 'run', '-PteamA={}'.format(teamA_arg), '-PteamB={}'.format(teamB_arg), '-PclassLocationA={}'.format(classLocationA_arg), '-PclassLocationB={}'.format(classLocationB_arg), '-PpackageNameA={}'.format(packageNameA_arg), '-PpackageNameB={}'.format(packageNameB_arg), '-Pmaps={}'.format(maps_arg), '-Preplay=replay.bc21' ], cwd=rootdir, timeout=TIMEOUT_GAME) if result[0] != 0: game_log_error(gametype, gameid, 'Game execution had non-zero return code') # Upload replay file # In tour mode, we create the replay link by appending the match number to the replay hex replay_id = replay if tourmode: replay_id += '-' + str(game_number) replay_ids.append(replay_id) bucket = client.get_bucket(GCLOUD_BUCKET_REPLAY) try: with open(os.path.join(rootdir, 'replay.bc21'), 'rb') as file_obj: bucket.blob(os.path.join('replays', '{}.bc21'.format(replay_id))).upload_from_file(file_obj) except: game_log_error(gametype, gameid, 'Could not send replay file to bucket') # Interpret game result server_output = result[1].split('\n') wins = [0, 0] try: # Read the winner of each game from the engine for line in server_output: if re.fullmatch(GAME_WINNER, line): game_winner = line[line.rfind('wins')-3] assert (game_winner == 'A' or game_winner == 'B') if game_winner == 'A': wins[0] += 1 elif game_winner == 'B': wins[1] += 1 # We should have as many game wins as games played assert (wins[0] + wins[1] == len(maps_arg.split(','))) logging.info('Game ended. Result {}:{}'.format(wins[0], wins[1])) except: game_log_error(gametype, gameid, 'Could not determine winner') else: # Tally up these wins # wins_overall is in order [player1, player2] # wins is in order [teamA, teamB] if teamA_is_player1: wins_overall[0] += wins[0] wins_overall[1] += wins[1] else: wins_overall[0] += wins[1] wins_overall[1] += wins[0] # Find the overall winner logging.info('Match ended. Result {}:{}'.format(wins_overall[0], wins_overall[1])) replay_ids_string = ','.join(replay_ids) if wins_overall[0] > wins_overall[1]: game_report_result(gametype, gameid, GAME_REDWON, wins_overall[0], wins_overall[1], replay_ids_string) elif wins_overall[1] > wins_overall[0]: game_report_result(gametype, gameid, GAME_BLUEWON, wins_overall[1], wins_overall[0], replay_ids_string) else: game_log_error(gametype, gameid, 'Ended in draw, which should not happen') finally: # Clean up working directory try: shutil.rmtree(classdir) shutil.rmtree(builddir) os.remove(os.path.join(rootdir, 'replay.bc21')) except: logging.warning('Could not clean up game execution directory')