def parse_replay_file(file): with open(file, 'rb') as f: content = zstd.loads(f.read()) try: return json.loads(content) except TypeError: return json.loads(content.decode('utf-8'))
def _unzip(game_id, game_binary): """ Takes a zstd file and unzips it :param game_id: The unique id for the game object (name of resulting file) :param game_binary: The zipped binary :return: the file unzipped if possible """ try: return zstd.loads(game_binary).decode() except Exception: raise ValueError("Could not unzip file at: {}!".format(game_id))
def jsonLoad(src: Any) -> Any: ''' Load object from Json file handler, binary buffer, or file path. ''' if isinstance(src, str): gz, zst = src.endswith('.gz'), src.endswith('.zst') if zst: with open(src, 'rb') as f: return json.loads(zstd.loads(f.read())) else: with (gzip.open(src, 'rb') if gz else open(src, 'r')) as f: return json.loads(f.read()) else: return json.load(src)
def load_replay(self, path: str, meta_only=False): with open(path, 'rb') as infile: raw_replay = zstd.loads(infile.read()).decode() replay = json.loads(raw_replay) self.path = path self.replay = replay meta_keys = ['ENGINE_VERSION', 'GAME_CONSTANTS', 'REPLAY_FILE_VERSION', 'game_statistics', 'map_generator_seed', 'number_of_players', 'players'] self.meta_data = {key: replay[key] for key in meta_keys} #self.meta_data['GAME_CONSTANTS']['DEFAULT_MAP_HEIGHT'] = 64 if meta_only: return self.events = [self.parse_events(f) for f in replay['full_frames']] self.factories = self.parse_factories() self.production = self.load_production(replay) self.moves, self.generate = self.load_moves(replay) self.entities, self.ship_ids = self.load_entities(replay) self.energy = self.load_energy(replay) self.deposited = self.load_deposited(replay) self.dropoffs = self.load_dropoffs() # player_names = [x['name'].split(' ')[0] for x in meta_data['players']] # Some of these need to be trimmed # First is just an init frame self.events = self.events[1:] # Last reflects what the production will be if moves were made on last frame # (but there aren't any on last). Also, we don't care what the production looks like # on the last frame (because we will have already made our last move). # The indexing is weird because the replays show what production would be after moves # are made. self.production = self.production[:-2] self.dropoffs = self.dropoffs[:-2] # As if moved after last frame, but there are no moves self.energy = self.energy[:-1] self.deposited = self.deposited[:-1]
def get_winner_name(file_path): #print("Loading:" + file_path) with open(file_path, "rb") as f: data = json.loads(zstd.loads(f.read())) total_turns = data["game_statistics"]["number_turns"] players = [{ "id": item["player_id"], "rank": item["rank"] } for item in data["game_statistics"]["player_statistics"]] for rank in players: for player_object in data["players"]: if player_object["player_id"] == rank["id"]: rank = {**player_object, **rank} if rank["rank"] == 1: winner = rank #print("winer:", winner["name"]) return winner["name"]
def parse_replay_file(file_name, player_name): print("Load Replay: " + file_name) with open(file_name, 'rb') as f: data = json.loads(zstd.loads(f.read()).decode('utf-8')) print("Load Basic Information") player = [p for p in data['players'] if p['name'] == player_name][0] player_id = int(player['player_id']) my_shipyard = hlt.Shipyard( player_id, ARBITRARY_ID, hlt.Position(player['factory_location']['x'], player['factory_location']['y'])) other_shipyards = [ hlt.Shipyard( p['player_id'], ARBITRARY_ID, hlt.Position(p['factory_location']['x'], p['factory_location']['y'])) for p in data['players'] if int(p['player_id']) != player_id ] width = data['production_map']['width'] height = data['production_map']['height'] print("Load Cell Information") first_cells = [] for x in range(len(data['production_map']['grid'])): row = [] for y in range(len(data['production_map']['grid'][x])): row += [ hlt.MapCell(hlt.Position(x, y), data['production_map']['grid'][x][y]['energy']) ] first_cells.append(row) frames = [] for f in data['full_frames']: prev_cells = first_cells if len(frames) == 0 else frames[-1]._cells new_cells = copy.deepcopy(prev_cells) for c in f['cells']: new_cells[c['y']][c['x']].halite_amount = c['production'] frames.append(hlt.GameMap(new_cells, width, height)) print("Load Player Ships") moves = [{} if str(player_id) not in f['moves'] else { m['id']: m['direction'] for m in f['moves'][str(player_id)] if m['type'] == "m" } for f in data['full_frames']] ships = [{} if str(player_id) not in f['entities'] else { int(sid): hlt.Ship(player_id, int(sid), hlt.Position(ship['x'], ship['y']), ship['energy']) for sid, ship in f['entities'][str(player_id)].items() } for f in data['full_frames']] print("Load Other Player Ships") other_ships = [{ int(sid): hlt.Ship(int(pid), int(sid), hlt.Position(ship['x'], ship['y']), ship['energy']) for pid, p in f['entities'].items() if int(pid) != player_id for sid, ship in p.items() } for f in data['full_frames']] print("Load Droppoff Information") first_my_dropoffs = [my_shipyard] first_them_dropoffs = other_shipyards my_dropoffs = [] them_dropoffs = [] for f in data['full_frames']: new_my_dropoffs = copy.deepcopy( first_my_dropoffs if len(my_dropoffs) == 0 else my_dropoffs[-1]) new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len( them_dropoffs) == 0 else them_dropoffs[-1]) for e in f['events']: if e['type'] == 'construct': if int(e['owner_id']) == player_id: new_my_dropoffs.append( hlt.Dropoff( player_id, ARBITRARY_ID, hlt.Position(e['location']['x'], e['location']['y']))) else: new_them_dropoffs.append( hlt.Dropoff( e['owner_id'], ARBITRARY_ID, hlt.Position(e['location']['x'], e['location']['y']))) my_dropoffs.append(new_my_dropoffs) them_dropoffs.append(new_them_dropoffs) return list( zip(frames, moves, ships, other_ships, my_dropoffs, them_dropoffs))
def load_replay(file_name, player_id, mus_are_known=True): player_id = str(player_id) parsed_frames = [] with open(file_name, 'rb') as f: data = json.loads(zstd.loads(f.read()).decode("utf-8")) if mus_are_known: game_mus, went_home = read_mus(player_id) else: game_mus, went_home = None, None #fake_mus(player_id, data) ''' Notes on replay format (keys of data['full_frames']): entities: ships on the map at the BEGINNING of the frame moves: moves taken by those entities THIS turn events: events that happened THIS turn cells: changes to cells THIS turn deposited: total depositied for all of time as of END of this frame energy: player energy at END of turn ''' # ***** Unchanging constants ***** board_size = data['production_map']['width'] max_turns = data['GAME_CONSTANTS']['MAX_TURNS'] actual_turns = len(data['full_frames']) # ***** Load initial state ***** halite = np.zeros((board_size, board_size), dtype=np.int32) friendly_dropoffs = np.zeros((board_size, board_size), dtype=np.int32) enemy_dropoffs = np.zeros((board_size, board_size), dtype=np.int32) for y in range(board_size): #halite for x in range(board_size): halite[y][x] = data['production_map']['grid'][y][x]['energy'] for player_info in data['players']: # dropoffs player = str(player_info['player_id']) x = player_info['factory_location']['x'] y = player_info['factory_location']['y'] if (player == player_id): # friendly friendly_dropoffs[y][x] = 1 else: #enemy enemy_dropoffs[y][x] = 1 ship_info = {} # ***** Update for each frame ***** for t in range(actual_turns): frame = data['full_frames'][t] if mus_are_known: frame_mus = ([] if t == 0 else game_mus[t - 1]) if t <= len(game_mus) else \ {j:None for i in frame["entities"] for j in frame["entities"][i]} # Mus are all none if no moves were made frame_went_home = ([] if t == 0 else went_home[t - 1]) if t <= len(went_home) else \ {j:None for i in frame["entities"] for j in frame["entities"][i]} # Generate matrices for this turn old_halite = np.copy(halite) friendly_ships = np.zeros((board_size, board_size), dtype=np.int32) friendly_ships_halite = np.zeros((board_size, board_size), dtype=np.int32) old_friendly_dropoffs = np.copy(friendly_dropoffs) enemy_ships = np.zeros((board_size, board_size), dtype=np.int32) enemy_ships_halite = np.zeros((board_size, board_size), dtype=np.int32) old_enemy_dropoffs = np.copy(enemy_dropoffs) ship_info = {} # move info keyed by ship id moves = {str(d['id']): d for d in data['full_frames'][t]['moves'][player_id] if 'id' in d} \ if player_id in data['full_frames'][t]['moves'] else {} """ Note to Nate and Kent: don't give a f**k about masks buffer is going to be flat, don't forget to grab one extra obs at the end when sampling it's ok that the obs after a ship dies is unrelated, next obs only matters when done == False (probably) don't aggregate retrace at the end yet because we think it ignores cutoffs and that's hard ? everything seems shifted one timestep in the replay. ship as entity shows up on the same frame its move shows up in and we moved the moves so ...? figure out what to do about this Monday December 3rd 2018 """ rounds_left = max_turns - t player_energy = data['full_frames'][t - 1]['energy'][ player_id] if t > 0 else 5000 # energy at END of frame energy_delta = frame['energy'][player_id] - player_energy for changed_cell in frame['cells']: # update halite x = changed_cell['x'] y = changed_cell['y'] halite[y][x] = changed_cell['production'] for player in frame['entities']: # update ships player_entities = frame['entities'][player] for entity_id in player_entities: entity = player_entities[entity_id] x = entity['x'] y = entity['y'] energy = entity['energy'] if (player == player_id): # friendly ship_info[entity_id] = {'pos': {'x': x, 'y': y}} ship_info[entity_id]['energy'] = energy ship_info[entity_id][ 'energy_delta'] = 0 # will be overridden once we know what actually happened # note: create supply depot move looks like this: {'id': 5, 'type': 'c'} assert (not mus_are_known or t == 0 or t >= actual_turns - 1 or frame_went_home[entity_id] is not None), "frame_went_home is None!! :(" if mus_are_known and frame_went_home[entity_id]: ship_info[entity_id]['action'] = 'h' else: ship_info[entity_id]['action'] = 'o' if not entity_id in moves else \ moves[entity_id]['direction'] if 'direction' in moves[entity_id] else moves[entity_id]['type'] friendly_ships[y][x] = 1 friendly_ships_halite[y][x] = energy assert (not mus_are_known or t == 0 or t >= actual_turns - 1 or frame_mus[entity_id] is not None), "mus are None!! :(" ship_info[entity_id]["mus"] = np.array( frame_mus[entity_id], dtype=np.float32) if mus_are_known else np.array( [np.nan]) else: # enemy enemy_ships[y][x] = 1 enemy_ships_halite[y][x] = energy if t > 0: # compute ship halite deltas for previous turn for idee in parsed_frames[t - 1]['ship_info']: parsed_frames[t-1]['ship_info'][idee]['energy_delta'] = \ (ship_info[idee]['energy'] if idee in ship_info else 0) \ - parsed_frames[t-1]['ship_info'][idee]['energy'] for event in frame['events']: # update dropoffs if event['type'] == 'construct': x = event['location']['x'] y = event['location']['y'] player = str(event['owner_id']) if (player == player_id): # friendly friendly_dropoffs[y][x] = 1 else: # enemy enemy_dropoffs[y][x] = 1 halite_left = old_halite.sum( ) # This is different than the halite available on the # online thing. Mine doesn't count onboard ship halite. # remember that old mean "beginning of turn" aka current state during which actions were taken turn_results = { "halite_map": old_halite, # beginning of turn "friendly_ships": friendly_ships, # indicator variable # beginning of turn "friendly_ships_halite": friendly_ships_halite, # beginning of turn "friendly_dropoffs": old_friendly_dropoffs, # indicator # beginning of turn "enemy_ships": enemy_ships, # beginning of turn "enemy_ships_halite": enemy_ships_halite, # beginning of turn "enemy_dropoffs": old_enemy_dropoffs, # beginning of turn "total_halite": old_halite.sum(), # total on the board # beginning of turn "player_energy": player_energy, # how much you have in the bank # beginning of turn "rounds_left": rounds_left, # beginning of turn "board_size": board_size, "energy_delta": energy_delta, #ON THIS TURN ************** # ** AUGMENTED WITH DELTA RETROACTIVELY "ship_info": ship_info, # dictionary keyed by ship_id, contains (x,y) pos, energy, energy-delta, action and mu # Beg. Beg. on turn on-turn on-turn } parsed_frames.append(turn_results) parsed_frames = parsed_frames[1:-1] return parsed_frames
def parse_replay_file(self, file_name, player_name): ARBITRARY_ID = -1 with open(file_name, 'rb') as f: data = json.loads(zstd.loads(f.read())) player = [ p for p in data['players'] if p['name'].split(" ")[0] == player_name ][0] player_id = int(player['player_id']) my_shipyard = entity.Shipyard( player_id, ARBITRARY_ID, positionals.Position(player['factory_location']['x'], player['factory_location']['y'])) other_shipyards = [ entity.Shipyard( p['player_id'], ARBITRARY_ID, positionals.Position(p['factory_location']['x'], p['factory_location']['y'])) for p in data['players'] if int(p['player_id']) != player_id ] width = data['production_map']['width'] height = data['production_map']['height'] first_cells = [] for x in range(len(data['production_map']['grid'])): row = [] for y in range(len(data['production_map']['grid'][x])): row += [ game_map.MapCell( positionals.Position(x, y), data['production_map']['grid'][x][y]['energy']) ] first_cells.append(row) frames = [] frames.append(game_map.GameMap(first_cells, width, height)) first_my_dropoffs = [my_shipyard] first_them_dropoffs = other_shipyards my_dropoffs = [] them_dropoffs = [] for f in data['full_frames']: new_my_dropoffs = copy.deepcopy(first_my_dropoffs if len( my_dropoffs) == 0 else my_dropoffs[-1]) new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len( them_dropoffs) == 0 else them_dropoffs[-1]) for e in f['events']: if e['type'] == 'construct': if int(e['owner_id']) == player_id: new_my_dropoffs.append( entity.Dropoff( player_id, ARBITRARY_ID, positionals.Position(e['location']['x'], e['location']['y']))) else: new_them_dropoffs.append( entity.Dropoff( e['owner_id'], ARBITRARY_ID, positionals.Position(e['location']['x'], e['location']['y']))) my_dropoffs.append(new_my_dropoffs) them_dropoffs.append(new_them_dropoffs) return frames[0], my_dropoffs[-1], them_dropoffs[-1]
def test_random(self): DATA = os.urandom(128 * 1024) # Read 128kb self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def helper_compression_random(self): DATA = os.urandom(128 * 1024) # Read 128kb self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def parse_replay_file(file_name): with open(file_name, 'rb') as f: data = json.loads(zstd.loads(f.read())) constants.set_dimensions(data["production_map"]["width"], data["production_map"]["height"]) player_id = int(file_name.split("-")[-1][:-4]) player_name = name_dict[player_id] if player_id in name_dict else None player = [p for p in data['players'] if " ".join(p['name'].split(" ")[:-1]) == player_name] if len(player) > 0: player = player[0] else: print(f"Skipping {file_name}") return [] player_id = int(player["player_id"]) my_shipyard = entity.Shipyard(player_id, ARBITRARY_ID, positionals.Position(player['factory_location']['x'], player['factory_location']['y'])) other_shipyards = [ entity.Shipyard(p['player_id'], ARBITRARY_ID, positionals.Position(p['factory_location']['x'], p['factory_location']['y'])) for p in data['players'] if int(p['player_id']) != player_id] width = data['production_map']['width'] height = data['production_map']['height'] first_cells = [] for x in range(len(data['production_map']['grid'])): row = [] for y in range(len(data['production_map']['grid'][x])): row += [game_map.MapCell(positionals.Position(x, y), data['production_map']['grid'][x][y]['energy'])] first_cells.append(row) frames = [] for f in data['full_frames']: prev_cells = first_cells if len(frames) == 0 else frames[-1]._cells new_cells = copy.deepcopy(prev_cells) for c in f['cells']: new_cells[c['y']][c['x']].halite_amount = c['production'] frames.append(game_map.GameMap(new_cells, width, height)) moves = [{} if str(player_id) not in f['moves'] else {m['id']: m['direction'] for m in f['moves'][str(player_id)] if m['type'] == "m"} for f in data['full_frames']] ships = [{} if str(player_id) not in f['entities'] else { int(sid): entity.Ship(player_id, int(sid), positionals.Position(ship['x'], ship['y']), ship['energy']) for sid, ship in f['entities'][str(player_id)].items()} for f in data['full_frames']] other_ships = [ {int(sid): entity.Ship(int(pid), int(sid), positionals.Position(ship['x'], ship['y']), ship['energy']) for pid, p in f['entities'].items() if int(pid) != player_id for sid, ship in p.items()} for f in data['full_frames']] first_my_dropoffs = [my_shipyard] first_them_dropoffs = other_shipyards my_dropoffs = [] them_dropoffs = [] for f in data['full_frames']: new_my_dropoffs = copy.deepcopy(first_my_dropoffs if len(my_dropoffs) == 0 else my_dropoffs[-1]) new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len(them_dropoffs) == 0 else them_dropoffs[-1]) for e in f['events']: if e['type'] == 'construct': if int(e['owner_id']) == player_id: new_my_dropoffs.append( entity.Dropoff(player_id, ARBITRARY_ID, positionals.Position(e['location']['x'], e['location']['y']))) else: new_them_dropoffs.append( entity.Dropoff(e['owner_id'], ARBITRARY_ID, positionals.Position(e['location']['x'], e['location']['y']))) my_dropoffs.append(new_my_dropoffs) them_dropoffs.append(new_them_dropoffs) return list(zip(frames, moves, ships, other_ships, my_dropoffs, them_dropoffs))
def read_replay(filelike): return json.loads(zstd.loads(filelike.read()))
def parse_compressed_replay_file(file_name, player_id): with open(file_name, 'rb') as f: data = json.loads(zstd.loads(f.read()).decode()) return parse_replay_data(data, player_id)
def process_file(replay_file): """Generate states for all players in the game. Yields ------ List of GameFrames """ out_path = replay_file.replace("training_replays", "processed_replays") if os.path.exists(out_path): return replay = json.loads(zstd.loads(open(replay_file, "rb").read())) production_arr = parse_initial_production(replay["production_map"]) width, height = production_arr.shape factory_locations = {} for player in replay["players"]: player_id = player["player_id"] factory_loc = player["factory_location"] factory_locations[player_id] = (factory_loc["x"], factory_loc["y"]) game_constants = replay["GAME_CONSTANTS"] map_width = game_constants["DEFAULT_MAP_WIDTH"] num_players = replay["number_of_players"] num_turns = game_constants["MAX_TURNS"] state_actions = [] dropoff_locs = {player_id: [] for player_id in range(num_players)} for frame_num, frame in enumerate(replay["full_frames"]): # Look carefully at the JSON to understand this. # To get the state for (0-indexed) turn i, use the energy from frame i # and the entities from frame i + 1 if frame_num == len(replay["full_frames"]) - 1: break next_frame = replay["full_frames"][frame_num + 1] # TODO: see why the max is needed turns_remaining = max(num_turns - frame_num, 0) # player -> (ship_id -> position) ship_ids = {player_id: {} for player_id in range(num_players)} # player -> ship_id mutable player_ship_set = {player_id: set() for player_id in range(num_players)} # player_id -> (position -> energy) ship_pos_energy = {player_id: {} for player_id in range(num_players)} for player_id, entities in next_frame["entities"].items(): player_id = int(player_id) for ship_id, ship_info in entities.items(): ship_id = int(ship_id) player_ship_set[player_id].update([ship_id]) x, y = ship_info["x"], ship_info["y"] ship_ids[player_id][ship_id] = x, y ship_pos_energy[player_id][(x, y)] = ship_info["energy"] spawns = [] moves = defaultdict(dict) constructions = {player_id: [] for player_id in range(num_players)} scores = { int(player_id_str): energy for player_id_str, energy in frame["energy"].items() } # Update the Halite map. for cell in frame["cells"]: production_arr[cell["x"], cell["y"]] = cell["production"] for player_id_str, player_moves in next_frame["moves"].items(): player_id = int(player_id_str) for move in player_moves: if move["type"] == "g": assert scores[player_id] >= 1000 spawns.append(player_id) elif move["type"] == "m": ship_id = move["id"] player_ship_set[player_id].remove(ship_id) position = ship_ids[player_id][ship_id] moves[player_id][position] = actions.ACTION_CHR_DIR[ move["direction"] ] elif move["type"] == "c": loc = frame["entities"][player_id_str][str(move["id"])] constructions[player_id].append((loc["x"], loc["y"])) for event in frame["events"]: if event["type"] == "spawn": # These events are handled as moves. continue elif event["type"] == "shipwreck": # We don't care about these. continue player_id = int(event["owner_id"]) assert event["type"] == "construct" location = event["location"] dropoff_locs[player_id].append((location["x"], location["y"])) if frame_num <= num_turns: for player_id, p_ship_ids in player_ship_set.items(): for ship_id in p_ship_ids: position = ship_ids[player_id][ship_id] # some players rely on the halite engine collecting for # them automatically moves[player_id][position] = (0, 0) action = Actions( spawns=spawns, moves={0: moves[0]}, # only consider moves for p0 constructions={0: constructions[0]}, num_players=num_players, map_width=map_width, ) state = State( halite_map=production_arr, ships=ship_pos_energy, turns_remaining=turns_remaining, factory_locs=factory_locations, scores=scores, dropoff_locs=dropoff_locs, num_players=num_players, ) state_actions.append((state, action)) # TODO: add rewards sar = state_actions out_dir = os.path.dirname(out_path) os.makedirs(out_dir, exist_ok=True) dump(sar, open(out_path, "wb"))