def parse_replay_file(file):
with open(file, 'rb') as f:
content = zstd.loads(f.read())
try:
return json.loads(content)
except TypeError:
return json.loads(content.decode('utf-8'))
def _unzip(game_id, game_binary):
"""
Takes a zstd file and unzips it
:param game_id: The unique id for the game object (name of resulting file)
:param game_binary: The zipped binary
:return: the file unzipped if possible
"""
try:
return zstd.loads(game_binary).decode()
except Exception:
raise ValueError("Could not unzip file at: {}!".format(game_id))
def jsonLoad(src: Any) -> Any:
'''
Load object from Json file handler, binary buffer, or file path.
'''
if isinstance(src, str):
gz, zst = src.endswith('.gz'), src.endswith('.zst')
if zst:
with open(src, 'rb') as f:
return json.loads(zstd.loads(f.read()))
else:
with (gzip.open(src, 'rb') if gz else open(src, 'r')) as f:
return json.loads(f.read())
else:
return json.load(src)
def load_replay(self, path: str, meta_only=False):
with open(path, 'rb') as infile:
raw_replay = zstd.loads(infile.read()).decode()
replay = json.loads(raw_replay)
self.path = path
self.replay = replay
meta_keys = ['ENGINE_VERSION', 'GAME_CONSTANTS', 'REPLAY_FILE_VERSION',
'game_statistics', 'map_generator_seed',
'number_of_players', 'players']
self.meta_data = {key: replay[key] for key in meta_keys}
#self.meta_data['GAME_CONSTANTS']['DEFAULT_MAP_HEIGHT'] = 64
if meta_only:
return
self.events = [self.parse_events(f) for f in replay['full_frames']]
self.factories = self.parse_factories()
self.production = self.load_production(replay)
self.moves, self.generate = self.load_moves(replay)
self.entities, self.ship_ids = self.load_entities(replay)
self.energy = self.load_energy(replay)
self.deposited = self.load_deposited(replay)
self.dropoffs = self.load_dropoffs()
# player_names = [x['name'].split(' ')[0] for x in meta_data['players']]
# Some of these need to be trimmed
# First is just an init frame
self.events = self.events[1:]
# Last reflects what the production will be if moves were made on last frame
# (but there aren't any on last). Also, we don't care what the production looks like
# on the last frame (because we will have already made our last move).
# The indexing is weird because the replays show what production would be after moves
# are made.
self.production = self.production[:-2]
self.dropoffs = self.dropoffs[:-2]
# As if moved after last frame, but there are no moves
self.energy = self.energy[:-1]
self.deposited = self.deposited[:-1]
def get_winner_name(file_path):
#print("Loading:" + file_path)
with open(file_path, "rb") as f:
data = json.loads(zstd.loads(f.read()))
total_turns = data["game_statistics"]["number_turns"]
players = [{
"id": item["player_id"],
"rank": item["rank"]
} for item in data["game_statistics"]["player_statistics"]]
for rank in players:
for player_object in data["players"]:
if player_object["player_id"] == rank["id"]:
rank = {**player_object, **rank}
if rank["rank"] == 1:
winner = rank
#print("winer:", winner["name"])
return winner["name"]
def parse_replay_file(file_name, player_name):
print("Load Replay: " + file_name)
with open(file_name, 'rb') as f:
data = json.loads(zstd.loads(f.read()).decode('utf-8'))
print("Load Basic Information")
player = [p for p in data['players'] if p['name'] == player_name][0]
player_id = int(player['player_id'])
my_shipyard = hlt.Shipyard(
player_id, ARBITRARY_ID,
hlt.Position(player['factory_location']['x'],
player['factory_location']['y']))
other_shipyards = [
hlt.Shipyard(
p['player_id'], ARBITRARY_ID,
hlt.Position(p['factory_location']['x'],
p['factory_location']['y'])) for p in data['players']
if int(p['player_id']) != player_id
]
width = data['production_map']['width']
height = data['production_map']['height']
print("Load Cell Information")
first_cells = []
for x in range(len(data['production_map']['grid'])):
row = []
for y in range(len(data['production_map']['grid'][x])):
row += [
hlt.MapCell(hlt.Position(x, y),
data['production_map']['grid'][x][y]['energy'])
]
first_cells.append(row)
frames = []
for f in data['full_frames']:
prev_cells = first_cells if len(frames) == 0 else frames[-1]._cells
new_cells = copy.deepcopy(prev_cells)
for c in f['cells']:
new_cells[c['y']][c['x']].halite_amount = c['production']
frames.append(hlt.GameMap(new_cells, width, height))
print("Load Player Ships")
moves = [{} if str(player_id) not in f['moves'] else {
m['id']: m['direction']
for m in f['moves'][str(player_id)] if m['type'] == "m"
} for f in data['full_frames']]
ships = [{} if str(player_id) not in f['entities'] else {
int(sid): hlt.Ship(player_id, int(sid),
hlt.Position(ship['x'], ship['y']), ship['energy'])
for sid, ship in f['entities'][str(player_id)].items()
} for f in data['full_frames']]
print("Load Other Player Ships")
other_ships = [{
int(sid): hlt.Ship(int(pid), int(sid),
hlt.Position(ship['x'], ship['y']), ship['energy'])
for pid, p in f['entities'].items() if int(pid) != player_id
for sid, ship in p.items()
} for f in data['full_frames']]
print("Load Droppoff Information")
first_my_dropoffs = [my_shipyard]
first_them_dropoffs = other_shipyards
my_dropoffs = []
them_dropoffs = []
for f in data['full_frames']:
new_my_dropoffs = copy.deepcopy(
first_my_dropoffs if len(my_dropoffs) == 0 else my_dropoffs[-1])
new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len(
them_dropoffs) == 0 else them_dropoffs[-1])
for e in f['events']:
if e['type'] == 'construct':
if int(e['owner_id']) == player_id:
new_my_dropoffs.append(
hlt.Dropoff(
player_id, ARBITRARY_ID,
hlt.Position(e['location']['x'],
e['location']['y'])))
else:
new_them_dropoffs.append(
hlt.Dropoff(
e['owner_id'], ARBITRARY_ID,
hlt.Position(e['location']['x'],
e['location']['y'])))
my_dropoffs.append(new_my_dropoffs)
them_dropoffs.append(new_them_dropoffs)
return list(
zip(frames, moves, ships, other_ships, my_dropoffs, them_dropoffs))
def load_replay(file_name, player_id, mus_are_known=True):
player_id = str(player_id)
parsed_frames = []
with open(file_name, 'rb') as f:
data = json.loads(zstd.loads(f.read()).decode("utf-8"))
if mus_are_known:
game_mus, went_home = read_mus(player_id)
else:
game_mus, went_home = None, None #fake_mus(player_id, data)
'''
Notes on replay format (keys of data['full_frames']):
entities: ships on the map at the BEGINNING of the frame
moves: moves taken by those entities THIS turn
events: events that happened THIS turn
cells: changes to cells THIS turn
deposited: total depositied for all of time as of END of this frame
energy: player energy at END of turn
'''
# ***** Unchanging constants *****
board_size = data['production_map']['width']
max_turns = data['GAME_CONSTANTS']['MAX_TURNS']
actual_turns = len(data['full_frames'])
# ***** Load initial state *****
halite = np.zeros((board_size, board_size), dtype=np.int32)
friendly_dropoffs = np.zeros((board_size, board_size), dtype=np.int32)
enemy_dropoffs = np.zeros((board_size, board_size), dtype=np.int32)
for y in range(board_size): #halite
for x in range(board_size):
halite[y][x] = data['production_map']['grid'][y][x]['energy']
for player_info in data['players']: # dropoffs
player = str(player_info['player_id'])
x = player_info['factory_location']['x']
y = player_info['factory_location']['y']
if (player == player_id): # friendly
friendly_dropoffs[y][x] = 1
else: #enemy
enemy_dropoffs[y][x] = 1
ship_info = {}
# ***** Update for each frame *****
for t in range(actual_turns):
frame = data['full_frames'][t]
if mus_are_known:
frame_mus = ([] if t == 0 else game_mus[t - 1]) if t <= len(game_mus) else \
{j:None for i in frame["entities"] for j in frame["entities"][i]} # Mus are all none if no moves were made
frame_went_home = ([] if t == 0 else went_home[t - 1]) if t <= len(went_home) else \
{j:None for i in frame["entities"] for j in frame["entities"][i]}
# Generate matrices for this turn
old_halite = np.copy(halite)
friendly_ships = np.zeros((board_size, board_size), dtype=np.int32)
friendly_ships_halite = np.zeros((board_size, board_size),
dtype=np.int32)
old_friendly_dropoffs = np.copy(friendly_dropoffs)
enemy_ships = np.zeros((board_size, board_size), dtype=np.int32)
enemy_ships_halite = np.zeros((board_size, board_size), dtype=np.int32)
old_enemy_dropoffs = np.copy(enemy_dropoffs)
ship_info = {}
# move info keyed by ship id
moves = {str(d['id']): d for d in data['full_frames'][t]['moves'][player_id] if 'id' in d} \
if player_id in data['full_frames'][t]['moves'] else {}
""" Note to Nate and Kent:
don't give a f**k about masks
buffer is going to be flat, don't forget to grab one extra obs at the end when sampling
it's ok that the obs after a ship dies is unrelated, next obs only matters when done == False (probably)
don't aggregate retrace at the end yet because we think it ignores cutoffs and that's hard ?
everything seems shifted one timestep in the replay. ship as entity shows up on the same frame its move shows up in
and we moved the moves so ...? figure out what to do about this Monday December 3rd 2018
"""
rounds_left = max_turns - t
player_energy = data['full_frames'][t - 1]['energy'][
player_id] if t > 0 else 5000 # energy at END of frame
energy_delta = frame['energy'][player_id] - player_energy
for changed_cell in frame['cells']: # update halite
x = changed_cell['x']
y = changed_cell['y']
halite[y][x] = changed_cell['production']
for player in frame['entities']: # update ships
player_entities = frame['entities'][player]
for entity_id in player_entities:
entity = player_entities[entity_id]
x = entity['x']
y = entity['y']
energy = entity['energy']
if (player == player_id): # friendly
ship_info[entity_id] = {'pos': {'x': x, 'y': y}}
ship_info[entity_id]['energy'] = energy
ship_info[entity_id][
'energy_delta'] = 0 # will be overridden once we know what actually happened
# note: create supply depot move looks like this: {'id': 5, 'type': 'c'}
assert (not mus_are_known or t == 0
or t >= actual_turns - 1
or frame_went_home[entity_id]
is not None), "frame_went_home is None!! :("
if mus_are_known and frame_went_home[entity_id]:
ship_info[entity_id]['action'] = 'h'
else:
ship_info[entity_id]['action'] = 'o' if not entity_id in moves else \
moves[entity_id]['direction'] if 'direction' in moves[entity_id] else moves[entity_id]['type']
friendly_ships[y][x] = 1
friendly_ships_halite[y][x] = energy
assert (not mus_are_known or t == 0
or t >= actual_turns - 1 or frame_mus[entity_id]
is not None), "mus are None!! :("
ship_info[entity_id]["mus"] = np.array(
frame_mus[entity_id],
dtype=np.float32) if mus_are_known else np.array(
[np.nan])
else: # enemy
enemy_ships[y][x] = 1
enemy_ships_halite[y][x] = energy
if t > 0:
# compute ship halite deltas for previous turn
for idee in parsed_frames[t - 1]['ship_info']:
parsed_frames[t-1]['ship_info'][idee]['energy_delta'] = \
(ship_info[idee]['energy'] if idee in ship_info else 0) \
- parsed_frames[t-1]['ship_info'][idee]['energy']
for event in frame['events']: # update dropoffs
if event['type'] == 'construct':
x = event['location']['x']
y = event['location']['y']
player = str(event['owner_id'])
if (player == player_id): # friendly
friendly_dropoffs[y][x] = 1
else: # enemy
enemy_dropoffs[y][x] = 1
halite_left = old_halite.sum(
) # This is different than the halite available on the
# online thing. Mine doesn't count onboard ship halite.
# remember that old mean "beginning of turn" aka current state during which actions were taken
turn_results = {
"halite_map": old_halite, # beginning of turn
"friendly_ships":
friendly_ships, # indicator variable # beginning of turn
"friendly_ships_halite":
friendly_ships_halite, # beginning of turn
"friendly_dropoffs":
old_friendly_dropoffs, # indicator # beginning of turn
"enemy_ships": enemy_ships, # beginning of turn
"enemy_ships_halite": enemy_ships_halite, # beginning of turn
"enemy_dropoffs": old_enemy_dropoffs, # beginning of turn
"total_halite":
old_halite.sum(), # total on the board # beginning of turn
"player_energy":
player_energy, # how much you have in the bank # beginning of turn
"rounds_left": rounds_left, # beginning of turn
"board_size": board_size,
"energy_delta": energy_delta, #ON THIS TURN **************
# ** AUGMENTED WITH DELTA RETROACTIVELY
"ship_info":
ship_info, # dictionary keyed by ship_id, contains (x,y) pos, energy, energy-delta, action and mu
# Beg. Beg. on turn on-turn on-turn
}
parsed_frames.append(turn_results)
parsed_frames = parsed_frames[1:-1]
return parsed_frames
def parse_replay_file(self, file_name, player_name):
ARBITRARY_ID = -1
with open(file_name, 'rb') as f:
data = json.loads(zstd.loads(f.read()))
player = [
p for p in data['players']
if p['name'].split(" ")[0] == player_name
][0]
player_id = int(player['player_id'])
my_shipyard = entity.Shipyard(
player_id, ARBITRARY_ID,
positionals.Position(player['factory_location']['x'],
player['factory_location']['y']))
other_shipyards = [
entity.Shipyard(
p['player_id'], ARBITRARY_ID,
positionals.Position(p['factory_location']['x'],
p['factory_location']['y']))
for p in data['players'] if int(p['player_id']) != player_id
]
width = data['production_map']['width']
height = data['production_map']['height']
first_cells = []
for x in range(len(data['production_map']['grid'])):
row = []
for y in range(len(data['production_map']['grid'][x])):
row += [
game_map.MapCell(
positionals.Position(x, y),
data['production_map']['grid'][x][y]['energy'])
]
first_cells.append(row)
frames = []
frames.append(game_map.GameMap(first_cells, width, height))
first_my_dropoffs = [my_shipyard]
first_them_dropoffs = other_shipyards
my_dropoffs = []
them_dropoffs = []
for f in data['full_frames']:
new_my_dropoffs = copy.deepcopy(first_my_dropoffs if len(
my_dropoffs) == 0 else my_dropoffs[-1])
new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len(
them_dropoffs) == 0 else them_dropoffs[-1])
for e in f['events']:
if e['type'] == 'construct':
if int(e['owner_id']) == player_id:
new_my_dropoffs.append(
entity.Dropoff(
player_id, ARBITRARY_ID,
positionals.Position(e['location']['x'],
e['location']['y'])))
else:
new_them_dropoffs.append(
entity.Dropoff(
e['owner_id'], ARBITRARY_ID,
positionals.Position(e['location']['x'],
e['location']['y'])))
my_dropoffs.append(new_my_dropoffs)
them_dropoffs.append(new_them_dropoffs)
return frames[0], my_dropoffs[-1], them_dropoffs[-1]
def test_random(self): DATA = os.urandom(128 * 1024) # Read 128kb self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def helper_compression_random(self):
DATA = os.urandom(128 * 1024) # Read 128kb
self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def parse_replay_file(file_name):
with open(file_name, 'rb') as f:
data = json.loads(zstd.loads(f.read()))
constants.set_dimensions(data["production_map"]["width"], data["production_map"]["height"])
player_id = int(file_name.split("-")[-1][:-4])
player_name = name_dict[player_id] if player_id in name_dict else None
player = [p for p in data['players'] if " ".join(p['name'].split(" ")[:-1]) == player_name]
if len(player) > 0:
player = player[0]
else:
print(f"Skipping {file_name}")
return []
player_id = int(player["player_id"])
my_shipyard = entity.Shipyard(player_id, ARBITRARY_ID,
positionals.Position(player['factory_location']['x'],
player['factory_location']['y']))
other_shipyards = [
entity.Shipyard(p['player_id'], ARBITRARY_ID,
positionals.Position(p['factory_location']['x'], p['factory_location']['y']))
for p in data['players'] if int(p['player_id']) != player_id]
width = data['production_map']['width']
height = data['production_map']['height']
first_cells = []
for x in range(len(data['production_map']['grid'])):
row = []
for y in range(len(data['production_map']['grid'][x])):
row += [game_map.MapCell(positionals.Position(x, y), data['production_map']['grid'][x][y]['energy'])]
first_cells.append(row)
frames = []
for f in data['full_frames']:
prev_cells = first_cells if len(frames) == 0 else frames[-1]._cells
new_cells = copy.deepcopy(prev_cells)
for c in f['cells']:
new_cells[c['y']][c['x']].halite_amount = c['production']
frames.append(game_map.GameMap(new_cells, width, height))
moves = [{} if str(player_id) not in f['moves'] else {m['id']: m['direction'] for m in f['moves'][str(player_id)] if
m['type'] == "m"} for f in data['full_frames']]
ships = [{} if str(player_id) not in f['entities'] else {
int(sid): entity.Ship(player_id, int(sid), positionals.Position(ship['x'], ship['y']), ship['energy']) for
sid, ship in
f['entities'][str(player_id)].items()} for f in data['full_frames']]
other_ships = [
{int(sid): entity.Ship(int(pid), int(sid), positionals.Position(ship['x'], ship['y']), ship['energy']) for
pid, p in
f['entities'].items() if
int(pid) != player_id for sid, ship in p.items()} for f in data['full_frames']]
first_my_dropoffs = [my_shipyard]
first_them_dropoffs = other_shipyards
my_dropoffs = []
them_dropoffs = []
for f in data['full_frames']:
new_my_dropoffs = copy.deepcopy(first_my_dropoffs if len(my_dropoffs) == 0 else my_dropoffs[-1])
new_them_dropoffs = copy.deepcopy(first_them_dropoffs if len(them_dropoffs) == 0 else them_dropoffs[-1])
for e in f['events']:
if e['type'] == 'construct':
if int(e['owner_id']) == player_id:
new_my_dropoffs.append(
entity.Dropoff(player_id, ARBITRARY_ID,
positionals.Position(e['location']['x'], e['location']['y'])))
else:
new_them_dropoffs.append(
entity.Dropoff(e['owner_id'], ARBITRARY_ID,
positionals.Position(e['location']['x'], e['location']['y'])))
my_dropoffs.append(new_my_dropoffs)
them_dropoffs.append(new_them_dropoffs)
return list(zip(frames, moves, ships, other_ships, my_dropoffs, them_dropoffs))
def read_replay(filelike):
return json.loads(zstd.loads(filelike.read()))
def parse_compressed_replay_file(file_name, player_id):
with open(file_name, 'rb') as f:
data = json.loads(zstd.loads(f.read()).decode())
return parse_replay_data(data, player_id)
def process_file(replay_file):
"""Generate states for all players in the game.
Yields
------
List of GameFrames
"""
out_path = replay_file.replace("training_replays", "processed_replays")
if os.path.exists(out_path):
return
replay = json.loads(zstd.loads(open(replay_file, "rb").read()))
production_arr = parse_initial_production(replay["production_map"])
width, height = production_arr.shape
factory_locations = {}
for player in replay["players"]:
player_id = player["player_id"]
factory_loc = player["factory_location"]
factory_locations[player_id] = (factory_loc["x"], factory_loc["y"])
game_constants = replay["GAME_CONSTANTS"]
map_width = game_constants["DEFAULT_MAP_WIDTH"]
num_players = replay["number_of_players"]
num_turns = game_constants["MAX_TURNS"]
state_actions = []
dropoff_locs = {player_id: [] for player_id in range(num_players)}
for frame_num, frame in enumerate(replay["full_frames"]):
# Look carefully at the JSON to understand this.
# To get the state for (0-indexed) turn i, use the energy from frame i
# and the entities from frame i + 1
if frame_num == len(replay["full_frames"]) - 1:
break
next_frame = replay["full_frames"][frame_num + 1]
# TODO: see why the max is needed
turns_remaining = max(num_turns - frame_num, 0)
# player -> (ship_id -> position)
ship_ids = {player_id: {} for player_id in range(num_players)}
# player -> ship_id mutable
player_ship_set = {player_id: set() for player_id in range(num_players)}
# player_id -> (position -> energy)
ship_pos_energy = {player_id: {} for player_id in range(num_players)}
for player_id, entities in next_frame["entities"].items():
player_id = int(player_id)
for ship_id, ship_info in entities.items():
ship_id = int(ship_id)
player_ship_set[player_id].update([ship_id])
x, y = ship_info["x"], ship_info["y"]
ship_ids[player_id][ship_id] = x, y
ship_pos_energy[player_id][(x, y)] = ship_info["energy"]
spawns = []
moves = defaultdict(dict)
constructions = {player_id: [] for player_id in range(num_players)}
scores = {
int(player_id_str): energy
for player_id_str, energy in frame["energy"].items()
}
# Update the Halite map.
for cell in frame["cells"]:
production_arr[cell["x"], cell["y"]] = cell["production"]
for player_id_str, player_moves in next_frame["moves"].items():
player_id = int(player_id_str)
for move in player_moves:
if move["type"] == "g":
assert scores[player_id] >= 1000
spawns.append(player_id)
elif move["type"] == "m":
ship_id = move["id"]
player_ship_set[player_id].remove(ship_id)
position = ship_ids[player_id][ship_id]
moves[player_id][position] = actions.ACTION_CHR_DIR[
move["direction"]
]
elif move["type"] == "c":
loc = frame["entities"][player_id_str][str(move["id"])]
constructions[player_id].append((loc["x"], loc["y"]))
for event in frame["events"]:
if event["type"] == "spawn":
# These events are handled as moves.
continue
elif event["type"] == "shipwreck":
# We don't care about these.
continue
player_id = int(event["owner_id"])
assert event["type"] == "construct"
location = event["location"]
dropoff_locs[player_id].append((location["x"], location["y"]))
if frame_num <= num_turns:
for player_id, p_ship_ids in player_ship_set.items():
for ship_id in p_ship_ids:
position = ship_ids[player_id][ship_id]
# some players rely on the halite engine collecting for
# them automatically
moves[player_id][position] = (0, 0)
action = Actions(
spawns=spawns,
moves={0: moves[0]}, # only consider moves for p0
constructions={0: constructions[0]},
num_players=num_players,
map_width=map_width,
)
state = State(
halite_map=production_arr,
ships=ship_pos_energy,
turns_remaining=turns_remaining,
factory_locs=factory_locations,
scores=scores,
dropoff_locs=dropoff_locs,
num_players=num_players,
)
state_actions.append((state, action))
# TODO: add rewards
sar = state_actions
out_dir = os.path.dirname(out_path)
os.makedirs(out_dir, exist_ok=True)
dump(sar, open(out_path, "wb"))
