def get_unique_unit_replay():
game_list = []
counter = 0
# loop over replay master list, grab first "num_games_to_parse"
for full_filename in replays_master:
replay = replayer.load(full_filename)
if counter >= num_games_to_parse:
break
counter += 1
print("loaded replay: " + full_filename + " of length:" +
str(len(replay)))
# ids of units I care about:
for frame_number in range(len(replay)):
frame = replay.getFrame(frame_number)
units = frame.units
for player_id, unit_arr in units.iteritems():
if player_id < 0:
# This is a map resource / object, not an army unit
continue
# For each unit in this frame:
for unit in unit_arr:
# Make sure it's a type that I care about
if unit.type in valid_types:
unit_data_arr = unit_to_arr(unit)
# Don't add if I already have this exact state/label combo
if (unit_data_arr in game_list) or (unit_data_arr[1]
== 133):
continue
else:
game_list.append(unit_data_arr)
return game_list
def game_over_time(full_replay_path, valid_types, playerid=0, step_size=4):
"""
turns a .tcr into an array of dictionaries for visualizing / processing
:param full_replay_path: path to replay file
:param valid_types: list of unit_type attributes to consider
:param playerid: player to follow (default 0)
:return: array of dictionaries of unit data over time
"""
# Load in the replay:
replay = replayer.load(full_replay_path)
print("loaded replay: " + full_replay_path + " of length:" +
str(len(replay)))
game_data = []
starting_positions = {}
for frame_number in range(0, len(replay), step_size):
frame = replay.getFrame(frame_number)
units = frame.units
this_frame = {}
for player_id, unit_arr in units.iteritems():
# I only want to look at the player i pass in (default 0)
if player_id != playerid and playerid < 2:
continue
# For each unit in this frame:
for unit in unit_arr:
# Make sure it's a type that I care about
if unit.type in valid_types:
if unit.id in starting_positions:
this_frame[unit.id] = unit_to_dict(
unit, starting_positions[unit.id], unit_arr,
valid_types)
else:
this_frame[unit.id] = unit_to_dict(
unit, (-1, -1), unit_arr, valid_types)
starting_positions[unit.id] = (unit.x, unit.y)
# End iteration over units for player_id in this frame
# End iteration over units for all players in this frame
# TODO: this is a hack for only TvT matchups. remove this later:
# if len(this_frame) == 0 and frame_number == 0:
# # no valid units in this frame, we can assume this was not a terran player
# return -1
game_data.append(this_frame)
# End iteration over all frames
player0psi = frame.resources[0].used_psi
player1psi = frame.resources[1].used_psi
if player0psi > player1psi:
print("I think player 0 won with " + str(player0psi - player1psi) +
" more psi than player 1")
else:
print("I think player 1 won with " + str(player1psi - player0psi) +
" more psi than player 0")
return game_data
def featurize(fn):
savep = "{}/{}".format(path.basename(path.dirname(fn)), path.basename(fn))
savep = path.join(base, savep) + '.npz'
os.makedirs(path.dirname(savep), exist_ok=True)
if args.skip_existing and path.exists(savep):
logging.info("Found existing {}, skipping".format(savep))
return True
if not os.path.exists(fn):
raise RuntimeError("No such replay")
rep = replayer.load(fn)
map = rep.getMap()
map_size = map['walkability'].shape
sl = make_start_loc(map['start_locations'], map_size)
batch = []
for i in range(0, len(rep)):
batch.append(rep.getFrame(i))
reduced = featurizer.reduce(batch)
map_features = np.stack([
map['walkability'],
map['buildability'],
map['ground_height'],
sl,
],
axis=0).astype(dtype='uint8')
info_hash = chain(*get_info(rep)) # 6 ints, p0race, p0slx, p0sly, p1...
map_hash = hash_map(map) # 4 ints for the map hash
info = np.asarray(list(chain(info_hash, map_hash)), dtype=np.int32)
mfdata = io.BytesIO()
with gzip.GzipFile(fileobj=mfdata, mode='w') as f:
np.save(f, map_features)
to_save = {
'reduced': reduced,
'map': mfdata.getvalue(),
'info': info,
}
# TODO: Both reduced data and overall data may be compressed
np.savez(savep, **to_save)
return True
def get_temporal_replays():
step_size = 8 # 1 frame per second if step size = 8
for full_filename in replays_master:
replay = replayer.load(full_filename)
print("loaded replay: " + full_filename + " of length:" +
str(len(replay)))
units_over_time = {}
for frame_number in range(0, len(replay), step_size):
frame = replay.getFrame(frame_number)
units = frame.units
for player_id, unit_arr in units.iteritems():
if player_id < 0:
# This is a map resource / object, not an army unit
continue
# For each unit in this frame:
for unit in unit_arr:
# Make sure it's a type that I care about
if unit.type in valid_types:
# get nearest neighbors
neighbors = get_nearest_units(unit,
unit_arr,
threshold=200,
n_nearest=11)
# create matrix with me at top, my neighbors under me
unit_and_neighbors = np.vstack(
(unit_to_np(unit), neighbors))
if unit.id in units_over_time:
units_over_time[unit.id].append(unit_and_neighbors)
else:
units_over_time[unit.id] = [unit_and_neighbors]
# End iteration over units for player_id in this frame
# End iteration over units for all players in this frame
new_fn = '/media/asilva/HD_home/my_sc_data/temporal_pickles/' + full_filename.split(
'/')[-1] + '.pkl'
pickle.dump(units_over_time, open(new_fn, 'wb'))
print('dumped to ' + new_fn)
# End iteration over all frames
return True
def featurize(fn):
raise RuntimeError("This doesn't work anymore, add in game name, visibility")
rep = replayer.load(fn)
map = rep.getMap()
map_size = map['walkability'].shape
race = [x[0] for x in get_info(rep)]
sl = make_start_loc(map['start_locations'], map_size)
batch = []
fromt = args.from_time if args.until_time != 0 else 0
until = args.from_time if args.until_time != 0 else len(rep)
for i in range(fromt, min(len(rep), until), args.skip_frames):
frames = [rep.getFrame(k) for k in
range(i, min(i + args.combine_frames, len(rep)))]
batch.append(frames)
featurized = (
# map
th.from_numpy(
np.stack([
map['walkability'],
map['buildability'],
map['ground_height'],
sl,
], axis=0)[np.newaxis, :]).type(th.FloatTensor),
# race
th.LongTensor([race if pers == 0 else list(reversed(race))]),
# input
th.from_numpy(
normalize(featurizer.featurize(
batch, map_size, perspective=pers
).transpose(0, 3, 1, 2))[:-1]).type(th.FloatTensor),
# targets
th.from_numpy(
normalize(featurizer.featurize(
batch, map_size, perspective=pers, full=True
).transpose(0, 3, 1, 2))[1:]).type(th.FloatTensor),
)
assert featurized[2].size(0) > 1
return featurized
def autoencode_dataset(data_replays,
valid_types,
step_size,
window_size,
feature_set,
add_orders,
num_games):
games_collected = 0
X = None
for replay_path in data_replays:
if games_collected >= num_games:
break
replay = replayer.load(replay_path)
print("loaded replay: " + replay_path + " of length:" + str(len(replay)))
input_data = generate_role_datasets.game_over_time(replay=replay,
valid_types=valid_types,
playerid=2,
step_size=step_size,
feature_set=feature_set,
add_orders=add_orders)
if not input_data[0]:
print("No valid units in " + replay_path)
continue
print("Data collected")
x_in = generate_role_datasets.hmm_data(input_data, n_timesteps=window_size, for_drawing=False)
x_in = np.array(x_in)
print("temporally stitched")
if X is None:
X = x_in
else:
X = np.vstack((X, x_in))
games_collected += 1
return X
'other': -1
}
reverse_role_dict = dict()
for key1, val in role_dict.iteritems():
reverse_role_dict[val] = key1
test_set_replays = params['replays_master']
visualize = False
pkl_dir = '/home/asilva/Documents/stardata_analysis/gt_pkls'
labelled_games = 0
games_to_label = 200
for replay_path in test_set_replays:
if labelled_games >= games_to_label:
break
replay = replayer.load(replay_path)
print("loaded replay: " + replay_path + " of length:" + str(len(replay)))
valid_role_units = params['terran_valid_types'] # + params['zerg_valid_types'] + params['protoss_valid_types']
role_data = game_role_data(replay_data=replay,
valid_types=valid_role_units,
playerid=2,
step_size=1)
if not role_data[0]:
print("No valid units in " + replay_path)
continue
print("Roles assigned")
ffilled = forward_fill(role_data)
print("Forward filled")
pkl_name = os.path.join(pkl_dir, replay_path.split('/')[-1].split('.')[0] + 'GT.pkl')
# Created by Andrew Silva on 3/30/18
from torchcraft import replayer
import glob
# python port of the cpp script for getting corrupted replays
# best I can tell it is the same?
input_pattern = '/media/asilva/HD_home/StarData/dumped_replays/**/*.tcr'
all_files = glob.glob(input_pattern)
bad_files = []
for filename in all_files:
try:
rep = replayer.load(filename)
ore = 0
gas = 0
used_ore = 0
used_gas = 0
for i in range(len(rep)):
frame = rep.getFrame(i)
cur_ore = 0
cur_gas = 0
res = frame.resources
for r in res.values():
cur_ore += r.ore
cur_gas += r.gas
if i > len(rep) / 2:
used_ore += max(0, ore - cur_ore)
used_gas += max(0, gas - cur_gas)
ore = cur_ore
gas = cur_gas
percent_used_ore = 1.0 * used_ore / (used_ore + ore)
def featurize(fn):
rep = replayer.load(fn)
map = rep.getMap()
map_size = map['walkability'].shape
batch = []
for i in range(0, len(rep), args.skip_frames):
frames = [
rep.getFrame(k)
for k in range(i, min(i + args.combine_frames, len(rep)))
]
batch.append(frames)
if method == 'prev':
return [
featurizer.featurize(batch, map_size, perspective=0),
featurizer.featurize(batch, map_size, perspective=1),
featurizer.featurize(batch, map_size, perspective=0,
full=True),
featurizer.featurize(batch, map_size, perspective=1,
full=True),
]
elif method == 'stat':
features = featurizer.featurize(batch, map_size, full=True)
map_hash = hash_map(map)
'''
Two players with the same info is considered to be the same 'game'
for this baseline: All ZvT mu that start at the same location
should be hashed to a single info. Should be irrespective of p1
or p2, so we sort.
'''
info_hash = tuple(sorted(get_info(rep)))
if ret:
return (map_hash, info_hash, features)
key = (map_hash, info_hash)
with global_lock:
if key in all_data:
data_tup = all_data[key]
else:
all_data[key] = [np.zeros(features.shape), 0]
locks[key] = Lock()
with locks[key]:
data_tup = all_data[key]
acc = data_tup[0]
if acc.shape[0] < features.shape[0]:
acc = np.pad(acc, ((0, features.shape[0] - acc.shape[0]),
(0, 0), (0, 0), (0, 0)),
mode='constant',
constant_values=0)
view = acc[tuple(
slice(0, int(axis)) for axis in features.shape)]
view += features
view += features
data_tup[0] = acc
data_tup[1] += 1
all_data[key] = data_tup
return True
else:
raise RuntimeError("No such way to calculate a baseline. "
"Check your --method")
num_games_to_parse = 1
def get_orders(unit_in):
orders = []
for order in unit_in.orders:
orders.append(order.type)
return orders
# master list of games
game_list = None
for full_filename in replays_master:
replay = replayer.load(full_filename)
if counter >= num_games_to_parse:
break
counter += 1
print("loaded replay: " + full_filename + " of length:" + str(len(replay)))
# ids of units I care about:
units_i_want = {}
for frame_number in range(0, len(replay), int(N_secs * 8)):
frame = replay.getFrame(frame_number)
units = frame.units
this_frame = []
for player_id, unit_arr in units.iteritems():
if player_id < 0:
# This is a map resource / object, not an army unit
# game_info = game_over_time(replay_path, valid_types=valid_units, playerid=2, step_size=4)
# if game_info != -1:
# all_games.append(game_info)
color_dict = [(0, 0, 0), (255, 0, 0), (0, 255, 0), (0, 0, 255),
(100, 100, 100), (100, 100, 255), (100, 255, 255),
(10, 151, 226), (0, 255, 255), (0, 0, 255), (255, 100, 255),
(0, 0, 0), (160, 0, 255), (120, 0, 180), (0, 0, 100)]
fps = 20
params = hyper_params()
save_vid = True
play_vid = False
draw_text = False
# Load in the replay:
replay = replayer.load(params['replay_path'])
print("loaded replay: " + params['replay_path'] + " of length:" +
str(len(replay)))
game_data = game_data_for_drawing(replay=replay,
valid_types=params['valid_units'],
playerid=params['playerid'],
step_size=params['step_frames'])
print("draw data extracted")
game_info = game_over_time(replay=replay,
valid_types=params['valid_units'],
playerid=params['playerid'],
step_size=params['step_frames'],
feature_set=params['feature_set'],
add_orders=params['add_orders'])
print("raw data extracted")
