def analyze_replay_file(replay_path: str,
output_path: str,
overwrite=True,
controls: ControlsCreator = None,
sanity_check: SanityChecker = None):
"""
Decompile and analyze a replay file.
:param replay_path: Path to replay file
:param output_path: Path to write JSON
:param overwrite: If to overwrite JSON (suggest True if speed is not an issue)
:param controls: Generate controls from the replay using our best guesses (ALPHA)
:param sanity_check: Run sanity check to make sure we analyzed correctly (BETA)
:return: AnalysisManager of game with analysis.
"""
_json = decompile_replay(replay_path, output_path, overwrite=overwrite)
game = Game(loaded_json=_json)
# get_controls(game) # TODO: enable and optimise.
if controls is not None:
controls.get_controls(game)
if sanity_check is not None:
sanity_check.check_game(game)
analysis = AnalysisManager(game)
analysis.create_analysis()
return analysis
def analyze_replay_file(replay_path: str, output_path: str = None, overwrite=True, controls: ControlsCreator = None,
sanity_check: SanityChecker = None, analysis_per_goal=False, rattletrap_path: str = None):
"""
Decompile and analyze a replay file.
:param replay_path: Path to replay file
:param output_path: Path to write JSON
:param overwrite: If to overwrite JSON (suggest True if speed is not an issue)
:param controls: Generate controls from the replay using our best guesses (ALPHA)
:param sanity_check: Run sanity check to make sure we analyzed correctly (BETA)
:param analysis_per_goal: Runs the analysis per a goal instead of the replay as a whole
:param rattletrap_path: Custom location for rattletrap executable. Path to folder.
:param force_full_analysis: If True full analysis will be performed even if checks say it should not.
:return: AnalysisManager of game with analysis.
"""
_json = decompile_replay(replay_path, output_path=output_path, overwrite=overwrite, rattletrap_path=rattletrap_path)
game = Game()
game.initialize(loaded_json=_json)
# get_controls(game) # TODO: enable and optimise.
if sanity_check is not None:
sanity_check.check_game(game)
if analysis_per_goal:
analysis = PerGoalAnalysis(game)
else:
analysis = AnalysisManager(game)
analysis.create_analysis()
if controls is not None:
controls.get_controls(game)
return analysis
def convert_json_to_game_frames(filename):
with open(filename, encoding='utf-8', errors='ignore') as json_file:
_json = json.load(json_file)
game = Game()
game.initialize(loaded_json=_json)
analysis = AnalysisManager(game)
analysis.create_analysis()
x = ControlsCreator()
x.get_controls(game)
frames = []
total = 0
duplicates = 0
previous_frame = None
for col, row in analysis.data_frame.iterrows():
frame = {}
frame["GameState"] = {}
frame["GameState"]["time"] = NaN_fixer(row["game"]["time"])
frame["GameState"]["seconds_remaining"] = NaN_fixer(
row["game"]["seconds_remaining"])
frame["GameState"]["deltatime"] = NaN_fixer(row["game"]["delta"])
frame["GameState"]["ball"] = {}
frame["GameState"]["ball"]["position"] = [
NaN_fixer(row["ball"]["pos_x"]),
NaN_fixer(row["ball"]["pos_y"]),
NaN_fixer(row["ball"]["pos_z"])
]
frame["GameState"]["ball"]["velocity"] = [
velocity_scaler(NaN_fixer(row["ball"]["vel_x"])),
velocity_scaler(NaN_fixer(row["ball"]["vel_y"])),
velocity_scaler(NaN_fixer(row["ball"]["vel_z"]))
]
frame["GameState"]["ball"]["rotation"] = [
NaN_fixer(row["ball"]["rot_x"]),
NaN_fixer(row["ball"]["rot_y"]),
NaN_fixer(row["ball"]["rot_z"])
]
frame["PlayerData"] = []
for i in range(len(game.players)):
frame["PlayerData"].append(
getPlayerFrame(game.players[i], i, col, row))
if previous_frame != None:
if duplicateFrameCheck(frame, previous_frame):
total += 1
duplicates += 1
continue
previous_frame = frame
frames.append(frame)
total += 1
return frames
def analyze_replay_file(replay_path: str,
output_path: str = None,
overwrite=True,
controls: ControlsCreator = None,
sanity_check: SanityChecker = None,
analysis_per_goal=False,
rattletrap_path: str = None,
logging_level=logging.NOTSET,
calculate_intensive_events: bool = False):
"""
Decompile and analyze a replay file.
:param replay_path: Path to replay file
:param output_path: Path to write JSON
:param overwrite: If to overwrite JSON (suggest True if speed is not an issue)
:param controls: Generate controls from the replay using our best guesses (ALPHA)
:param sanity_check: Run sanity check to make sure we analyzed correctly (BETA)
:param analysis_per_goal: Runs the analysis per a goal instead of the replay as a whole
:param rattletrap_path: Custom location for rattletrap executable. Path to folder.
:param force_full_analysis: If True full analysis will be performed even if checks say it should not.
:param logging_level: Sets the logging level globally across carball
:param calculate_intensive_events: Indicates if expensive calculations should run to include additional stats.
:return: AnalysisManager of game with analysis.
"""
if logging_level != logging.NOTSET:
logging.getLogger('carball').setLevel(logging_level)
_json = decompile_replay(replay_path,
output_path=output_path,
overwrite=overwrite,
rattletrap_path=rattletrap_path)
game = Game()
game.initialize(loaded_json=_json)
# get_controls(game) # TODO: enable and optimise.
if sanity_check is not None:
sanity_check.check_game(game)
if analysis_per_goal:
analysis = PerGoalAnalysis(game)
else:
analysis = AnalysisManager(game)
analysis.create_analysis(
calculate_intensive_events=calculate_intensive_events)
if controls is not None:
controls.get_controls(game)
return analysis
def parseReplayToGameData(replayID):
replayPath = os.path.join("replays", "%s.replay" % replayID)
print("Loading...\n\t%s" % replayPath)
json = carball.decompile_replay(replayPath)
game = Game()
game.initialize(loaded_json=json)
# Parses action-related properties into their actual actions
ControlsCreator().get_controls(game)
return game
def predict_boost_values(rp, model):
if isinstance(rp, str):
rp = cb.analyze_replay_file(rp,
controls=ControlsCreator(),
logging_level=logging.CRITICAL)
pp = replay_to_dfs(rp, skip_ties=False)
cpx, cpy = convert_dfs(pp, frame_mode=1)
normalize(cpx)
results = np.zeros((cpx[0].shape[0], len(pp["players"]), 101))
# xs = np.zeros((cpx[0].shape[0], len(pp["players"])))
# ys = np.zeros((cpx[0].shape[0], len(pp["players"])))
model.eval()
with torch.no_grad():
for p, row in pp["players"].iterrows():
print(p)
cpxc = [np.copy(v) for v in cpx]
# # For making histogram
# preds = model(*[torch.from_numpy(v).float().cuda() for v in cpxc])
# preds = preds[0].cpu().detach().numpy()
# preds = preds[:, 1] - preds[:, 0]
# if row["color"] == "blue":
# preds = -preds
# xs[:, p] = cpxc[2][:, p, 19]
# ys[:, p] = preds
# For changing the boost amount at every step
for b in range(101):
print("\t", b)
cpxc[2][:, p, 19] = b / 100
cpxc[2][:, p, 19][cpxc[2][:, p, 20] == 1] = 0
preds = model(
*[torch.from_numpy(v).float().cuda() for v in cpxc])
preds = preds[0].cpu().detach().numpy()
preds = preds[:, 1] - preds[:, 0]
if row["color"] == "blue":
preds = -preds
results[:, p, b] = preds
return results
def replay_to_dfs(replay, skip_ties=True, skip_kickoffs=True):
import pandas as pd
import carball as cb
if isinstance(replay, (str, bytes)):
from carball.controls.controls import ControlsCreator
replay = cb.analyze_replay_file(replay, controls=ControlsCreator(), logging_level=logging.CRITICAL)
blue_team, orange_team = replay.game.teams
if blue_team.is_orange:
blue_team, orange_team = orange_team, blue_team
frames = pd.DataFrame(index=replay.data_frame.index)
frames[f"ball/pos_x"] = replay.game.ball["pos_x"].fillna(0.)
frames[f"ball/pos_y"] = replay.game.ball["pos_y"].fillna(0.)
frames[f"ball/pos_z"] = replay.game.ball["pos_z"].fillna(0.)
frames[f"ball/vel_x"] = replay.game.ball["vel_x"].fillna(0.) / 10
frames[f"ball/vel_y"] = replay.game.ball["vel_y"].fillna(0.) / 10
frames[f"ball/vel_z"] = replay.game.ball["vel_z"].fillna(0.) / 10
frames[f"ball/ang_vel_x"] = replay.game.ball["ang_vel_x"].fillna(0.) / 1000
frames[f"ball/ang_vel_y"] = replay.game.ball["ang_vel_y"].fillna(0.) / 1000
frames[f"ball/ang_vel_z"] = replay.game.ball["ang_vel_z"].fillna(0.) / 1000
boost_grabs = pd.DataFrame(columns=["frame", "boost_id", "player"])
player_index = {}
player_df = pd.DataFrame(columns=["color", "online_id", "name"])
i = 0
for color, team in ("blue", blue_team), ("orange", orange_team):
for player in team.players:
player_index[player.online_id] = i
frames[f"{i}/pos_x"] = player.data["pos_x"].fillna(0.)
frames[f"{i}/pos_y"] = player.data["pos_y"].fillna(0.)
frames[f"{i}/pos_z"] = player.data["pos_z"].fillna(0.)
yaw = player.data["rot_y"].fillna(0.)
pitch = player.data["rot_x"].fillna(0.)
roll = player.data["rot_z"].fillna(0.)
forward, up = rotator_to_matrix(yaw, pitch, roll)
frames[f"{i}/forward_x"] = forward[0]
frames[f"{i}/forward_y"] = forward[1]
frames[f"{i}/forward_z"] = forward[2]
frames[f"{i}/up_x"] = up[0]
frames[f"{i}/up_y"] = up[1]
frames[f"{i}/up_z"] = up[2]
frames[f"{i}/vel_x"] = player.data["vel_x"].fillna(0.) / 10
frames[f"{i}/vel_y"] = player.data["vel_y"].fillna(0.) / 10
frames[f"{i}/vel_z"] = player.data["vel_z"].fillna(0.) / 10
frames[f"{i}/ang_vel_x"] = player.data["ang_vel_x"].fillna(0.) / 1000
frames[f"{i}/ang_vel_y"] = player.data["ang_vel_y"].fillna(0.) / 1000
frames[f"{i}/ang_vel_z"] = player.data["ang_vel_z"].fillna(0.) / 1000
frames[f"{i}/boost_amount"] = player.data["boost"].fillna(0.) / 2.55
for col in player.controls.columns:
frames[f"{i}/{col}_control"] = player.controls[col].astype(float)
boost_pickups = player.data["boost"].diff() > 0
for frame, row in player.data[boost_pickups].iterrows():
xyz = (row["pos_x"], row["pos_y"], row["pos_z"])
# TODO use boost ID instead
closest_boost = min(range(len(boost_locations)),
key=lambda j: sum((a - b) ** 2 for a, b in zip(xyz, boost_locations[j])))
boost_grabs.loc[len(boost_grabs)] = [frame, closest_boost, i]
player_df.loc[len(player_df)] = [color, player.online_id, player.name]
i += 1
rallies = pd.DataFrame(columns=["start_frame", "end_frame", "team"])
for kf1, kf2 in zip(replay.game.kickoff_frames, replay.game.kickoff_frames[1:] + [replay.game.frames.index[-1]]):
for goal in replay.game.goals:
if kf1 < goal.frame_number < kf2:
rallies.loc[len(rallies)] = [kf1, goal.frame_number, ["blue", "orange"][goal.player_team]]
break
else: # No goal between kickoffs
rallies.loc[len(rallies)] = [kf1, kf2, None]
demos = pd.DataFrame(columns=["frame", "attacker", "victim"])
for demo in replay.game.demos:
frame = demo.get("frame_number", None)
attacker = demo.get("attacker", None)
victim = demo.get("victim", None)
if frame and attacker and victim:
demos.loc[len(demos)] = [frame, player_index[attacker.online_id], player_index[victim.online_id]]
touches = pd.DataFrame(columns=["frame", "player"])
for touch in replay.protobuf_game.game_stats.hits:
touches.loc[len(touches)] = [touch.frame_number, player_index[touch.player_id.id]]
if skip_ties:
goal_rallies = rallies[rallies["team"].notna()]
frames = frames.loc[
np.r_[tuple(slice(row["start_frame"], row["end_frame"]) for _, row in goal_rallies.iterrows())]]
elif skip_kickoffs:
goal_rallies = rallies
frames = frames.loc[
np.r_[tuple(slice(row["start_frame"], row["end_frame"]) for _, row in goal_rallies.iterrows())]]
# if isinstance(frame_mode, int):
# frames = frames.iloc[np.random.randint(0, frame_mode)::frame_mode]
# else:
# frames = frames.sample(frac=frame_mode)
return {"frames": frames, "rallies": rallies, "touches": touches, "boost_grabs": boost_grabs, "demos": demos,
"players": player_df}
def plot_replay(rp_path,
model,
plot_players=True,
invert=None,
plot_goal_distance=False):
if plot_players is True:
plot_players = ("blue", "orange")
elif plot_players is False:
plot_players = ()
elif plot_players == "blue":
plot_players = ("blue", )
elif plot_players == "orange":
plot_players = ("orange", )
else:
plot_players = ()
rp = cb.analyze_replay_file(rp_path,
controls=ControlsCreator(),
logging_level=logging.CRITICAL)
id_team = {
p.online_id: ["blue", "orange"][p.team.is_orange]
for p in rp.game.players
}
id_name = {p.online_id: p.name for p in rp.game.players}
pred_df, dfs = make_summary(rp, model)
if invert is None:
d = 0
for goal in rp.game.goals:
if goal.player.is_orange:
d += 1
else:
d -= 1
invert = d < 0
# pp["pred"][pp["ball/ball/vel_y"] == 0] = float("nan")
# pp["pred"][pp["ball/ball/vel_y"].isna()] = float("nan")
fig, ax = plt.subplots(figsize=((6.4 * dfs["frames"].index[-1] + 1000) /
5000, 4.8),
dpi=400,
constrained_layout=True)
ax.hlines(0.5,
xmin=0,
xmax=dfs["frames"].index[-1],
color="black",
linestyle="dashed")
blue_shots = []
orange_shots = []
blue_saves = []
orange_saves = []
for hit in rp.protobuf_game.game_stats.hits:
if hit.shot:
if id_team[hit.player_id.id] == "blue":
blue_shots.append(hit.frame_number)
else:
orange_shots.append(hit.frame_number)
if hit.save:
if id_team[hit.player_id.id] == "blue":
blue_saves.append(hit.frame_number)
else:
orange_saves.append(hit.frame_number)
blue_ymin, blue_ymax, orange_ymin, orange_ymax = (0.5, 1, 0,
0.5) if invert else (0,
0.5,
0.5,
1)
ax.vlines(blue_shots,
ymin=blue_ymin,
ymax=blue_ymax,
color="m",
linestyle="dotted",
label="Shot")
ax.vlines(orange_shots,
ymin=orange_ymin,
ymax=orange_ymax,
color="m",
linestyle="dotted")
ax.vlines(blue_saves,
ymin=orange_ymin,
ymax=orange_ymax,
color="c",
linestyle="dotted",
label="Save")
ax.vlines(orange_saves,
ymin=blue_ymin,
ymax=blue_ymax,
color="c",
linestyle="dotted")
if plot_goal_distance:
goal_location = np.array([0, -5120 if invert else 5120, 642 / 2])
distances = np.linalg.norm(
np.tile(goal_location, len(dfs["frames"])).reshape(
(len(dfs["frames"]), 3)) -
dfs["frames"].filter(regex="ball/pos_").values,
axis=-1)
ax.plot(dfs["frames"].index,
distances / 11216,
label="Ball-Goal Distance")
pred = sigmoid(pred_df["pred"])
ax.plot(dfs["frames"].index,
pred if invert else 1 - pred,
color="blue",
label="Advantage")
blue_colors = ["springgreen", "mediumseagreen", "green"]
orange_colors = ["lightcoral", "indianred", "brown"]
# plt.show()
b = o = 0
for i, row in dfs["players"].iterrows():
name = row["name"]
color = row["color"]
avg_score = np.exp(pred_df[f"{i}/pred"].mean())
print(name, avg_score)
# pred_df[f"{identifier}/pred"].plot.hist(title=name, bins=100)
# plt.show()
pred_col = sigmoid(pred_df[f"{i}/pred"])
if color == "blue" and "blue" in plot_players:
ax.plot(dfs["frames"].index,
pred_col if invert else 1 - pred_col,
color=blue_colors[b],
label=name,
linewidth=0.5)
b += 1
elif color == "orange" and "orange" in plot_players:
ax.plot(dfs["frames"].index,
1 - pred_col if invert else pred_col,
color=orange_colors[o],
label=name,
linewidth=0.5)
o += 1
rallies = dfs["rallies"]
ax.vlines(rallies[rallies["team"].notna()]["end_frame"],
ymin=0,
ymax=1,
color="red",
label="Goal")
ax.set_xlim(0, dfs["frames"].index[-1])
ax.set_ylim(0, 1)
ax.set_xlabel("Frame")
ax.set_ylabel("Orange <-> Blue" if invert else "Blue <-> Orange")
ax.set_title(rp.game.name)
ax.legend(bbox_to_anchor=(1.01, 1), loc='upper left')
secax = ax.twiny()
new_tick_locations = []
new_ticks = []
taken = None
for frame in reversed(rp.game.frames.index):
left = rp.game.frames["seconds_remaining"].loc[frame]
if np.isnan(left):
continue
left = int(left)
if left % 10 == 0 and left != taken:
new_tick_locations.append(frame)
new_ticks.append("{0:.0f}:{1:02.0f}".format(
*divmod(rp.game.frames["seconds_remaining"].loc[frame], 60)))
taken = left
secax.set_xlim(*ax.get_xlim())
secax.set_xticks(new_tick_locations)
secax.set_xticklabels(new_ticks)
secax.set_xlabel("Time")
ax.grid()
# df["pred"].plot(x=df["game/info/frame"], color="blue")
return fig, dfs
def __test_replays(BASE_DIR):
import logging
ROOT_DIR = os.path.dirname(BASE_DIR)
OUTPUT_DIR = os.path.join(ROOT_DIR, 'output')
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
MOVE_WORKING = True
DEBUGGING = True
success = 0
failure = 0
create_dir(OUTPUT_DIR)
sanity_check = SanityChecker()
sanity_check = None
for filepath in glob.iglob(ROOT_DIR + '/**carball/replays/*.replay', recursive=True):
logger.info('decompiling %s', filepath)
if "output" in filepath:
continue
base_file = os.path.basename(filepath)
json_path = os.path.join(OUTPUT_DIR, 'replays/decompiled/{}'.format(base_file.replace(".replay", ".json")))
proto_path = os.path.join(OUTPUT_DIR, 'replays/protos/{}'.format(base_file.replace(".replay", ".pts")))
pandas_path = os.path.join(OUTPUT_DIR, 'replays/pandas/{}'.format(base_file.replace(".replay", ".gzip")))
create_dir(os.path.dirname(json_path))
create_dir(os.path.dirname(proto_path))
create_dir(os.path.dirname(pandas_path))
if DEBUGGING:
try:
analysis_manager = analyze_replay_file(filepath, json_path,
controls=ControlsCreator(), analysis_per_goal=False,
sanity_check=sanity_check)
id = analysis_manager.protobuf_game.game_metadata.id
with open(os.path.join(OUTPUT_DIR, id + 'game.json'), 'w') as f:
f.write(MessageToJson(analysis_manager.protobuf_game))
bytes = PandasManager.write_pandas_to_buffer_for_tooling(analysis_manager.get_data_frame(),
analysis_manager.get_protobuf_data().players)
with open(pandas_path, 'wb') as fo:
fo.write(bytes)
data_frame = PandasManager.safe_read_pandas_to_memory(BytesIO(analysis_manager.df_bytes))
logger.info('length of decoded pandas %i', len(data_frame))
except subprocess.CalledProcessError as e:
traceback.print_exc()
else:
try:
analysis_manager = analyze_replay_file(filepath, json_path)
with open(proto_path, 'wb') as fo:
analysis_manager.write_proto_out_to_file(fo)
with gzip.open(pandas_path, 'wb') as fo:
analysis_manager.write_pandas_out_to_file(fo)
if MOVE_WORKING:
shutil.move(filepath, os.path.join('replays', 'working', filepath))
success += 1
except Exception as e:
traceback.print_exc()
failure += 1
if not DEBUGGING:
if float(success + failure) == 0:
print("NO REPLAYS WERE RUN.")
print("Need files in: " + BASE_DIR)
ratio = success / float(success + failure)
print('success ratio:', ratio)
def __test_replays(BASE_DIR):
import logging
ROOT_DIR = os.path.dirname(BASE_DIR)
OUTPUT_DIR = os.path.join(ROOT_DIR, 'output')
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
MOVE_WORKING = True
DEBUGGING = True
success = 0
failure = 0
create_dir(OUTPUT_DIR)
for filepath in glob.iglob(ROOT_DIR + '/**/*.replay', recursive=True):
logger.info('decompiling %s', filepath)
base_file = os.path.basename(filepath)
json_path = os.path.join(
OUTPUT_DIR, 'replays/decompiled/{}'.format(
base_file.replace(".replay", ".json")))
proto_path = os.path.join(
OUTPUT_DIR,
'replays/protos/{}'.format(base_file.replace(".replay", ".pts")))
pandas_path = os.path.join(
OUTPUT_DIR,
'replays/pandas/{}'.format(base_file.replace(".replay", ".gzip")))
create_dir(os.path.dirname(json_path))
create_dir(os.path.dirname(proto_path))
create_dir(os.path.dirname(pandas_path))
if DEBUGGING:
try:
analysis_manager = analyze_replay_file(
filepath,
json_path,
sanity_check=SanityChecker(),
controls=ControlsCreator())
with open(os.path.join(OUTPUT_DIR, 'game.json'), 'w') as f:
f.write(MessageToJson(analysis_manager.protobuf_game))
except subprocess.CalledProcessError as e:
traceback.print_exc()
else:
try:
analysis_manager = analyze_replay_file(filepath, json_path)
with open(proto_path, 'wb') as fo:
analysis_manager.write_proto_out_to_file(fo)
with gzip.open(pandas_path, 'wb') as fo:
analysis_manager.write_pandas_out_to_file(fo)
if MOVE_WORKING:
shutil.move(filepath,
os.path.join('replays', 'working', filepath))
success += 1
except Exception as e:
traceback.print_exc()
failure += 1
if not DEBUGGING:
if float(success + failure) == 0:
print("NO REPLAYS WERE RUN.")
print("Need files in: " + BASE_DIR)
ratio = success / float(success + failure)
print('success ratio:', ratio)