def setUp(self):
if not os.path.isdir(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
for filename in [f for f in os.listdir(OUTPUT_DIR) if os.path.isfile(os.path.join(OUTPUT_DIR, f))]:
if OUTPUT_FORMAT in filename:
filepath = os.path.join(OUTPUT_DIR, filename)
with gzip.open(filepath, 'rb') as file:
self.pandas = PandasManager.read_numpy_from_memory(file)
def get_pandas(self) -> pd.DataFrame:
"""
Gets the pandas lazily decompiles it
:return:
"""
if self.decoded_pandas is not None:
return self.decoded_pandas
self.decoded_pandas = PandasManager.safe_read_pandas_to_memory(zlib.decompress(self.pandas))
self.pandas = None
return self.decoded_pandas
def get_replay_df(self, replay_id: str) -> pd.DataFrame:
url = self.BASE_URL + f'parsed/{replay_id}.replay.gzip'
r = requests.get(url, params={'key': self.query_params.key})
r.raise_for_status()
gzip_file = gzip.GzipFile(fileobj=io.BytesIO(r.content), mode='rb')
df = PandasManager.safe_read_pandas_to_memory(gzip_file)
if df is None:
raise BrokenDataFrameError
logger.debug(f"Loaded {replay_id} df from site.")
return df
def get_df(self, id_) -> pd.DataFrame:
if id_ in self.BROKEN:
raise BrokenDataError
url = BASE_URL + 'parsed/{}.replay.gzip?key=1'.format(id_)
r = requests.get(url)
gzip_file = gzip.GzipFile(fileobj=io.BytesIO(r.content), mode='rb')
try:
pandas_ = PandasManager.safe_read_pandas_to_memory(gzip_file)
except:
self.BROKEN.append(id_)
raise BrokenDataError
return pandas_
def get_df(self, id_: str) -> pd.DataFrame:
"""
Loads df from cache if available, else loads the df from the server and saves it to the cache folder.
:param id_: replay id
:return: pd.DataFrame
"""
if id_ in self.known_bad_ids:
raise BrokenDataError
cached_filepath = os.path.join(self.cache_path, id_ + '.replay.gzip')
if os.path.exists(cached_filepath):
gzip_file = gzip.GzipFile(cached_filepath, mode='rb')
logger.debug(f"Loaded {id_} df from cache.")
else:
url = BASE_URL + f'parsed/{id_}.replay.gzip?key=1'
r = requests.get(url)
file = io.BytesIO(r.content)
gzip_file = gzip.GzipFile(fileobj=file, mode='rb')
logger.debug(f"Loaded {id_} df from site.")
try:
pandas_ = PandasManager.safe_read_pandas_to_memory(gzip_file)
if pandas_ is None:
raise BrokenDataError
if not os.path.exists(cached_filepath):
with open(cached_filepath, 'wb') as f:
f.write(r.content)
except:
self.add_bad_id(id_)
raise BrokenDataError
try:
gzip_file.close()
file.close()
except NameError:
pass
return pandas_
def process_replay(replay: ReplayRecord, directory: str):
try:
replay['mmrs'] = json.loads(replay['mmrs'])
replay['ranks'] = json.loads(replay['ranks'])
rank_mmr = list(zip(replay['mmrs'], replay['ranks']))
rank_mmr = list(
filter(lambda x: x[0] is not None and x[0] > 0, rank_mmr))
replay_data = {
'hash': replay['hash'],
'map': None,
'avg_mmr': None,
'avg_rank': None
}
if len(rank_mmr) == 0:
return None, replay_data
if all(map(lambda x: x[1] == 0, rank_mmr)):
return None, replay_data
avg_mmr = np.average(list(map(lambda x: x[0], rank_mmr))).item()
avg_rank = round(
np.average(
list(filter(lambda x: x > 0, map(lambda x: x[1],
rank_mmr)))).item())
replay_data['avg_mmr'] = avg_mmr
replay_data['avg_rank'] = avg_rank
with gzip.open(os.path.join(directory, f'df/{replay["hash"]}.gzip'),
'rb') as f:
frames = PandasManager.read_numpy_from_memory(f)
with open(os.path.join(directory, f'stats/{replay["hash"]}.pts'),
'rb') as f:
proto = ProtobufManager.read_proto_out_from_file(f)
replay_data['map'] = maps.inverse[proto.game_metadata.map]
ids = dict(
map(lambda x: (x.id.id, (x.name, x.is_orange)), proto.players))
events = []
for event in proto.game_stats.rumble_items:
df = frames[ids[event.player_id.id][0]]
item_event = {
'player_id':
event.player_id.id,
'frame_get':
event.frame_number_get,
'frame_use':
event.frame_number_use,
'item':
event.item,
'use_x':
None,
'use_y':
None,
'use_z':
None,
'wait_time':
None,
'is_kickoff':
is_kickoff_item(event.frame_number_get, proto,
event.player_id.id),
'is_orange':
ids[event.player_id.id][1] == 1,
}
if event.frame_number_use > -1:
item_event['use_x'] = df.loc[
event.frame_number_use]['pos_x'].item()
item_event['use_y'] = df.loc[
event.frame_number_use]['pos_y'].item()
item_event['use_z'] = df.loc[
event.frame_number_use]['pos_z'].item()
item_event['wait_time'] = frames['game'].loc[event.frame_number_use]['time'].item() - \
frames['game'].loc[event.frame_number_get]['time'].item()
events.append(item_event)
return events, replay_data
except Exception as e:
log.error(f'Failed to handle {replay["hash"]}', exc_info=e)
return None, None
def __test_replays(BASE_DIR):
import logging
ROOT_DIR = os.path.dirname(BASE_DIR)
OUTPUT_DIR = os.path.join(ROOT_DIR, 'output')
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
MOVE_WORKING = True
DEBUGGING = True
success = 0
failure = 0
create_dir(OUTPUT_DIR)
sanity_check = SanityChecker()
sanity_check = None
for filepath in glob.iglob(ROOT_DIR + '/**carball/replays/*.replay', recursive=True):
logger.info('decompiling %s', filepath)
if "output" in filepath:
continue
base_file = os.path.basename(filepath)
json_path = os.path.join(OUTPUT_DIR, 'replays/decompiled/{}'.format(base_file.replace(".replay", ".json")))
proto_path = os.path.join(OUTPUT_DIR, 'replays/protos/{}'.format(base_file.replace(".replay", ".pts")))
pandas_path = os.path.join(OUTPUT_DIR, 'replays/pandas/{}'.format(base_file.replace(".replay", ".gzip")))
create_dir(os.path.dirname(json_path))
create_dir(os.path.dirname(proto_path))
create_dir(os.path.dirname(pandas_path))
if DEBUGGING:
try:
analysis_manager = analyze_replay_file(filepath, json_path,
controls=ControlsCreator(), analysis_per_goal=False,
sanity_check=sanity_check)
id = analysis_manager.protobuf_game.game_metadata.id
with open(os.path.join(OUTPUT_DIR, id + 'game.json'), 'w') as f:
f.write(MessageToJson(analysis_manager.protobuf_game))
bytes = PandasManager.write_pandas_to_buffer_for_tooling(analysis_manager.get_data_frame(),
analysis_manager.get_protobuf_data().players)
with open(pandas_path, 'wb') as fo:
fo.write(bytes)
data_frame = PandasManager.safe_read_pandas_to_memory(BytesIO(analysis_manager.df_bytes))
logger.info('length of decoded pandas %i', len(data_frame))
except subprocess.CalledProcessError as e:
traceback.print_exc()
else:
try:
analysis_manager = analyze_replay_file(filepath, json_path)
with open(proto_path, 'wb') as fo:
analysis_manager.write_proto_out_to_file(fo)
with gzip.open(pandas_path, 'wb') as fo:
analysis_manager.write_pandas_out_to_file(fo)
if MOVE_WORKING:
shutil.move(filepath, os.path.join('replays', 'working', filepath))
success += 1
except Exception as e:
traceback.print_exc()
failure += 1
if not DEBUGGING:
if float(success + failure) == 0:
print("NO REPLAYS WERE RUN.")
print("Need files in: " + BASE_DIR)
ratio = success / float(success + failure)
print('success ratio:', ratio)