def get_proto(self, id_: str) -> Game:
"""
Loads game proto from cache if available, else loads it from the server and saves it to the cache folder.
:param id_: replay id
:return: Game protobuf
"""
if id_ in self.known_bad_ids:
raise BrokenDataError
cached_filepath = os.path.join(self.cache_path, id_ + '.replay.pts')
if os.path.exists(cached_filepath):
file = open(cached_filepath, mode='rb')
logger.debug(f"Loaded {id_} proto from cache.")
else:
url = BASE_URL + f'parsed/{id_}.replay.pts?key=1'
r = requests.get(url)
file = io.BytesIO(r.content)
logger.debug(f"Loaded {id_} proto from site.")
with open(cached_filepath, 'wb') as f:
f.write(r.content)
proto = ProtobufManager.read_proto_out_from_file(file)
file.close()
return proto
def get_replay_proto(self, replay_id: str) -> Game:
url = self.BASE_URL + f'parsed/{replay_id}.replay.pts'
r = requests.get(url, params={'key': self.query_params.key})
r.raise_for_status()
file = io.BytesIO(r.content)
proto = ProtobufManager.read_proto_out_from_file(file)
logger.debug(f"Loaded {replay_id} proto from site.")
return proto
def get_replay_proto(self, replay_id: str) -> Game:
url = self.PROTO_URL + f'{replay_id}.replay.pts'
r = requests.get(url)
r.raise_for_status()
file = io.BytesIO(r.content)
proto = ProtobufManager.read_proto_out_from_file(file)
logger.debug(f"Loaded {replay_id} proto from site.")
return proto
def get_proto(self, id_):
if id_ in self.PROTO_MAP:
return self.PROTO_MAP[id_]
url = BASE_URL + 'parsed/{}.replay.pts?key=1'.format(id_)
r = requests.get(url)
# file_obj = io.BytesIO()
# for chunk in r.iter_content(chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# file_obj.write(chunk)
proto = ProtobufManager.read_proto_out_from_file(io.BytesIO(r.content))
self.PROTO_MAP[id_] = proto
return proto
def api_upload_proto(query_params=None):
# Convert to byte files from base64
response = request.get_json()
proto_in_memory = io.BytesIO(
zlib.decompress(base64.b64decode(response['proto'])))
protobuf_game = ProtobufManager.read_proto_out_from_file(proto_in_memory)
# Process
try:
parsed_replay_processing(protobuf_game, query_params=query_params)
except Exception as e:
log_error(e, logger=logger)
return jsonify({'Success': True})
def api_upload_proto(session=None, query_params=None):
# Convert to byte files from base64
payload = request.get_json()
proto_in_memory = io.BytesIO(zlib.decompress(base64.b64decode(payload['proto'])))
protobuf_game = ProtobufManager.read_proto_out_from_file(proto_in_memory)
# Process
try:
match_exists = parsed_replay_processing(protobuf_game, query_params=query_params)
add_saved_replay(match_exists)
except Exception as e:
payload['stack'] = traceback.format_exc()
payload['error_type'] = type(e).__name__
ErrorLogger.log_replay_error(payload, query_params, proto_game=protobuf_game)
ErrorLogger.log_error(e, logger=logger)
return jsonify({'Success': False})
ErrorLogger.log_replay(payload, query_params, proto_game=protobuf_game)
return jsonify({'Success': True})
def api_upload_proto():
print('Proto uploaded')
# Convert to byte files from base64
response = request.get_json()
proto_in_memory = io.BytesIO(
base64.b64decode(gzip.decompress(response['proto'])))
pandas_in_memory = io.BytesIO(
base64.b64decode(gzip.decompress(response['pandas'])))
protobuf_game = ProtobufManager.read_proto_out_from_file(proto_in_memory)
# Path creation
filename = protobuf_game.game_metadata.match_guid
if filename == '':
filename = protobuf_game.game_metadata.id
filename += '.replay'
parsed_path = os.path.join(os.path.dirname(__file__), '..', '..', '..',
'data', 'parsed', filename)
id_replay_path = os.path.join(os.path.dirname(__file__), '..', '..', '..',
'data', 'rlreplays',
protobuf_game.game_metadata.id + '.replay')
guid_replay_path = os.path.join(os.path.dirname(__file__), '..', '..',
'..', 'data', 'rlreplays', filename)
# Process
add_objects(protobuf_game)
# Write to disk
proto_in_memory.seek(0)
pandas_in_memory.seek(0)
with open(parsed_path + '.pts', 'wb') as f:
f.write(proto_in_memory.read())
with open(parsed_path + '.gzip', 'wb') as f:
f.write(pandas_in_memory.read())
# Cleanup
if os.path.isfile(id_replay_path):
shutil.move(id_replay_path, guid_replay_path)
return jsonify({'Success': True})
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--directory', type=str, required=True)
parser.add_argument('-p', '--processes', type=int, default=1)
args = parser.parse_args()
db = Database(args.directory)
total = db.Session().query(ItemsExtracted).count()
processed = 0
for replay in db.Session().query(ItemsExtracted):
try:
with open(os.path.join(args.directory, f'stats/{replay.hash}.pts'),
'rb') as f:
proto = ProtobufManager.read_proto_out_from_file(f)
for goal in proto.game_metadata.goals:
player = next(
filter(lambda x: x.id.id == goal.player_id.id,
proto.players))
goal_record = GoalRecord(
parent_id=replay.id,
player_id=goal.player_id.id,
frame=goal.frame_number,
pre_item=goal.extra_mode_info.pre_items,
is_orange=player.is_orange)
if goal.extra_mode_info.scored_with_item:
goal_record.item = goal.extra_mode_info.used_item
def get_proto(self, id_) -> Game:
url = BASE_URL + 'parsed/{}.replay.pts?key=1'.format(id_)
r = requests.get(url)
proto = ProtobufManager.read_proto_out_from_file(io.BytesIO(r.content))
return proto
def process_replay(replay: ReplayRecord, directory: str):
try:
replay['mmrs'] = json.loads(replay['mmrs'])
replay['ranks'] = json.loads(replay['ranks'])
rank_mmr = list(zip(replay['mmrs'], replay['ranks']))
rank_mmr = list(
filter(lambda x: x[0] is not None and x[0] > 0, rank_mmr))
replay_data = {
'hash': replay['hash'],
'map': None,
'avg_mmr': None,
'avg_rank': None
}
if len(rank_mmr) == 0:
return None, replay_data
if all(map(lambda x: x[1] == 0, rank_mmr)):
return None, replay_data
avg_mmr = np.average(list(map(lambda x: x[0], rank_mmr))).item()
avg_rank = round(
np.average(
list(filter(lambda x: x > 0, map(lambda x: x[1],
rank_mmr)))).item())
replay_data['avg_mmr'] = avg_mmr
replay_data['avg_rank'] = avg_rank
with gzip.open(os.path.join(directory, f'df/{replay["hash"]}.gzip'),
'rb') as f:
frames = PandasManager.read_numpy_from_memory(f)
with open(os.path.join(directory, f'stats/{replay["hash"]}.pts'),
'rb') as f:
proto = ProtobufManager.read_proto_out_from_file(f)
replay_data['map'] = maps.inverse[proto.game_metadata.map]
ids = dict(
map(lambda x: (x.id.id, (x.name, x.is_orange)), proto.players))
events = []
for event in proto.game_stats.rumble_items:
df = frames[ids[event.player_id.id][0]]
item_event = {
'player_id':
event.player_id.id,
'frame_get':
event.frame_number_get,
'frame_use':
event.frame_number_use,
'item':
event.item,
'use_x':
None,
'use_y':
None,
'use_z':
None,
'wait_time':
None,
'is_kickoff':
is_kickoff_item(event.frame_number_get, proto,
event.player_id.id),
'is_orange':
ids[event.player_id.id][1] == 1,
}
if event.frame_number_use > -1:
item_event['use_x'] = df.loc[
event.frame_number_use]['pos_x'].item()
item_event['use_y'] = df.loc[
event.frame_number_use]['pos_y'].item()
item_event['use_z'] = df.loc[
event.frame_number_use]['pos_z'].item()
item_event['wait_time'] = frames['game'].loc[event.frame_number_use]['time'].item() - \
frames['game'].loc[event.frame_number_get]['time'].item()
events.append(item_event)
return events, replay_data
except Exception as e:
log.error(f'Failed to handle {replay["hash"]}', exc_info=e)
return None, None