def parse_replay_task(self,
fn,
preserve_upload_date=False,
custom_file_location: str = None,
force_reparse=False):
if custom_file_location is None:
pickled = os.path.join(os.path.dirname(__file__), '..', '..', 'data',
'parsed', os.path.basename(fn))
else:
pickled = os.path.join(custom_file_location, os.path.basename(fn))
if custom_file_location is None:
failed_dir = os.path.join(os.path.dirname(os.path.dirname(pickled)),
'failed')
else:
failed_dir = custom_file_location
if os.path.isfile(pickled) and not force_reparse:
return
# try:
try:
analysis_manager = analyze_replay_file(fn) # type: ReplayGame
except Exception as e:
if not os.path.isdir(failed_dir):
os.makedirs(failed_dir)
shutil.move(fn, os.path.join(failed_dir, os.path.basename(fn)))
with open(os.path.join(failed_dir,
os.path.basename(fn) + '.txt'), 'a') as f:
f.write(str(e))
f.write(traceback.format_exc())
raise e
with open(pickled + '.pts', 'wb') as fo:
analysis_manager.write_proto_out_to_file(fo)
with gzip.open(pickled + '.gzip', 'wb') as fo:
analysis_manager.write_pandas_out_to_file(fo)
g = analysis_manager.protobuf_game
sess = self.session()
game, player_games, players, teamstats = convert_pickle_to_db(g)
add_objs_to_db(game,
player_games,
players,
teamstats,
sess,
preserve_upload_date=preserve_upload_date)
sess.commit()
sess.close()
replay_id = g.game_metadata.match_guid
if replay_id == '':
replay_id = g.game_metadata.id
shutil.move(fn, os.path.join(os.path.dirname(fn), replay_id + '.replay'))
shutil.move(
pickled + '.pts',
os.path.join(os.path.dirname(pickled), replay_id + '.replay.pts'))
shutil.move(
pickled + '.gzip',
os.path.join(os.path.dirname(pickled), replay_id + '.replay.gzip'))
return replay_id
def parse_pickle(p):
s = session() # type: Session
with open(p, 'rb') as f:
try:
g = pickle.load(f) # type: ReplayGame
if g.api_game.id in games:
print('skipping', g.api_game.id)
return
except EOFError:
print ('what error')
return
try:
game, player_games, players, teamstats = convert_pickle_to_db(g, offline_redis=r)
add_objs_to_db(game, player_games, players, teamstats, s)
except Exception as e:
print(e)
traceback.print_exc()
s.commit()