def parse_replay_task(self, fn, preserve_upload_date=False, custom_file_location: str = None, force_reparse=False): if custom_file_location is None: pickled = os.path.join(os.path.dirname(__file__), '..', '..', 'data', 'parsed', os.path.basename(fn)) else: pickled = os.path.join(custom_file_location, os.path.basename(fn)) if custom_file_location is None: failed_dir = os.path.join(os.path.dirname(os.path.dirname(pickled)), 'failed') else: failed_dir = custom_file_location if os.path.isfile(pickled) and not force_reparse: return # try: try: analysis_manager = analyze_replay_file(fn) # type: ReplayGame except Exception as e: if not os.path.isdir(failed_dir): os.makedirs(failed_dir) shutil.move(fn, os.path.join(failed_dir, os.path.basename(fn))) with open(os.path.join(failed_dir, os.path.basename(fn) + '.txt'), 'a') as f: f.write(str(e)) f.write(traceback.format_exc()) raise e with open(pickled + '.pts', 'wb') as fo: analysis_manager.write_proto_out_to_file(fo) with gzip.open(pickled + '.gzip', 'wb') as fo: analysis_manager.write_pandas_out_to_file(fo) g = analysis_manager.protobuf_game sess = self.session() game, player_games, players, teamstats = convert_pickle_to_db(g) add_objs_to_db(game, player_games, players, teamstats, sess, preserve_upload_date=preserve_upload_date) sess.commit() sess.close() replay_id = g.game_metadata.match_guid if replay_id == '': replay_id = g.game_metadata.id shutil.move(fn, os.path.join(os.path.dirname(fn), replay_id + '.replay')) shutil.move( pickled + '.pts', os.path.join(os.path.dirname(pickled), replay_id + '.replay.pts')) shutil.move( pickled + '.gzip', os.path.join(os.path.dirname(pickled), replay_id + '.replay.gzip')) return replay_id
def parse_pickle(p): s = session() # type: Session with open(p, 'rb') as f: try: g = pickle.load(f) # type: ReplayGame if g.api_game.id in games: print('skipping', g.api_game.id) return except EOFError: print ('what error') return try: game, player_games, players, teamstats = convert_pickle_to_db(g, offline_redis=r) add_objs_to_db(game, player_games, players, teamstats, s) except Exception as e: print(e) traceback.print_exc() s.commit()