def parse_replay(replay_player_path, sampled_action_path, reward): if os.path.isfile(os.path.join(FLAGS.parsed_replay_path, 'GlobalFeatures', replay_player_path)): return # Global Info with open(os.path.join(FLAGS.parsed_replay_path, 'GlobalInfos', replay_player_path)) as f: global_info = json.load(f) units_info = static_data.StaticData(Parse(global_info['data_raw'], sc_pb.ResponseData())).units feat = features.Features(Parse(global_info['game_info'], sc_pb.ResponseGameInfo())) # Sampled Actions with open(sampled_action_path) as f: sampled_action = json.load(f) sampled_action_id = [id // FLAGS.step_mul + 1 for id in sampled_action] # Actions with open(os.path.join(FLAGS.parsed_replay_path, 'Actions', replay_player_path)) as f: actions = json.load(f) actions = [None if len(actions[idx]) == 0 else Parse(actions[idx][0], sc_pb.Action()) for idx in sampled_action_id] # Observations observations = [obs for obs in stream.parse(os.path.join(FLAGS.parsed_replay_path, 'SampledObservations', replay_player_path), sc_pb.ResponseObservation)] assert len(sampled_action) == len(sampled_action_id) == len(actions) == len(observations) states = process_replay(sampled_action, actions, observations, feat, units_info, reward) with open(os.path.join(FLAGS.parsed_replay_path, 'GlobalFeatures', replay_player_path), 'w') as f: json.dump(states, f)
def from_vg_graph_file(cls, vg_graph_file_name, only_read_nodes=False, use_cache_if_available=False): nodes = {} paths = [] edges = [] i = 0 for line in stream.parse(vg_graph_file_name, vg_pb2.Graph): i += 1 if hasattr(line, "node"): for node in line.node: nodes[node.id] = node.sequence if only_read_nodes: continue if hasattr(line, "path"): for path in line.path: paths.append(path) if hasattr(line, "edge"): for edge in line.edge: assert edge.overlap == 0 graph = cls(nodes, edges, paths) graph._cache_nodes() return graph
def parse_replay(replay_player_path, sampled_frame_path, reward): if os.path.isfile(os.path.join(FLAGS.parsed_replay_path, 'GlobalFeatures', replay_player_path)): return # Global Info with open(os.path.join(FLAGS.parsed_replay_path, 'GlobalInfos', replay_player_path)) as f: global_info = json.load(f) units_info = static_data.StaticData(Parse(global_info['data_raw'], sc_pb.ResponseData())).units feat = features.features_from_game_info(Parse(global_info['game_info'], sc_pb.ResponseGameInfo())) # Sampled Frames with open(sampled_frame_path) as f: sampled_frames = json.load(f) sampled_actions_idx = [frame // FLAGS.step_mul - 1 for frame in sampled_frames] # Create index to retrieve actions corresponding to sampled frames # Actions with open(os.path.join(FLAGS.parsed_replay_path, 'Actions', replay_player_path)) as f: actions = json.load(f) sampled_actions = [None if len(actions[idx]) == 0 else Parse(actions[idx][0], sc_pb.Action()) for idx in sampled_actions_idx] # Get first action executed after each sampled frame # Observations observations = [obs for obs in stream.parse(os.path.join(FLAGS.parsed_replay_path, 'SampledObservations', replay_player_path), sc_pb.ResponseObservation)] assert len(sampled_frames) == len(sampled_actions_idx) == len(sampled_actions) == len(observations) states = process_replay(sampled_frames, sampled_actions, observations, feat, units_info, reward) with open(os.path.join(FLAGS.parsed_replay_path, 'GlobalFeatures', replay_player_path), 'w') as f: json.dump(states, f)
def load_single_ticker_market_event_data_bundle( universe, atype, market, ticker, start_dt_str, end_dt_str, db_path, pb_cls=kfinformat_pb2.kfinevent): start_date, end_date = start_dt_str.split('T')[0], end_dt_str.split('T')[0] to_do_file_path_list, t_init = list(), True if atype in ('STOCK', 'ETF'): file_name = '{}_{}_RM0_{}_*-*-*.gz'.format(atype, market, ticker) file_path = '{}\\{}\\TICK\\{}\\{}\\{}'.format(db_path, universe, atype, ticker, file_name) file_list = glob.glob(file_path) for fp in file_list: file_date = fp.split('_')[-1][:-3] if (file_date >= start_date) & (file_date <= end_date): to_do_file_path_list.append(fp) to_do_file_path_list.sort(reverse=False) for fp_inx, fp_val in enumerate(to_do_file_path_list): if t_init: t, t_init = stream.parse(ifp=fp_val, pb_cls=pb_cls), False continue t = chain(t, stream.parse(ifp=fp_val, pb_cls=pb_cls)) elif atype in ('FUTURES', ): for inx, date in enumerate( generate_day_list(start_date=start_date, end_date=end_date)): file_name = '{}_{}_*_{}.gz'.format(atype, ticker, date) fp_val = '{}\\{}\\TICK\\{}\\{}\\{}\\{}'.format( db_path, universe, atype, ticker.split('_')[0], date, file_name) file_list = glob.glob(fp_val) if len(file_list) != 0: if t_init: t, t_init = stream.parse(ifp=file_list[0], pb_cls=pb_cls), False continue t = chain(t, stream.parse(ifp=file_list[0], pb_cls=pb_cls)) return t
def parse_replay(replay_player_path, sampled_action_path, reward, race, enemy_race, stat): with open(os.path.join(FLAGS.parsed_replay_path, 'GlobalInfos', replay_player_path)) as f: global_info = json.load(f) feat = SpatialFeatures(Parse(global_info['game_info'], sc_pb.ResponseGameInfo())) states = [obs for obs in stream.parse(os.path.join(FLAGS.parsed_replay_path, 'SampledObservations', replay_player_path), sc_pb.ResponseObservation)] # Sampled Actions with open(sampled_action_path) as f: sampled_action = json.load(f) sampled_action_id = [id // FLAGS.step_mul + 1 for id in sampled_action] # Actions with open(os.path.join(FLAGS.parsed_replay_path, 'Actions', replay_player_path)) as f: actions = json.load(f) actions = [None if len(actions[idx]) == 0 else Parse(actions[idx][0], sc_pb.Action()) for idx in sampled_action_id] assert len(states) == len(actions) spatial_states_np, global_states_np = [], [] for state, action in zip(states, actions): action_id = -1 if action is not None: try: func_id = feat.reverse_action(action).function func_name = FUNCTIONS[func_id].name if func_name.split('_')[0] in {'Build', 'Train', 'Research', 'Morph', 'Cancel', 'Halt', 'Stop'}: action_id = func_id except: pass obs = feat.transform_obs(state.observation) spatial_states_np.append(np.concatenate([obs['screen'], obs['minimap']], axis=0)) global_states_np.append(np.hstack([obs['player']/(stat['max']+1e-5), obs['score'], [reward], [stat['action_id'][action_id]]])) spatial_states_np = np.asarray(spatial_states_np) global_states_np = np.asarray(global_states_np) spatial_states_np = spatial_states_np.reshape([len(states), -1]) sparse.save_npz(os.path.join(FLAGS.parsed_replay_path, 'SpatialFeatureTensor', replay_player_path+'@S'), sparse.csc_matrix(spatial_states_np)) sparse.save_npz(os.path.join(FLAGS.parsed_replay_path, 'SpatialFeatureTensor', replay_player_path+'@G'), sparse.csc_matrix(global_states_np))
def from_vg_snarls_file(cls, vg_snarls_file_name): snarls = (snarl for snarl in stream.parse(vg_snarls_file_name, vg_pb2.Snarl)) return cls(snarls)
def get_protobuf_message(self, path, doc_id): """ Retrieve protocol buffer message matching 'doc_id' from binary fire """ return [ d for d in stream.parse(path, document_pb2.Document) if d.doc_id == doc_id ][0]
def get_list_protobuf_messages(self, path): """ Retrieve list of protocol buffer messages from binary fire """ return [d for d in stream.parse(path, document_pb2.Document)]
import stream from s2clientprotocol import sc2api_pb2 as sc_pb SAMPLED_OBSERVATION_PATH = 'parsed_replays/SampledObservations/Terran_vs_Terran/Terran/1@0008edd10656cc66b14b367400bfe5bd50ddd6bb6f5941c2921e144ef0c01f18.SC2Replay' OBS = [ obs for obs in stream.parse(SAMPLED_OBSERVATION_PATH, sc_pb.ResponseObservation) ] print(OBS)
import stream from s2clientprotocol import sc2api_pb2 as sc_pb PATH = 'parsed_replays/SampledActions/Terran_vs_Terran/Terran/7cc6fe85694768dbab7987344196ab44615842bd896f9172ee1177a2b899ba58.SC2Replay' # OBS = [obs for obs in stream.parse(PATH), sc_pb.ResponseObservation)] # print(OBS) for obs in stream.parse(PATH): print(sc_pb.ResponseObservation(obs))