def fix_entities(self, replay, actual_players): # Change the players that aren't playing into observers for p in [p for p in replay.players if p.pid not in actual_players]: # Fix the slot data to be accurate p.slot_data["observe"] = 1 p.slot_data["team_id"] = None obs = Observer(p.sid, p.slot_data, p.uid, p.init_data, p.pid) # Because these obs start the game as players the client # creates various Beacon units for them. obs.units = p.units # Remove all references to the old player del replay.player[p.pid] del replay.entity[p.pid] del replay.human[p.uid] replay.players.remove(p) replay.entities.remove(p) replay.humans.remove(p) # Create all the necessary references for the new observer replay.observer[obs.uid] = obs replay.entity[obs.pid] = obs replay.human[obs.uid] = obs replay.observers.append(obs) replay.entities.append(obs) replay.humans.append(obs) # Maintain order, just in case someone is depending on it replay.observers = sorted(replay.observers, key=lambda o: o.sid) replay.entities = sorted(replay.entities, key=lambda o: o.sid) replay.humans = sorted(replay.humans, key=lambda o: o.sid) # Assume one player per team, should be valid for GameHeart games replay.team = dict() replay.teams = list() for index, player in enumerate(replay.players): team_id = index + 1 team = Team(team_id) replay.team[team_id] = team replay.teams.append(team) player.team = team team.result = player.result player.pick_race = actual_players[player.pid] player.play_race = player.pick_race team.players = [player] team.result = player.result if team.result == "Win": replay.winner = team
def fix_entities(self, replay, actual_players): # Change the players that aren't playing into observers for p in [p for p in replay.players if p.pid not in actual_players]: # Fix the slot data to be accurate p.slot_data['observe'] = 1 p.slot_data['team_id'] = None obs = Observer(p.sid, p.slot_data, p.uid, p.init_data, p.pid) # Because these obs start the game as players the client # creates various Beacon units for them. obs.units = p.units # Remove all references to the old player del replay.player[p.pid] del replay.entity[p.pid] del replay.human[p.uid] replay.players.remove(p) replay.entities.remove(p) replay.humans.remove(p) # Create all the necessary references for the new observer replay.observer[obs.uid] = obs replay.entity[obs.pid] = obs replay.human[obs.uid] = obs replay.observers.append(obs) replay.entities.append(obs) replay.humans.append(obs) # Maintain order, just in case someone is depending on it replay.observers = sorted(replay.observers, key=lambda o: o.sid) replay.entities = sorted(replay.entities, key=lambda o: o.sid) replay.humans = sorted(replay.humans, key=lambda o: o.sid) # Assume one player per team, should be valid for GameHeart games replay.team = dict() replay.teams = list() for index, player in enumerate(replay.players): team_id = index+1 team = Team(team_id) replay.team[team_id] = team replay.teams.append(team) player.team = team team.result = player.result player.pick_race = actual_players[player.pid] player.play_race = player.pick_race team.players = [player] team.result = player.result if team.result == 'Win': replay.winner = team
def load_players(self): #If we don't at least have details and attributes_events we can go no further if 'replay.details' not in self.raw_data: return if 'replay.attributes.events' not in self.raw_data: return if 'replay.initData' not in self.raw_data: return self.clients = list() self.client = dict() # For players, we can use the details file to look up additional # information. detail_id marks the current index into this data. detail_id = 0 player_id = 1 details = self.raw_data['replay.details'] initData = self.raw_data['replay.initData'] # Assume that the first X map slots starting at 1 are player slots # so that we can assign player ids without the map self.entities = list() for slot_id, slot_data in enumerate(initData['lobby_state']['slots']): user_id = slot_data['user_id'] if slot_data['control'] == 2: if slot_data['observe'] == 0: self.entities.append( Participant(slot_id, slot_data, user_id, initData['user_initial_data'][user_id], player_id, details['players'][detail_id], self.attributes.get(player_id, dict()))) detail_id += 1 player_id += 1 else: self.entities.append( Observer(slot_id, slot_data, user_id, initData['user_initial_data'][user_id], player_id)) player_id += 1 elif slot_data['control'] == 3: self.entities.append( Computer(slot_id, slot_data, player_id, details['players'][detail_id], self.attributes.get(player_id, dict()))) detail_id += 1 player_id += 1 def get_team(team_id): if team_id is not None and team_id not in self.team: team = Team(team_id) self.team[team_id] = team self.teams.append(team) return self.team[team_id] # Set up all our cross reference data structures for entity in self.entities: if entity.is_observer is False: entity.team = get_team(entity.team_id) entity.team.players.append(entity) self.players.append(entity) self.player[entity.pid] = entity else: self.observers.append(entity) self.observer[entity.uid] = entity if entity.is_human: self.humans.append(entity) self.human[entity.uid] = entity else: self.computers.append(entity) self.computer[entity.pid] = entity # Index by pid so that we can match events to players in pre-HotS replays self.entity[entity.pid] = entity # Pull results up for teams for team in self.teams: results = set([p.result for p in team.players]) if len(results) == 1: team.result = list(results)[0] if team.result == 'Win': self.winner = team else: self.logger.warn( "Conflicting results for Team {0}: {1}".format( team.number, results)) team.result = 'Unknown' self.teams.sort(key=lambda t: t.number) # These are all deprecated self.clients = self.humans self.people = self.entities self.client = self.human self.person = self.entity self.real_type = utils.get_real_type(self.teams) # Assign the default region to computer players for consistency # We know there will be a default region because there must be # at least 1 human player or we wouldn't have a replay. default_region = self.humans[0].region for entity in self.entities: if not entity.region: entity.region = default_region # Pretty sure this just never worked, forget about it for now self.recorder = None entity_names = sorted(map(lambda p: p.name, self.entities)) hash_input = self.gateway + ":" + ','.join(entity_names) self.people_hash = hashlib.sha256( hash_input.encode('utf8')).hexdigest() # The presence of observers and/or computer players makes this not actually ladder # This became an issue in HotS where Training, vs AI, Unranked, and Ranked # were all marked with "amm" => Ladder if len(self.observers) > 0 or len(self.humans) != len(self.players): self.is_ladder = False
def load_players(self): #If we don't at least have details and attributes_events we can go no further if 'replay.details' not in self.raw_data: return if 'replay.attributes.events' not in self.raw_data: return # Create and add the players based on attribute and details information player_index, obs_index, default_region = 0, 1, '' player_data = self.raw_data['replay.details'].players for pid, attributes in sorted(self.attributes.iteritems()): # We've already processed the global attributes if pid == 16: continue # Open Slots are skipped because it doesn't seem useful to store # an "Open" player to fill a spot that would otherwise be empty. if attributes['Player Type'] == 'Open': continue # Get the player data from the details file, increment the index to # Keep track of which player we are processing pdata = player_data[player_index] player_index += 1 # If this is a human player, push back the initial observer index in # the list of all human players we gathered from the initdata file. if attributes['Player Type'] == 'Human': obs_index += 1 # Create the player using the current pid and the player name from # The details file. This works because players are stored in order # of pid inside of the details file. Attributes and Details must be # processed together because Details doesn't index players by or # store their player ids; Attributes can provide that information # and allow for accurate pid mapping even with computers, observers, # and open open slots. # # General information re: each player comes from the following files # * replay.initData # * replay.details # * replay.attribute.events # # TODO: recognize current locale and use that instead of western # TODO: fill in the LOCALIZED_RACES table player = Player(pid,pdata.name) # Cross reference the player and team lookups team_number = attributes['Teams'+self.type] if not team_number in self.team: self.team[team_number] = Team(team_number) self.teams.append(self.team[team_number]) self.team[team_number].players.append(player) player.team = self.team[team_number] # Do basic win/loss processing from details data if pdata.result == 1: player.team.result = "Win" self.winner = player.team elif pdata.result == 2: player.team.result = "Loss" player.pick_race = attributes['Race'] player.play_race = LOCALIZED_RACES.get(pdata.race, pdata.race) player.difficulty = attributes['Difficulty'] player.is_human = (attributes['Player Type'] == 'Human') player.uid = pdata.bnet.uid player.subregion = pdata.bnet.subregion player.handicap = pdata.handicap # We need initData for the gateway portion of the url! if 'replay.initData' in self.raw_data and self.gateway: player.gateway = self.gateway if player.is_human and player.subregion: player.region = REGIONS[self.gateway][player.subregion] default_region = player.region # Conversion instructions to the new color object: # color_rgba is the color object itself # color_hex is color.hex # color is str(color) player.color = utils.Color(**pdata.color._asdict()) # Each player can be referenced in a number of different ways, # primarily for convenience of access in any given situation. self.people.append(player) self.players.append(player) self.player[pid] = player self.person[pid] = player #Create an store an ordered lineup string for team in self.teams: team.lineup = sorted(player.play_race[0].upper() for player in team) if 'replay.initData' in self.raw_data: # Assign the default region to computer players for consistency # We know there will be a default region because there must be # at least 1 human player or we wouldn't have a self. for player in self.players: if not player.is_human: player.region = default_region # Create observers out of the leftover names gathered from initData all_players = [p.name for p in self.players] all_people = self.raw_data['replay.initData'].player_names for obs_name in all_people: if obs_name in all_players: continue observer = Observer(obs_index,obs_name) observer.gateway = self.gateway self.observers.append(observer) self.people.append(observer) self.person[obs_index] = observer obs_index += 1 # Miscellaneous people processing self.humans = filter(lambda p: p.is_human, self.people) if 'replay.message.events' in self.raw_data: # Figure out recorder self.packets = self.raw_data['replay.message.events'].packets packet_senders = map(lambda p: p.pid, self.packets) human_pids = map(lambda p: p.pid, self.humans) recorders = list(set(human_pids) - set(packet_senders)) if len(recorders) == 1: self.recorder = self.person[recorders[0]] self.recorder.recorder = True else: raise ValueError("Get Recorder algorithm is broken!") player_names = sorted(map(lambda p: p.name, self.people)) hash_input = self.gateway+":"+','.join(player_names) self.people_hash = hashlib.sha256(hash_input).hexdigest()
def Full(replay): # Populate replay with details if 'initData' in replay.raw and replay.raw.initData.map_data: replay.gateway = replay.raw.initData.map_data[0].gateway replay.map = Map(replay.gateway, replay.raw.initData.map_data[-1].map_hash) #Expand this special case mapping if replay.gateway == 'sg': replay.gateway = 'sea' if 'details' in replay.raw: if replay.map: replay.map.name = replay.raw.details.map replay.file_time = replay.raw.details.file_time replay.unix_timestamp = windows_to_unix(replay.file_time) replay.date = datetime.fromtimestamp(replay.unix_timestamp) replay.utc_date = datetime.utcfromtimestamp(replay.unix_timestamp) if 'attributes_events' in replay.raw: # Organize the attribute data to be useful attribute_data = defaultdict(dict) for attr in replay.raw.attributes_events: attribute_data[attr.player][attr.name] = attr.value # Populate replay with attributes replay.speed = attribute_data[16]['Game Speed'] replay.category = attribute_data[16]['Category'] replay.type = attribute_data[16]['Game Type'] replay.is_ladder = (replay.category == "Ladder") replay.is_private = (replay.category == "Private") #If we don't at least have details and attributes_events we can go no further if not ('details' in replay.raw and 'attributes_events' in replay.raw): return replay # Create and add the players based on attribute and details information player_index, observer_index, default_region = 0, 0, '' player_data = replay.raw.details.players for pid, attributes in sorted(attribute_data.iteritems()): # We've already processed the global attributes if pid == 16: continue # Open Slots are skipped because it doesn't seem useful to store # an "Open" player to fill a spot that would otherwise be empty. if attributes['Player Type'] == 'Open': continue # Get the player data from the details file, increment the index to # Keep track of which player we are processing pdata = player_data[player_index] player_index += 1 # If this is a human player, push back the initial observer index in # the list of all human players we gathered from the initdata file. if attributes['Player Type'] == 'Human': observer_index += 1 # Create the player using the current pid and the player name from # The details file. This works because players are stored in order # of pid inside of the details file. Attributes and Details must be # processed together because Details doesn't index players by or # store their player ids; Attributes can provide that information # and allow for accurate pid mapping even with computers, observers, # and open open slots. # # General information re: each player comes from the following files # * replay.initData # * replay.details # * replay.attribute.events # # TODO: recognize current locale and use that instead of western # TODO: fill in the LOCALIZED_RACES table player = Player(pid, pdata.name, replay) # Cross reference the player and team lookups team_number = attributes['Teams' + replay.type] if not team_number in replay.team: replay.team[team_number] = Team(team_number) replay.teams.append(replay.team[team_number]) replay.team[team_number].players.append(player) player.team = replay.team[team_number] # Do basic win/loss processing from details data if pdata.result == 1: player.team.result = "Win" elif pdata.result == 2: player.team.result = "Loss" player.pick_race = attributes['Race'] player.play_race = LOCALIZED_RACES.get(pdata.race, pdata.race) player.difficulty = attributes['Difficulty'] player.type = attributes['Player Type'] player.uid = pdata.bnet.uid player.subregion = pdata.bnet.subregion player.handicap = pdata.handicap # We need initData for the gateway which is required to build the url! if 'initData' in replay.raw and replay.gateway: player.gateway = replay.gateway if player.type == 'Human' and player.subregion: player.region = REGIONS[replay.gateway][player.subregion] default_region = player.region # Conversion instructions to the new color object: # color_rgba is the color object itself # color_hex is color.hex # color is str(color) player.color = Color(**pdata.color._asdict()) # Each player can be referenced in a number of different ways, # primarily for convenience of access in any given situation. replay.people.append(player) replay.players.append(player) replay.player[pid] = player replay.person[pid] = player #Create an store an ordered lineup string for team in replay.teams: team.lineup = sorted(player.play_race[0].upper() for player in team) if 'initData' in replay.raw: # Assign the default region to computer players for consistency # We know there will be a default region because there must be # at least 1 human player or we wouldn't have a replay. for player in replay.players: if player.type == 'Computer': player.region = default_region # Create observers out of the leftover names gathered from initData all_players = replay.raw.initData.player_names for i in range(observer_index, len(all_players)): observer = Observer(i + 1, all_players[i], replay) replay.observers.append(observer) replay.people.append(observer) replay.person[i + 1] = observer # Miscellaneous people processing replay.humans = filter(lambda p: p.type == 'Human', replay.people) if 'message_events' in replay.raw: # Process the packets for time, pid, flags, data in replay.raw.message_events.packets: replay.packets.append(Packet(time, replay.person[pid], data)) # Process the messages # TODO: process the message into the Message class for time, pid, flags, target, text in replay.raw.message_events.messages: message = Message(time, replay.person[pid], target, text) replay.messages.append(message) replay.person[pid].messages.append(message) # Figure out recorder packet_senders = map(lambda p: p.player, replay.packets) recorders = list(set(replay.humans) - set(packet_senders)) if len(recorders) == 1: replay.recorder = recorders[0] replay.recorder.recorder = True else: raise ValueError("Get Recorder algorithm is broken!") #If we don't have game events, this is as far as we can go if not 'game_events' in replay.raw: return replay # Copy the events over # TODO: the events need to be fixed both on the reader and processor side try: data = config.build_data[replay.build] except KeyError: # If we don't have data for this version, we can't do any more. return replay replay.events = replay.raw.game_events for event in replay.events: if event.is_local: # Set up the object cross references event.player = replay.person[event.pid] event.player.events.append(event) # Because applying the events is slow, make it configurable try: if replay.opt.apply: event.apply(data) except Exception, e: print "Ouch, exception while applying", e traceback.print_exc() l = replay.events_by_type[event.name] l.append(event)
def createObserver(pid, name, attributes): # TODO: Make use of that attributes, new in HotS observer = Observer(pid, name) return observer
def Full(replay): # Populate replay with details if 'initData' in replay.raw and replay.raw.initData.map_data: replay.gateway = replay.raw.initData.map_data[0].gateway #Expand this special case mapping if replay.gateway == 'sg': replay.gateway = 'sea' if 'details' in replay.raw: replay.map = replay.raw.details.map replay.file_time = replay.raw.details.file_time replay.unix_timestamp = windows_to_unix(replay.file_time) replay.date = datetime.fromtimestamp(replay.unix_timestamp) replay.utc_date = datetime.utcfromtimestamp(replay.unix_timestamp) if 'attributes_events' in replay.raw: # Organize the attribute data to be useful attribute_data = defaultdict(dict) for attr in replay.raw.attributes_events: attribute_data[attr.player][attr.name] = attr.value # Populate replay with attributes replay.speed = attribute_data[16]['Game Speed'] replay.category = attribute_data[16]['Category'] replay.type = attribute_data[16]['Game Type'] replay.is_ladder = (replay.category == "Ladder") replay.is_private = (replay.category == "Private") #If we don't at least have details and attributes_events we can go no further if not ('details' in replay.raw and 'attributes_events' in replay.raw): return replay # Create and add the players based on attribute and details information player_index, observer_index, default_region = 0, 0, '' player_data = replay.raw.details.players for pid, attributes in sorted(attribute_data.iteritems()): # We've already processed the global attributes if pid == 16: continue # Open Slots are skipped because it doesn't seem useful to store # an "Open" player to fill a spot that would otherwise be empty. if attributes['Player Type'] == 'Open': continue # Get the player data from the details file, increment the index to # Keep track of which player we are processing pdata = player_data[player_index] player_index += 1 # If this is a human player, push back the initial observer index in # the list of all human players we gathered from the initdata file. if attributes['Player Type'] == 'Human': observer_index += 1 # Create the player using the current pid and the player name from # The details file. This works because players are stored in order # of pid inside of the details file. Attributes and Details must be # processed together because Details doesn't index players by or # store their player ids; Attributes can provide that information # and allow for accurate pid mapping even with computers, observers, # and open open slots. # # General information re: each player comes from the following files # * replay.initData # * replay.details # * replay.attribute.events # # TODO: recognize current locale and use that instead of western # TODO: fill in the LOCALIZED_RACES table player = Player(pid,pdata.name,replay) # Cross reference the player and team lookups team_number = attributes['Teams'+replay.type] if not team_number in replay.team: replay.team[team_number] = Team(team_number) replay.teams.append(replay.team[team_number]) replay.team[team_number].players.append(player) player.team = replay.team[team_number] # Do basic win/loss processing from details data if pdata.result == 1: player.team.result = "Win" elif pdata.result == 2: player.team.result = "Loss" player.pick_race = attributes['Race'] player.play_race = LOCALIZED_RACES.get(pdata.race, pdata.race) player.difficulty = attributes['Difficulty'] player.type = attributes['Player Type'] player.uid = pdata.bnet.uid player.subregion = pdata.bnet.subregion player.handicap = pdata.handicap # We need initData for the gateway which is required to build the url! if 'initData' in replay.raw and replay.gateway: player.gateway = replay.gateway if player.type == 'Human': player.region = REGIONS[replay.gateway][player.subregion] default_region = player.region # Conversion instructions to the new color object: # color_rgba is the color object itself # color_hex is color.hex # color is str(color) player.color = Color(**pdata.color._asdict()) # Each player can be referenced in a number of different ways, # primarily for convenience of access in any given situation. replay.people.append(player) replay.players.append(player) replay.player[pid] = player replay.person[pid] = player #Create an store an ordered lineup string for team in replay.teams: team.lineup = sorted(player.play_race[0].upper() for player in team) if 'initData' in replay.raw: # Assign the default region to computer players for consistency # We know there will be a default region because there must be # at least 1 human player or we wouldn't have a replay. for player in replay.players: if player.type == 'Computer': player.region = default_region # Create observers out of the leftover names gathered from initData all_players = replay.raw.initData.player_names for i in range(observer_index,len(all_players)): observer = Observer(i+1,all_players[i],replay) replay.observers.append(observer) replay.people.append(observer) replay.person[i+1] = observer # Miscellaneous people processing replay.humans = filter(lambda p: p.type == 'Human', replay.people) if 'message_events' in replay.raw: # Process the packets for time, pid, flags, data in replay.raw.message_events.packets: replay.packets.append(Packet(time, replay.person[pid], data)) # Process the messages # TODO: process the message into the Message class for time, pid, flags, target, text in replay.raw.message_events.messages: message = Message(time, replay.person[pid], target, text) replay.messages.append(message) replay.person[pid].messages.append(message) # Figure out recorder packet_senders = map(lambda p: p.player, replay.packets) recorders = list(set(replay.humans) - set(packet_senders)) if len(recorders) == 1: replay.recorder = recorders[0] replay.recorder.recorder = True else: raise ValueError("Get Recorder algorithm is broken!") #If we don't have game events, this is as far as we can go if not 'game_events' in replay.raw: return replay # Copy the events over # TODO: the events need to be fixed both on the reader and processor side replay.events = replay.raw.game_events for event in replay.events: if event.is_local: # Set up the object cross references event.player = replay.person[event.pid] event.player.events.append(event) # Because applying the events is slow, make it configurable if replay.opt.apply: event.apply() l = replay.events_by_type[event.name] l.append(event) # Gather data for APM measurements for event in replay.events: if event.is_local and event.is_player_action: player = event.player if not player.is_observer: # Count up the APS, APM second, minute = event.second, event.second/60 player.aps[event.second] += 1 player.apm[minute] += 1 player.avg_apm += 1 # Average the APM for actual players for player in replay.players: if player.events: event_minutes = player.events[-1].second/60.0 if event_minutes: player.avg_apm /= event_minutes # If the details file didn't have all the results, try to figure them out. # Our first approach simulates the teams losing one by one by stepping # through the recorded player leave events and marking losses as the teams # run out of players. # # TODO: See if this ever tells us more than details file does by default! if filter(lambda t: t.result == "Unknown", replay.teams): #Get the total players for each team so we can start the count down pcount = dict([team.number, len(team.players)] for team in replay.teams) for event in replay.events_by_type['PlayerLeave']: team = replay.player[event.pid].team # Observer leave events are recorded too, filter them out if event.pid in replay.player: # If this was the last person on the team, that team loses if pcount[team.number] == 1: team.result = "Loss" del pcount[team.number] #If only one team has players left, they have won! if len(pcount) == 1: replay.team[pcount.keys()[0]].result = "Win" break else: pcount[team.number] -= 1 # Our player-leave approach didn't determine a winner. If there are only # two remaining teams and the recorder is the only player on one of them # then his team must have lost since his leave event isn't recorded. # # The above definitely true if another there are two or more teams left # or the other team has two people left. If there is one team left with # only one person remaining then, if he wins by destroying all the # opponent's buildings it doesn't generate leave events for the other # player. # # Also if there is a tie in any situation there may not be leave events. # # TODO: Figure out how to handle ties, if we even can # TODO: Find some replays that actually have ties? else: #We can't determine anything else without the recorder if 'message_events' in replay.raw: # If the recorder is the last person on his team, this obviously # doesn't apply for recorders who are observers team = replay.recorder.team if replay.recorder.pid in replay.players and pcount[team.number] == 1: # Get all other teams with at least 2 players left: conditions = lambda p: p[0] != team.number and p[1] > 1 big_teams = filter(conditions, pcount.iteritems()) # If there are any big teams left or more than one 1 person # team left this is definitely either a loss or a really # awkward tie between 3 or more people on 2 or more teams. # # Since I don't think ties can be detected, ignore that case if len(pcount) > 2 or big_teams: replay.recorder.team.result = "Loss" del pcount[replay.recorder.team.number] # If that leaves only one team left standing, they won! if len(pcount) == 1: replay.team[pcount.keys()[0]].result = "Win" # Since there were only two teams of 1 person each remaining # we can't know if this was a win by building destruction, # loss, or a tie, so leave them as unknown result for now. # TODO: verify that this is the right thing to do else: pass return replay