def get_lobby_chat(header, encoding, diplomacy_type, players):
"""Get lobby chat."""
chats = []
for message in header.lobby.messages:
if not message.message:
continue
try:
chats.append(parse_chat(
message.message.decode(encoding), 0, players, diplomacy_type, origination='lobby'
))
except UnicodeDecodeError:
LOGGER.warning('could not decode lobby chat')
return chats
async def get_extracted_data( # pylint: disable=too-many-arguments, too-many-locals
header, encoding, diplomacy_type, players, start_time, duration,
playback, handle):
"""Get extracted data."""
timeseries = []
research = defaultdict(dict)
market = []
objects = {}
state = []
last = {}
chats = get_lobby_chat(header, encoding, diplomacy_type, players)
client = await Client.create(playback, handle.name, start_time, duration)
async for tick, source, message in client.sync(timeout=120):
if source == Source.MGZ and message[0] == fast.Operation.CHAT:
try:
chats.append(
parse_chat(message[1].decode(encoding), tick, players,
diplomacy_type))
except UnicodeDecodeError:
LOGGER.warning('could not decode chat')
elif source == Source.MEMORY:
update_market(tick, message.MarketCoefficients(), market)
for i in range(0, message.ObjectsLength()):
update_objects(tick, message.Objects(i), objects, state, last)
for i in range(0, message.PlayersLength()):
player = message.Players(i)
timeseries.append(build_timeseries_record(tick, player))
for j in range(0, player.TechsLength()):
update_research(player.Id(), player.Techs(j), research)
handle.close()
return {
'chat': chats,
'timeseries': timeseries,
'research': flatten_research(research),
'market': market,
'objects': [dict(obj, instance_id=i) for i, obj in objects.items()],
'state': state
}
def _process_body(self): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
"""Process rec body."""
start_time = time.time()
ratings = {}
checksums = []
ladder = None
voobly = False
rated = None
i = 0
duration = self._header.initial.restore_time
fast.meta(self._handle)
self._actions = []
while True:
try:
operation, payload = fast.operation(self._handle)
if operation == fast.Operation.SYNC:
i += 1
duration += payload[0]
if payload[1] and len(checksums) < CHECKSUMS:
checksums.append(payload[1].to_bytes(8,
'big',
signed=True))
elif operation == fast.Operation.ACTION:
self._actions.append((duration, *payload))
if payload[0] == fast.Action.POSTGAME:
self._cache[
'postgame'] = mgz.body.actions.postgame.parse(
payload[1]['bytes'])
elif payload[0] == fast.Action.RESIGN:
self._cache['resigned'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.TRIBUTE and payload[1][
'player_id_to'] == 0:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.TRIBUTE and payload[1][
'player_id'] == 0:
self._cache['cheaters'].add(payload[1]['player_id_to'])
elif payload[0] == fast.Action.CREATE:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.BUILD and payload[1][
'building_id'] not in VALID_BUILDINGS:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.GAME and payload[1][
'mode_id'] in [2, 4, 6]:
self._cache['cheaters'].add(payload[1]['player_id'])
elif operation == fast.Operation.CHAT:
text = payload
if text is None:
continue
try:
parsed = parse_chat(text, self.get_encoding(),
duration, self.get_players(),
self.get_diplomacy().get('type'))
self._chats.append(parsed)
if parsed['type'] == Chat.RATING:
ratings[parsed['player']] = parsed['rating']
elif parsed['type'] == Chat.LADDER:
ladder = parsed['ladder']
elif parsed['type'] == Chat.VOOBLY:
voobly = True
except UnicodeDecodeError:
pass
except EOFError:
break
self._cache['duration'] = duration
if voobly:
rated = len(ratings) > 0 and set(ratings.values()) != {1600}
if self._header.version == Version.DE:
self._cache['hash'] = hashlib.sha1(self._header.de.guid)
else:
self._cache['hash'] = hashlib.sha1(b''.join(checksums)) \
if len(checksums) == CHECKSUMS else None
self._cache['from_voobly'] = voobly
if voobly:
self._cache['platform_id'] = 'voobly'
if self._header.version == Version.DE and self._header.de.multiplayer:
self._cache['platform_id'] = 'de'
self._cache['ladder'] = ladder
self._cache['rated'] = rated
self._cache['ratings'] = ratings if rated else {}
LOGGER.info("parsed body in %.2f seconds", time.time() - start_time)
def parse_match(handle):
"""Parse a match.
This is one big function because the dependency graph between
the variables is dense.
"""
data = parse(handle)
consts = get_consts()
dataset_id, dataset = get_dataset(data['version'], data['mod'])
# self._header.hd.selected_map_id if self._header.hd else self._header.scenario.game_settings.map_id
map_data, encoding, language = get_map_data(
data['hd']['map_id']
if data['version'] is Version.HD else data['scenario']['map_id'],
data['scenario']['instructions'],
data['map']['dimension'],
data['version'],
dataset_id,
dataset,
data['map']['tiles'],
de_seed=data['lobby']['seed'])
# Handle DE-specific data
if data['de']:
de_players = {
player['number']: player
for player in data['de']['players']
}
lobby = data['de']['lobby']
guid = data['de']['guid']
else:
de_players = dict()
lobby = None
guid = None
# Parse gaia objects
gaia = [
Object(dataset['objects'].get(str(obj['object_id'])),
obj['instance_id'],
Position(obj['position']['x'], obj['position']['y']))
for obj in data['players'][0]['objects']
]
inputs = Inputs({o.instance_id: o.name for o in gaia})
# Parse players
players = dict()
allies = dict()
for player in data['players'][1:]:
allies[player['number']] = set([player['number']])
for i, stance in enumerate(player['diplomacy']):
if stance == 2:
allies[player['number']].add(i)
de_player = de_players.get(player['number'])
if de_player:
player.update(de_player)
pos_x = None
pos_y = None
for obj in player['objects']:
if obj['object_id'] in TC_IDS:
pos_x = obj['position']['x']
pos_y = obj['position']['y']
players[player['number']] = Player(
player['color_id'] + 1, player['name'].decode(encoding),
consts['player_colors'][str(player['color_id'])],
dataset['civilizations'][str(player['civilization_id'])]['name'],
Position(pos_x, pos_y), [
Object(dataset['objects'].get(str(obj['object_id'])),
obj['instance_id'],
Position(obj['position']['x'], obj['position']['y']))
for obj in player['objects']
], player.get('profile_id'))
# Assign teams
team_ids = set([frozenset(s) for s in allies.values()])
teams = []
for team in team_ids:
t = [players[x] for x in team]
for x in team:
players[x].team = t
teams.append(t)
# Compute diplomacy
diplomacy_type = get_diplomacy_type(teams, players)
# Extract lobby chat
pd = [dict(name=p.name, number=n) for n, p in players.items()]
chats = []
for c in data['lobby']['chat']:
chat = parse_chat(c, encoding, 0, pd, diplomacy_type, 'lobby')
if chat['player_number'] not in players:
continue
chats.append(
Chat(timedelta(milliseconds=chat['timestamp']), chat['message'],
players[chat['player_number']]))
inputs.add_chat(chats[-1])
# Parse player actions
fast.meta(handle)
timestamp = 0
resigned = []
actions = []
viewlocks = []
last_viewlock = None
while True:
try:
op_type, op_data = fast.operation(handle)
if op_type is fast.Operation.SYNC:
timestamp += op_data[0]
elif op_type is fast.Operation.VIEWLOCK:
if op_data == last_viewlock:
continue
viewlock = Viewlock(timedelta(milliseconds=timestamp),
Position(*op_data),
players[data['metadata']['owner_id']])
viewlocks.append(viewlock)
last_viewlock = op_data
elif op_type is fast.Operation.CHAT:
chat = parse_chat(op_data, encoding, timestamp, pd,
diplomacy_type, 'game')
if chat['type'] == ChatEnum.MESSAGE:
chats.append(
Chat(timedelta(milliseconds=chat['timestamp']),
chat['message'], players[chat['player_number']]))
inputs.add_chat(chats[-1])
elif op_type is fast.Operation.ACTION:
action_type, action_data = op_data
action = Action(timedelta(milliseconds=timestamp), action_type,
action_data)
if action_type is fast.Action.RESIGN:
resigned.append(players[action_data['player_id']])
if 'player_id' in action_data:
action.player = players[action_data['player_id']]
del action.payload['player_id']
enrich_action(action, action_data, dataset, consts)
actions.append(action)
inputs.add_action(action)
except EOFError:
break
# Compute winner(s)
for team in teams:
winner = not any([player for player in team if player in resigned])
for player in team:
player.winner = winner
return Match(
list(players.values()), teams, gaia,
Map(map_data['name'], map_data['dimension'],
consts['map_sizes'][str(map_data['dimension'])],
map_data['custom'], map_data['seed'], [
Tile(tile['terrain_id'], tile['elevation'],
Position(tile['x'], tile['y']))
for tile in map_data['tiles']
]),
File(codecs.lookup(encoding), language,
players[data['metadata']['owner_id']], viewlocks),
consts['speeds'][str(int(round(data['metadata']['speed'], 2) * 100))],
data['metadata']['cheats'], data['lobby']['lock_teams'],
data['lobby']['population'], chats, guid, lobby,
dataset['dataset']['name'],
consts['game_types'][str(data['lobby']['game_type_id'])],
consts['map_reveal_choices'][str(data['lobby']['reveal_map_id'])],
timedelta(milliseconds=timestamp), diplomacy_type, data['version'],
actions, inputs.inputs)