def parse_file_fast(path):
with open(path, 'rb') as f:
f.seek(0, 2)
eof = f.tell()
f.seek(0)
h = header.parse_stream(f)
fast.meta(f)
while f.tell() < eof:
fast.operation(f)
def print_histogram(path):
"""Show operation and action histogram."""
with open(path, 'rb') as handle:
size = os.fstat(handle.fileno()).st_size
mgz.header.parse_stream(handle)
operations = defaultdict(int)
actions = defaultdict(int)
labels = {}
while handle.tell() < size:
op_type, payload = fast.operation(handle)
operations[op_type.name] += 1
if op_type == fast.Operation.ACTION:
action_id = payload[1]
action_id = '{0:#0{1}x}'.format(payload[0].value, 4)
labels[action_id] = payload[0].name
actions[action_id] += 1
print('Operations')
print(
tabulate([[operation, operations[operation]] for operation in
sorted(operations, key=operations.get, reverse=True)],
headers=['Name', 'Count'],
tablefmt='simple'))
print()
print('Actions')
print(
tabulate(
[[action, labels[action], actions[action]]
for action in sorted(actions, key=actions.get, reverse=True)],
headers=['ID', 'Name', 'Count'],
tablefmt='simple'))
async def sync(self):
"""Synchronize state and body messages."""
mgz_time = self.start_time
ws_time = 0
mgz_done = False
ws_done = False
LOGGER.info("starting synchronization")
start = time.time()
keep_reading = True
fast.meta(self._handle)
while not mgz_done or not ws_done:
if self.total_timeout and time.time() - start > self.total_timeout:
LOGGER.warning(
"playback time exceeded timeout (%d%% completed)",
int(mgz_time / self.duration * 100))
break
try:
while not mgz_done and (mgz_time <= ws_time or ws_done):
op_type, payload = fast.operation(self._handle)
if op_type == fast.Operation.SYNC:
mgz_time += payload[0]
elif op_type == fast.Operation.CHAT and payload:
yield mgz_time, Source.MGZ, (op_type,
payload.strip(b'\x00'))
elif op_type == fast.Operation.ACTION:
yield mgz_time, Source.MGZ, (op_type, payload)
except EOFError:
LOGGER.info("MGZ parsing stream finished")
mgz_done = True
while not ws_done and (ws_time <= mgz_time or mgz_done):
try:
data = await asyncio.wait_for(
self.read_state().__anext__(), timeout=self.io_timeout)
except (asyncio.TimeoutError,
asyncio.streams.IncompleteReadError):
LOGGER.warning("state reader timeout")
keep_reading = False
break
ws_time = data.WorldTime()
if data.GameFinished():
LOGGER.info("state reader stream finished")
ws_done = True
yield ws_time, Source.MEMORY, data
if not keep_reading:
break
if ws_done and mgz_done:
LOGGER.info("synchronization finished in %.2f seconds",
time.time() - start)
else:
raise RuntimeError("synchronization timeout encountered")
await self.session.close()
self._handle.close()
async def sync(self, timeout=None):
"""Synchronize state and body messages."""
mgz_time = self.start_time
ws_time = 0
mgz_done = False
ws_done = False
LOGGER.info("starting synchronization")
start = time.time()
while not mgz_done or not ws_done:
if timeout and time.time() - start > timeout:
LOGGER.warning("synchronization timeout encountered")
self.close_all()
raise RuntimeError("synchronization timeout encountered")
try:
while not mgz_done and (mgz_time <= ws_time or ws_done):
op_type, payload = fast.operation(self._handle)
if op_type == fast.Operation.SYNC:
mgz_time += payload[0]
elif op_type == fast.Operation.CHAT and payload:
yield mgz_time, Source.MGZ, (op_type,
payload.strip(b'\x00'))
except (ConstructError, ValueError, EOFError):
LOGGER.info("MGZ parsing stream finished")
mgz_done = True
while not ws_done and (ws_time <= mgz_time or mgz_done):
try:
data = await asyncio.wait_for(
self.read_state().__anext__(), timeout=timeout)
except (asyncio.TimeoutError,
asyncio.streams.IncompleteReadError):
LOGGER.warning(
"socket timeout or read failure encountered")
self.close_all()
raise RuntimeError(
"socket timeout or read failure encountered")
ws_time = data.WorldTime()
if data.GameFinished():
LOGGER.info("state reader stream finished")
ws_done = True
yield ws_time, Source.MEMORY, data
LOGGER.info("synchronization finished in %.2f seconds",
time.time() - start)
self.close_all()
def _process_body(self): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
"""Process rec body."""
start_time = time.time()
ratings = {}
checksums = []
ladder = None
voobly = False
rated = None
i = 0
duration = self._header.initial.restore_time
fast.meta(self._handle)
self._actions = []
while True:
try:
operation, payload = fast.operation(self._handle)
if operation == fast.Operation.SYNC:
i += 1
duration += payload[0]
if payload[1] and len(checksums) < CHECKSUMS:
checksums.append(payload[1].to_bytes(8,
'big',
signed=True))
elif operation == fast.Operation.ACTION:
self._actions.append((duration, *payload))
if payload[0] == fast.Action.POSTGAME:
self._cache[
'postgame'] = mgz.body.actions.postgame.parse(
payload[1]['bytes'])
elif payload[0] == fast.Action.RESIGN:
self._cache['resigned'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.TRIBUTE and payload[1][
'player_id_to'] == 0:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.TRIBUTE and payload[1][
'player_id'] == 0:
self._cache['cheaters'].add(payload[1]['player_id_to'])
elif payload[0] == fast.Action.CREATE:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.BUILD and payload[1][
'building_id'] not in VALID_BUILDINGS:
self._cache['cheaters'].add(payload[1]['player_id'])
elif payload[0] == fast.Action.GAME and payload[1][
'command_id'] in [2, 4, 6]:
self._cache['cheaters'].add(payload[1]['player_id'])
elif operation == fast.Operation.CHAT:
text = payload
if text is None:
continue
try:
parsed = parse_chat(text, self.get_encoding(),
duration, self.get_players(),
self.get_diplomacy().get('type'))
self._chats.append(parsed)
if parsed['type'] == Chat.RATING:
ratings[parsed['player']] = parsed['rating']
elif parsed['type'] == Chat.LADDER:
ladder = parsed['ladder']
elif parsed['type'] == Chat.VOOBLY:
voobly = True
except UnicodeDecodeError:
pass
except EOFError:
break
self._cache['duration'] = duration
if voobly:
rated = len(ratings) > 0 and set(ratings.values()) != {1600}
if self._header.version == Version.DE:
self._cache['hash'] = hashlib.sha1(self._header.de.guid)
elif self._header.version == Version.HD and self._header.save_version >= 12.49:
self._cache['hash'] = hashlib.sha1(self._header.hd.guid)
else:
self._cache['hash'] = hashlib.sha1(b''.join(checksums)) \
if len(checksums) == CHECKSUMS else None
if self._header.de:
rated = self._header.de.ranked
self._cache['from_voobly'] = voobly
if voobly:
self._cache['platform_id'] = 'voobly'
if self._header.version == Version.DE and self._header.de.multiplayer:
self._cache['platform_id'] = 'de'
if self._header.version == Version.HD and self._header.hd.multiplayer:
self._cache['platform_id'] = 'hd'
self._cache['ladder'] = ladder
self._cache['rated'] = rated
self._cache['ratings'] = ratings if rated else {}
LOGGER.info("parsed body in %.2f seconds", time.time() - start_time)
def parse_match(handle):
"""Parse a match.
This is one big function because the dependency graph between
the variables is dense.
"""
data = parse(handle)
body_pos = handle.tell() - 4 # log version
consts = get_consts()
dataset_id, dataset = get_dataset(data['version'], data['mod'])
map_id = data['hd']['map_id'] if data['version'] is Version.HD else data[
'scenario']['map_id']
try:
map_data, encoding, language = get_map_data(
map_id,
data['scenario']['instructions'],
data['map']['dimension'],
data['version'],
dataset_id,
dataset,
data['map']['tiles'],
de_seed=data['lobby']['seed'])
except ValueError:
raise RuntimeError("could not get map data")
# Handle DE-specific data
if data['de']:
de_players = {
player['number']: player
for player in data['de']['players']
}
lobby = data['de']['lobby']
guid = data['de']['guid']
else:
de_players = dict()
lobby = None
guid = None
# Parse gaia objects
gaia = [
Object(dataset['objects'].get(str(obj['object_id'])), obj['class_id'],
obj['object_id'], obj['instance_id'], obj['index'],
Position(obj['position']['x'], obj['position']['y']))
for obj in data['players'][0]['objects']
]
inputs = Inputs({o.instance_id: o.name for o in gaia})
# Parse players
players = dict()
allies = dict()
for player in data['players'][1:]:
allies[player['number']] = set([player['number']])
for i, stance in enumerate(player['diplomacy']):
if stance == 2:
allies[player['number']].add(i)
de_player = de_players.get(player['number'])
if de_player:
player.update(de_player)
pos_x = None
pos_y = None
for obj in player['objects']:
if obj['object_id'] in TC_IDS:
pos_x = obj['position']['x']
pos_y = obj['position']['y']
players[player['number']] = Player(
player['number'], player['name'].decode(encoding),
consts['player_colors'][str(player['color_id'])],
player['color_id'],
dataset['civilizations'][str(player['civilization_id'])]['name'],
player['civilization_id'], Position(pos_x, pos_y), [
Object(dataset['objects'].get(str(
obj['object_id'])), obj['class_id'], obj['object_id'],
obj['instance_id'], obj['index'],
Position(obj['position']['x'], obj['position']['y']))
for obj in player['objects']
], player.get('profile_id'), player.get('prefer_random'))
# Assign teams
if de_players:
by_team = collections.defaultdict(list)
for number, player in de_players.items():
if player['team_id'] > 1:
by_team[player['team_id']].append(number)
elif player['team_id'] == 1:
by_team[number + 9].append(number)
team_ids = by_team.values()
else:
team_ids = set([frozenset(s) for s in allies.values()])
teams = []
for team in team_ids:
t = [players[x] for x in team]
for x in team:
players[x].team = t
teams.append(t)
# Compute diplomacy
diplomacy_type = get_diplomacy_type(teams, players)
# Extract lobby chat
pd = [dict(name=p.name, number=n) for n, p in players.items()]
chats = []
for c in data['lobby']['chat']:
chat = parse_chat(c, encoding, 0, pd, diplomacy_type, 'lobby')
if chat['type'] == ChatEnum.DISCARD or chat[
'player_number'] not in players:
continue
chats.append(
Chat(timedelta(milliseconds=chat['timestamp']), chat['message'],
chat['origination'], chat['audience'],
players[chat['player_number']]))
inputs.add_chat(chats[-1])
# Parse player actions
fast.meta(handle)
timestamp = 0
resigned = []
actions = []
viewlocks = []
last_viewlock = None
while True:
try:
op_type, op_data = fast.operation(handle)
if op_type is fast.Operation.SYNC:
timestamp += op_data[0]
elif op_type is fast.Operation.VIEWLOCK:
if op_data == last_viewlock:
continue
viewlock = Viewlock(timedelta(milliseconds=timestamp),
Position(*op_data),
players[data['metadata']['owner_id']])
viewlocks.append(viewlock)
last_viewlock = op_data
elif op_type is fast.Operation.CHAT:
chat = parse_chat(op_data, encoding, timestamp, pd,
diplomacy_type, 'game')
if chat['type'] == ChatEnum.MESSAGE:
chats.append(
Chat(
timedelta(milliseconds=chat['timestamp'] +
data['map']['restore_time']),
chat['message'], chat['origination'],
chat['audience'], players[chat['player_number']]))
inputs.add_chat(chats[-1])
elif op_type is fast.Operation.ACTION:
action_type, action_data = op_data
action = Action(timedelta(milliseconds=timestamp), action_type,
action_data)
if action_type is fast.Action.RESIGN:
resigned.append(players[action_data['player_id']])
if 'player_id' in action_data and action_data[
'player_id'] in players:
action.player = players[action_data['player_id']]
del action.payload['player_id']
enrich_action(action, action_data, dataset, consts)
actions.append(action)
inputs.add_action(action)
except EOFError:
break
# Compute winner(s)
for team in teams:
winner = not any([player for player in team if player in resigned])
if resigned:
for player in team:
player.winner = winner
handle.seek(body_pos)
file_bytes = handle.read()
file_size = body_pos + 4 + len(file_bytes)
file_hash = hashlib.sha1(file_bytes).hexdigest()
return Match(
list(players.values()), teams, gaia,
Map(
map_id, map_data['name'], map_data['dimension'],
consts['map_sizes'][str(map_data['dimension'])],
map_data['custom'], map_data['seed'], data['de']['rms_mod_id']
if data['version'] is Version.DE else None,
map_data['name'].startswith('ZR@'), map_data['modes'], [
Tile(tile['terrain_id'], tile['elevation'],
Position(tile['x'], tile['y']))
for tile in map_data['tiles']
]),
File(codecs.lookup(encoding), language, file_hash, file_size,
players[data['metadata']['owner_id']],
viewlocks), data['map']['restore_time'] > 0,
timedelta(milliseconds=data['map']['restore_time']),
consts['speeds'][str(int(round(data['metadata']['speed'], 2) * 100))],
int(round(data['metadata']['speed'], 2) * 100),
data['metadata']['cheats'], data['lobby']['lock_teams'],
data['lobby']['population'], chats, guid, lobby,
dataset['dataset']['name'], consts['game_types'][str(
data['lobby']['game_type_id'])], data['lobby']['game_type_id'],
consts['map_reveal_choices'][str(data['lobby']['reveal_map_id'])],
data['lobby']['reveal_map_id'],
consts['difficulties'].get(str(get_difficulty(data))),
get_difficulty(data),
consts['starting_ages'].get(str(get_starting_age(data))),
get_starting_age(data), get_team_together(data), get_lock_speed(data),
get_all_technologies(data),
True if data['version'] is Version.DE else None,
timedelta(milliseconds=timestamp + data['map']['restore_time']),
diplomacy_type, bool(resigned), data['version'], data['game_version'],
data['save_version'], data['log_version'],
data['de']['build'] if data['version'] is Version.DE else None,
datetime.fromtimestamp(data['de']['timestamp'])
if data['version'] is Version.DE and data['de']['timestamp'] else None,
timedelta(seconds=data['de']['spec_delay'])
if data['version'] is Version.DE else None,
data['de']['allow_specs'] if data['version'] is Version.DE else None,
data['de']['hidden_civs'] if data['version'] is Version.DE else None,
data['de']['visibility_id'] == 2 if data['version'] is Version.DE else
None, get_hash(data), actions, inputs.inputs)
def _process_body(self): # pylint: disable=too-many-locals, too-many-statements, too-many-branches
"""Process rec body."""
start_time = time.time()
ratings = {}
checksums = []
ladder = None
voobly = False
rated = None
i = 0
duration = self._header.initial.restore_time
while True:
try:
operation, payload = fast.operation(self._handle)
if operation == fast.Operation.SYNC:
i += 1
duration += payload[0]
if payload[1] and len(checksums) < CHECKSUMS:
checksums.append(payload[1].to_bytes(8,
'big',
signed=True))
elif operation == fast.Operation.ACTION and payload[0] == 255:
self._cache['postgame'] = mgz.body.actions.postgame.parse(
payload[1]['bytes'])
elif operation == fast.Operation.ACTION and payload[0] == 11:
self._cache['resigned'].add(payload[1]['player_id'])
elif operation == fast.Operation.CHAT:
text = payload
if text is None:
continue
text = text.strip(b'\x00')
try:
text = text.decode(self.get_encoding())
if text.find('Voobly: Ratings provided') > 0:
start = text.find("'") + 1
end = text.find("'", start)
ladder = text[start:end]
voobly = True
elif text.find('<Rating>') > 0:
player_start = text.find('>') + 2
player_end = text.find(':', player_start)
player = text[player_start:player_end]
ratings[player] = int(text[player_end +
2:len(text)])
elif text.find('No ratings are available') > 0:
voobly = True
elif text.find(
'This match was played at Voobly.com') > 0:
voobly = True
except UnicodeDecodeError:
pass
except EOFError:
break
self._cache['duration'] = duration
if voobly:
rated = ratings and set(ratings.values()) != {1600}
if self._header.de:
self._cache['hash'] = hashlib.sha1(self._header.de.guid)
else:
self._cache['hash'] = hashlib.sha1(b''.join(checksums)) \
if len(checksums) == CHECKSUMS else None
self._cache['from_voobly'] = voobly
if voobly:
self._cache['platform_id'] = 'voobly'
if self._header.de and self._header.de.multiplayer:
self._cache['platform_id'] = 'de'
self._cache['ladder'] = ladder
self._cache['rated'] = rated
self._cache['ratings'] = ratings if rated else {}
LOGGER.info("parsed body in %.2f seconds", time.time() - start_time)
def parse_match(handle):
"""Parse a match.
This is one big function because the dependency graph between
the variables is dense.
"""
data = parse(handle)
consts = get_consts()
dataset_id, dataset = get_dataset(data['version'], data['mod'])
# self._header.hd.selected_map_id if self._header.hd else self._header.scenario.game_settings.map_id
map_data, encoding, language = get_map_data(
data['hd']['map_id']
if data['version'] is Version.HD else data['scenario']['map_id'],
data['scenario']['instructions'],
data['map']['dimension'],
data['version'],
dataset_id,
dataset,
data['map']['tiles'],
de_seed=data['lobby']['seed'])
# Handle DE-specific data
if data['de']:
de_players = {
player['number']: player
for player in data['de']['players']
}
lobby = data['de']['lobby']
guid = data['de']['guid']
else:
de_players = dict()
lobby = None
guid = None
# Parse gaia objects
gaia = [
Object(dataset['objects'].get(str(obj['object_id'])),
obj['instance_id'],
Position(obj['position']['x'], obj['position']['y']))
for obj in data['players'][0]['objects']
]
inputs = Inputs({o.instance_id: o.name for o in gaia})
# Parse players
players = dict()
allies = dict()
for player in data['players'][1:]:
allies[player['number']] = set([player['number']])
for i, stance in enumerate(player['diplomacy']):
if stance == 2:
allies[player['number']].add(i)
de_player = de_players.get(player['number'])
if de_player:
player.update(de_player)
pos_x = None
pos_y = None
for obj in player['objects']:
if obj['object_id'] in TC_IDS:
pos_x = obj['position']['x']
pos_y = obj['position']['y']
players[player['number']] = Player(
player['color_id'] + 1, player['name'].decode(encoding),
consts['player_colors'][str(player['color_id'])],
dataset['civilizations'][str(player['civilization_id'])]['name'],
Position(pos_x, pos_y), [
Object(dataset['objects'].get(str(obj['object_id'])),
obj['instance_id'],
Position(obj['position']['x'], obj['position']['y']))
for obj in player['objects']
], player.get('profile_id'))
# Assign teams
team_ids = set([frozenset(s) for s in allies.values()])
teams = []
for team in team_ids:
t = [players[x] for x in team]
for x in team:
players[x].team = t
teams.append(t)
# Compute diplomacy
diplomacy_type = get_diplomacy_type(teams, players)
# Extract lobby chat
pd = [dict(name=p.name, number=n) for n, p in players.items()]
chats = []
for c in data['lobby']['chat']:
chat = parse_chat(c, encoding, 0, pd, diplomacy_type, 'lobby')
if chat['player_number'] not in players:
continue
chats.append(
Chat(timedelta(milliseconds=chat['timestamp']), chat['message'],
players[chat['player_number']]))
inputs.add_chat(chats[-1])
# Parse player actions
fast.meta(handle)
timestamp = 0
resigned = []
actions = []
viewlocks = []
last_viewlock = None
while True:
try:
op_type, op_data = fast.operation(handle)
if op_type is fast.Operation.SYNC:
timestamp += op_data[0]
elif op_type is fast.Operation.VIEWLOCK:
if op_data == last_viewlock:
continue
viewlock = Viewlock(timedelta(milliseconds=timestamp),
Position(*op_data),
players[data['metadata']['owner_id']])
viewlocks.append(viewlock)
last_viewlock = op_data
elif op_type is fast.Operation.CHAT:
chat = parse_chat(op_data, encoding, timestamp, pd,
diplomacy_type, 'game')
if chat['type'] == ChatEnum.MESSAGE:
chats.append(
Chat(timedelta(milliseconds=chat['timestamp']),
chat['message'], players[chat['player_number']]))
inputs.add_chat(chats[-1])
elif op_type is fast.Operation.ACTION:
action_type, action_data = op_data
action = Action(timedelta(milliseconds=timestamp), action_type,
action_data)
if action_type is fast.Action.RESIGN:
resigned.append(players[action_data['player_id']])
if 'player_id' in action_data:
action.player = players[action_data['player_id']]
del action.payload['player_id']
enrich_action(action, action_data, dataset, consts)
actions.append(action)
inputs.add_action(action)
except EOFError:
break
# Compute winner(s)
for team in teams:
winner = not any([player for player in team if player in resigned])
for player in team:
player.winner = winner
return Match(
list(players.values()), teams, gaia,
Map(map_data['name'], map_data['dimension'],
consts['map_sizes'][str(map_data['dimension'])],
map_data['custom'], map_data['seed'], [
Tile(tile['terrain_id'], tile['elevation'],
Position(tile['x'], tile['y']))
for tile in map_data['tiles']
]),
File(codecs.lookup(encoding), language,
players[data['metadata']['owner_id']], viewlocks),
consts['speeds'][str(int(round(data['metadata']['speed'], 2) * 100))],
data['metadata']['cheats'], data['lobby']['lock_teams'],
data['lobby']['population'], chats, guid, lobby,
dataset['dataset']['name'],
consts['game_types'][str(data['lobby']['game_type_id'])],
consts['map_reveal_choices'][str(data['lobby']['reveal_map_id'])],
timedelta(milliseconds=timestamp), diplomacy_type, data['version'],
actions, inputs.inputs)
