async def player_stats_helper(player_stats: dict):
"""Extract relevant stats for player histogram from raw player stats"""
try:
if player_stats == None:
return None
stats = {}
hist_fields = histogram_fields.keys()
if 'battles' not in hist_fields:
bu.error('\'battles\' must be defined in \'histogram_fields\'')
sys.exit(1)
for field in hist_fields:
stats[field] = 0
for field in hist_fields:
stats[field] += max(0, player_stats['statistics']['all'][field])
if stats['battles'] == 0:
return None
for field in hist_fields:
if field == 'battles': continue
stats[field] = stats[field] / stats['battles']
return stats
except Exception as err:
bu.error(exception=err)
return None
def process_player_dist(results: list, player_stats: dict, stat_id_map: dict): """Process player distribution""" hist_stats = dict() try: for field in histogram_fields: hist_stats[field] = PlayerHistogram(field, histogram_fields[field][0], histogram_fields[field][1], histogram_fields[field][2], histogram_fields[field][3] ) for result in results: for player in result['allies'] | result['enemies']: # uniton of Sets player_remapped = stat_id_map[player] if player_remapped in player_stats: if player in result['allies']: for stat_field in hist_stats: hist_stats[stat_field].record_ally(player_stats[player_remapped][stat_field]) else: for stat_field in hist_stats: hist_stats[stat_field].record_enemy(player_stats[player_remapped][stat_field]) for stat_field in hist_stats: hist_stats[stat_field].print() except Exception as err: bu.error(exception=err) return None
async def get_db_tank_tier_stats(db : motor.motor_asyncio.AsyncIOMotorDatabase, stat_id_str: str) -> dict:
"""Get player stats from MongoDB (you are very unlikely to have it, unless you have set it up)"""
if db == None:
return None
try:
dbc = db[DB_C_TANK_STATS]
( account_id, tier, battletime ) = str2ints(stat_id_str)
tier_tanks = wg.get_tanks_by_tier(tier)
time_buffer = 2*7*24*3600
#bu.debug('Tank_ids: ' + ','.join(map(str, tier_tanks)))
# if battletime == None:
# pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'tank_id' : {'$in': tier_tanks} } ]}},
# { '$sort': { 'last_battle_time': -1 }},
# { '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}},
# { '$replaceRoot': { 'newRoot': '$doc' }},
# { '$project': { '_id': 0 }} ]
# else:
pipeline = [ { '$match': { '$and': [ { 'account_id': account_id }, { 'last_battle_time': { '$lte': battletime + time_buffer }}, { 'tank_id' : {'$in': tier_tanks} } ]}},
{ '$sort': { 'last_battle_time': -1 }},
{ '$group': { '_id': '$tank_id', 'doc': { '$first': '$$ROOT' }}},
{ '$replaceRoot': { 'newRoot': '$doc' }},
{ '$project': { '_id': 0 }} ]
# cursor = dbc.aggregate(pipeline, allowDiskUse=True)
cursor = dbc.aggregate(pipeline)
return await tank_stats_helper(await cursor.to_list(1000))
except Exception as err:
bu.error('account_id: ' + str(account_id) + ' Error', err)
return None
async def tank_stats_helper(stat_list: list):
"""Helpher func got get_db_tank_tier_stats() and get_wg_tank_tier_stats()"""
try:
if stat_list == None:
return None
stats = {}
hist_fields = histogram_fields.keys()
if 'battles' not in hist_fields:
bu.error('\'battles\' must be defined in \'histogram_fields\'')
sys.exit(1)
for field in hist_fields:
stats[field] = 0
for tank_stat in stat_list:
tank_stat = tank_stat['all']
for field in hist_fields:
stats[field] += max(0, tank_stat[field])
if stats['battles'] == 0:
return None
for field in hist_fields:
if field == 'battles': continue
stats[field] = stats[field] / stats['battles']
return stats
except Exception as err:
bu.error(exception=err)
return None
async def read_user_strs(blitz_app_base: str) -> dict:
"""Read user strings to convert map and tank names"""
tank_strs = {}
map_strs = {}
filename = blitz_app_base + BLITZAPP_STRINGS
bu.debug('Opening file: ' + filename + ' for reading UserStrings')
try:
async with aiofiles.open(filename, 'r', encoding="utf8") as f:
p_tank = re.compile('^"(#\\w+?_vehicles:.+?)": "(.+)"$')
p_map = re.compile('^"#maps:(.+?):.+?: "(.+?)"$')
async for l in f:
m = p_tank.match(l)
if m != None:
tank_strs[m.group(1)] = m.group(2)
m = p_map.match(l)
if m != None and m.group(2) != 'Macragge':
map_strs[m.group(1)] = m.group(2)
except Exception as err:
bu.error(err)
sys.exit(1)
return tank_strs, map_strs
async def convert_tank_names(tanklist: list, tank_strs: dict) -> dict:
"""Convert tank names for Tankopedia"""
tankopedia = {}
userStrs = {}
try:
for tank in tanklist:
tank['name'] = tank_strs[tank['userStr']]
userStrs[tank['userStr'].split(':')[1]] = tank['name']
tank.pop('userStr', None)
tankopedia[str(tank['tank_id'])] = tank
# sorting
tankopedia_sorted = collections.OrderedDict()
for tank_id in sorted(tankopedia.keys(), key=int):
tankopedia_sorted[str(tank_id)] = tankopedia[str(tank_id)]
userStrs_sorted = collections.OrderedDict()
for userStr in sorted(userStrs.keys()):
userStrs_sorted[userStr] = userStrs[userStr]
except Exception as err:
bu.error(err)
sys.exit(1)
return tankopedia_sorted, userStrs_sorted
async def extract_tanks(blitz_app_base: str, nation: str):
tanks = []
list_xml_file = blitz_app_base + BLITZAPP_VEHICLES_DIR + nation + BLITZAPP_VEHICLE_FILE
if not os.path.isfile(list_xml_file):
print('ERROR: cannot open ' + list_xml_file)
return None
bu.debug('Opening file: ' + list_xml_file + ' (Nation: ' + nation + ')')
async with aiofiles.open(list_xml_file, 'r', encoding="utf8") as f:
try:
tankList = xmltodict.parse(await f.read())
for data in tankList['root'].keys():
tank_xml = tankList['root'][data]
tank = {}
tank['tank_id'] = await get_tank_id(nation,
int(tank_xml['id']))
tank['userStr'] = tank_xml['userString']
tank['nation'] = nation
tank['tier'] = int(tank_xml['level'])
#debug(tank_xml['price'])
tank['is_premium'] = issubclass(type(tank_xml['price']), dict)
tank['type'] = await get_tank_type(tank_xml['tags'])
tanks.append(tank)
except Exception as err:
bu.error(err)
sys.exit(2)
return tanks
def getTitle(replayfile: str, title: str = None, i : int = 0) -> str:
global wg
if title == None:
try:
filename = os.path.basename(replayfile)
bu.debug(filename)
tank = None
map_name = None
with zipfile.ZipFile(replayfile, 'r') as archive:
# bu.debug('Replay file: ' + replayfile + ' opened')
with io.TextIOWrapper(archive.open('meta.json')) as meta:
# bu.debug('Replay file\'s metadata: opened')
try:
metadata_json = json.load(meta)
#player = metadata_json['playerName']
tank = wg.tank_str2name(metadata_json['playerVehicleName'])
map_name = wg.get_map(metadata_json['mapName'])
bu.debug('Tank: ' + tank + ' Map: ' + map_name)
except Exception as err:
bu.error(exception = err)
if (tank != None) and (map_name != None):
title = tank + ' @ ' + map_name
else:
title = re.sub('\\.wotbreplay$', '', filename)
except Exception as err:
bu.error(err)
else:
title.replace('NN', str(i))
bu.debug('Returning: ' + title)
return title
def calcTeamStats(result_list: list, player_stats : dict, account_id: int) -> list:
return_list = []
stat_types = player_stats[list(player_stats.keys())[0]].keys()
for result in result_list:
try:
n_allies = collections.defaultdict(defaultvalueZero)
allies_stats = collections.defaultdict(defaultvalueZero)
for ally in result['allies']:
# Player itself is not in 'allies': see readReplayJSON()
for stat in stat_types:
if player_stats[ally][stat] != None:
allies_stats[stat] += player_stats[ally][stat]
n_allies[stat] += 1
for stat in stat_types:
result['allies_' + str(stat)] = allies_stats[stat] / n_allies[stat]
n_enemies = collections.defaultdict(defaultvalueZero)
enemies_stats = collections.defaultdict(defaultvalueZero)
for enemy in result['enemies']:
for stat in stat_types:
if player_stats[enemy][stat] != None:
enemies_stats[stat] += player_stats[enemy][stat]
n_enemies[stat] += 1
for stat in stat_types:
result['enemies_' + str(stat)] = enemies_stats[stat] / n_enemies[stat]
return_list.append(result)
except KeyError as err:
bu.error('Key :' + str(err) + ' not found')
except Exception as err:
bu.error(err)
return return_list
def set_histogram_buckets(json: dict): global histogram_fields try: for field in json.keys(): histogram_fields[field][1] = json[field] except (KeyError, Exception) as err: bu.error(exception=err) return histogram_fields
def get_stat_id_player(stat_id_str: str) -> str:
"""Return stat_id = account_id:tank_tier"""
try:
stat_id = stat_id_str.split(':')
battle_time = (int(stat_id[2]) // BATTLE_TIME_BUCKET) * BATTLE_TIME_BUCKET
return ':'.join([ stat_id[0], str(battle_time) ])
except Exception as err:
bu.error('Stats_id: ' + stat_id_str)
bu.error(exception=err)
return None
def recordResult(self, result : dict) -> bool:
try:
for field in self.avg_fields | self.ratio_fields:
if result[field] != None:
self.results[field] += result[field]
self.battles += 1
return True
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
return False
def printResults(self):
try:
print(' '.join(self.getHeaders()))
for row in self.getResults():
print(' : '.join(row))
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
except Exception as err:
bu.error(str(err))
return None
def print_results(self):
try:
first_btl_record = list(self.category.values())[0]
print(' '.join(first_btl_record.get_headers(self.get_category_name())))
for row in self.get_results():
print(' : '.join(row))
except KeyError as err:
bu.error('Key not found', err)
except Exception as err:
bu.error(exception = err)
return None
def getHeaders(self):
try:
headers = [ result_cat_header_frmt.format(result_categories[self.category_name][0]) ]
for field in result_fields.keys():
print_format = '{:^' + str(result_fields[field][2]) + 's}'
headers.append(print_format.format(result_fields[field][0]))
return headers
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
except Exception as err:
bu.error(str(err))
return None
def __init__(self):
try:
self.battles = 0
self.missing_stats = 0
self.n_players = 0
self.ratios_ready = False
self.results_ready = False
self.results = collections.defaultdict(def_value_zero)
except KeyError as err:
bu.error('Key not found', err)
def get_headers(self, cat_name: str):
try:
headers = [ self.RESULT_CAT_HEADER_FRMT.format(cat_name) ]
for field in self.result_fields:
print_format = '{:^' + str(self._result_fields[field][2]) + 's}'
headers.append(print_format.format(self._result_fields[field][0]))
return headers
except KeyError as err:
bu.error('Key not found', err)
except Exception as err:
bu.error(str(err))
return None
def getResults(self):
if not self.results_ready:
self.calcResults()
results = []
try:
for field in result_fields.keys():
results.append( result_fields[field][3].format(self.results[field]) )
return results
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
except Exception as err:
bu.error(str(err))
return None
async def process_player_stats(players, N_workers: int, args : argparse.Namespace, db : motor.motor_asyncio.AsyncIOMotorDatabase) -> dict:
"""Start stat_workers to retrieve and store players' stats"""
## Create queue of player-tank pairs to find stats for
try:
statsQ = asyncio.Queue()
bu.debug('Create player stats queue: ' + str(len(players)) + ' players')
stat_id_map = {}
stat_ids = set()
bu.set_progress_bar('Fetching player stats', len(players), 25, True)
stat_id_map_func = globals()[STAT_FUNC[args.stat_func][0]]
for player in players:
stat_id_map[player] = stat_id_map_func(player)
stat_ids.add(stat_id_map[player])
# create statsQ
for stat_id in stat_ids:
await statsQ.put(stat_id)
# Process player WR / battle stats
stats_tasks = []
bu.debug('Starting player stats workers')
for i in range(N_workers):
bu.debug("Starting worker " + str(i))
stats_tasks.append(asyncio.create_task(stat_worker(statsQ, i, args, db)))
bu.debug('Waiting stats workers to finish')
await statsQ.join()
bu.debug('Cancelling stats workers')
for task in stats_tasks:
task.cancel()
player_stats = {}
stat_id_remap = {}
bu.debug('Gathering stats worker outputs')
for (stats, id_remap) in await asyncio.gather(*stats_tasks):
player_stats = {**player_stats, **stats}
stat_id_remap = {**stat_id_remap, **id_remap}
bu.finish_progress_bar()
## DO REMAPPING
stat_id_map = remap_stat_id(stat_id_map, stat_id_remap)
bu.debug('Returning player_stats')
return (player_stats, stat_id_map)
except Exception as err:
bu.error(exception=err)
return None
def calc_ratios(self) -> bool:
if self.ratios_ready:
return True
try:
for field in self.result_fields_ratio:
if self.results[self._result_ratios[field][1]] != 0:
self.results[field] = self.results[self._result_ratios[field][0]] / self.results[self._result_ratios[field][1]]
else:
self.results[field] = float('Inf')
self.ratios_ready = True
return True
except KeyError as err:
bu.error('Key not found', err)
return False
async def replayWorker(queue: asyncio.Queue, workerID: int, account_id: int, priv = False):
"""Async Worker to process the replay queue"""
global SKIPPED_N
global ERROR_N
while True:
item = await queue.get()
filename = item[0]
N = item[1]
title = item[2]
replay_json_fn = filename + '.json'
msg_str = 'Replay[' + str(N) + ']: '
#bu.debug(msg_str + replay_json_fn)
try:
if os.path.isfile(replay_json_fn):
async with aiofiles.open(replay_json_fn) as fp:
replay_json = json.loads(await fp.read())
#if replay_json['status'] == 'ok':
if wi.chk_JSON_replay(replay_json):
bu.verbose_std(msg_str + title + ' has already been posted. Skipping.' )
else:
os.remove(replay_json_fn)
bu.debug(msg_str + "Replay JSON not valid: Deleting " + replay_json_fn, id=workerID)
SKIPPED_N += 1
queue.task_done()
continue
except asyncio.CancelledError as err:
raise err
except Exception as err:
bu.error(msg_str + 'Unexpected error: ' + str(type(err)) + ' : '+ str(err))
try:
#bu.debug('Opening file [' + str(N) +']: ' + filename)
async with aiofiles.open(filename,'rb') as fp:
filename = os.path.basename(filename)
bu.debug(msg_str + 'File: ' + filename)
json_resp = await wi.post_replay(await fp.read(), filename, account_id, title, priv, N)
if json_resp != None:
if (await bu.save_JSON(replay_json_fn,json_resp)):
if wi.chk_JSON_replay(json_resp):
if not bu.debug(msg_str + 'Replay saved OK: ' + filename):
bu.verbose_std(msg_str + title + ' posted')
else:
bu.warning(msg_str +'Replay file is not valid: ' + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Error saving replay: ' + filename)
ERROR_N += 1
else:
bu.error(msg_str + 'Replay file is not valid: ' + filename)
ERROR_N += 1
except Exception as err:
bu.error(msg_str + 'Unexpected Exception: ' + str(type(err)) + ' : ' + str(err) )
bu.debug(msg_str + 'Marking task done')
queue.task_done()
return None
def calcResults(self):
if self.results_ready == True:
return True
if not self.ratios_ready:
self.calcRatios()
try:
for field in self.avg_fields:
self.results[field] = self.results[field] / self.battles
self.results['battles'] = self.battles
self.results_ready = True
return True
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
except Exception as err:
bu.error(str(err))
return False
def calcRatios(self) -> bool:
if self.ratios_ready:
return True
try:
ratio_result_flds = set(result_ratios.keys()) & set(result_fields.keys())
for field in ratio_result_flds:
if self.results[result_ratios[field][1]] != 0:
self.results[field] = self.results[result_ratios[field][0]] / self.results[result_ratios[field][1]]
else:
self.results[field] = float('Inf')
self.ratios_ready = True
return True
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
return False
def __init__(self):
try:
self.battles = 0
self.ratios_ready = False
self.results_ready = False
self.results = collections.defaultdict(defaultvalueZero)
self.avg_fields = set(result_fields.keys())
self.avg_fields.remove('battles')
self.avg_fields.difference_update(result_ratios.keys())
self.ratio_fields = set()
for ratio in result_ratios.keys():
self.ratio_fields.add(result_ratios[ratio][0])
self.ratio_fields.add(result_ratios[ratio][1])
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
def recordResult(self, result: dict):
try:
if self.type == 'total':
cat = 'Total'
elif self.type == 'number':
cat = str(result[self.category_name])
elif self.type == 'string':
cat = result[self.category_name]
else:
cat = result_categories[self.category_name][1][result[self.category_name]]
self.category[cat].recordResult(result)
return True
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
except Exception as err:
bu.error(str(err))
return False
async def convert_tank_names(tanklist : list, tank_strs: dict) -> dict:
"""Convert tank names for Tankopedia"""
tankopedia = {}
userStrs = {}
try:
for tank in tanklist:
tank['name'] = tank_strs[tank['userStr']]
#userStrs[tank['userStr'].split(':')[1]] = tank['name']
tank.pop('userStr', None)
tank_tmp = collections.OrderedDict()
for key in sorted(tank.keys()):
tank_tmp[key] = tank[key]
tankopedia[str(tank['tank_id'])] = tank_tmp
for tank_str in tank_strs:
skip = False
key = tank_str.split(':')[1]
bu.debug('Tank string: ' + key + ' = ' + tank_strs[tank_str])
re_strs = [r'^Chassis_', r'^Turret_', r'^_', r'_short$' ]
for re_str in re_strs:
p = re.compile(re_str)
if p.match(key):
skip = True
break
if skip:
continue
userStrs[key] = tank_strs[tank_str]
# sorting
tankopedia_sorted = collections.OrderedDict()
for tank_id in sorted(tankopedia.keys(), key=int):
tankopedia_sorted[str(tank_id)] = tankopedia[str(tank_id)]
userStrs_sorted = collections.OrderedDict()
for userStr in sorted(userStrs.keys()):
userStrs_sorted[userStr] = userStrs[userStr]
# bu.debug('Tank strings: ' + str(len(userStrs_sorted)))
except Exception as err:
bu.error(err)
sys.exit(1)
return tankopedia_sorted, userStrs_sorted
def calc_results(self, total_battles: int = None):
if self.results_ready == True:
return True
if not self.ratios_ready:
self.calc_ratios()
try:
for field in self.avg_fields:
self.results[field] = self.results[field] / max(self.battles,1)
self.results['battles'] = self.battles
self.results['battles%'] = self.battles / total_battles
self.results[MISSING_STATS] = self.missing_stats / max(self.n_players, 1)
self.results_ready = True
return True
except KeyError as err:
bu.error('Key not found', err)
except Exception as err:
bu.error(exception=err)
return False
async def replayReader(queue: asyncio.Queue, readerID: int,
args: argparse.Namespace):
"""Async Worker to process the replay queue"""
#global SKIPPED_N
account_id = args.accountID
results = []
playerstanks = set()
try:
while True:
item = await queue.get()
filename = item[0]
replayID = item[1]
try:
replay_json = await bu.readJSON(filename, wi.chkJSONreplay)
if replay_json == None:
bu.verbose('Replay[' + str(replayID) + ']: ' + filename +
' is invalid. Skipping.')
#SKIPPED_N += 1
queue.task_done()
continue
## Read the replay JSON
bu.debug('[' + str(readerID) + '] reading replay: ' + filename)
result = await readReplayJSON(replay_json, args)
if result == None:
bu.error('[' + str(readerID) + '] Error reading replay: ' +
filename)
queue.task_done()
continue
if (account_id != None):
playerstanks.update(set(result['allies']))
playerstanks.update(set(result['enemies']))
results.append(result)
bu.debug('Marking task ' + str(replayID) + ' done')
except Exception as err:
bu.error(str(err))
queue.task_done()
except asyncio.CancelledError:
return results, playerstanks
return None
async def replay_reader(queue: asyncio.Queue, readerID: int, args : argparse.Namespace):
"""Async Worker to process the replay queue"""
#global SKIPPED_N
results = []
playerstanks = set()
try:
while True:
item = await queue.get()
replay_json = item[0]
replayID = item[1]
replay_file = item[2]
try:
msg_str = 'Replay[' + str(replayID) + ']: '
if replay_json == None:
bu.warning(msg_str + 'Invalid replay. Skipping: ' + (replay_file if replay_file != None else '') )
#SKIPPED_N += 1
queue.task_done()
continue
## Read the replay JSON
bu.debug('reading replay', readerID)
result = await read_replay_JSON(replay_json, args)
bu.print_progress()
if result == None:
bu.warning(msg_str + 'Invalid replay' + (replay_file if replay_file != None else '') )
queue.task_done()
continue
# if (account_id != None):
playerstanks.update(set(result['allies']))
playerstanks.update(set(result['enemies']))
playerstanks.update(set([ result['player'] ]))
results.append(result)
bu.debug('Marking task ' + str(replayID) + ' done')
except Exception as err:
bu.error(exception=err)
queue.task_done()
except (asyncio.CancelledError, concurrent.futures.CancelledError):
bu.debug( str(len(results)) + ' replays, ' + str(len(playerstanks)) + ' player/tanks', readerID)
return results, playerstanks
return None
def getResults(self):
try:
results = []
# results.append(self.getHeaders())
if self.type == 'number':
for cat in sorted( [ int(s) for s in self.category.keys() ] ):
cat = str(cat)
row = [ result_cat_frmt.format(cat) ]
row.extend(self.category[cat].getResults())
results.append(row)
else:
for cat in sorted(self.category.keys() , key=str.casefold):
row = [ result_cat_frmt.format(cat) ]
row.extend(self.category[cat].getResults())
results.append(row)
return results
except KeyError as err:
bu.error('Key ' + str(err) + ' not found')
return None
