def gen_api_ladder(members=None,
team_size=1,
url='http://fake-url',
gd=True,
**kwargs):
""" Generate api ladder object from members and other data. Can generalte legacy data or gamedata version. """
if members is None:
members = [gen_member(**kwargs)]
if gd:
return ApiLadder(gen_ladder_data(members, team_size=team_size), url)
else:
# Skipping races for now, since it is not needed for current set of tests.
# races["favoriteRaceP%d" % (i + 1)] = Race.key_by_ids[m['race']].upper()
return ApiLadder(
{
'ladderMembers': [{
"points": m['points'],
"previousRank": 0,
"wins": m['wins'],
"losses": m['losses'],
"highestRank": 0,
"joinTimestamp": to_unix(m['join_time']),
"character": {
"realm": m['realm'],
"profilePath": None,
"clanName": m['clan'],
"id": m['bid'],
"clanTag": m['tag'],
"displayName": m['name'],
}
} for m in members]
}, url)
def test_survives_missing_legacy_name(self):
l = deepcopy(API_LADDER_1V1)
del l['team'][0]['member'][0]['legacy_link']['name']
al = ApiLadder(l)
self.assertEqual([{
'join_time': 1479018088,
'bid': 6205328,
'realm': 1,
'name': '',
'tag': 'Partha',
'clan': 'boom boom long time',
'points': 381,
'mmr': 2136,
'wins': 12,
'losses': 24,
'race': Race.PROTOSS,
}], al.members_for_ranking(1))
def gen_api_data(l_bid,
*p_bids,
mmq="HOTS_SOLO",
league="SILVER",
race="ZERG",
join_time=None,
join_times=None,
season="currentSeason",
team_size=None):
""" Generate api data from blizzard. Returns <ladder_data,
player_ladder_data>. Player ladder data will contain one team. """
join_time = join_time or utcnow()
join_times = join_times or [join_time]
team_size = team_size or len(p_bids)
names = [uniqueid(10) for _ in p_bids]
pld = {
"ladderId": l_bid,
"league": league,
"matchMakingQueue": mmq,
"wins": 20,
"losses": 22
}
ppds = [{
"id": p_bid,
"realm": 1,
"displayName": name,
"profilePath": "/profile/%d/1/%s/" % (p_bid, name)
} for p_bid, name in zip(p_bids, names)]
members = [{
"character": {
"id": p_bid,
"realm": 1,
"profilePath": "/profile/%d/1/%s/" % (p_bid, name)
},
"joinTimestamp": int(jt.strftime("%s")),
"points": 102.2,
"wins": 20,
"losses": 22,
"favoriteRaceP1": race
} for p_bid, jt, name in zip(p_bids, itertools.cycle(join_times), names)]
for member in members:
if team_size > 1: member['favoriteRaceP2'] = race
if team_size > 2: member['favoriteRaceP3'] = race
if team_size > 3: member['favoriteRaceP4'] = race
l = {"ladderMembers": members}
p = {season: [{"ladder": [pld], "characters": ppds}]}
# To simplify l_bid = p_bid.
return ApiLadder(l, '/ladder/%d/' % l_bid), ApiPlayerLadders(
p, '/profile/%d/' % l_bid)
def test_handles_empty(self):
l = deepcopy(API_LADDER_1V1)
del l['team']
al = ApiLadder(l)
self.assertEqual(None, al.first_join())
self.assertEqual(None, al.last_join())
self.assertEqual(0, al.member_count())
self.assertEqual(None, al.max_points())
self.assertEqual([], al.members_for_ranking(1))
def test_parsing_of_4v4_ladder_works(self):
al = ApiLadder(API_LADDER_4V4)
self.assertEqual(from_unix(1479927625), al.first_join())
self.assertEqual(from_unix(1479927625), al.last_join())
self.assertEqual(4, al.member_count())
self.assertEqual(0, al.max_points())
self.assertEqual([{
'join_time': 1479927625,
'bid': 897866,
'realm': 1,
'name': 'AaxeoiS',
'tag': '',
'clan': '',
'race': Race.TERRAN,
'points': 0,
'mmr': 3965,
'wins': 4,
'losses': 1,
}, {
'join_time': 1479927625,
'bid': 1048851,
'realm': 1,
'name': 'Hodn',
'tag': '',
'clan': '',
'race': Race.RANDOM,
'points': 0,
'mmr': 3965,
'wins': 4,
'losses': 1,
}, {
'join_time': 1479927625,
'bid': 1371993,
'realm': 1,
'name': 'Tsakal',
'tag': 'GROF',
'clan': 'Greek Operation Forces',
'race': Race.ZERG,
'points': 0,
'mmr': 3965,
'wins': 4,
'losses': 1,
}, {
'join_time': 1479927625,
'bid': 2972548,
'realm': 1,
'name': 'sakis',
'tag': 'munaki',
'clan': 'munaki',
'race': Race.RANDOM,
'points': 0,
'mmr': 3965,
'wins': 4,
'losses': 1,
}], al.members_for_ranking(4))
def test_skips_empty_member(self):
l = deepcopy(API_LADDER_1V1)
l['team'][0]['member'] = []
al = ApiLadder(l)
self.assertEqual(from_unix(1479018088), al.first_join())
self.assertEqual(from_unix(1479018088), al.last_join())
self.assertEqual(0, al.member_count())
self.assertEqual(381, al.max_points())
self.assertEqual([], al.members_for_ranking(1))
def test_parsing_of_1v1_ladder_works(self):
al = ApiLadder(API_LADDER_1V1)
self.assertEqual(from_unix(1479018088), al.first_join())
self.assertEqual(from_unix(1479018088), al.last_join())
self.assertEqual(1, al.member_count())
self.assertEqual(381, al.max_points())
self.assertEqual([{
'join_time': 1479018088,
'bid': 6205328,
'realm': 1,
'name': 'peimon',
'tag': 'Partha',
'clan': 'boom boom long time',
'points': 381,
'mmr': 2136,
'wins': 12,
'losses': 24,
'race': Race.PROTOSS,
}], al.members_for_ranking(1))
def test_parsing_of_1v1_legacy_ladder_works(self):
al = ApiLadder(LEGACY_API_LADDER_1V1)
self.assertEqual(from_unix(1468138637), al.first_join())
self.assertEqual(from_unix(1468138637), al.last_join())
self.assertEqual(1, al.member_count())
self.assertEqual(101, al.max_points())
self.assertEqual([{
'join_time': 1468138637,
'bid': 6061640,
'realm': 1,
'name': 'QueenOFpaiN',
'tag': 'PIN',
'clan': 'Pain',
'race': Race.ZERG,
'points': 101.0,
'mmr': NO_MMR,
'wins': 4,
'losses': 1,
}], al.members_for_ranking(1))
def countinously_update(regions=None,
check_stop=None,
update_manager=None,
switch_hour=10):
update_manager = update_manager or UpdateManager()
ranking = Ranking.objects.order_by('-id').first()
if ranking.status != Ranking.COMPLETE_WITH_DATA:
raise Exception("ranking %d is not in a good state, clean up" %
ranking.id)
season = ranking.season
cpp = sc2.RankingData(get_db_name(), Enums.INFO)
while not check_stop(throw=False):
# Check if we want to switch to new season.
current_season = Season.get_current_season()
if current_season.id != season.id:
if current_season.id != season.get_next().id:
raise Exception(
"something is really wrong, current season is not next")
if Ladder.objects.filter(season=current_season,
strangeness=Ladder.GOOD).count() > 8:
season = current_season
logger.info(
"switching to rank new season %d multiple new season ladders was detected"
% season.id)
# Do we want to create new ranking? We want to switch around switch_hour UTC every day but not if ranking is
# too young. If too old, switch anyway.
now = utcnow()
if season.id != ranking.season_id:
# Create new ranking based on new season.
ranking = Ranking.objects.create(season=season,
created=now,
data_time=season.start_time(),
min_data_time=season.start_time(),
max_data_time=season.start_time(),
status=Ranking.CREATED)
logger.info("created new ranking %d based on new season %d" %
(ranking.id, season.id))
cpp.clear_team_ranks()
cpp.reconnect_db()
update_manager.save_ranking(cpp, ranking, 0)
elif ((ranking.created + timedelta(hours=48) < now or
(ranking.created + timedelta(hours=12) < now
and now.hour == switch_hour))
and not ranking.season.near_start(now, days=4)):
# Create a new ranking within the season.
cpp.clear_team_ranks()
cpp.reconnect_db()
with transaction.atomic():
new_ranking = Ranking.objects.create(
season=season,
created=now,
data_time=ranking.data_time,
min_data_time=ranking.min_data_time,
max_data_time=ranking.max_data_time,
status=Ranking.CREATED)
# Copy all caches of old ranking to new ranking. Also remake the full ranking while doing so to get
# rid of leave leaguers.
logger.info(
"created new ranking %d basing it on copy of ranking %d, seaons %d"
% (new_ranking.id, ranking.id, season.id))
count = ranking.sources.count()
logger.info(
"copying %d cached ladders from ranking %d to ranking %d and adding them to ranking"
% (count, ranking.id, new_ranking.id))
for i, lc in enumerate(ranking.sources.all(), start=1):
lc.pk = None
lc.created = utcnow()
lc.ladder = None
lc.ranking = ranking
lc.save()
new_ranking.sources.add(lc)
ladder = Ladder.objects.get(region=lc.region, bid=lc.bid)
team_size = Mode.team_size(ladder.mode)
stats = cpp.update_with_ladder(
ladder.id, lc.id, ladder.region, ladder.mode,
ladder.league, ladder.tier, ladder.version,
ladder.season_id, to_unix(lc.updated), team_size,
ApiLadder(lc.data,
lc.url).members_for_ranking(team_size))
if i % 100 == 0:
logger.info(
"copied and added cache %d/%d, player cache size %d, team cache size %d"
% (i, count, stats['player_cache_size'],
stats['team_cache_size']))
ranking = new_ranking
update_manager.save_ranking(cpp, ranking, 0)
else:
logger.info("continuing with ranking %d, season %d" %
(ranking.id, season.id))
cpp.reconnect_db()
cpp.load(ranking.id)
now = utcnow()
until = now.replace(hour=switch_hour, minute=0, second=0)
if until < now:
until += timedelta(hours=24)
update_manager.update_until(ranking=ranking,
cpp=cpp,
regions=regions,
until=until,
check_stop=check_stop)
def run(self, args, logger):
logger.info(
"NOTE: fetching needs to be turned off if repairing latest rank")
ranking = Ranking.objects.get(pk=args.ranking_id)
if ranking.status not in [Ranking.CREATED, Ranking.COMPLETE_WITH_DATA]:
raise Exception("ranking with status %s can not be repaired" %
ranking.status)
# If last in season use all available ladders, not only those connected to ranking.
last_in_season = Ranking.objects.filter(
season=ranking.season).order_by('-id').first()
if last_in_season == ranking:
cursor = connection.cursor()
cursor.execute(
"SELECT id FROM ("
" SELECT DISTINCT ON (c.bid, c.region) c.id, c.updated FROM cache c JOIN ladder l"
" ON c.bid = l.bid AND c.region = l.region"
" WHERE l.strangeness = %s AND l.season_id = %s"
" ORDER BY c.bid, c.region, c.updated DESC) s"
" ORDER by updated", [Ladder.GOOD, ranking.season_id])
cache_ids = [row[0] for row in cursor.fetchall()]
cursor.execute(
"UPDATE cache SET ranking_id = NULL WHERE ranking_id = %s",
[ranking.id])
else:
cache_ids = [
c['id']
for c in ranking.sources.values('id').order_by('updated')
]
cpp = sc2.RankingData(get_db_name(), Enums.INFO)
count = len(cache_ids)
for i, id_ in enumerate(cache_ids, start=1):
cache = Cache.objects.get(id=id_)
self.check_stop()
try:
ladder = Ladder.objects.get(season=ranking.season,
region=cache.region,
bid=cache.bid)
except Ladder.DoesNotExist:
raise Exception(
"ladder region %s, bid %s missing in ladder table" %
(cache.region, cache.bid))
if cache.ranking is None and cache.ladder is None:
cache.ranking = ranking
cache.save()
elif cache.ranking != ranking:
logger.info("cache %s was not included in ranking copying" %
cache.id)
cache.id = None
cache.ladder = None
cache.ranking = ranking
cache.save()
logger.info("adding cache %s, ladder %s, %d/%d" %
(cache.id, ladder.id, i, count))
team_size = Mode.team_size(ladder.mode)
cpp.update_with_ladder(
ladder.id, cache.id, ladder.region, ladder.mode, ladder.league,
ladder.tier, ladder.version, ladder.season_id,
to_unix(cache.updated), team_size,
ApiLadder(cache.data).members_for_ranking(team_size))
ranking.set_data_time(ranking.season, cpp)
ranking.save()
self.check_stop()
cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow()))
self.check_stop()
cpp.save_stats(ranking.id, to_unix(utcnow()))
return 0
def test_name_is_truncated_to_12_chars(self):
l = deepcopy(API_LADDER_1V1)
l['team'][0]['member'][0]['legacy_link'][
'name'] = '12345678901234567890#123'
al = ApiLadder(l)
self.assertEqual('123456789012', al.members_for_ranking(1)[0]['name'])
def test_parsing_of_4v4_legacy_ladder_works(self):
al = ApiLadder(LEGACY_API_LADDER_4V4)
self.assertEqual(from_unix(1478313972), al.first_join())
self.assertEqual(from_unix(1478313972), al.last_join())
self.assertEqual(4, al.member_count())
self.assertEqual(0.0, al.max_points())
self.assertEqual([
{
'join_time': 1478313972,
'bid': 6539206,
'realm': 1,
'name': 'Maximus',
'tag': '',
'clan': '',
'race': Race.PROTOSS,
'points': 0.0,
'mmr': NO_MMR,
'wins': 0,
'losses': 5,
},
{
'join_time': 1478313972,
'bid': 6714244,
'realm': 1,
'name': 'Herakles',
'tag': '',
'clan': '',
'race': Race.TERRAN,
'points': 0.0,
'mmr': NO_MMR,
'wins': 0,
'losses': 5,
},
{
'join_time': 1478313972,
'bid': 6718054,
'realm': 1,
'name': 'Amazone',
'tag': '',
'clan': '',
'race': Race.TERRAN,
'points': 0.0,
'mmr': NO_MMR,
'wins': 0,
'losses': 5,
},
{
'join_time': 1478313972,
'bid': 6742389,
'realm': 1,
'name': 'philipp',
'tag': 'TAG',
'clan': 'CLAN',
'race': Race.ZERG,
'points': 0.0,
'mmr': NO_MMR,
'wins': 0,
'losses': 5,
},
], al.members_for_ranking(4))
def test_converts_high_mmr_as_no_mmr(self):
l = deepcopy(API_LADDER_1V1)
l['team'][0]['rating'] = 500342
al = ApiLadder(l)
self.assertEqual(NO_MMR, al.members_for_ranking(1)[0]['mmr'])
def run(self, args, logger):
seasons = list(Season.objects.filter(pk__gte=args.ss_id, pk__lte=args.es_id))
for region in args.regions:
logger.info("processing region %s" % region)
for ladder in Ladder.objects.filter(season__in=seasons, region=region, bid__gte=args.bid).order_by('bid'):
self.check_stop()
context = "ladder %d, region %s, bid %d, %s" %\
(ladder.id, Region.key_by_ids[region], ladder.bid, ladder.info())
try:
lcs = ladder.cached_raw.filter(type=Cache.LADDER).order_by('id')
if len(lcs) != 1:
raise Exception("expected one ladder cache for ladder %d, found %s" %
(ladder.id, [c.id for c in lcs]))
lc = lcs[0]
if lc.bid != ladder.bid or lc.region != ladder.region:
raise Exception("bid or region did not match lc on %s" % ladder.id)
al = ApiLadder(loads(lc.data), lc.url)
pcs = ladder.cached_raw.filter(type=Cache.PLAYER_LADDERS)
if len(pcs) > 1:
raise Exception("expected one player ladders cache for ladder %d, found %s" %
(ladder.id, [c.id for c in pcs]))
pc = pcs[0] if pcs else None
if pc:
if pc.region != ladder.region:
raise Exception("region did not match pc on %s, %s" % ladder.id)
ap = ApiPlayerLadders(loads(pc.data), pc.url)
else:
ap = None
new = Ladder()
join_season, join_valid = get_season_based_on_join_times(al.first_join(), al.last_join())
if ap:
match = ap.refers(ladder.bid)
if not match:
logger.error("%s: failed match" % context)
continue
context += ", match %s" % match
new.season, valid = determine_season(fetch_time=lc.updated, match=match,
join_season=join_season, join_valid=join_valid)
else:
new.season, valid = join_season, join_valid
context += ", match None"
new.version, new.mode, new.league = get_version_mode_league(ladder.bid, new.season, al, ap)
new.strangeness = get_strangeness(lc.updated, al, ap)
for what in args.verify.split(','):
nv = getattr(new, what)
ov = getattr(ladder, what)
if nv != ov:
if new.strangeness in (Ladder.GOOD, Ladder.NOP):
log = logger.error
else:
log = logger.warning
log("%s: failed %s, old %s, new %s" % (context, what, ov, nv))
break
else:
logger.info("%s: success" % context)
except Exception as e:
logger.error("%s: %s(\"%s\")" % (context, e.__class__.__name__, e))
return 0