def save_ranking(self, cpp, ranking, queue_length): ranking.set_data_time(ranking.season.reload(), cpp) logger.info( "saving ranking %d, %d updates left in queue not included, new data_time is %s" % (ranking.id, queue_length, ranking.data_time)) cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) cpp.save_stats(ranking.id, to_unix(utcnow())) ranking.status = Ranking.COMPLETE_WITH_DATA ranking.save() # Ping server to reload ranking. try: raw = request_tcp('localhost', 4747, json.dumps({ 'cmd': 'refresh' }).encode('utf-8'), timeout=self.server_ping_timeout) response = json.loads(raw.decode('utf-8')) code = response.get('code') if code == 'ok': logger.info("refresh ping returned ok") else: logger.warning("refresh ping returned %s" % code) except OSError as e: logger.warning("refresh ping to server failed: " + str(e))
def get(self, request, mode_id=None): mode_id = int(mode_id) if not (mode_id in Mode.stat_v1_ids): return HttpResponse(status=404) last_updated = to_unix( cache_value("ranking_stats_last_modified", 600, ranking_stats_last_modified)) now = to_unix(utcnow()) try: if_modified_since = parse_http_date( request.META['HTTP_IF_MODIFIED_SINCE']) except (ValueError, KeyError): if_modified_since = 0 if if_modified_since >= last_updated: response = HttpResponse("", content_type="application/json", status=304) else: response = HttpResponse(cache_value("ranking_stats_%d" % mode_id, 600, rankings_view_client, 'ranking_stats', mode_id), content_type="application/json") response['Cache-Control'] = "max-age=86400" response['Date'] = http_date(now) response['Expires'] = http_date(now + 86400) response['Last-Modified'] = http_date(last_updated) return response
def _default_team_rank(self, team_rank): """ Update a team_rank dict with defaults. """ for k, v in self.default_ranking_data__data.items(): team_rank.setdefault(k, v) team_rank.setdefault("team_id", self.team.id) team_rank.setdefault("data_time", to_unix(self.ranking.data_time)) team_rank.setdefault("version", Version.HOTS) team_rank.setdefault("region", Region.EU) team_rank.setdefault("mode", Mode.TEAM_1V1) team_rank.setdefault("league", League.GOLD) team_rank.setdefault("tier", 0) team_rank.setdefault("ladder_id", self.ladder.id) team_rank.setdefault("join_time", to_unix(self.ranking.data_time)) team_rank.setdefault("source_id", self.cache.id) team_rank.setdefault("mmr", 1000) team_rank.setdefault("points", 100.0) team_rank.setdefault("wins", 10) team_rank.setdefault("losses", 10) team_rank.setdefault("race0", Race.ZERG) team_rank.setdefault("race1", Race.UNKNOWN) team_rank.setdefault("race2", Race.UNKNOWN) team_rank.setdefault("race3", Race.UNKNOWN) team_rank.setdefault("ladder_rank", 1) team_rank.setdefault("ladder_count", 1) team_rank.setdefault("league_rank", 1) team_rank.setdefault("league_count", 1) team_rank.setdefault("region_rank", 1) team_rank.setdefault("region_count", 1) team_rank.setdefault("world_rank", 1) team_rank.setdefault("world_count", 1)
def run(self, args, logger): cpp = sc2.RankingData(get_db_name(), Enums.INFO) for ranking in Ranking.objects.filter(season_id__gte=28): cpp.load(ranking.id) cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) cpp.save_stats(ranking.id, to_unix(utcnow())) return 0
def gen_api_ladder(members=None, team_size=1, url='http://fake-url', gd=True, **kwargs): """ Generate api ladder object from members and other data. Can generalte legacy data or gamedata version. """ if members is None: members = [gen_member(**kwargs)] if gd: return ApiLadder(gen_ladder_data(members, team_size=team_size), url) else: # Skipping races for now, since it is not needed for current set of tests. # races["favoriteRaceP%d" % (i + 1)] = Race.key_by_ids[m['race']].upper() return ApiLadder( { 'ladderMembers': [{ "points": m['points'], "previousRank": 0, "wins": m['wins'], "losses": m['losses'], "highestRank": 0, "joinTimestamp": to_unix(m['join_time']), "character": { "realm": m['realm'], "profilePath": None, "clanName": m['clan'], "id": m['bid'], "clanTag": m['tag'], "displayName": m['name'], } } for m in members] }, url)
def _get_season_list(): return json.dumps([{ "id": season.id, "number": season.number, "year": season.year, "start": to_unix(season.start_time()), "end": to_unix(season.end_time()) if season.end_date else to_unix(utcnow()), "color": "#ff6666" if season.id % 2 == 0 else "#6666ff" } for season in Season.objects.filter( id__gt=14, start_date__isnull=False).order_by('id')])
def process_ladder(self, load=False, save=False, region=Region.EU, fetch_time=None, mode=Mode.TEAM_1V1, version=Version.HOTS, league=League.GOLD, season=None, tier=0, members=None, **kwargs): """ Update a ranking building single member with kwargs or use members if set. """ season = season or self.db.season fetch_time = fetch_time or utcnow() members = members or [gen_member(**kwargs)] if not getattr(self, 'cpp', None): self.cpp = sc2.RankingData(self.db.db_name, Enums.INFO) if load: self.load() self.cpp.update_with_ladder(0, # bid 0, # source_id region, mode, league, tier, version, season.id, to_unix(fetch_time), fetch_time.date().isoformat(), Mode.team_size(mode), members) if save: self.save_to_ranking()
def mock_current_season(self, status=200, season_id=None, start_time=None, fetch_time=None): self.bnet.fetch_current_season = \ Mock(return_value=SeasonResponse(status, ApiSeason({'seasonId': season_id or self.db.season.id, 'startDate': to_unix(start_time or utcnow())}, 'http://fake-url'), fetch_time or utcnow(), 0))
def update_ranking_stats(self, ranking_id=None): """ Will build ranking stats based of the ranking by calling c++. """ if ranking_id is None: ranking_id = self.ranking.id cpp = sc2.RankingData(self.db_name, Enums.INFO) cpp.load(ranking_id) cpp.save_stats(ranking_id, to_unix(utcnow())) cpp.release()
def update_age(data): """ Update age of ladder data to present it as correctly as possible (to not be cached in outer layers). """ now = to_unix(utcnow()) for t in data['teams']: delta = now - int(t["data_time"]) if delta < 3600: t["age"] = "%dm" % max((delta + 60) // 60, 1) else: t["age"] = "%dh" % (delta // 3600)
def gen_member(**kwargs): """ Generate member as returned from ApiLadder and used by process_ladder. """ return merge_args( { "bid": randint(1, 1e6), "realm": 0, "name": uniqueid(length=12), "clan": uniqueid(length=32), "tag": uniqueid(length=6), "join_time": int(to_unix(utcnow())), "points": float(randint(0, 2000)), "wins": randint(0, 200), "mmr": randint(1000, 5000), "losses": randint(0, 200), "race": choice([Race.ZERG, Race.PROTOSS, Race.TERRAN, Race.RANDOM]) }, **kwargs)
def update_ladder_cache(cpp, ranking, ladder, status, api_ladder, fetch_time): """ Update cache and link/unlink. The fetch is done for a specific ranking and ladder, both provided. Since this is a refetch of a present GOOD ladder (or it is becoming GOOD) only 200 responses are allowed. Transaction should be spanning call to make transaction abortion possible. """ try: lc = ranking.sources.get(region=ladder.region, bid=ladder.bid, type=Cache.LADDER) except Cache.DoesNotExist: lc = None lc = lc or Cache(region=ladder.region, bid=ladder.bid, type=Cache.LADDER, created=fetch_time) lc.ranking = ranking lc.data = api_ladder.to_text() lc.url = api_ladder.url lc.updated = fetch_time lc.status = status lc.save() ladder.updated = fetch_time ladder.last_join = api_ladder.last_join() ladder.max_points = api_ladder.max_points() ladder.member_count = api_ladder.member_count() ladder.strangeness = Ladder.GOOD ladder.save() team_size = Mode.team_size(ladder.mode) return cpp.update_with_ladder(ladder.id, lc.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(lc.updated), lc.updated.date().isoformat(), team_size, api_ladder.members_for_ranking(team_size))
def create_ranking_data(self, raw=True, **kwargs): kwargs = merge_args(dict(ranking=self.ranking, updated=utcnow()), kwargs) data = kwargs.pop('data', []) ranking = kwargs['ranking'] for team_rank in data: self._default_team_rank(team_rank) ranking.sources.add(self.get(Cache, pk=team_rank['source_id'])) self.ranking_data = RankingData.objects.create(**kwargs) sc2.save_ranking_data_raw(self.db_name, ranking.id, 0, data, True) if not raw: cpp = sc2.RankingData(self.db_name, Enums.INFO) cpp.load(ranking.id) cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) cpp.release() return self.ranking_data
def archive_unused_caches(self, check_stop=lambda: None): """ One off archive of cache data that is no lunger used or updated save each all data in a data dir file. """ with transaction.atomic(): filename = "%s/archive-ladder-caches-%d.gz" % (self.dir, to_unix(utcnow())) logger.info(f"archiving unused caches to {filename}") with gzip.open(filename, mode='wb') as file: def move_to_file(caches): self.write_caches(file, caches, check_stop) caches.delete() move_to_file(Cache.objects.filter(type=Cache.PLAYER_LADDERS)) move_to_file(Cache.objects.filter(ladder__season_id__lt=LAST_AVAILABLE_SEASON, ranking__isnull=True, type=Cache.LADDER)) move_to_file(Cache.objects.filter(ladder__isnull=True, ranking__isnull=True, type=Cache.LADDER)) return filename
def purge_player_data(check_stop=lambda: None): rankings = list( Ranking.objects.filter( status__in=(Ranking.COMPLETE_WITH_DATA, Ranking.COMPLETE_WITOUT_DATA)).order_by('id')) cpp_purger = sc2.Purger(get_db_name()) for ranking in rankings[:-1]: if check_stop(): return logger.info(f"Purge teams from ranking {ranking.id}.") cpp_purger.purge_removed_teams_from_ranking( ranking.id, to_unix(utcnow()), api_data_purge_date().isoformat()) # Maybe would be better to null here instead of delete, we will see what happens. Team.all_objects.filter(last_seen__lt=api_data_purge_date()).delete() Player.all_objects.filter(last_seen__lt=api_data_purge_date()).delete()
def refetch_past_season(season, now, check_stop, bnet_client): """ Refetch ladders for past seasons. """ start = perf_counter() need_refetch_limit = now - timedelta( days=Season.REFETCH_PAST_REFRESH_WHEN_OLDER_THAN_DAYS) with transaction.atomic(): ranking = Ranking.objects.filter( season=season).order_by('-data_time').first() if ranking is None: logger.warning( f"season {season.id} has no ranking to check refetch past for, this is strange, skipping" ) return # Prevent refetch of sea since sea api endpoints are no longer present, the data will be purged eventually. ladders_query = Ladder.objects.filter( season=season, strangeness=Ladder.GOOD).exclude(region=Region.SEA) last_updated = ladders_query.aggregate(Min('updated'))['updated__min'] if need_refetch_limit < last_updated: logger.info( f"skipping refetch of season {season.id}, it was refetched {last_updated.date()}" ) return ladders = list(ladders_query.filter(updated__lt=need_refetch_limit)) ladders_count = ladders_query.count() logger.info( f"{len(ladders)} (of {ladders_count}) to refetch for season {season.id}" ) # if not ladders: # return # This is kind of bad but since c++ works in it's own db connection we can't fetch ladders and update # ranking in same transaction, which in turn means that if the code fails here ranking needs to be repaired. # TODO Move updating of cache to cpp? How is this done in update? cpp = sc2.RankingData(get_db_name(), Enums.INFO) cpp.load(ranking.id) fetch_time = 0 try: for i, ladder in enumerate(ladders, start=1): check_stop() with transaction.atomic(), LogContext( region=Region.key_by_ids[ladder.region]): status, api_ladder, fetch_time, fetch_duration = \ bnet_client.fetch_ladder(ladder.region, ladder.bid, timeout=20) logger.info("fetched %s got %d in %.2fs, %s (%d/%d)" % (api_ladder.url, status, fetch_duration, ladder.info(), i, len(ladders))) if status == 503: logger.warning( "got 503, skipping refetch past for rest of this season" ) raise SystemExit() if status != 200: logger.info("refetching %d returned %d, skipping ladder" % (ladder.id, status)) continue update_ladder_cache(cpp, ranking, ladder, status, api_ladder, fetch_time) logger.info( "saved updated ladder %d and added data to ranking %d" % (ladder.id, ranking.id)) except SystemExit: pass except Exception as e: raise Exception( "failure while refetching past, you will need to repair ranking %d:" % ranking.id) from e if fetch_time: logger.info("saving ranking data and ranking stats for ranking %d" % ranking.id) cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) cpp.save_stats(ranking.id, to_unix(utcnow())) ranking.set_data_time(season, cpp) ranking.save() else: logger.info( "skipping save of ranking data and ranking stats for ranking %d, nothing changed" % ranking.id) logger.info( f"completed refetch of season {season.id} in {human_i_split(int(perf_counter() - start))} seconds" )
def save_to_ranking(self): self.cpp.save_data(self.db.ranking.id, self.db.ranking.season_id, to_unix(utcnow()))
def unix_time(self, **kwargs): return to_unix(self.now + timedelta(**kwargs))
def refetch_past_season(season, now, check_stop, bnet_client): """ Refetch ladders for past seasons. """ with transaction.atomic(): ranking = Ranking.objects.filter( season=season).order_by('-data_time').first() if ranking is None: logger.warning( "season %d has no ranking to check refetch past for, this is strange, skipping" % season.id) return too_old_limit = now - timedelta( days=Season.REFETCH_PAST_DAYS_AGE_LIMIT) need_refetch_limit = season.end_time() + timedelta( days=Season.REFETCH_PAST_UNTIL_DAYS_AFTER_SEASON_END) ladders = list( Ladder.objects.filter(season=season, strangeness=Ladder.GOOD, updated__lt=need_refetch_limit, updated__gt=too_old_limit)) logger.info("%d ladders to refetch for season %d" % (len(ladders), season.id)) if not ladders: return # This is kind of bad but since c++ works in it's own db connection we can't fetch ladders and update # ranking in same transaction, which in turn means that if the code fails here ranking needs to be repaired. cpp = sc2.RankingData(get_db_name(), Enums.INFO) cpp.load(ranking.id) fetch_time = 0 try: for i, ladder in enumerate(ladders, start=1): check_stop() with transaction.atomic(), LogContext( region=Region.key_by_ids[ladder.region]): status, api_ladder, fetch_time, fetch_duration = \ bnet_client.fetch_ladder(ladder.region, ladder.bid, timeout=20) logger.info("fetched %s got %d in %.2fs, %s (%d/%d)" % (api_ladder.url, status, fetch_duration, ladder.info(), i, len(ladders))) if status == 503: logger.warning( "got 503, skipping refetch past for rest of this season" ) raise SystemExit() if status != 200: logger.info("refetching %d returned %d, skipping" % (ladder.id, status)) continue update_ladder_cache(cpp, ranking, ladder, status, api_ladder, fetch_time) logger.info( "saved updated ladder %d and added data to ranking %d" % (ladder.id, ranking.id)) except SystemExit: pass except Exception as e: raise Exception( "failure while refetching past, you will need to repair ranking %d:" % ranking.id) from e if fetch_time: logger.info("saving ranking data and ranking stats for ranking %d" % ranking.id) cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) cpp.save_stats(ranking.id, to_unix(utcnow())) ranking.data_time = min(season.end_time(), fetch_time) ranking.save()
def countinously_update(regions=None, check_stop=None, update_manager=None, switch_hour=10): update_manager = update_manager or UpdateManager() ranking = Ranking.objects.order_by('-id').first() if ranking.status != Ranking.COMPLETE_WITH_DATA: raise Exception("ranking %d is not in a good state, clean up" % ranking.id) season = ranking.season cpp = sc2.RankingData(get_db_name(), Enums.INFO) while not check_stop(throw=False): # Check if we want to switch to new season. current_season = Season.get_current_season() if current_season.id != season.id: if current_season.id != season.get_next().id: raise Exception( "something is really wrong, current season is not next") if Ladder.objects.filter(season=current_season, strangeness=Ladder.GOOD).count() > 8: season = current_season logger.info( "switching to rank new season %d multiple new season ladders was detected" % season.id) # Do we want to create new ranking? We want to switch around switch_hour UTC every day but not if ranking is # too young. If too old, switch anyway. now = utcnow() if season.id != ranking.season_id: # Create new ranking based on new season. ranking = Ranking.objects.create(season=season, created=now, data_time=season.start_time(), min_data_time=season.start_time(), max_data_time=season.start_time(), status=Ranking.CREATED) logger.info("created new ranking %d based on new season %d" % (ranking.id, season.id)) cpp.clear_team_ranks() cpp.reconnect_db() update_manager.save_ranking(cpp, ranking, 0) elif ((ranking.created + timedelta(hours=48) < now or (ranking.created + timedelta(hours=12) < now and now.hour == switch_hour)) and not ranking.season.near_start(now, days=4)): # Create a new ranking within the season. cpp.clear_team_ranks() cpp.reconnect_db() with transaction.atomic(): new_ranking = Ranking.objects.create( season=season, created=now, data_time=ranking.data_time, min_data_time=ranking.min_data_time, max_data_time=ranking.max_data_time, status=Ranking.CREATED) # Copy all caches of old ranking to new ranking. Also remake the full ranking while doing so to get # rid of leave leaguers. logger.info( "created new ranking %d basing it on copy of ranking %d, seaons %d" % (new_ranking.id, ranking.id, season.id)) count = ranking.sources.count() logger.info( "copying %d cached ladders from ranking %d to ranking %d and adding them to ranking" % (count, ranking.id, new_ranking.id)) for i, lc in enumerate(ranking.sources.all(), start=1): lc.pk = None lc.created = utcnow() lc.ladder = None lc.ranking = ranking lc.save() new_ranking.sources.add(lc) ladder = Ladder.objects.get(region=lc.region, bid=lc.bid) team_size = Mode.team_size(ladder.mode) stats = cpp.update_with_ladder( ladder.id, lc.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(lc.updated), team_size, ApiLadder(lc.data, lc.url).members_for_ranking(team_size)) if i % 100 == 0: logger.info( "copied and added cache %d/%d, player cache size %d, team cache size %d" % (i, count, stats['player_cache_size'], stats['team_cache_size'])) ranking = new_ranking update_manager.save_ranking(cpp, ranking, 0) else: logger.info("continuing with ranking %d, season %d" % (ranking.id, season.id)) cpp.reconnect_db() cpp.load(ranking.id) now = utcnow() until = now.replace(hour=switch_hour, minute=0, second=0) if until < now: until += timedelta(hours=24) update_manager.update_until(ranking=ranking, cpp=cpp, regions=regions, until=until, check_stop=check_stop)
def run(self, args, logger): logger.info( "NOTE: fetching needs to be turned off if repairing latest rank") ranking = Ranking.objects.get(pk=args.ranking_id) if ranking.status not in [Ranking.CREATED, Ranking.COMPLETE_WITH_DATA]: raise Exception("ranking with status %s can not be repaired" % ranking.status) # If last in season use all available ladders, not only those connected to ranking. last_in_season = Ranking.objects.filter( season=ranking.season).order_by('-id').first() if last_in_season == ranking: cursor = connection.cursor() cursor.execute( "SELECT id FROM (" " SELECT DISTINCT ON (c.bid, c.region) c.id, c.updated FROM cache c JOIN ladder l" " ON c.bid = l.bid AND c.region = l.region" " WHERE l.strangeness = %s AND l.season_id = %s" " ORDER BY c.bid, c.region, c.updated DESC) s" " ORDER by updated", [Ladder.GOOD, ranking.season_id]) cache_ids = [row[0] for row in cursor.fetchall()] cursor.execute( "UPDATE cache SET ranking_id = NULL WHERE ranking_id = %s", [ranking.id]) else: cache_ids = [ c['id'] for c in ranking.sources.values('id').order_by('updated') ] cpp = sc2.RankingData(get_db_name(), Enums.INFO) count = len(cache_ids) for i, id_ in enumerate(cache_ids, start=1): cache = Cache.objects.get(id=id_) self.check_stop() try: ladder = Ladder.objects.get(season=ranking.season, region=cache.region, bid=cache.bid) except Ladder.DoesNotExist: raise Exception( "ladder region %s, bid %s missing in ladder table" % (cache.region, cache.bid)) if cache.ranking is None and cache.ladder is None: cache.ranking = ranking cache.save() elif cache.ranking != ranking: logger.info("cache %s was not included in ranking copying" % cache.id) cache.id = None cache.ladder = None cache.ranking = ranking cache.save() logger.info("adding cache %s, ladder %s, %d/%d" % (cache.id, ladder.id, i, count)) team_size = Mode.team_size(ladder.mode) cpp.update_with_ladder( ladder.id, cache.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(cache.updated), team_size, ApiLadder(cache.data).members_for_ranking(team_size)) ranking.set_data_time(ranking.season, cpp) ranking.save() self.check_stop() cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) self.check_stop() cpp.save_stats(ranking.id, to_unix(utcnow())) return 0