def process_ladder(self, load=False, save=False, region=Region.EU, fetch_time=None, mode=Mode.TEAM_1V1, version=Version.HOTS, league=League.GOLD, season=None, tier=0, members=None, **kwargs): """ Update a ranking building single member with kwargs or use members if set. """ season = season or self.db.season fetch_time = fetch_time or utcnow() members = members or [gen_member(**kwargs)] if not getattr(self, 'cpp', None): self.cpp = sc2.RankingData(self.db.db_name, Enums.INFO) if load: self.load() self.cpp.update_with_ladder(0, # bid 0, # source_id region, mode, league, tier, version, season.id, to_unix(fetch_time), fetch_time.date().isoformat(), Mode.team_size(mode), members) if save: self.save_to_ranking()
def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['last_updated'], _ = last_updated_info() context['versions'] = list(reversed(Version.ranking_ids)) context['leagues'] = list(reversed(League.ranking_ids)) context['races'] = [r for r in Race.ranking_ids if r != Race.UNKNOWN ] + [Race.UNKNOWN] # Make better order. context['regions'] = [Region.ALL] + Region.ranking_ids context['seasons'] = get_season_list() context['mode_id'] = self.mode_id context['mode_got_race_stats'] = Mode.team_size( self.mode_id) == 1 or self.mode_id == Mode.ARCHON return context
def update_ladder_cache(cpp, ranking, ladder, status, api_ladder, fetch_time): """ Update cache and link/unlink. The fetch is done for a specific ranking and ladder, both provided. Since this is a refetch of a present GOOD ladder (or it is becoming GOOD) only 200 responses are allowed. Transaction should be spanning call to make transaction abortion possible. """ try: lc = ranking.sources.get(region=ladder.region, bid=ladder.bid, type=Cache.LADDER) except Cache.DoesNotExist: lc = None lc = lc or Cache(region=ladder.region, bid=ladder.bid, type=Cache.LADDER, created=fetch_time) lc.ranking = ranking lc.data = api_ladder.to_text() lc.url = api_ladder.url lc.updated = fetch_time lc.status = status lc.save() ladder.updated = fetch_time ladder.last_join = api_ladder.last_join() ladder.max_points = api_ladder.max_points() ladder.member_count = api_ladder.member_count() ladder.strangeness = Ladder.GOOD ladder.save() team_size = Mode.team_size(ladder.mode) return cpp.update_with_ladder(ladder.id, lc.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(lc.updated), lc.updated.date().isoformat(), team_size, api_ladder.members_for_ranking(team_size))
def get(self, request): try: mode = request.GET.get('mode', '').strip() mode_id = Mode.id_by_keys.get(mode, Mode.UNKNOWN_ID) if mode_id == Mode.UNKNOWN_ID: raise BadRequestException("uknown mode '%s'" % mode) players = sorted((self.get_player(url) for url in request.GET.getlist('player')), key=lambda p: p.id) if len(players) != Mode.team_size(mode_id): raise BadRequestException( "team size and player count mismatch") players.extend([None, None, None]) try: team = Team.objects.get(mode=mode_id, member0=players[0], member1=players[1], member2=players[2], member3=players[3]) return HttpResponse(json.dumps({'team_id': team.id}), content_type="application/json", status=200) except Team.DoesNotExist: raise Http404("could not find team") except Http404 as e: return HttpResponse(json.dumps({'message': str(e)}), content_type="application/json", status=404) except BadRequestException as e: return HttpResponse(json.dumps({'message': str(e)}), content_type="application/json", status=400)
def get(self, request, version=None, mode=None, reverse=None, sort_key=None): if sort_key == 'ladder-rank': return redirect( self.redirect_mmr_url(request, 'ladder', version=version, mode=mode, reverse=reverse)) context = self.get_context_data() # # Parse parameters. # region_id, race_id, league_id = self.extract_filters(request) is_reverse, sort_key_id = self.extract_common(reverse, sort_key) version_id, mode_id, team_id, request_offset = self.extract_rest( request, version, mode) # # Fetch data. # team_size = Mode.team_size(mode_id) filter_race = None if mode_id != Mode.ARCHON and team_size > 1 else race_id try: if (request_offset is None or request_offset == 0) and not team_id: key = "%s-%s-%s-%s-%s-%s-%s" % (sort_key_id, version_id, mode_id, is_reverse, league_id, region_id, filter_race) data = cache_value(key, 40, self.fetch_data, sort_key_id, version_id, mode_id, is_reverse=is_reverse, league_id=league_id, region_id=region_id, race_id=filter_race, limit=PAGE_SIZE) else: data = self.fetch_data(sort_key_id, version_id, mode_id, is_reverse=is_reverse, league_id=league_id, region_id=region_id, race_id=filter_race, offset=request_offset, team_id=team_id, limit=PAGE_SIZE) except ClientError as e: logger.exception("Fetch from client error: %s" % str(e)) context[ 'error'] = 'The server is not feeling well, try again later.' return self.render_to_response(context) LadderCommon.update_age(data) context['ladder'] = data context['highlight_team_id'] = team_id context['team_size'] = team_size context['mode_id'] = mode_id # # Build navigation based on current parameters. # paths = { 'version': version, 'mode': mode, 'reverse': reverse, 'sort_key': sort_key, } args = { 'region': Region.key_by_ids.get(region_id, Region.ALL_KEY), 'league': League.key_by_ids.get(league_id), 'race': Race.key_by_ids.get(race_id), 'team_id': team_id, } values = [(Mode.name_by_ids[i], Mode.key_by_ids[i]) for i in sorted(Mode.ids) if i != Mode.UNKNOWN] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='mode', values=values) values = [(Version.name_by_ids[i], Version.key_by_ids[i]) for i in reversed(Version.ids) if i != Version.UNKNOWN] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='version', values=values) context['reverse_visiting'] = 'visiting' if reverse else '' context['reverse_href'] = ladder_url(request, paths, args, 'reverse', '' if reverse else '-') values = [('MMR', 'mmr'), ('League points', 'league-points'), ('Games played', 'played'), ('Wins', 'wins'), ('Losses', 'losses'), ('Win rate', 'win-rate')] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='sort_key', values=values) values = [('All', None)] + [(Race.name_by_ids[i], Race.key_by_ids[i]) for i in Race.ids if i != Race.UNKNOWN] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='race', values=values, highlight=Race.key_by_ids.get(filter_race)) values = [('All', None)] + [ (League.name_by_ids[i], League.key_by_ids[i]) for i in League.ids if i not in (League.UNKNOWN, League.ALL) ] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='league', values=values) values = [(Region.name_by_ids[i], Region.key_by_ids[i]) for i in Region.ids if i != Region.UNKNOWN] LadderCommon.set_nav(context, request, ladder_url, paths, args, name='region', values=values) # # Buld pagination, page is 0 indexed (+1 when displayed). # pages = [] count = data['count'] data_offset = data['offset'] # page size: 4 # ladder: 0 1 2 3| 4 5 6 7| 8 9 10 11|12 13 # offset: ^ => current_page = 3, page_offset = 3 # pages: 0| 1 2 3 4| 5 6 7 8| 9 10 11 12|13 if data_offset >= count: current_page = -1 else: current_page = math.ceil(data_offset / PAGE_SIZE) page_offset = -data_offset % PAGE_SIZE page_count = math.ceil((count + page_offset) / PAGE_SIZE) BEG_SIZE = 4 MID_SIZE = 7 # Should be uneven. END_SIZE = 4 start = 0 end = min(BEG_SIZE, page_count) i = 0 for i in range(start, end): page_index = PAGE_SIZE * i - page_offset pages.append({ 'index': i, 'href': ladder_url(request, paths, args, name='offset', key=page_index) }) start = min(max(i + 1, current_page - MID_SIZE // 2), page_count) end = min(current_page + 1 + MID_SIZE // 2, page_count) if start > i + 1: pages.append({'gap': True}) for i in range(start, end): page_index = PAGE_SIZE * i - page_offset pages.append({ 'index': i, 'href': ladder_url(request, paths, args, name='offset', key=page_index) }) start = max(i + 1, page_count - END_SIZE) end = page_count if start > i + 1: pages.append({'gap': True}) for i in range(start, end): page_index = PAGE_SIZE * i - page_offset pages.append({ 'index': i, 'href': ladder_url(request, paths, args, name='offset', key=page_index) }) context['pages'] = pages if len(pages) > 1 else None context['current_page'] = current_page context['page_offset'] = page_offset context['page_count'] = page_count return self.render_to_response(context)
def countinously_update(regions=None, check_stop=None, update_manager=None, switch_hour=10): update_manager = update_manager or UpdateManager() ranking = Ranking.objects.order_by('-id').first() if ranking.status != Ranking.COMPLETE_WITH_DATA: raise Exception("ranking %d is not in a good state, clean up" % ranking.id) season = ranking.season cpp = sc2.RankingData(get_db_name(), Enums.INFO) while not check_stop(throw=False): # Check if we want to switch to new season. current_season = Season.get_current_season() if current_season.id != season.id: if current_season.id != season.get_next().id: raise Exception( "something is really wrong, current season is not next") if Ladder.objects.filter(season=current_season, strangeness=Ladder.GOOD).count() > 8: season = current_season logger.info( "switching to rank new season %d multiple new season ladders was detected" % season.id) # Do we want to create new ranking? We want to switch around switch_hour UTC every day but not if ranking is # too young. If too old, switch anyway. now = utcnow() if season.id != ranking.season_id: # Create new ranking based on new season. ranking = Ranking.objects.create(season=season, created=now, data_time=season.start_time(), min_data_time=season.start_time(), max_data_time=season.start_time(), status=Ranking.CREATED) logger.info("created new ranking %d based on new season %d" % (ranking.id, season.id)) cpp.clear_team_ranks() cpp.reconnect_db() update_manager.save_ranking(cpp, ranking, 0) elif ((ranking.created + timedelta(hours=48) < now or (ranking.created + timedelta(hours=12) < now and now.hour == switch_hour)) and not ranking.season.near_start(now, days=4)): # Create a new ranking within the season. cpp.clear_team_ranks() cpp.reconnect_db() with transaction.atomic(): new_ranking = Ranking.objects.create( season=season, created=now, data_time=ranking.data_time, min_data_time=ranking.min_data_time, max_data_time=ranking.max_data_time, status=Ranking.CREATED) # Copy all caches of old ranking to new ranking. Also remake the full ranking while doing so to get # rid of leave leaguers. logger.info( "created new ranking %d basing it on copy of ranking %d, seaons %d" % (new_ranking.id, ranking.id, season.id)) count = ranking.sources.count() logger.info( "copying %d cached ladders from ranking %d to ranking %d and adding them to ranking" % (count, ranking.id, new_ranking.id)) for i, lc in enumerate(ranking.sources.all(), start=1): lc.pk = None lc.created = utcnow() lc.ladder = None lc.ranking = ranking lc.save() new_ranking.sources.add(lc) ladder = Ladder.objects.get(region=lc.region, bid=lc.bid) team_size = Mode.team_size(ladder.mode) stats = cpp.update_with_ladder( ladder.id, lc.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(lc.updated), team_size, ApiLadder(lc.data, lc.url).members_for_ranking(team_size)) if i % 100 == 0: logger.info( "copied and added cache %d/%d, player cache size %d, team cache size %d" % (i, count, stats['player_cache_size'], stats['team_cache_size'])) ranking = new_ranking update_manager.save_ranking(cpp, ranking, 0) else: logger.info("continuing with ranking %d, season %d" % (ranking.id, season.id)) cpp.reconnect_db() cpp.load(ranking.id) now = utcnow() until = now.replace(hour=switch_hour, minute=0, second=0) if until < now: until += timedelta(hours=24) update_manager.update_until(ranking=ranking, cpp=cpp, regions=regions, until=until, check_stop=check_stop)
def team_size(mode): return Mode.team_size(mode)
def run(self, args, logger): logger.info( "NOTE: fetching needs to be turned off if repairing latest rank") ranking = Ranking.objects.get(pk=args.ranking_id) if ranking.status not in [Ranking.CREATED, Ranking.COMPLETE_WITH_DATA]: raise Exception("ranking with status %s can not be repaired" % ranking.status) # If last in season use all available ladders, not only those connected to ranking. last_in_season = Ranking.objects.filter( season=ranking.season).order_by('-id').first() if last_in_season == ranking: cursor = connection.cursor() cursor.execute( "SELECT id FROM (" " SELECT DISTINCT ON (c.bid, c.region) c.id, c.updated FROM cache c JOIN ladder l" " ON c.bid = l.bid AND c.region = l.region" " WHERE l.strangeness = %s AND l.season_id = %s" " ORDER BY c.bid, c.region, c.updated DESC) s" " ORDER by updated", [Ladder.GOOD, ranking.season_id]) cache_ids = [row[0] for row in cursor.fetchall()] cursor.execute( "UPDATE cache SET ranking_id = NULL WHERE ranking_id = %s", [ranking.id]) else: cache_ids = [ c['id'] for c in ranking.sources.values('id').order_by('updated') ] cpp = sc2.RankingData(get_db_name(), Enums.INFO) count = len(cache_ids) for i, id_ in enumerate(cache_ids, start=1): cache = Cache.objects.get(id=id_) self.check_stop() try: ladder = Ladder.objects.get(season=ranking.season, region=cache.region, bid=cache.bid) except Ladder.DoesNotExist: raise Exception( "ladder region %s, bid %s missing in ladder table" % (cache.region, cache.bid)) if cache.ranking is None and cache.ladder is None: cache.ranking = ranking cache.save() elif cache.ranking != ranking: logger.info("cache %s was not included in ranking copying" % cache.id) cache.id = None cache.ladder = None cache.ranking = ranking cache.save() logger.info("adding cache %s, ladder %s, %d/%d" % (cache.id, ladder.id, i, count)) team_size = Mode.team_size(ladder.mode) cpp.update_with_ladder( ladder.id, cache.id, ladder.region, ladder.mode, ladder.league, ladder.tier, ladder.version, ladder.season_id, to_unix(cache.updated), team_size, ApiLadder(cache.data).members_for_ranking(team_size)) ranking.set_data_time(ranking.season, cpp) ranking.save() self.check_stop() cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow())) self.check_stop() cpp.save_stats(ranking.id, to_unix(utcnow())) return 0