def process_ladder(self, load=False, save=False, region=Region.EU, fetch_time=None,
mode=Mode.TEAM_1V1, version=Version.HOTS, league=League.GOLD, season=None, tier=0,
members=None, **kwargs):
""" Update a ranking building single member with kwargs or use members if set. """
season = season or self.db.season
fetch_time = fetch_time or utcnow()
members = members or [gen_member(**kwargs)]
if not getattr(self, 'cpp', None):
self.cpp = sc2.RankingData(self.db.db_name, Enums.INFO)
if load:
self.load()
self.cpp.update_with_ladder(0, # bid
0, # source_id
region,
mode,
league,
tier,
version,
season.id,
to_unix(fetch_time),
fetch_time.date().isoformat(),
Mode.team_size(mode),
members)
if save:
self.save_to_ranking()
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['last_updated'], _ = last_updated_info()
context['versions'] = list(reversed(Version.ranking_ids))
context['leagues'] = list(reversed(League.ranking_ids))
context['races'] = [r for r in Race.ranking_ids if r != Race.UNKNOWN
] + [Race.UNKNOWN] # Make better order.
context['regions'] = [Region.ALL] + Region.ranking_ids
context['seasons'] = get_season_list()
context['mode_id'] = self.mode_id
context['mode_got_race_stats'] = Mode.team_size(
self.mode_id) == 1 or self.mode_id == Mode.ARCHON
return context
def update_ladder_cache(cpp, ranking, ladder, status, api_ladder, fetch_time):
"""
Update cache and link/unlink. The fetch is done for a specific ranking and ladder, both provided. Since this is a
refetch of a present GOOD ladder (or it is becoming GOOD) only 200 responses are allowed. Transaction should be
spanning call to make transaction abortion possible.
"""
try:
lc = ranking.sources.get(region=ladder.region, bid=ladder.bid, type=Cache.LADDER)
except Cache.DoesNotExist:
lc = None
lc = lc or Cache(region=ladder.region,
bid=ladder.bid,
type=Cache.LADDER,
created=fetch_time)
lc.ranking = ranking
lc.data = api_ladder.to_text()
lc.url = api_ladder.url
lc.updated = fetch_time
lc.status = status
lc.save()
ladder.updated = fetch_time
ladder.last_join = api_ladder.last_join()
ladder.max_points = api_ladder.max_points()
ladder.member_count = api_ladder.member_count()
ladder.strangeness = Ladder.GOOD
ladder.save()
team_size = Mode.team_size(ladder.mode)
return cpp.update_with_ladder(ladder.id,
lc.id,
ladder.region,
ladder.mode,
ladder.league,
ladder.tier,
ladder.version,
ladder.season_id,
to_unix(lc.updated),
lc.updated.date().isoformat(),
team_size,
api_ladder.members_for_ranking(team_size))
def get(self, request):
try:
mode = request.GET.get('mode', '').strip()
mode_id = Mode.id_by_keys.get(mode, Mode.UNKNOWN_ID)
if mode_id == Mode.UNKNOWN_ID:
raise BadRequestException("uknown mode '%s'" % mode)
players = sorted((self.get_player(url)
for url in request.GET.getlist('player')),
key=lambda p: p.id)
if len(players) != Mode.team_size(mode_id):
raise BadRequestException(
"team size and player count mismatch")
players.extend([None, None, None])
try:
team = Team.objects.get(mode=mode_id,
member0=players[0],
member1=players[1],
member2=players[2],
member3=players[3])
return HttpResponse(json.dumps({'team_id': team.id}),
content_type="application/json",
status=200)
except Team.DoesNotExist:
raise Http404("could not find team")
except Http404 as e:
return HttpResponse(json.dumps({'message': str(e)}),
content_type="application/json",
status=404)
except BadRequestException as e:
return HttpResponse(json.dumps({'message': str(e)}),
content_type="application/json",
status=400)
def get(self,
request,
version=None,
mode=None,
reverse=None,
sort_key=None):
if sort_key == 'ladder-rank':
return redirect(
self.redirect_mmr_url(request,
'ladder',
version=version,
mode=mode,
reverse=reverse))
context = self.get_context_data()
#
# Parse parameters.
#
region_id, race_id, league_id = self.extract_filters(request)
is_reverse, sort_key_id = self.extract_common(reverse, sort_key)
version_id, mode_id, team_id, request_offset = self.extract_rest(
request, version, mode)
#
# Fetch data.
#
team_size = Mode.team_size(mode_id)
filter_race = None if mode_id != Mode.ARCHON and team_size > 1 else race_id
try:
if (request_offset is None or request_offset == 0) and not team_id:
key = "%s-%s-%s-%s-%s-%s-%s" % (sort_key_id, version_id,
mode_id, is_reverse, league_id,
region_id, filter_race)
data = cache_value(key,
40,
self.fetch_data,
sort_key_id,
version_id,
mode_id,
is_reverse=is_reverse,
league_id=league_id,
region_id=region_id,
race_id=filter_race,
limit=PAGE_SIZE)
else:
data = self.fetch_data(sort_key_id,
version_id,
mode_id,
is_reverse=is_reverse,
league_id=league_id,
region_id=region_id,
race_id=filter_race,
offset=request_offset,
team_id=team_id,
limit=PAGE_SIZE)
except ClientError as e:
logger.exception("Fetch from client error: %s" % str(e))
context[
'error'] = 'The server is not feeling well, try again later.'
return self.render_to_response(context)
LadderCommon.update_age(data)
context['ladder'] = data
context['highlight_team_id'] = team_id
context['team_size'] = team_size
context['mode_id'] = mode_id
#
# Build navigation based on current parameters.
#
paths = {
'version': version,
'mode': mode,
'reverse': reverse,
'sort_key': sort_key,
}
args = {
'region': Region.key_by_ids.get(region_id, Region.ALL_KEY),
'league': League.key_by_ids.get(league_id),
'race': Race.key_by_ids.get(race_id),
'team_id': team_id,
}
values = [(Mode.name_by_ids[i], Mode.key_by_ids[i])
for i in sorted(Mode.ids) if i != Mode.UNKNOWN]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='mode',
values=values)
values = [(Version.name_by_ids[i], Version.key_by_ids[i])
for i in reversed(Version.ids) if i != Version.UNKNOWN]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='version',
values=values)
context['reverse_visiting'] = 'visiting' if reverse else ''
context['reverse_href'] = ladder_url(request, paths, args, 'reverse',
'' if reverse else '-')
values = [('MMR', 'mmr'), ('League points', 'league-points'),
('Games played', 'played'), ('Wins', 'wins'),
('Losses', 'losses'), ('Win rate', 'win-rate')]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='sort_key',
values=values)
values = [('All', None)] + [(Race.name_by_ids[i], Race.key_by_ids[i])
for i in Race.ids if i != Race.UNKNOWN]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='race',
values=values,
highlight=Race.key_by_ids.get(filter_race))
values = [('All', None)] + [
(League.name_by_ids[i], League.key_by_ids[i])
for i in League.ids if i not in (League.UNKNOWN, League.ALL)
]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='league',
values=values)
values = [(Region.name_by_ids[i], Region.key_by_ids[i])
for i in Region.ids if i != Region.UNKNOWN]
LadderCommon.set_nav(context,
request,
ladder_url,
paths,
args,
name='region',
values=values)
#
# Buld pagination, page is 0 indexed (+1 when displayed).
#
pages = []
count = data['count']
data_offset = data['offset']
# page size: 4
# ladder: 0 1 2 3| 4 5 6 7| 8 9 10 11|12 13
# offset: ^ => current_page = 3, page_offset = 3
# pages: 0| 1 2 3 4| 5 6 7 8| 9 10 11 12|13
if data_offset >= count:
current_page = -1
else:
current_page = math.ceil(data_offset / PAGE_SIZE)
page_offset = -data_offset % PAGE_SIZE
page_count = math.ceil((count + page_offset) / PAGE_SIZE)
BEG_SIZE = 4
MID_SIZE = 7 # Should be uneven.
END_SIZE = 4
start = 0
end = min(BEG_SIZE, page_count)
i = 0
for i in range(start, end):
page_index = PAGE_SIZE * i - page_offset
pages.append({
'index':
i,
'href':
ladder_url(request, paths, args, name='offset', key=page_index)
})
start = min(max(i + 1, current_page - MID_SIZE // 2), page_count)
end = min(current_page + 1 + MID_SIZE // 2, page_count)
if start > i + 1: pages.append({'gap': True})
for i in range(start, end):
page_index = PAGE_SIZE * i - page_offset
pages.append({
'index':
i,
'href':
ladder_url(request, paths, args, name='offset', key=page_index)
})
start = max(i + 1, page_count - END_SIZE)
end = page_count
if start > i + 1: pages.append({'gap': True})
for i in range(start, end):
page_index = PAGE_SIZE * i - page_offset
pages.append({
'index':
i,
'href':
ladder_url(request, paths, args, name='offset', key=page_index)
})
context['pages'] = pages if len(pages) > 1 else None
context['current_page'] = current_page
context['page_offset'] = page_offset
context['page_count'] = page_count
return self.render_to_response(context)
def countinously_update(regions=None,
check_stop=None,
update_manager=None,
switch_hour=10):
update_manager = update_manager or UpdateManager()
ranking = Ranking.objects.order_by('-id').first()
if ranking.status != Ranking.COMPLETE_WITH_DATA:
raise Exception("ranking %d is not in a good state, clean up" %
ranking.id)
season = ranking.season
cpp = sc2.RankingData(get_db_name(), Enums.INFO)
while not check_stop(throw=False):
# Check if we want to switch to new season.
current_season = Season.get_current_season()
if current_season.id != season.id:
if current_season.id != season.get_next().id:
raise Exception(
"something is really wrong, current season is not next")
if Ladder.objects.filter(season=current_season,
strangeness=Ladder.GOOD).count() > 8:
season = current_season
logger.info(
"switching to rank new season %d multiple new season ladders was detected"
% season.id)
# Do we want to create new ranking? We want to switch around switch_hour UTC every day but not if ranking is
# too young. If too old, switch anyway.
now = utcnow()
if season.id != ranking.season_id:
# Create new ranking based on new season.
ranking = Ranking.objects.create(season=season,
created=now,
data_time=season.start_time(),
min_data_time=season.start_time(),
max_data_time=season.start_time(),
status=Ranking.CREATED)
logger.info("created new ranking %d based on new season %d" %
(ranking.id, season.id))
cpp.clear_team_ranks()
cpp.reconnect_db()
update_manager.save_ranking(cpp, ranking, 0)
elif ((ranking.created + timedelta(hours=48) < now or
(ranking.created + timedelta(hours=12) < now
and now.hour == switch_hour))
and not ranking.season.near_start(now, days=4)):
# Create a new ranking within the season.
cpp.clear_team_ranks()
cpp.reconnect_db()
with transaction.atomic():
new_ranking = Ranking.objects.create(
season=season,
created=now,
data_time=ranking.data_time,
min_data_time=ranking.min_data_time,
max_data_time=ranking.max_data_time,
status=Ranking.CREATED)
# Copy all caches of old ranking to new ranking. Also remake the full ranking while doing so to get
# rid of leave leaguers.
logger.info(
"created new ranking %d basing it on copy of ranking %d, seaons %d"
% (new_ranking.id, ranking.id, season.id))
count = ranking.sources.count()
logger.info(
"copying %d cached ladders from ranking %d to ranking %d and adding them to ranking"
% (count, ranking.id, new_ranking.id))
for i, lc in enumerate(ranking.sources.all(), start=1):
lc.pk = None
lc.created = utcnow()
lc.ladder = None
lc.ranking = ranking
lc.save()
new_ranking.sources.add(lc)
ladder = Ladder.objects.get(region=lc.region, bid=lc.bid)
team_size = Mode.team_size(ladder.mode)
stats = cpp.update_with_ladder(
ladder.id, lc.id, ladder.region, ladder.mode,
ladder.league, ladder.tier, ladder.version,
ladder.season_id, to_unix(lc.updated), team_size,
ApiLadder(lc.data,
lc.url).members_for_ranking(team_size))
if i % 100 == 0:
logger.info(
"copied and added cache %d/%d, player cache size %d, team cache size %d"
% (i, count, stats['player_cache_size'],
stats['team_cache_size']))
ranking = new_ranking
update_manager.save_ranking(cpp, ranking, 0)
else:
logger.info("continuing with ranking %d, season %d" %
(ranking.id, season.id))
cpp.reconnect_db()
cpp.load(ranking.id)
now = utcnow()
until = now.replace(hour=switch_hour, minute=0, second=0)
if until < now:
until += timedelta(hours=24)
update_manager.update_until(ranking=ranking,
cpp=cpp,
regions=regions,
until=until,
check_stop=check_stop)
def team_size(mode):
return Mode.team_size(mode)
def run(self, args, logger):
logger.info(
"NOTE: fetching needs to be turned off if repairing latest rank")
ranking = Ranking.objects.get(pk=args.ranking_id)
if ranking.status not in [Ranking.CREATED, Ranking.COMPLETE_WITH_DATA]:
raise Exception("ranking with status %s can not be repaired" %
ranking.status)
# If last in season use all available ladders, not only those connected to ranking.
last_in_season = Ranking.objects.filter(
season=ranking.season).order_by('-id').first()
if last_in_season == ranking:
cursor = connection.cursor()
cursor.execute(
"SELECT id FROM ("
" SELECT DISTINCT ON (c.bid, c.region) c.id, c.updated FROM cache c JOIN ladder l"
" ON c.bid = l.bid AND c.region = l.region"
" WHERE l.strangeness = %s AND l.season_id = %s"
" ORDER BY c.bid, c.region, c.updated DESC) s"
" ORDER by updated", [Ladder.GOOD, ranking.season_id])
cache_ids = [row[0] for row in cursor.fetchall()]
cursor.execute(
"UPDATE cache SET ranking_id = NULL WHERE ranking_id = %s",
[ranking.id])
else:
cache_ids = [
c['id']
for c in ranking.sources.values('id').order_by('updated')
]
cpp = sc2.RankingData(get_db_name(), Enums.INFO)
count = len(cache_ids)
for i, id_ in enumerate(cache_ids, start=1):
cache = Cache.objects.get(id=id_)
self.check_stop()
try:
ladder = Ladder.objects.get(season=ranking.season,
region=cache.region,
bid=cache.bid)
except Ladder.DoesNotExist:
raise Exception(
"ladder region %s, bid %s missing in ladder table" %
(cache.region, cache.bid))
if cache.ranking is None and cache.ladder is None:
cache.ranking = ranking
cache.save()
elif cache.ranking != ranking:
logger.info("cache %s was not included in ranking copying" %
cache.id)
cache.id = None
cache.ladder = None
cache.ranking = ranking
cache.save()
logger.info("adding cache %s, ladder %s, %d/%d" %
(cache.id, ladder.id, i, count))
team_size = Mode.team_size(ladder.mode)
cpp.update_with_ladder(
ladder.id, cache.id, ladder.region, ladder.mode, ladder.league,
ladder.tier, ladder.version, ladder.season_id,
to_unix(cache.updated), team_size,
ApiLadder(cache.data).members_for_ranking(team_size))
ranking.set_data_time(ranking.season, cpp)
ranking.save()
self.check_stop()
cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow()))
self.check_stop()
cpp.save_stats(ranking.id, to_unix(utcnow()))
return 0
