def countinously_update(regions=None,
check_stop=None,
update_manager=None,
switch_hour=10):
update_manager = update_manager or UpdateManager()
ranking = Ranking.objects.order_by('-id').first()
if ranking.status != Ranking.COMPLETE_WITH_DATA:
raise Exception("ranking %d is not in a good state, clean up" %
ranking.id)
season = ranking.season
cpp = sc2.RankingData(get_db_name(), Enums.INFO)
while not check_stop(throw=False):
# Check if we want to switch to new season.
current_season = Season.get_current_season()
if current_season.id != season.id:
if current_season.id != season.get_next().id:
raise Exception(
"something is really wrong, current season is not next")
if Ladder.objects.filter(season=current_season,
strangeness=Ladder.GOOD).count() > 8:
season = current_season
logger.info(
"switching to rank new season %d multiple new season ladders was detected"
% season.id)
# Do we want to create new ranking? We want to switch around switch_hour UTC every day but not if ranking is
# too young. If too old, switch anyway.
now = utcnow()
if season.id != ranking.season_id:
# Create new ranking based on new season.
ranking = Ranking.objects.create(season=season,
created=now,
data_time=season.start_time(),
min_data_time=season.start_time(),
max_data_time=season.start_time(),
status=Ranking.CREATED)
logger.info("created new ranking %d based on new season %d" %
(ranking.id, season.id))
cpp.clear_team_ranks()
cpp.reconnect_db()
update_manager.save_ranking(cpp, ranking, 0)
elif ((ranking.created + timedelta(hours=48) < now or
(ranking.created + timedelta(hours=12) < now
and now.hour == switch_hour))
and not ranking.season.near_start(now, days=4)):
# Create a new ranking within the season.
cpp.clear_team_ranks()
cpp.reconnect_db()
with transaction.atomic():
new_ranking = Ranking.objects.create(
season=season,
created=now,
data_time=ranking.data_time,
min_data_time=ranking.min_data_time,
max_data_time=ranking.max_data_time,
status=Ranking.CREATED)
# Copy all caches of old ranking to new ranking. Also remake the full ranking while doing so to get
# rid of leave leaguers.
logger.info(
"created new ranking %d basing it on copy of ranking %d, seaons %d"
% (new_ranking.id, ranking.id, season.id))
count = ranking.sources.count()
logger.info(
"copying %d cached ladders from ranking %d to ranking %d and adding them to ranking"
% (count, ranking.id, new_ranking.id))
for i, lc in enumerate(ranking.sources.all(), start=1):
lc.pk = None
lc.created = utcnow()
lc.ladder = None
lc.ranking = ranking
lc.save()
new_ranking.sources.add(lc)
ladder = Ladder.objects.get(region=lc.region, bid=lc.bid)
team_size = Mode.team_size(ladder.mode)
stats = cpp.update_with_ladder(
ladder.id, lc.id, ladder.region, ladder.mode,
ladder.league, ladder.tier, ladder.version,
ladder.season_id, to_unix(lc.updated), team_size,
ApiLadder(lc.data,
lc.url).members_for_ranking(team_size))
if i % 100 == 0:
logger.info(
"copied and added cache %d/%d, player cache size %d, team cache size %d"
% (i, count, stats['player_cache_size'],
stats['team_cache_size']))
ranking = new_ranking
update_manager.save_ranking(cpp, ranking, 0)
else:
logger.info("continuing with ranking %d, season %d" %
(ranking.id, season.id))
cpp.reconnect_db()
cpp.load(ranking.id)
now = utcnow()
until = now.replace(hour=switch_hour, minute=0, second=0)
if until < now:
until += timedelta(hours=24)
update_manager.update_until(ranking=ranking,
cpp=cpp,
regions=regions,
until=until,
check_stop=check_stop)
def update_until(self,
ranking=None,
cpp=None,
regions=None,
until=None,
check_stop=None,
fetch_manager=None,
bnet_client=None):
""" Update until time until (utc) has passed and code outisde will decide if season and/or ranking switching
should be done. """
bnet_client = bnet_client or BnetClient()
fetch_manager = fetch_manager or FetchManager(ranking, regions,
bnet_client)
try:
logger.info(
"updating season %d, ranking %d, regions %s, until %s" %
(ranking.season_id, ranking.id, regions, until))
last_save = utcnow()
last_season_check = utcnow()
last_gm = utcnow(days=-20)
last_plat = utcnow(days=-20)
last_rest = utcnow(days=-20)
while not check_stop(throw=False):
now = utcnow()
if now > until:
logger.info(
"we reached until time %s, pausing to switch season/ranking"
% until)
break
if now - last_season_check > timedelta(minutes=30):
last_season_check = now
if ranking.season_id != Season.get_current_season().id:
logger.info(
"current season %d is closed, pausing to give chance for season switch"
% ranking.season_id)
break
if now - last_save > timedelta(seconds=60):
self.save_ranking(cpp, ranking,
len(fetch_manager.fetched_queue))
last_save = utcnow(
) # This can take a long time, so get new now again.
if now - last_gm > timedelta(seconds=60):
last_gm = now
fetch_manager.start_gm()
if now - last_plat > timedelta(minutes=10):
last_plat = now
fetch_manager.start_plat()
if now - last_rest > timedelta(minutes=60):
last_rest = now
fetch_manager.start_rest()
try:
ladder, status, api_ladder, fetch_time = fetch_manager.pop(
)
with transaction.atomic():
stats = update_ladder_cache(cpp, ranking, ladder,
status, api_ladder,
fetch_time)
with LogContext(region=ladder.region):
logger.info(
"saved updated ladder %d and added data to ranking %d, "
"updated %d players %d teams, inserted %d players %d teams, "
"cache sizes %d players %d teams" % (
ladder.id,
ranking.id,
stats["updated_player_count"],
stats["updated_team_count"],
stats["inserted_player_count"],
stats["inserted_team_count"],
stats["player_cache_size"],
stats["team_cache_size"],
))
except IndexError:
sleep(0.04)
logger.info("stopped fetching, saving")
fetch_manager.stop()
fetch_manager.join()
self.save_ranking(cpp, ranking, len(fetch_manager.fetched_queue))
except Exception:
fetch_manager.stop()
raise
cpp.release()
def get(self, request, *args, tag=None, reverse=None, sort_key=None, **kwargs):
if sort_key == 'ladder-rank':
return redirect(self.redirect_mmr_url(request, 'clan', tag=tag, reverse=reverse))
context = self.get_context_data()
json_response = 'json' in request.GET
if tag is None:
raise Http404()
region_id, race_id, league_id = self.extract_filters(request)
is_reverse, sort_key_id = self.extract_common(reverse, sort_key)
player = Player.objects.filter(tag=tag, mode=Mode.TEAM_1V1, season=Season.get_current_season()).first()
if not player:
raise Http404()
context['tag'] = tag
context['clan'] = player.clan
try:
data = self.fetch_data(tag, sort_key_id, is_reverse=is_reverse,
league_id=league_id, region_id=region_id, race_id=race_id)
except ClientError as e:
logger.exception("Fetch from client error: %s" % str(e))
context['error'] = 'The server is not feeling well, try again later.'
return self.respond(context, json_response)
LadderCommon.update_age(data)
context['ladder'] = data
context['team_size'] = 1
paths = {
'tag': tag,
'sort_key': sort_key,
'reverse': reverse,
}
args = {
'region': Region.key_by_ids.get(region_id, Region.ALL_KEY),
'league': League.key_by_ids.get(league_id),
'race': Race.key_by_ids.get(race_id),
}
values = [('All', None)] + [(Race.name_by_ids[i], Race.key_by_ids[i])
for i in Race.ids if i != Race.UNKNOWN]
LadderCommon.set_nav(context, request, clan_url, paths, args, name='race', values=values)
values = [('All', None)] + [(League.name_by_ids[i], League.key_by_ids[i])
for i in League.ids if i not in (League.UNKNOWN, League.ALL)]
LadderCommon.set_nav(context, request, clan_url, paths, args, name='league', values=values)
values = [(Region.name_by_ids[i], Region.key_by_ids[i])
for i in Region.ids if i != Region.UNKNOWN]
LadderCommon.set_nav(context, request, clan_url, paths, args, name='region', values=values)
values = [('MMR', 'mmr'), ('League points', 'league-points'), ('Games played', 'played'), ('Wins', 'wins'),
('Losses', 'losses'), ('Win rate', 'win-rate')]
LadderCommon.set_nav(context, request, clan_url, paths, args, name='sort_key', values=values)
context['reverse_visiting'] = 'visiting' if reverse else ''
context['reverse_href'] = clan_url(request, paths, args, 'reverse', '' if reverse else '-')
return self.respond(context, json_response)
def get_top_clans():
current_season = Season.get_current_season()
return list(Player.objects
.filter(season=current_season, mode=Mode.TEAM_1V1, tag__gt='')
.values('clan', 'tag').annotate(count=Count('tag')).order_by('-count')[:32])
def fetch_new(region=None, check_stop=lambda: None, bnet_client=None):
with transaction.atomic():
current_season = Season.get_current_season()
for count in range(1, 6):
check_stop()
res = bnet_client.fetch_current_season(region)
api_season = res.api_season
if region == Region.SEA:
if res.status != 200:
logger.info(
"could not get season info from %s, status %s, bailing"
% (api_season.url, res.status))
return
else:
logger.warning("sea is alive")
break
if res.status == 200:
break
else:
level = INFO if region == Region.CN else WARNING
logger.log(
level,
"could not get season info from %s after %d tries, status %s, bailing"
% (api_season.url, count, res.status))
return
# Update or create the season cache, just for bookkeeping.
try:
cache = Cache.objects.get(region=region, type=Cache.SEASON, bid=1)
cache.updated = res.fetch_time
cache.data = api_season.to_text()
cache.save()
except Cache.DoesNotExist:
Cache.objects.create(url=api_season.url,
bid=1,
type=Cache.SEASON,
region=region,
status=res.status,
created=res.fetch_time,
updated=res.fetch_time,
data=api_season.to_text())
# Check season.
if current_season.id == api_season.season_id():
# We already have latest season.
pass
elif current_season.id + 1 == api_season.season_id():
# New season detected, create new season.
create_new_season(current_season, api_season.start_date())
return
elif current_season.near_start(utcnow(), days=2):
logger.info(
"current season %d near start, blizzard says %d, probably cached or other region, bailing"
% (current_season.id, api_season.season_id()))
return
else:
raise Exception(
"season mismatch blizzard says %d, current in db is %d" %
(api_season.season_id(), current_season.id))
fetch_new_in_region(check_stop, bnet_client, current_season, region)
