def run(self, args, logger):
for region in args.regions:
fetch_new(region=region,
check_stop=self.check_stop,
bnet_client=BnetClient())
return 0
def refetch_past_seasons(check_stop=lambda: None,
bnet_client=None,
now=None,
skip_fetch_new=False):
bnet_client = bnet_client or BnetClient()
now = now or utcnow()
# Wait for this date before refetching season.
season_end_limit = now - timedelta(
days=Season.REFETCH_PAST_MIN_DAYS_AFTER_SEASON_END)
# Fetch new for a while after season close to make sure we got all ladders. Since we don't save non 200 leagues
# we can still miss ladders here, but unlikely. There is no point in continuing after need_refetch_limit since they
# will not be picked up in the ranking anyway.
prev_season = Season.get_current_season().get_prev()
need_refetch_limit = prev_season.end_time() + timedelta(
days=Season.REFETCH_PAST_UNTIL_DAYS_AFTER_SEASON_END)
if not skip_fetch_new and prev_season.end_time(
) < now <= need_refetch_limit:
for region in Region.ranking_ids:
fetch_new_in_region(check_stop, bnet_client, prev_season, region)
# Refetch all past seasons.
for season in Season.objects.filter(
id__gt=14, end_date__lt=season_end_limit).order_by('-id'):
refetch_past_season(season, now, check_stop, bnet_client)
check_stop()
def run(self, args, logger):
client = BnetClient()
season_id = 38
region = Region.EU
logger.info(Region.key_by_ids[region])
logger.info(client.fetch_current_season(region))
logger.info(
client.fetch_league(region, season_id, Version.LOTV, Mode.TEAM_1V1,
League.MASTER))
logger.info(client.fetch_ladder(region, 210556))
return 0
def update_until(self,
ranking=None,
cpp=None,
regions=None,
until=None,
check_stop=None,
fetch_manager=None,
bnet_client=None):
""" Update until time until (utc) has passed and code outisde will decide if season and/or ranking switching
should be done. """
bnet_client = bnet_client or BnetClient()
fetch_manager = fetch_manager or FetchManager(ranking, regions,
bnet_client)
try:
logger.info(
"updating season %d, ranking %d, regions %s, until %s" %
(ranking.season_id, ranking.id, regions, until))
last_save = utcnow()
last_season_check = utcnow()
last_gm = utcnow(days=-20)
last_plat = utcnow(days=-20)
last_rest = utcnow(days=-20)
while not check_stop(throw=False):
now = utcnow()
if now > until:
logger.info(
"we reached until time %s, pausing to switch season/ranking"
% until)
break
if now - last_season_check > timedelta(minutes=30):
last_season_check = now
if ranking.season_id != Season.get_current_season().id:
logger.info(
"current season %d is closed, pausing to give chance for season switch"
% ranking.season_id)
break
if now - last_save > timedelta(seconds=60):
self.save_ranking(cpp, ranking,
len(fetch_manager.fetched_queue))
last_save = utcnow(
) # This can take a long time, so get new now again.
if now - last_gm > timedelta(seconds=60):
last_gm = now
fetch_manager.start_gm()
if now - last_plat > timedelta(minutes=10):
last_plat = now
fetch_manager.start_plat()
if now - last_rest > timedelta(minutes=60):
last_rest = now
fetch_manager.start_rest()
try:
ladder, status, api_ladder, fetch_time = fetch_manager.pop(
)
with transaction.atomic():
stats = update_ladder_cache(cpp, ranking, ladder,
status, api_ladder,
fetch_time)
with LogContext(region=ladder.region):
logger.info(
"saved updated ladder %d and added data to ranking %d, "
"updated %d players %d teams, inserted %d players %d teams, "
"cache sizes %d players %d teams" % (
ladder.id,
ranking.id,
stats["updated_player_count"],
stats["updated_team_count"],
stats["inserted_player_count"],
stats["inserted_team_count"],
stats["player_cache_size"],
stats["team_cache_size"],
))
except IndexError:
sleep(0.04)
logger.info("stopped fetching, saving")
fetch_manager.stop()
fetch_manager.join()
self.save_ranking(cpp, ranking, len(fetch_manager.fetched_queue))
except Exception:
fetch_manager.stop()
raise
cpp.release()
def refetch_missing(region=None,
max_retries=None,
min_age=None,
check_stop=lambda: None,
bnet_client=None):
""" Refetch missing ladders. Note that these ladders will not be added to any ranking if passed current season. """
bnet_client = bnet_client or BnetClient()
# Find the latest ladders.
bids = list(
Cache.objects.values_list('bid', flat=True).filter(
region=region,
type=Cache.LADDER,
retry_count__lt=max_retries,
ladder__strangeness=Ladder.MISSING).exclude(
updated__gt=utcnow() -
timedelta(hours=min_age)).order_by('-bid').all()[:800])
bid_count = len(bids)
logger.info("refetching %d (max 800) non 200 ladders" % bid_count)
for count, bid in enumerate(bids, start=1):
check_stop()
logger.info("refetching missing ladder bid %d, %d/%d" %
(bid, count, bid_count))
res = bnet_client.fetch_ladder(region=region, bid=bid)
al = res.api_ladder
if res.status == 503:
logger.warning("got status 503, breaking for this region")
return
with transaction.atomic():
cache = Cache.objects.get(region=region,
bid=bid,
type=Cache.LADDER)
ladder = cache.ladder
cache.status = res.status
cache.data = al.to_text()
cache.updated = res.fetch_time
ladder.updated = res.fetch_time
if res.status != 200:
cache.retry_count += 1
else:
ladder.strangeness = Ladder.GOOD
ladder.max_points = al.max_points()
ladder.first_join = al.first_join()
ladder.last_join = al.last_join()
ladder.member_count = al.member_count()
cache.retry_count = 0
cache.save()
ladder.save()
if ladder.strangeness == Ladder.GOOD:
logger.info("ladder %s no longer missing, saved as %s" %
(ladder.id, ladder.info()))
else:
logger.info("status is %d, updated cache, retry_count %d" %
(cache.status, cache.retry_count))
def setUp(self):
super().setUp()
self.bnet = BnetClient()