def create_ladder(self, **kwargs):
kwargs = merge_args(
{
'bid': 1,
'region': Region.EU,
'strangeness': Ladder.GOOD,
'league': League.GOLD,
'tier': 0,
'version': Version.HOTS,
'mode': Mode.TEAM_1V1,
'season': self.season,
'first_join': utcnow(),
'last_join': utcnow(),
'created': utcnow(),
'updated': utcnow(),
'max_points': 20
}, **kwargs)
self.ladder = Ladder(**kwargs)
self.ladder.save()
return self.ladder
class Db(object):
""" Helper class to create test data in db. """
def __init__(self):
self.db_name = connection.settings_dict['NAME']
print(self.db_name)
self.team = None
self.cache = None
self.player = None
self.season = None
self.ladder = None
self.ranking = None
self.ranking_data = None
self.default_ranking_data__data = {}
self.clear_defaults()
def clear_defaults(self):
self.default_ranking_data__data = {}
@staticmethod
def filter(klass, *args, **kwargs):
return klass.objects.filter(*args, **kwargs)
@staticmethod
def count(klass, *args, **kwargs):
return klass.objects.filter(*args, **kwargs).count()
@staticmethod
def get(klass, *args, **kwargs):
return klass.objects.get(*args, **kwargs)
@staticmethod
def objects(klass):
return klass.objects
@staticmethod
def all(klass):
return klass.objects.all()
@staticmethod
def execute(sql):
cursor = connection.cursor()
cursor.execute(sql)
return cursor
def delete_all(self, keep=None, delete=None):
for d in (delete or [
RankingData, RankingStats, Cache, Ranking, Team, Player,
Ladder, Season
]):
if d not in (keep or []):
if d == Ranking:
cursor = connection.cursor()
cursor.execute("update cache set ranking_id = NULL")
if d == Ladder:
cursor = connection.cursor()
cursor.execute("update cache set ladder_id = NULL")
self.all(d).delete()
def create_season(self, **kwargs):
kwargs = merge_args(
{
'id': 16,
'start_date': '2013-11-11',
'end_date': '2014-01-03',
'name': '2013 Season 6',
'year': 2013,
'number': 6,
'version': Version.HOTS
}, **kwargs)
try:
self.get(Season, id=kwargs['id'])
raise AssertionError("Season with id %d already exists." %
kwargs['id'])
except Season.DoesNotExist:
pass
self.season = Season(**kwargs)
self.season.save()
return self.season
def create_cache(self, type=Cache.LADDER, members=None, **kwargs):
data = kwargs.pop('data', None)
if data is None and members is not None:
data = gen_ladder_data(members)
kwargs = merge_args(
{
'bid': randint(1, 1e6),
'url': 'http://bnet/' + uniqueid(10),
'type': type,
'region': Region.EU,
'created': utcnow(),
'updated': utcnow(),
'status': 200,
'retry_count': 0
}, **kwargs)
kwargs['data'] = json.dumps(data)
self.cache = Cache(**kwargs)
self.cache.save()
return self.cache
def create_ladder(self, **kwargs):
kwargs = merge_args(
{
'bid': 1,
'region': Region.EU,
'strangeness': Ladder.GOOD,
'league': League.GOLD,
'tier': 0,
'version': Version.HOTS,
'mode': Mode.TEAM_1V1,
'season': self.season,
'first_join': utcnow(),
'last_join': utcnow(),
'created': utcnow(),
'updated': utcnow(),
'max_points': 20
}, **kwargs)
self.ladder = Ladder(**kwargs)
self.ladder.save()
return self.ladder
def create_player(self, **kwargs):
kwargs = merge_args(
{
'bid': randint(0, 1e9),
'region': Region.EU,
'realm': 1,
'mode': Mode.TEAM_1V1,
'season': self.season,
'race': Race.ZERG,
'name': uniqueid(12),
'clan': uniqueid(32),
'tag': uniqueid(6)
}, **kwargs)
self.player = Player(**kwargs)
self.player.save()
return self.player
def create_team(self, **kwargs):
kwargs = merge_args(
dict(region=Region.EU,
mode=Mode.TEAM_1V1,
season=self.season,
version=Version.HOTS,
league=League.GOLD,
member0=self.player,
member1=None,
member2=None,
member3=None,
race0=Race.ZERG,
race1=Race.UNKNOWN,
race2=Race.UNKNOWN,
race3=Race.UNKNOWN), **kwargs)
self.team = Team(**kwargs)
self.team.save()
return self.team
def create_teams(self, count=1, **kwargs):
teams = []
for i in range(count):
self.create_player(name="%s-%d" % (uniqueid(8), i))
teams.append(self.create_team(**kwargs))
return teams
def get_teams_by_member0_bids(self, *bids, mode=Mode.TEAM_1V1):
tids = []
for bid in bids:
p = self.get(Player, bid=bid)
tids.append(self.get(Team, member0=p, mode=mode).id)
return tids
def create_ranking(self, **kwargs):
kwargs = merge_args(
dict(created=utcnow(),
data_time=utcnow(),
min_data_time=utcnow(),
max_data_time=utcnow(),
status=Ranking.COMPLETE_WITH_DATA,
season=self.season), **kwargs)
self.ranking = Ranking.objects.create(**kwargs)
return self.ranking
def _default_team_rank(self, team_rank):
""" Update a team_rank dict with defaults. """
for k, v in self.default_ranking_data__data.items():
team_rank.setdefault(k, v)
team_rank.setdefault("team_id", self.team.id)
team_rank.setdefault("data_time", to_unix(self.ranking.data_time))
team_rank.setdefault("version", Version.HOTS)
team_rank.setdefault("region", Region.EU)
team_rank.setdefault("mode", Mode.TEAM_1V1)
team_rank.setdefault("league", League.GOLD)
team_rank.setdefault("tier", 0)
team_rank.setdefault("ladder_id", self.ladder.id)
team_rank.setdefault("join_time", to_unix(self.ranking.data_time))
team_rank.setdefault("source_id", self.cache.id)
team_rank.setdefault("mmr", 1000)
team_rank.setdefault("points", 100.0)
team_rank.setdefault("wins", 10)
team_rank.setdefault("losses", 10)
team_rank.setdefault("race0", Race.ZERG)
team_rank.setdefault("race1", Race.UNKNOWN)
team_rank.setdefault("race2", Race.UNKNOWN)
team_rank.setdefault("race3", Race.UNKNOWN)
team_rank.setdefault("ladder_rank", 1)
team_rank.setdefault("ladder_count", 1)
team_rank.setdefault("league_rank", 1)
team_rank.setdefault("league_count", 1)
team_rank.setdefault("region_rank", 1)
team_rank.setdefault("region_count", 1)
team_rank.setdefault("world_rank", 1)
team_rank.setdefault("world_count", 1)
def create_ranking_data(self, raw=True, **kwargs):
kwargs = merge_args(dict(ranking=self.ranking, updated=utcnow()),
kwargs)
data = kwargs.pop('data', [])
ranking = kwargs['ranking']
for team_rank in data:
self._default_team_rank(team_rank)
ranking.sources.add(self.get(Cache, pk=team_rank['source_id']))
self.ranking_data = RankingData.objects.create(**kwargs)
sc2.save_ranking_data_raw(self.db_name, ranking.id, 0, data, True)
if not raw:
cpp = sc2.RankingData(self.db_name, Enums.INFO)
cpp.load(ranking.id)
cpp.save_data(ranking.id, ranking.season_id, to_unix(utcnow()))
cpp.release()
return self.ranking_data
def update_ranking_stats(self, ranking_id=None):
""" Will build ranking stats based of the ranking by calling c++. """
if ranking_id is None: ranking_id = self.ranking.id
cpp = sc2.RankingData(self.db_name, Enums.INFO)
cpp.load(ranking_id)
cpp.save_stats(ranking_id, to_unix(utcnow()))
cpp.release()
def fetch_new_ladder(bnet_client, season, region, version, mode, league, tier,
bid):
""" Fetch a previously unknown ladder, save ladder and cache. """
res = bnet_client.fetch_ladder(season.id, region, bid)
al = res.api_ladder
if res.status == 503:
logger.error("got 503 from %s, exiting" % al.url)
raise SystemExit()
ladder = Ladder(region=region,
bid=bid,
created=res.fetch_time,
updated=res.fetch_time)
ladder.season = season
ladder.version = version
ladder.mode = mode
ladder.league = league
ladder.tier = tier
ladder.strangeness = Ladder.MISSING
try:
# Sanity check, there should never be a cache already.
cache = Cache.objects.get(region=region, bid=bid, type=Cache.LADDER)
raise Exception(
"cache %d already exists for region %s, bid %s, did not expect this"
% (cache.id, region, bid))
except Cache.DoesNotExist:
cache = Cache(region=region,
bid=bid,
type=Cache.LADDER,
url=al.url,
created=res.fetch_time,
retry_count=0)
cache.status = res.status
cache.data = al.to_text()
cache.updated = res.fetch_time
if res.status == 200:
ladder.strangeness = Ladder.GOOD
ladder.max_points = al.max_points()
ladder.first_join = al.first_join()
ladder.last_join = al.last_join()
ladder.member_count = al.member_count()
ladder.save()
cache.ladder = ladder
cache.save()
logger.info("saved new ladder %s (bid %d) as %s with cache %d" %
(ladder.id, ladder.bid, ladder.info(), cache.id))
def run(self, args, logger):
seasons = list(Season.objects.filter(pk__gte=args.ss_id, pk__lte=args.es_id))
for region in args.regions:
logger.info("processing region %s" % region)
for ladder in Ladder.objects.filter(season__in=seasons, region=region, bid__gte=args.bid).order_by('bid'):
self.check_stop()
context = "ladder %d, region %s, bid %d, %s" %\
(ladder.id, Region.key_by_ids[region], ladder.bid, ladder.info())
try:
lcs = ladder.cached_raw.filter(type=Cache.LADDER).order_by('id')
if len(lcs) != 1:
raise Exception("expected one ladder cache for ladder %d, found %s" %
(ladder.id, [c.id for c in lcs]))
lc = lcs[0]
if lc.bid != ladder.bid or lc.region != ladder.region:
raise Exception("bid or region did not match lc on %s" % ladder.id)
al = ApiLadder(loads(lc.data), lc.url)
pcs = ladder.cached_raw.filter(type=Cache.PLAYER_LADDERS)
if len(pcs) > 1:
raise Exception("expected one player ladders cache for ladder %d, found %s" %
(ladder.id, [c.id for c in pcs]))
pc = pcs[0] if pcs else None
if pc:
if pc.region != ladder.region:
raise Exception("region did not match pc on %s, %s" % ladder.id)
ap = ApiPlayerLadders(loads(pc.data), pc.url)
else:
ap = None
new = Ladder()
join_season, join_valid = get_season_based_on_join_times(al.first_join(), al.last_join())
if ap:
match = ap.refers(ladder.bid)
if not match:
logger.error("%s: failed match" % context)
continue
context += ", match %s" % match
new.season, valid = determine_season(fetch_time=lc.updated, match=match,
join_season=join_season, join_valid=join_valid)
else:
new.season, valid = join_season, join_valid
context += ", match None"
new.version, new.mode, new.league = get_version_mode_league(ladder.bid, new.season, al, ap)
new.strangeness = get_strangeness(lc.updated, al, ap)
for what in args.verify.split(','):
nv = getattr(new, what)
ov = getattr(ladder, what)
if nv != ov:
if new.strangeness in (Ladder.GOOD, Ladder.NOP):
log = logger.error
else:
log = logger.warning
log("%s: failed %s, old %s, new %s" % (context, what, ov, nv))
break
else:
logger.info("%s: success" % context)
except Exception as e:
logger.error("%s: %s(\"%s\")" % (context, e.__class__.__name__, e))
return 0