def populate_static_table(endpoint='',
model=None,
payload={},
version=None):
entity_name = model.__name__.lower()
log.debug('Populating {0} table'.format(model.__tablename__))
try:
results = riot_static_api.get(endpoint=endpoint, payload=payload)
except ForbiddenException:
return
results_length = len(results['data'].keys())
log.info('Fetched {0} {1}s'.format(results_length, entity_name))
with DBManager.create_session_scope() as session:
status_cnt = 0
for entity_data in results['data'].values():
status_cnt += 1
print('[*] Status {0}/{1}'.format(status_cnt, results_length),
end='\r')
_process_ctx = getattr(DataProcessContext,
'process_{0}_data'.format(entity_name))
if _process_ctx is None:
raise Exception(
'No data process function for model {0} was '
'defined'.format(entity_name))
_processed_data = _process_ctx(entity_data)
session.merge(
model(**dict(_processed_data, patch_version=version)))
log.debug('Done!')
def populate_matchups(cls, limit=50, region='EUNE'):
session = DBManager.create_session(expire_on_commit=False)
matches = session.query(QueuedMatch).filter_by(region=region).filter(
~exists().where(QueuedMatch.id == CheckedMatch.id))\
.limit(limit).all()
for match in matches:
payload = {
'api_key': RIOT_API_KEY,
'includeTimeline': 'true',
}
log.debug('Fetching info for match {0}'.format(match.id))
url = 'https://{0}.api.pvp.net/api/lol/{0}'.format(region.lower())
endpoint = '/{0}/match/{1}'.format(api_versions['match'], match.id)
try:
result = riot_api.get(url=url,
endpoint=endpoint,
payload=payload)
except ForbiddenException:
continue
except NotFound:
continue
cls.populate_single_matchup(result)
checked_match = CheckedMatch({
'id':
match.id,
'region':
match.region,
'checked_at':
datetime.now(),
'match_timestamp':
match.match_timestamp
})
session.merge(checked_match)
session.commit()
def __init__(self, interval, action, *args, **kwargs):
log.debug('New Interval Worker {0}'.format(action))
self._timer = None
self.interval = interval
self.action = action
self.args = args
self.kwargs = kwargs
self.is_running = False
self.run()
def check_version(self):
latest_versions = StaticDataContext.get_api_version()['versions'][:10]
with open('/var/tmp/onevone/api_version.json', 'r+') as v:
versions = json.load(v)
current_version = versions['versions'][0]
if current_version is None or latest_versions[0] > current_version:
log.debug('Updating Version from {0} to {1}'.format(
current_version, latest_versions[0]))
v.seek(0)
v.truncate()
json.dump({'versions': latest_versions}, v, indent=4)
return False
return True
def populate_queued_matches(limit=50, region='EUNE'):
matches = {}
log.debug('Collecting match information for each player. This might '
'take a while...')
with DBManager.create_session_scope(expire_on_commit=False) as session:
payload = {
'api_key': RIOT_API_KEY,
'rankedQueues':
'TEAM_BUILDER_DRAFT_RANKED_5x5,RANKED_SOLO_5x5',
'seasons': 'SEASON2016,PRESEASON2016,PRESEASON2017',
'beginIndex': 0,
'endIndex': limit
}
latest_game = session.query(QueuedMatch).filter_by(
region=region).order_by(
QueuedMatch.match_timestamp.desc()).first()
if latest_game is not None:
payload['beginTime'] = latest_game.match_timestamp
players = session.query(ProPlayer).all()
url = 'https://{0}.api.pvp.net/api/lol/{0}'.format(region.lower())
for player in players:
endpoint = '/{0}/matchlist/by-summoner/{1}'\
.format(api_versions['matchlist'], player.id)
try:
result = riot_api.get(url=url,
endpoint=endpoint,
payload=payload)
except ForbiddenException:
continue
except NotFound:
continue
for match in result.get("matches", []):
match_data = {
'id': match.get('matchId'),
'region': match.get('region'),
'match_timestamp': match.get('timestamp'),
'added_at': datetime.now()
}
session.merge(QueuedMatch(match_data))
session.commit()
def run(self):
self.update_static_data()
version_checker = IntervalWorker(DAY, self.update_static_data)
log.debug('Found {0} Regions {1}'.format(len(REGIONS),
', '.join(REGIONS)))
log.debug('Booting {0} threads'.format(len(REGIONS) * 3))
for region in REGIONS:
player_worker = IntervalWorker(WEEK,
MatchContext.populate_players,
region=region,
player_limit=100)
match_worker = IntervalWorker(5 * HOUR,
MatchContext.populate_queued_matches,
region=region,
limit=30)
matchup_worker = IntervalWorker(30 * MINUTE,
MatchContext.populate_matchups,
region=region,
limit=125)
average_calc_worker = IntervalWorker(DAY,
MatchContext.populate_averages)
def populate_players(region='EUNE', league='challenger', player_limit=50):
# Get a list with the top 50 challenger players (if any)
log.debug('Fetching top {0} players from {1} in region '
'{2}'.format(player_limit, league, region))
payload = {'api_key': RIOT_API_KEY, 'type': 'RANKED_SOLO_5x5'}
url = 'https://{0}.api.pvp.net/api/lol/{0}'.format(region.lower())
endpoint = '/{0}/league/{1}'.format(api_versions['league'], league)
try:
ladder = riot_api.get(url=url, endpoint=endpoint, payload=payload)
except ForbiddenException:
return
if len(ladder.keys()) == 0:
return
players = [
player for _, player in sorted([(player['leaguePoints'], player)
for player in ladder['entries']],
reverse=True,
key=lambda x: x[0])[:player_limit]
]
results_length = len(players)
log.debug('[+] Fetched {0} players'.format(results_length))
with DBManager.create_session_scope() as session:
for player in players:
player_data = {
'id': player.get('playerOrTeamId'),
'region': region.lower(),
'tier': league.lower(),
'name': player.get('playerOrTeamName')
}
session.merge(ProPlayer(player_data))
log.debug('[+] Done!')
def populate_averages(cls):
with DBManager.create_session_scope(expire_on_commit=False) as session:
champions = [
c.id for c in session.query(Champion).options(load_only(
'id')).order_by('id').all()
]
total_champs = len(champions)
versions = StaticDataContext.get_api_version()['versions'][:2]
for version in versions:
# Keep only the 2 majon numbers of the patch version
patch_version = re.sub(r'^((?:\d+\.*){2})((?:\.\d*)*)$', r'\1',
version)
log.debug('Calculating Averages for patch : {0}'.format(
patch_version))
for i in range(0, total_champs):
print('Status {0}/{1}'.format(i, total_champs), end='\r')
for j in range(0, total_champs):
if i == j:
continue
idA = champions[i]
idB = champions[j]
matchups = session.query(Matchup, SpellTimeLine, ItemTimeline)\
.join(SpellTimeLine,
(Matchup.id == SpellTimeLine.matchup_id)) \
.join(ItemTimeline,
(Matchup.id == ItemTimeline.matchup_id)) \
.filter(
(Matchup.champion == idA) & (Matchup.enemy == idB) &
(Matchup.checked == False) & (Matchup.patch_version == patch_version)
).all()
total_games = len(matchups)
if total_games == 0:
continue
matchups, spell_timelines, item_timelines = zip(
*matchups)
kills = 0
deaths = 0
assists = 0
damage_dealt = 0
wins = 0
creep_score = 0
duration = 0
for matchup in matchups:
if matchup.won:
wins += 1
kills += matchup.kills
deaths += matchup.deaths
assists += matchup.assists
damage_dealt += matchup.damage_dealt
creep_score += matchup.creep_score
duration += matchup.duration
sts = [st.spell_timeline for st in spell_timelines]
avg_spells = cls.timelines_average(data=sts)
its = [it.item_timeline for it in item_timelines]
avg_items = cls.timelines_average(data=its)
mus = [(mu.masteries, mu.runes, mu.summoners)
for mu in matchups]
masteries, runes, summoners = zip(*mus)
avg_masteries = cls.timelines_average(masteries)
avg_runes = cls.timelines_average(runes)
summoners = [sorted(s.split(',')) for s in summoners]
avg_summoners = cls.timelines_average(summoners)
matchup_avgs = {
'champion': idA,
'enemy': idB,
'total_games': total_games,
'kills': float(kills / total_games),
'deaths': float(deaths / total_games),
'assists': float(assists / total_games),
'creep_score': float(creep_score / total_games),
'damage_dealt': float(damage_dealt / total_games),
'duration': float(duration / total_games),
'wins': wins,
'item_timeline': avg_items,
'spell_timeline': avg_spells,
'masteries': avg_masteries,
'runes': avg_runes,
'summoners': avg_summoners,
'patch_version': patch_version
}
try:
matchup_avgs = MatchupAverages(matchup_avgs)
session.merge(matchup_avgs)
session.commit()
except IntegrityError:
session.rollback()
log.warn('Average already {0}vs{1} patch {2}'
'exists. Updating'.format(
idA, idB, patch_version))
prev_matchup = session.query(
MatchupAverages).filter_by(
champion=idA,
enemy=idB,
patch_version=patch_version).first()
session.merge(matchup_avgs)
session.commit()