def begin_crawling(seed_summoner_id, seasons, ranked_queues):
'''
Breadth first crawling interations, Summoner -> Match -> Summoner...
'''
#seed intialization
try:
print('Seed initializing...')
seed_summoner = riotapi.get_summoner_by_id(seed_summoner_id)
conn = sqlite3.connect('lola.db')
conn.execute("INSERT INTO Summoner VALUES('{}','{}',{})".format(seed_summoner.id, seed_summoner.name, 0)) #watch out "" / ''
conn.commit()
conn.close()
print('\nInitialization completed.')
except Exception as e:
print('\nInitialization failed: ', e) # possibly because the seed is already in database
pass
# summoner queue interations
total_summoner_processed = 0
total_match_processed = 0
total_match_cralwed = 0
total_match_duplicate = 0
total_match_none = 0
iteration = 0
try:
conn = sqlite3.connect('lola.db')
queue_summoner_ids = pd.read_sql("SELECT summoner_id FROM Summoner WHERE is_crawled=0", conn)
except Exception as e:
raise(e)
finally:
conn.close()
while not queue_summoner_ids.empty:
print('\nSummoner Queue Length:', len(queue_summoner_ids))
iteration += 1 # only a relative number because of crawling restrarts
print ('Iteration', iteration, 'in the process...')
queue_summoner_ids_list = list(queue_summoner_ids['summoner_id'])
random.shuffle(queue_summoner_ids_list)
for summoner_id in queue_summoner_ids_list[:]: # pd.dataframe to list of list(queue_summoner_ids['summoner_id'])
conn = sqlite3.connect('lola.db')
summoner = riotapi.get_summoner_by_id(summoner_id)
match_reference_list = riotapi.get_match_list(summoner=summoner, seasons=seasons, ranked_queues=ranked_queues)
if match_reference_list is None: # TODO: tag this summoner to be 400/404 status (or the loop may happen quite rarely)
print("Summoner {} has None MatchList, skip..".format(summoner_id))
continue
print('\nSummoner {} ({}) in {}, {}: '.format(summoner.name, summoner.id, ranked_queues, seasons))
print('Total Match Number of the summoner: {}'.format(len(match_reference_list)))
match_no = 0 # crawled + duplicate + none
crawled_match_no = 0
duplicate_match_no = 0
none_match_no = 0
for mf in match_reference_list[:]:
if is_match_duplicate(mf, conn) == False:
try:
# TODO: urllib.error.URLError: <urlopen error [Errno 11001] getaddrinfo failed>
match = riotapi.get_match(mf) # match reference -> match
except Exception as e:
raise(e)
# may be None even if mf is not None, see https://github.com/meraki-analytics/cassiopeia/issues/57
# can not use != because of Match.__eq__ use Match.id
if match is None:
none_match_no += 1
continue # jump to the next interation
match_to_sqlite(match, summoner, conn)
# match is crawled
conn.execute("UPDATE Match SET is_crawled = 1 WHERE match_id='{}'".format(mf.id))
crawled_match_no += 1
else :
duplicate_match_no += 1
match_no += 1
if match_no % 10 == 0:
print (match_no, 'matches in', len(match_reference_list), 'processed.')
# summoner has been crawled
conn.execute("UPDATE Summoner SET is_crawled = 1 WHERE summoner_id='{}'".format(summoner_id))
conn.commit() # commit after every summoner finished
conn.close()
# sums of different kinds of matches
total_summoner_processed += 1
total_match_processed += match_no
total_match_cralwed += crawled_match_no
total_match_duplicate += duplicate_match_no
total_match_none += none_match_no
print('\ntotal processed summoner:', total_summoner_processed,'\ntotal processed match:', total_match_processed, \
'\ntotal crawled match', total_match_cralwed, '\ntotal duplicate match:', total_match_duplicate, \
'\ntotal none match:', total_match_none)
# read new queue for next iteration
try:
conn = sqlite3.connect('lola.db')
queue_summoner_ids = pd.read_sql("SELECT summoner_id FROM Summoner WHERE is_crawled=0", conn) #update queue
except Exception as e:
raise(e)
finally:
conn.close()
def test_stats():
int_test_handler.test_result(riotapi.get_stats(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_mastery_pages():
int_test_handler.test_result(riotapi.get_mastery_pages(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def insert_games(sum_id, region):
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.info(st + " - Inserting games for " + str(sum_id) + " from " + str(region))
try:
riotapi.set_region(region)
summoner = riotapi.get_summoner_by_id(sum_id)
games = summoner.match_list()
for g in games:
if not collgame.find({'_id': str(g.id)}).count() > 0:
try:
game = defaultdict(list)
match = g.match()
game.update({
"_id": str(g.id),
"date": str(match.creation),
"duration": str(match.duration),
"platform": str(match.platform),
"region": str(match.region),
"season": str(match.season),
"version": match.version,
"participants": []
})
try:
for p in g.match().participants:
try:
game['participants'].append({
"_id": str(p.summoner_id),
"name": p.summoner_name,
"champion": p.champion.name,
"won": p.stats.win,
"side": get_side(p.side),
"kills": p.stats.kills,
"deaths": p.stats.deaths,
"assists": p.stats.assists,
"kda": p.stats.kda,
"cs": p.stats.cs,
"sum_d": str(p.summoner_spell_d),
"sum_f": str(p.summoner_spell_f),
"champ_lvl": p.stats.champion_level,
"gold_earned": p.stats.gold_earned,
# "items": [item.name if item is not None else None for item in p.stats.items],
"m_dmg_dealt": p.stats.magic_damage_dealt,
"p_dmg_dealt": p.stats.physical_damage_dealt,
"dmg_2_champs": p.stats.damage_dealt_to_champions,
"lane": get_lane(p.timeline.lane)
})
except:
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.warning(st + " - APIError exception found while adding a participant for " + str(sum_id) + " in " + str(region))
except:
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.warning(st + " - APIError exception found while loading participants for " + str(sum_id) + " in " + str(region))
key = {"_id": str(g.id)}
data = {"$set": game}
collgame.update_one(key, data, upsert=True)
except:
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.warning(st + " - APIError exception found while loading game for " + str(sum_id) + " in " + str(region))
except:
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.error(st + " - APIError exception found while setting Region (Cassiopeia), getting summoner or getting " \
"match_list. Most likely it's match_list for " + str(sum_id) + " in " + str(region))
# Updating games count value
games_count = collgame.find({"participants._id": str(sum_id)}).count()
collsums.update({"_id": int(sum_id)}, {"$set": {"games_count": games_count}})
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.info(st + " - Analysing lane champion summoners wins for " + str(sum_id) + " in " + str(region))
lane_sums(str(sum_id))
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.info(st + " - Analysing lane summoners wins for " + str(sum_id) + " in " + str(region))
lane_champ_sums(str(sum_id))
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y %H:%M:%S')
logging.info(st + " - Games inserted for " + str(sum_id) + " in " + str(region))
# we will now send an email confirming the data is already collected if the person has subscribed to the list
summoner = riotapi.get_summoner_by_id(sum_id)
send_email(summoner.name, region)
def test_teams_by_summoner():
int_test_handler.test_result(riotapi.get_teams_by_summoner(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_current_game():
int_test_handler.test_result(riotapi.get_current_game(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_current_game():
int_test_handler.test_result(
riotapi.get_current_game(
riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_league_entries_by_summoner():
int_test_handler.test_result(
riotapi.get_league_entries_by_summoner(
riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_match_list():
int_test_handler.test_result(
riotapi.get_match_list(
riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def RecordTopMatchForSummoner(a_summoner):
global last_summoner_id
if last_summoner_id != -1:
if last_summoner_id != int(a_summoner['playerOrTeamId']):
return
else:
last_summoner_id = -1
with open("LastTopSummonerId.txt", "w") as f:
print "Write last top summoner id " + str(a_summoner['playerOrTeamId'])
f.write(str(a_summoner['playerOrTeamId']))
f.close()
summoner = riotapi.get_summoner_by_id(a_summoner['playerOrTeamId'])
match_list = summoner.match_list()
list_size = len(match_list)
match_count = 0
tier_array = []
for match_idx, match in enumerate(match_list):
#print 'Checking %d of %d' % (match_idx, list_size)
match_record = matches_checked.find_one({'matchId':match.id})
if match_record != None:
if match_record['shouldStop'] == False:
continue
else:
break
matches_checked.insert_one({'matchId':match.id, 'shouldStop':False})
if IsSoloRank(match.data.queue) == False:
print 'Continue for not being a solo rank with id ' + str(match.id)
continue
match_data = match.match()
a_match = json.loads(match_data.to_json())
if a_match['matchVersion'].startswith('6.24') == False:
print 'Break for not being in 6.24 with id ' + str(match.id)
matches_checked.update_one({'matchId':match.id}, {'$set': {'shouldStop': True}})
break
a_duration = a_match['matchDuration']
a_minute = a_duration/60
if a_minute < 20:
print 'Continue for not being a full game with id ' + str(match.id)
continue
match_count += 1
isTopMatch = True
#participants = a_match['participants']
#for a_participant_idx,a_participantIdentity in enumerate(a_match['participantIdentities']):
# a_id = a_participantIdentity['player']['summonerId']
# isTopMatch = str(a_id) in topIds
# if isTopMatch == False:
# print 'Break for not being a top player ' + str(a_id)
# break
if isTopMatch == True:
if matches_collection.find_one({'matchId':match.id}) == None:
matches_collection.insert_one(a_match)
print 'A match saved'
def match(request, match_num):
try:
m = Game.objects.get(Number=match_num)
print(m.filename)
except ObjectDoesNotExist:
return HttpResponse("There is no match with this ID")
with open(str(m.filename), 'rb') as infile:
game = pickle.load(infile)
#name = str(m)
riotapi.set_region("NA")
team1 = []
try:
team1.append(Player.objects.get(SummonerNumber=m.team1Player1))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team1Player1
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team1Player1)
team1.append(sub)
try:
team1.append(Player.objects.get(SummonerNumber=m.team1Player2))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team1Player2
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team1Player2)
team1.append(sub)
try:
team1.append(Player.objects.get(SummonerNumber=m.team1Player3))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team1Player3
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team1Player3)
team1.append(sub)
try:
team1.append(Player.objects.get(SummonerNumber=m.team1Player4))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team1Player4
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team1Player4)
team1.append(sub)
try:
team1.append(Player.objects.get(SummonerNumber=m.team1Player5))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team1Player5
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team1Player5)
team1.append(sub)
team2 = []
try:
team2.append(Player.objects.get(SummonerNumber=m.team2Player1))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team2Player1
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team2Player1)
team2.append(sub)
try:
team2.append(Player.objects.get(SummonerNumber=m.team2Player2))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team2Player2
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team2Player2)
team2.append(sub)
try:
team2.append(Player.objects.get(SummonerNumber=m.team2Player3))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team2Player3
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team2Player3)
team2.append(sub)
try:
team2.append(Player.objects.get(SummonerNumber=m.team2Player4))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team2Player4
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team2Player4)
team2.append(sub)
try:
team2.append(Player.objects.get(SummonerNumber=m.team2Player5))
except ObjectDoesNotExist:
sub = Player()
sub.SummonerNumber = m.team2Player5
sub.PlayerIGN = riotapi.get_summoner_by_id(m.team2Player5)
team2.append(sub)
return HttpResponse(render(request, 'Match.html', {'matchinfo' :m, 'game' : game, 't1': team1, 't2': team2}))
def test_league_entries_by_summoner():
int_test_handler.test_result(riotapi.get_league_entries_by_summoner(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
def test_rune_pages():
int_test_handler.test_result(
riotapi.get_rune_pages(
riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
- Participants [Participant]
- ParticipantStats
- Timeline
- Timeline
- Frames [Frame]
- Events [Event]
- Team
'''
riotapi.set_region("NA")
riotapi.set_api_key("") # set your key here
summoner_id='22005573'
seasons = 'PRESEASON2016'
ranked_queues = 'RANKED_SOLO_5x5'
summoner = riotapi.get_summoner_by_id(summoner_id)
#match
match_list = riotapi.get_match_list(summoner=summoner, seasons=seasons, ranked_queues=ranked_queues)
match_reference = match_list[0]
match = riotapi.get_match(match_reference)
match_reference_1 = match_list[1]
match_1 = riotapi.get_match(match_reference_1)
version = match.version
duration = math.ceil((match.duration).total_seconds() / 60) #minute
data = match.data
#team
team = match.red_team
team_participant = team[0]
team_bans = team.bans
team_win = team.win
def test_match_list():
int_test_handler.test_result(riotapi.get_match_list(riotapi.get_summoner_by_id(int_test_handler.summoner_id)))
