def teamConfiguration(regions):
start = time.time()
configQueue = Queue.Queue()
THREADS = 10
mergeLock = thread.allocate_lock()
mergeLock.acquire()
thread.start_new_thread(
mergeConfigs, (configQueue, mergeLock, THREADS * regions.__len__()))
total = 0
for region in regions:
conn = sqlite3.connect('lol-' + region + '.db')
c = conn.cursor()
c.execute("SELECT COUNT(*) FROM matchs")
total += c.fetchone()[0]
conn.close()
limits = rdb.gameIdsListing(100 / THREADS, region)
limits[0] = 0
limits[THREADS] = 2 * limits[THREADS]
for i in range(0, THREADS):
condition = " WHERE gameId >= " + str(
limits[i]) + " and gameId < " + str(limits[i + 1])
thread.start_new_thread(teamConfigurationChunk,
(region, condition, i + 1, configQueue))
print "Starting thread " + region.upper() + " " + str(
i + 1) + " with condition " + condition
mergeLock.acquire()
fullConfig = configQueue.get()
print pf.big_statement("Full config for " + str(regions) +
" computed in " +
pf.time_format(time.time() - start))
print pf.big_statement(fullConfig.__len__() + " team configuration in " +
str(total) + " games")
fullConfig.sort(key=lambda x: x.n)
print list(x.n for x in fullConfig[-100:])
def evaluate_model():
startTime = time.time()
classifier = classifier_construction()
probas_correct_label = raw_evaluation(classifier, 0)
weighted_accuracy_score = weighted_accuracy(classifier, probas_correct_label= probas_correct_label)
certainty_accuracy_scores = certainty_accuracy(classifier, probas_bounds=[0.55,0.60,0.65, 0.7, 0.75, 0.8, 0.85], probas_correct_label= probas_correct_label)
#TODO : improve
evaluation = classifier.evaluate(input_fn=lambda: get_test_inputs(FULL_TEST_BATCH, 0), steps=1)
accuracy_score = "Accuracy : {0:f}".format(evaluation['accuracy'])
print("\nTest Accuracy: {0:f}\n".format(evaluation['accuracy']))
timeDescription = "Evaluation done in {}".format(pf.time_format(time.time() - startTime))
fullEvaluation = [accuracy_score, weighted_accuracy_score]
fullEvaluation.extend(certainty_accuracy_scores)
fullEvaluation.append(timeDescription)
print (pf.big_statement('\n'.join(fullEvaluation)))
def train(shuffleOrder = 0, shuffling = 1):
global shuffle_index
shuffle_index = shuffleOrder
TRAINING_STEPS = 3000
startTime = time.time()
classifier = classifier_construction()
print ("Classifier ready")
conn = sqlite3.connect('lol-train.db')
c = conn.cursor()
c.execute("SELECT COUNT(*) FROM matchs")
total_train = c.fetchone()[0]
conn.close()
# Fit model.
iterations = total_train/TRAIN_BATCH
if total_train % TRAIN_BATCH != 0:
iterations += 1
probas = []
for j in range(0, shuffling):
if shuffle_index > 0:
TRAINING_STEPS = 2000
for i in range(0, iterations):
classifier.fit(input_fn=lambda: get_train_inputs(TRAIN_BATCH,i*TRAIN_BATCH ), steps = TRAINING_STEPS)
# Define the test inputs
# Evaluate accuracy.
accuracy_score = classifier.evaluate(input_fn=lambda: get_test_inputs(EVALUATE_BATCH, i* EVALUATE_BATCH),
steps=1)["accuracy"]
print("\nTest Accuracy: {0:f}\n".format(accuracy_score))
probas = proba_pedictions(probas, classifier)
weighted_accuracy(classifier, iterations*EVALUATE_BATCH)
shuffle_index += 1
for p in probas:
print(p)
print (pf.big_statement("Training on {} examples done in {}".format((shuffling*total_train), pf.time_format(time.time() - startTime))))
def average_rank(region="", precision=1.0, lowerLimit=0, higherLimit=36):
# stats = []
if region == "":
stats = average_rank_region("na", precision)
statsEuw = average_rank_region("euw", precision)
for i in range(0, stats.__len__()):
stats[i] += statsEuw[i]
else:
stats = average_rank_region(region, precision)
intro = "Games by average rank"
if region is not "":
intro += " for " + region.upper()
output = pf.big_statement(intro)
games = 0
totaldb = sum(stats)
for i in range(0, stats.__len__()):
if (MIN_RANK + i * precision) >= lowerLimit and (
MIN_RANK + (i + 1) * precision) <= higherLimit:
output += (str(MIN_RANK + i*precision) + " <= x < " + str(MIN_RANK + (i+1)*precision) + " : " + str(stats[i]) \
+ " (" + str(format(float(stats[i]*100)/totaldb,'.3f')) + " %)\n")
games += stats[i]
output += "\n\n" + str(games) + " games (" + str(
format(float(games) * 100 / totaldb,
'.2f')) + "%) with average rank between " + str(
lowerLimit) + " and " + str(higherLimit)
# if precision == 1.0 :
#
# statsFile = open('games_stats.txt', 'w')
# statsFile.writelines(map(str, stats))
# statsFile.close()
return output
def run_region(self):
try:
print "Search on " + self.region.upper() + " : " + str(
self.minRankLimit) + " to " + str(self.maxRankLimit) + "\n"
self.erroLog = open('errorlog-' + self.region + '.txt', 'w')
INTERMEDIATE_TIME_REPORT = max(15, min(self.runTime / 5, 900))
random.seed()
distribution = rdb.average_rank_region(self.region, 1)
self.calculate_probabilities(distribution)
print self.region.upper() + " Probabilities calculated"
conn = sqlite3.connect("lol-" + self.region + ".db")
c = conn.cursor()
self.load_keys()
self.load_challengers()
self.load_summoners()
#cheating :
self.summonersDone = struct.Locked_BST(["5"])
gameIds = list(
c.execute("SELECT gameId FROM matchs WHERE gameId > " +
str(self.gameIdCutoff)))
if gameIds:
self.recordedGameIds = struct.Locked_BST(gameIds)
else:
self.recordedGameIds = struct.Locked_BST([5])
if LOG:
log = open('log' + self.region + '.txt', 'w')
# log.write(str(self.summoners[-(SUMMONER_BATCH_SIZE + 5):]) + "\n")
startTime = time.time()
print self.region.upper() + " " + str(self.recordedGameIds.__len__(
)) + " gameIds ready, starting threads"
thread.start_new_thread(self.raw_games_collection, (startTime, ))
thread.start_new_thread(self.games_filtering, ())
thread.start_new_thread(self.record_games, ())
active = True
lastTimeReport = time.time()
while active:
# Check for error preventing ending
if (time.time() - startTime) > (self.runTime + 600):
print "We had a problem, the program would not stop"
if (time.time() - lastTimeReport) > INTERMEDIATE_TIME_REPORT:
lastTimeReport = time.time()
self.summonersLock.acquire()
self.recordsLock.acquire()
self.statisticsLock.acquire()
print pf.big_statement(self.region.upper() + " Summoners : " + str(self.summoners.__len__()) + ", toBeTreated : " +
str(self.summonersToBeTreated.qsize()) + ", rawGames : " + str(self.rawGames.qsize())
+ ", filteredGames : " + str(self.filteredGames.qsize())
+ " , records : " + str(self.records.__len__()) + ", in flight : " + str(self.playersInflight) + "\n" +
self.region.upper() + " " + str(self.success) + " games recorded so far (" + str(self.duplicate) + " duplicates and " \
+ str(self.invalid) + " invalid games) from " + str(self.playersDone) + " players in "
+ pf.time_format(lastTimeReport - startTime))
self.statisticsLock.release()
self.recordsLock.release()
self.summonersLock.release()
newRecords = []
self.recordsLock.acquire()
newRecords.extend(self.records)
self.records = []
self.recordsLock.release()
if newRecords:
for r in newRecords:
sqlAction = "INSERT INTO matchs VALUES(" + ','.join(
map(str, r)) + ")"
c.execute(sqlAction)
conn.commit()
else:
#Check for stop condition
self.recordsLock.acquire()
if not self.recordGamesActive and not self.records:
active = False
self.recordsLock.release()
else:
self.recordsLock.release()
time.sleep(5)
self.statisticsLock.acquire()
finalStatement = pf.big_statement("All done : " +
self.region.upper() + " Summoners : " + str(self.summoners.__len__()) + ", toBeTreated : " +
str(self.summonersToBeTreated.qsize()) + ", rawGames : " + str(self.rawGames.qsize())
+ ", filteredGames : " + str(self.filteredGames.qsize())
+ " , records : " + str(self.records.__len__()) + ", in flight : " + str(self.playersInflight) + "\n" +
str(self.success) + " games recorded (" + str(self.duplicate) + " duplicates, " \
+ str(self.invalid) + " invalid) from " + str(self.playersDone) + " players in "
+ pf.time_format(time.time() - startTime) +" and sleeping for " + pf.time_format(self.totalSleepTime) + \
" (" + str(format(self.totalSleepTime * 100/(time.time() - startTime), '.2f')) + " %)"
+ "\n" + str(self.totalApiCalls) \
+ " API calls (" +str(self.apiErrors) + " errors, 500 : " + str(self.apiError500) \
+ ", 429 : " + str(self.apiError429) + "), games per call ratio : " + str(format(float(self.success) /self.totalApiCalls, '.2f'))
+ ", " + str(int(self.success*600/(time.time() - startTime))) + " games per 10 minutes"\
+ "\nDictionnary of ranks of " + str(self.ranks.__len__()) + " players")
self.statisticsLock.release()
f = open('summoners-' + self.region + '.txt', 'w')
f.writelines("\n".join(self.summoners))
f.close()
print finalStatement
print self.region.upper() + " storing " + str(
self.summoners.__len__()) + " summonerIds for next time"
c.execute("SELECT COUNT(*) FROM matchs")
totaldb = c.fetchone()[0]
conn.close()
if LOG:
log.close()
self.erroLog.close()
print pf.big_statement("The " + self.region.upper() +
" database contains " + str(totaldb) +
" games")
except:
print "An error occured"
# finalStatement += rdb.average_rank(0.5)
# report(totaldb, successList, invalidList, duplicateList, finalStatement)
self.dataGathering.release()
if self.stopLock is not None:
self.stopLock.release()
return
def teamConfigurationByBlocks(regions):
start = time.time()
configQueue = Queue.Queue()
mergeLock = thread.allocate_lock()
mergeLock.acquire()
BLOCK_SIZE = 50000
total = 0
blocks = 0
for region in regions:
conn = sqlite3.connect('lol-' + region + '.db')
c = conn.cursor()
c.execute("SELECT COUNT(*) FROM matchs")
totalRegion = c.fetchone()[0]
total += totalRegion
blocks += int(math.ceil(float(totalRegion) / BLOCK_SIZE))
conn.close()
thread.start_new_thread(mergeConfigs, (configQueue, mergeLock, blocks))
start = time.time()
for region in regions:
BATCH = 800
n = 0
teams = []
rawTeams = []
startRegion = time.time()
conn = sqlite3.connect('lol-' + region + '.db')
c = conn.cursor()
games = c.execute("SELECT * FROM matchs")
for m in games:
if n % 10000 == 0 and n != 0:
print str(teams.__len__() + rawTeams.__len__()) + " team configurations over " + str(n) \
+ " games"
n += 1
t1 = Team(sorted(m[1:6]))
t2 = Team(sorted(m[6:11]))
if not checkKnownTeams(teams, rawTeams, t1):
rawTeams.append(t1)
if not checkKnownTeams(teams, rawTeams, t2):
rawTeams.append(t2)
if rawTeams.__len__() > BATCH:
teams.extend(rawTeams)
rawTeams = []
teams.sort(key=lambda x: x.fingerprint)
if n % BLOCK_SIZE == 0 and n != 0:
teams.extend(rawTeams)
configQueue.put(teams)
print("Adding a block of team configuration")
rawTeams = []
teams = []
teams.extend(rawTeams)
configQueue.put(teams)
print("Adding a block of team configuration")
# teams.sort(key=lambda x: x.n)
# print list(x.n for x in teams[-50:])
print region.upper() + " done, it contains " + str(
teams.__len__()) + " team configurations in " + str(n) \
+ " games, done in " + str(format(time.time() - startRegion, '.1f')) + " s"
# teams.sort(key=lambda x : x.fingerprint)
conn.close()
mergeLock.acquire()
fullConfig = configQueue.get()
print pf.big_statement("Full config for " + str(regions) +
" computed in " +
pf.time_format(time.time() - start))
print pf.big_statement(
str(fullConfig.__len__()) + " team configuration in " + str(total) +
" games")
fullConfig.sort(key=lambda x: x.n)
print list(x.n for x in fullConfig[-100:])