def performExperiments(numTrials, year, batchNumber, model):
correctVector = getActualBracketVector(year)
brackets = []
for n in range(numTrials):
newBracketVector = generateBracket(model, year)
newBracketScore = scoreBracket(newBracketVector, correctVector)
# numCorrectPicks = calcCorrectPicks(newBracketScore)
newBracketString = ''.join(str(bit) for bit in newBracketVector)
# brackets.append({'bracketVector': newBracketString, 'score': newBracketScore, 'correctPicks': numCorrectPicks, 'model': model['modelName']})
brackets.append({'bracketVector': newBracketString, 'score': newBracketScore})
bracketListDict = {'year': year, 'actualBracket': ''.join(str(bit) for bit in correctVector), 'brackets': brackets}
if numTrials < 1000:
folderName = 'Experiments/{0}Trials'.format(numTrials)
else:
folderName = 'Experiments/{0}kTrials'.format(int(numTrials / 1000))
batchFolderName = '{0}/Batch{1:02d}'.format(folderName, batchNumber)
outputFilename = '{2}/generatedBrackets_{0}_{1}.json'.format(model['modelName'], year, batchFolderName)
with open(outputFilename, 'w') as outputFile:
outputFile.write(json.dumps(bracketListDict))
def performExperiments(numTrials, year, batchNumber, model):
summarizer = RuntimeSummary(model)
correctVector = getActualBracketVector(year)
scores = [None] * numTrials
for n in range(numTrials):
newBracketVector = generateBracket(model, year)
summarizer.analyze_bracket(np.array(newBracketVector))
newBracketScore = scoreBracket(newBracketVector, correctVector)
# sys.stdout.write(''.join(str(bit) for bit in newBracketVector) + '\n')
# numCorrectPicks = calcCorrectPicks(newBracketScore)
scores[n] = newBracketScore[0]
bracketListDict = {
'year': year,
'actualBracket': ''.join(str(bit) for bit in correctVector),
'scores': scores
}
if numTrials < 1000:
folderName = 'Experiments/{0}Trials'.format(numTrials)
else:
folderName = 'Experiments/{0}kTrials'.format(int(numTrials / 1000))
batchFolderName = '{0}/Batch{1:02d}'.format(folderName, batchNumber)
outputFilename = '{2}/generatedScores_{0}_{1}.json'.format(
model['modelName'], year, batchFolderName)
summaryFilename = '{2}/vectorStats_{0}_{1}.json'.format(
model['modelName'], year, batchFolderName)
with open(outputFilename, 'w') as outputFile:
outputFile.write(json.dumps(bracketListDict))
summarizer.to_json(summaryFilename)
def summarizeBatch(numTrials, batchNumber, modelsList, outputFile):
for year in range(2013, 2020):
outputFile.write('{0} Tournament:,'.format(year))
numModels = len(modelsList)
maxScores = []
minScores = []
countAboveEspnMin = []
# maxCorrectPicks = []
percentile95 = []
percentile99 = []
proportionsAbovePF = []
if numTrials < MEAN_MEDIAN_VAR_CUTOFF:
meanScores = []
varianceScores = []
medianScores = []
for index in range(numModels):
modelName = modelsList[index]['modelName']
if numTrials < 1000:
folderName = 'Experiments/{0}Trials'.format(numTrials)
else:
folderName = 'Experiments/{0}kTrials'.format(
int(numTrials / 1000))
batchFolderName = '{0}/Batch{1:02d}'.format(folderName, batchNumber)
inputFilename = '{2}/generatedScores_{0}_{1}.json'.format(
modelName, year, batchFolderName)
if not os.path.exists(inputFilename):
continue
outputFile.write('{0},'.format(modelName))
with open(inputFilename, 'r') as inputFile:
dataJson = inputFile.read().replace('\n', '')
dataPyDict = json.loads(dataJson)
scores = dataPyDict['scores']
brackets = []
for score in scores:
newBracket = SimpleBracket(None, [score])
brackets.append(newBracket)
nBrackets = len(brackets)
# Determine max scoring bracket, as well as 95/99th percentiles
brackets.sort(key=lambda x: x.scores[0], reverse=True)
maxScoringBracket = brackets[0]
minScoringBracket = brackets[-1]
bracket95 = brackets[int(0.05 * nBrackets) - 1]
bracket99 = brackets[int(0.01 * nBrackets) - 1]
# Determine max correct picks
# brackets.sort(key=lambda x: x.correctPicks, reverse=True)
# maxCorrectPicksBracket = brackets[0]
# Determine score of Pick Favorite model
actualBracket = dataPyDict['actualBracket']
actualBracketVector = [int(actualBracket[i]) for i in
range(len(actualBracket))]
pickFavoriteString = '111111111000101111111111000101111111111000101111111111000101111'
pickFavoriteVector = [int(pickFavoriteString[i]) for i in range(63)]
pfScores = scoreBracket(pickFavoriteVector, actualBracketVector,
True)
pfTotalScore = pfScores[0]
bracketsAbovePickFavorite = [b for b in brackets if
b.scores[0] >= pfTotalScore]
nBracketsAbovePF = len(bracketsAbovePickFavorite)
proportionAbovePF = nBracketsAbovePF * 1.0 / nBrackets
bracketsInEspnLeaderboard = [b for b in brackets if
b.scores[0] >= espnMin[year - 2013]]
countAboveEspnMin.append(len(bracketsInEspnLeaderboard))
if numTrials < MEAN_MEDIAN_VAR_CUTOFF:
scores = [x.scores[0] for x in brackets]
meanScore = np.mean(scores)
varianceScore = np.var(scores)
medianScore = np.median(scores)
maxScores.append(maxScoringBracket.scores[0])
minScores.append(minScoringBracket.scores[0])
# maxCorrectPicks.append(maxCorrectPicksBracket.correctPicks)
percentile95.append(bracket95.scores[0])
percentile99.append(bracket99.scores[0])
proportionsAbovePF.append(proportionAbovePF)
if numTrials < MEAN_MEDIAN_VAR_CUTOFF:
meanScores.append(meanScore)
varianceScores.append(varianceScore)
medianScores.append(medianScore)
outputFile.write('\n')
outputFile.write('Max score,')
for val in maxScores:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('Min score,')
for val in minScores:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
if numTrials < MEAN_MEDIAN_VAR_CUTOFF:
outputFile.write('Median score,')
for val in medianScores:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('Mean score,')
for val in meanScores:
outputFile.write('{0},'.format(val))
outputFile.write('')
outputFile.write('\n')
outputFile.write('Var(scores),')
for val in varianceScores:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('No. in ESPN top 100,')
for val in countAboveEspnMin:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
# outputFile.write('Max correct picks,')
# for i in range(numModels):
# outputFile.write('{0},'.format(maxCorrectPicks[i]))
# outputFile.write('\n')
outputFile.write('95th percentile,')
for val in percentile95:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('99th percentile,')
for val in percentile99:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('Proportion >= PF ({0}),'.format(pfTotalScore))
for val in proportionsAbovePF:
outputFile.write('{0},'.format(val))
outputFile.write('\n')
outputFile.write('\n')
def performFixedAlphaExperiments(numTrials, year, isFixedFirstRoundAlphas, isFixedK, rangeK, batchNumber):
inputFilename = 'allBracketsTTT.json'
with open(inputFilename, 'r') as inputFile:
dataJson = inputFile.read().replace('\n', '')
dataPyDict = json.loads(dataJson)
bracketList = dataPyDict['brackets']
bracket = None
for bracketDict in bracketList:
bracket = buildBracketFromJson(bracketDict['bracket'])
if bracket.year == year:
break
correctVector = [int(bracket.fullVector[i]) for i in range(len(bracket.fullVector))]
# 0-th index is unused for easy indexing
alphaAvg = [0, 1.2, 1.0, 1.0, 0.3, 1.3, 1.8]
# Weighted average alpha values per round per year, rounded to two dec. places
if year == 2013:
alphaAvg = [0, 1.01, 1.10, 0.91, 0.36, 1.00, 1.54]
elif year == 2014:
alphaAvg = [0, 1.00, 1.03, 0.90, 0.23, 1.03, 1.54]
elif year == 2015:
alphaAvg = [0, 1.04, 1.03, 0.86, 0.19, 0.93, 1.55]
elif year == 2016:
alphaAvg = [0, 1.12, 1.02, 0.88, 0.22, 0.97, 1.57]
elif year == 2017:
alphaAvg = [0, 1.05, 1.01, 0.90, 0.14, 1.00, 1.35]
else:
alphaAvg = [0, 1.2, 1.0, 1.0, 0.3, 1.3, 1.8] # should not reach this; old guesses for alpha values
brackets = []
for n in range(numTrials):
# Sample to get new alpha values for each trial
if isFixedK:
alphaSwing = [rangeK for i in range(7)]
else:
alphaSwing = []
for i in range(7):
swingVal = random.random() * rangeK
alphaSwing.append(swingVal)
alphaVals = [0]
for roundNum in range(1, 7):
roundAlpha = alphaAvg[roundNum]
roundAlpha += (2 * random.random() - 1) * alphaSwing[roundNum]
alphaVals.append(roundAlpha)
newBracketVector = generateBracket(alphaVals, year, isFixedFirstRoundAlphas, alphaSwing[1]) # We pass in the K that will be used for R1, if needed
newBracketScore = scoreBracket(newBracketVector, correctVector)
numCorrectPicks = calcCorrectPicks(newBracketScore)
newBracketString = ''.join(str(outcome) for outcome in newBracketVector)
brackets.append({'bracketVector': newBracketString, 'score': newBracketScore, 'correctPicks': numCorrectPicks, 'alphaVals': ['{0:5.2f}'.format(alphaVals[j]) for j in range(1, 7)]})
bracketListDict = {'year': year, 'actualBracket': bracket.fullVector, 'brackets': brackets}
outputFilename = 'Experiments/OneMillionTrials/Batch{5:02d}/generatedBrackets_{0}_{1}_{2}_{3}_{4:.2f}.json'.format(numTrials, year, isFixedFirstRoundAlphas, isFixedK, rangeK, batchNumber)
with open(outputFilename, 'w') as outputFile:
outputFile.write(json.dumps(bracketListDict))
brackets.sort(key=lambda x: x.scores[0], reverse=True) maxScoringBracket = brackets[0] bracket95 = brackets[int(0.05 * nBrackets) - 1] bracket99 = brackets[int(0.01 * nBrackets) - 1] # Determine max correct picks brackets.sort(key=lambda x: x.correctPicks, reverse=True) maxCorrectPicksBracket = brackets[0] # Determine score of Pick Favorite model actualBracket = dataPyDict['actualBracket'] actualBracketVector = [int(actualBracket[i]) for i in range(len(actualBracket))] pickFavoriteString = '111111111000101111111111000101111111111000101111111111000101111' pickFavoriteVector = [int(pickFavoriteString[i]) for i in range(63)] pfScores = scoreBracket(pickFavoriteVector, actualBracketVector, True) pfTotalScore = pfScores[0] bracketsAbovePickFavorite = [b for b in brackets if b.scores[0] >= pfTotalScore] nBracketsAbovePF = len(bracketsAbovePickFavorite) proportionAbovePF = nBracketsAbovePF * 1.0 / nBrackets bracketsInEspnLeaderboard = [b for b in brackets if b.scores[0] >= espnMin[year-2013]] countAboveEspnMin.append(len(bracketsInEspnLeaderboard)) if numTrials < 50001: scores = [x.scores[0] for x in brackets] meanScore = np.mean(scores) varianceScore = np.var(scores) medianScore = np.median(scores)
from scoringUtils import scoreBracket
bracketsFilename = sys.argv[1]
# Will only print brackets at or above this minimum score
minScore = 0
if len(sys.argv) > 2:
minScore = int(sys.argv[2])
actualVector = getActualBracketVector(2018)
print 'Vector,Score,R64,R32,S16,E8,F4,NCG'
with open(bracketsFilename, 'rb') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
if len(row) == 0:
continue
bracketString = str(row[0])
bracketVector = [
int(bracketString[i]) for i in range(len(bracketString))
]
bracketScore = scoreBracket(bracketVector, actualVector)
if bracketScore[0] >= minScore:
print '"{0}",{1},{2},{3},{4},{5},{6},{7}'.format(
bracketString, bracketScore[0], bracketScore[1],
bracketScore[2], bracketScore[3], bracketScore[4],
bracketScore[5], bracketScore[6])
actualBracket = "111111111101111111110011010101111101110001000111100111110101000" # pickFavoriteString = '111111111000101111111111000101111111111000101111111111000101111' # Pick Favorite (with correct last three bits) pfBracket = '111111111000101111111111000101111111111000101111111111000101000' #111111111000101 #111111111000101 #111111111000101 #111111111000101 #000 actualVector = [int(actualBracket[i]) for i in range(63)] testVector = [int(pfBracket[i]) for i in range(63)] pfScores = scoreBracket(testVector, actualVector) print pfScores # Pick Favorite (with bits 11 and 14 of third region (East) flipped) modifiedBracket = "111111111000101111111111000101111111111001100111111111000101000" #111111111000101 #111111111000101 #111111111001100 #111111111000101 #000 testVector = [int(modifiedBracket[i]) for i in range(63)] modScores = scoreBracket(testVector, actualVector) print modScores