def scalesHiddenPlot(name='scales'):
matplotlib.rcParams.update({'font.size' : 20})
lw = 3
plt.hold(True)
if name == 'scalesPlus':
experimentName = 'Complex'
nLevels = 3
leveledFcData = turk.readScalesProblems('../../data/scale_plus_6stimuli_3levels_no_fam_24_january_SCAL.csv', name)
elif name == 'scales':
experimentName = 'Simple'
nLevels = 2
leveledFcData = turk.readScalesProblems('../../data/scales_6stimuli_3levels_no_fam_25_january_OSCA.csv', name)
else:
print '[forcedChoiceExperiments] Unknown experiment name: ', name
sizes = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80]
nModels = 10 # numbered 1 to 10
agents = [] # will be an array of arrays, one entry per hidden node, one entry per training iteration
# load the agents
for size in sizes:
sizeAgents = []
for agentNum in range(1,nModels + 1):
(listeners, speakers) = loadAllAgents('../../data/cogsci/agents-%d-%d.pickle' % (size, agentNum))
sizeAgents.append(listeners)
agents.append(sizeAgents)
for (levelProblems, lineColor) in zip(leveledFcData, colors):
dataset = forcedChoiceProblemsToDataset(levelProblems)
hiddenLayerAccuracies = []
hiddenLayerScores = []
yerrs = []
for (allListeners, size) in zip(agents, sizes): # for each # of hidden layers
sizeAccuracies = [] # accuracies for each independent trial for this # of hidden nodes and this level of problem. will be averaged.
sizeScores = []
for listeners in allListeners:
lastListener = listeners[3]
(correct, activations, scores) = evalListenerOnClassificationDataset(lastListener, dataset)
sizeAccuracies.append(float(correct) / len(scores))
sizeScores.append(scores)
averageAccuracy = np.array(sizeAccuracies).mean()
hiddenLayerAccuracies.append(averageAccuracy)
hiddenLayerScores.append(sizeScores)
interval = boot.ci(np.array(sizeScores), np.average)
err = (interval[1] - interval[0])/2.0
yerrs.append(err)
plt.errorbar(sizes, hiddenLayerAccuracies, yerr=yerrs, linewidth=lw, color=lineColor)
plt.axis([0, sizes[-1], 0, 1])
plt.title('ANN Accuracy on the %s Condition' % experimentName)
plt.xlabel('Number of Hidden Nodes')
plt.ylabel('Average Accuracy')
plt.legend(['Level %d' % i for i in range(nLevels)], loc='lower right')
plt.savefig('hidden%s.pdf' % name, format='pdf')
plt.show()
def forcedChoiceMultiExperiments(agentPrefix, nAgents, plotPath, name='scales', errorBars=False):
if name == 'scalesPlus':
condition = 'Complex'
leveledFcData = turk.readScalesProblems('../../data/scale_plus_6stimuli_3levels_no_fam_24_january_SCAL.csv', name)
elif name == 'scales':
condition = 'Simple'
leveledFcData = turk.readScalesProblems('../../data/scales_6stimuli_3levels_no_fam_25_january_OSCA.csv', name)
else:
print '[forcedChoiceExperiments] Unknown experiment name: ', name
agents = [] # list of list of listeners, where agents[i] is each independent trained list of agents, so for instance agents[0][0] would be the 1st agent trained on 1 iteration.
for i in range(1, nAgents + 1): # ugh, numbered then agents-20-1 and so on.
agentPath = '%s%d.pickle' % (agentPrefix, i)
(listeners, speakers) = loadAllAgents(agentPath)
agents.append(listeners)
allAccuracies = []
allActivations = []
allScores = []
for level, fcData in enumerate(leveledFcData):
dataset = forcedChoiceProblemsToDataset(fcData)
(literalCorrect, literalTargets, literalScores) = evalLiteralListenerOnClassificationDataset(literalListener, dataset)
levelAccuracies = [float(literalCorrect) / len(dataset)]
levelActivations = [literalTargets]
levelScores = [literalScores]
nIterations = len(agents[0])
for trainingIdx in range(nIterations):
iterationScores = []
iterationActivations = []
for agentIdx in range(nAgents):
listener = agents[agentIdx][trainingIdx]
(correct, activations, scores) = evalListenerOnClassificationDataset(listener, dataset)
iterationScores.append(scores)
iterationActivations.append(activations)
flattenedScores = [scores for agentScores in iterationScores for scores in agentScores]
levelScores.append(flattenedScores)
levelAccuracies.append(np.array(flattenedScores).mean())
levelActivations.append(np.array([activation for agentActivations in iterationActivations for activation in agentActivations]))
allAccuracies.append(np.array(levelAccuracies))
allActivations.append(levelActivations)
allScores.append(np.array(levelScores))
forcedChoicePlot(allAccuracies, allScores, [], [], plotPath, 'ANN Accuracy on the %s Condition' % condition, errorBars=errorBars)
def forcedChoiceExperiments(agentPath, plotPath, name='scales', errorBars=False):
""" name can be 'scales' or 'scalesPlus' """
if name == 'scalesPlus':
leveledFcData = turk.readScalesProblems('../../data/scale_plus_6stimuli_3levels_no_fam_24_january_SCAL.csv', name)
elif name == 'scales':
leveledFcData = turk.readScalesProblems('../../data/scales_6stimuli_3levels_no_fam_25_january_OSCA.csv', name)
else:
print '[forcedChoiceExperiments] Unknown experiment name: ', name
(listeners, speakers) = loadAllAgents(agentPath)
listenerAccuracies = []
listenerActivations = []
listenerScores = []
mturkAccuracies = []
mturkScores = []
for level, fcData in enumerate(leveledFcData):
dataset = forcedChoiceProblemsToDataset(fcData)
nProblems = len(dataset)
levelAccuracies = []
levelActivations = []
levelScores = []
(mturkAccuracy, turkScores) = evalTurkForcedChoices(fcData)
turkChoices = [problem[3] for problem in fcData]
mturkAccuracies.append(mturkAccuracy)
mturkScores.append(turkScores)
(literalCorrect, literalTargets, literalScores) = evalLiteralListenerOnClassificationDataset(literalListener, dataset)
levelAccuracies.append(float(literalCorrect) / nProblems)
levelActivations.append(literalTargets)
levelScores.append(literalScores)
for listener in listeners:
(correct, activations, scores) = evalListenerOnClassificationDataset(listener, dataset)
levelAccuracies.append(float(correct) / nProblems)
levelActivations.append(activations)
levelScores.append(scores)
listenerAccuracies.append(np.array(levelAccuracies))
listenerActivations.append(np.array(levelActivations))
listenerScores.append(np.array(levelScores))
forcedChoicePlot(listenerAccuracies, listenerScores, mturkAccuracies, mturkScores, plotPath, 'Model Performance on Forced Choice Data', errorBars=errorBars)
