def testAll(
modelId,
model,
dataFolder,
resultsFolder):
testAnswerFile = getAnswerFilename(modelId, resultsFolder)
testTruthFile = getTruthFilename(modelId, resultsFolder)
data = loadDataset(dataFolder)
outputTest = nn.test(model, data['testData'][0])
rate, correct, total = nn.calcRate(model, outputTest, data['testData'][1])
print 'rate: %.4f' % rate
resultsRank, \
resultsCategory, \
resultsWups = runAllMetrics(
data['testData'][0],
outputTest,
data['testData'][1],
data['ansIdict'],
data['questionTypeArray'],
testAnswerFile,
testTruthFile)
writeMetricsToFile(
modelId,
rate,
resultsRank,
resultsCategory,
resultsWups,
resultsFolder)
return outputTest
def runTests(params, model, trainer):
if params["testDataFilename"] is not None:
if params["imageqa"]:
imageqa_test.testAll(trainer.name, model, params["dataFolder"], params["outputFolder"])
else:
testData = np.load(params["testDataFilename"])
testInput = testData[0]
testTarget = testData[1]
model.loadWeights(np.load(trainer.modelFilename))
testOutput = nn.test(model, testInput)
testRate, c, t = nn.calcRate(model, testOutput, testTarget)
print "Test rate: ", testRate
with open(os.path.join(trainer.outputFolder, "result.txt"), "w+") as f:
f.write("Test rate: %f\n" % testRate)
def runTests(params, model, trainer):
if params['testDataFilename'] is not None:
if params['imageqa']:
imageqa_test.testAll(trainer.name, model, params['dataFolder'],
params['outputFolder'])
else:
testData = np.load(params['testDataFilename'])
testInput = testData[0]
testTarget = testData[1]
model.loadWeights(np.load(trainer.modelFilename))
testOutput = nn.test(model, testInput)
testRate, c, t = nn.calcRate(model, testOutput, testTarget)
print 'Test rate: ', testRate
with open(os.path.join(trainer.outputFolder, 'result.txt'),
'w+') as f:
f.write('Test rate: %f\n' % testRate)
def testEnsemble(
ensembleId,
models,
dataFolder,
classDataFolders,
resultsFolder):
"""
Test a class specific model in its original dataset.
"""
data = it.loadDataset(dataFolder)
inputTest = data['testData'][0]
targetTest = data['testData'][1]
ensembleOutputTest = runEnsemble(
inputTest,
models,
dataFolder,
classDataFolders,
data['questionTypeArray'])
ensembleAnswerFile = getAnswerFilename(ensembleId, resultsFolder)
ensembleTruthFile = getTruthFilename(ensembleId, resultsFolder)
rate, correct, total = nn.calcRate(
model, ensembleOutputTest, data['testData'][1])
print 'rate: %.4f' % rate
resultsRank, \
resultsCategory, \
resultsWups = it.runAllMetrics(
inputTest,
ensembleOutputTest,
targetTest,
data['ansIdict'],
data['questionTypeArray'],
ensembleAnswerFile,
ensembleTruthFile)
it.writeMetricsToFile(
ensembleId,
rate,
resultsRank,
resultsCategory,
resultsWups,
resultsFolder)
return ensembleOutputTest
modelId = sys.argv[i + 1]
elif flag == '-d' or flag == '-data':
dataFolder = sys.argv[i + 1]
elif flag == '-td' or flag == '-tdata':
testDataFolder = sys.argv[i + 1]
elif flag == '-reindex':
needReindex = True
elif flag == '-r' or flag == '-results':
resultsFolder = sys.argv[i + 1]
elif flag == '-dataset':
dataset = sys.argv[i + 1]
model = it.loadModel(modelId, resultsFolder)
data = it.loadDataset(dataFolder)
testdata = it.loadDataset(testDataFolder)
if needReindex:
testQuestions, testAnswers = reindexDataset(
testdata['testData'][0],
testdata['testData'][1],
testdata['questionIdict'],
data['questionDict'],
testdata['ansIdict'],
data['ansDict'])
else:
testQuestions = testdata['testData'][0]
testAnswers = testdata['testData'][1]
outputTest = nn.test(model, testQuestions)
rate, correct, total = nn.calcRate(model, outputTest, testAnswers)
print 'rate: %.4f' % rate
for i in range(0, 10):
trainInput_, trainTarget_, testInput_, testTarget_ = \
vt.splitData(trainInput, trainTarget, 0.1, i)
trainOpt['heldOutRatio'] = 0.1
trainOpt['xvalidNo'] = 0
trainOpt['needValid'] = True
model = nn.load(modelFilename)
trainer = nn.Trainer(
name=name + ('-%d-v' % i),
model=model,
trainOpt=trainOpt,
outputFolder=outputFolder
)
trainer.train(trainInput_, trainTarget_)
# Train again with all data, without validation
trainOpt['needValid'] = False
trainOpt['numEpoch'] = trainer.stoppedEpoch + 1
trainer = nn.Trainer(
name=name + ('-%d' % i),
model=model,
trainOpt=trainOpt,
outputFolder=outputFolder
)
trainer.train(trainInput_, trainTarget_)
testOutput = nn.test(model, testInput_)
testRate, correct, total = nn.calcRate(model, testOutput, testTarget_)
with open(os.path.join(trainer.outputFolder, 'result.txt'), 'w+') as f:
f.write('Test rate: %f' % testRate)
np.random.RandomState(2))
with open(configFilename) as f:
trainOpt = yaml.load(f)
for i in range(0, 10):
trainInput_, trainTarget_, testInput_, testTarget_ = \
vt.splitData(trainInput, trainTarget, 0.1, i)
trainOpt['heldOutRatio'] = 0.1
trainOpt['xvalidNo'] = 0
trainOpt['needValid'] = True
model = nn.load(modelFilename)
trainer = nn.Trainer(name=name + ('-%d-v' % i),
model=model,
trainOpt=trainOpt,
outputFolder=outputFolder)
trainer.train(trainInput_, trainTarget_)
# Train again with all data, without validation
trainOpt['needValid'] = False
trainOpt['numEpoch'] = trainer.stoppedEpoch + 1
trainer = nn.Trainer(name=name + ('-%d' % i),
model=model,
trainOpt=trainOpt,
outputFolder=outputFolder)
trainer.train(trainInput_, trainTarget_)
testOutput = nn.test(model, testInput_)
testRate, correct, total = nn.calcRate(model, testOutput, testTarget_)
with open(os.path.join(trainer.outputFolder, 'result.txt'), 'w+') as f:
f.write('Test rate: %f' % testRate)
for i, flag in enumerate(sys.argv):
if flag == '-m' or flag == '-model':
modelId = sys.argv[i + 1]
elif flag == '-d' or flag == '-data':
dataFolder = sys.argv[i + 1]
elif flag == '-td' or flag == '-tdata':
testDataFolder = sys.argv[i + 1]
elif flag == '-reindex':
needReindex = True
elif flag == '-r' or flag == '-results':
resultsFolder = sys.argv[i + 1]
elif flag == '-dataset':
dataset = sys.argv[i + 1]
model = it.loadModel(modelId, resultsFolder)
data = it.loadDataset(dataFolder)
testdata = it.loadDataset(testDataFolder)
if needReindex:
testQuestions, testAnswers = reindexDataset(testdata['testData'][0],
testdata['testData'][1],
testdata['questionIdict'],
data['questionDict'],
testdata['ansIdict'],
data['ansDict'])
else:
testQuestions = testdata['testData'][0]
testAnswers = testdata['testData'][1]
outputTest = nn.test(model, testQuestions)
rate, correct, total = nn.calcRate(model, outputTest, testAnswers)
print 'rate: %.4f' % rate
outputFolder=params['outputFolder']
)
trainer.train(trainInput, trainTarget, validInput, validTarget)
if params['testDataFilename'] is not None:
if params['imageqa']:
imageqa_test.testAll(
trainer.name, model, params['dataFolder'], params['outputFolder'])
else:
testData = np.load(params['testDataFilename'])
testInput = testData[0]
testTarget = testData[1]
model.loadWeights(np.load(trainer.modelFilename))
testOutput = nn.test(model, testInput)
testRate, c, t = nn.calcRate(model, testOutput, testTarget)
print 'Test rate: ', testRate
with open(os.path.join(
trainer.outputFolder, 'result.txt'), 'w+') as f:
f.write('Test rate: %f\n' % testRate)
# Send email
if trainOpt.has_key('sendEmail') and trainOpt['sendEmail']:
email.appendList(params['outputFolder'], trainer.name)
if params['testDataFilename'] is not None and\
params['validDataFilename'] is not None:
# Retrain with all the data
trainOpt['needValid'] = False
print 'Stopped score:', trainer.stoppedTrainScore
trainOpt['stopScore'] = trainer.stoppedTrainScore