def runEnsemble( inputTest, models, dataFolder, classDataFolders, questionTypeArray): """ Run a class specific model on any dataset. """ data = it.loadDataset(dataFolder) classAnsIdict = [] for df in classDataFolders: data_c = it.loadDataset(df) classAnsIdict.append(data_c['ansIdict']) ensembleOutputTest = __runEnsemble( inputTest, models, data['ansDict'], classAnsIdict, questionTypeArray) return ensembleOutputTest
def testEnsemble( ensembleId, models, dataFolder, classDataFolders, resultsFolder): """ Test a class specific model in its original dataset. """ data = it.loadDataset(dataFolder) inputTest = data['testData'][0] targetTest = data['testData'][1] ensembleOutputTest = runEnsemble( inputTest, models, dataFolder, classDataFolders, data['questionTypeArray']) ensembleAnswerFile = getAnswerFilename(ensembleId, resultsFolder) ensembleTruthFile = getTruthFilename(ensembleId, resultsFolder) rate, correct, total = nn.calcRate( model, ensembleOutputTest, data['testData'][1]) print 'rate: %.4f' % rate resultsRank, \ resultsCategory, \ resultsWups = it.runAllMetrics( inputTest, ensembleOutputTest, targetTest, data['ansIdict'], data['questionTypeArray'], ensembleAnswerFile, ensembleTruthFile) it.writeMetricsToFile( ensembleId, rate, resultsRank, resultsCategory, resultsWups, resultsFolder) return ensembleOutputTest
-k: Render top-K answers (default 1) -p[icture]: Picture folder, only required in LaTeX mode (default "img") -r[esults]: Results folder where trained models are stored (default "../results") -f[ile]: Output filename, only required in LaTex mode -dataset: Use DAQUAR/COCO-QA dataset (default "cocoqa") -format: Set output format to HTML/LaTeX (default "html") Input question list format: QID1,Question1,GroundTruthAnswer1 QID2,Question2,GroundTruthAnswer2 ... """ params = ir.parseComparativeParams(sys.argv) urlDict = ir.loadImgUrl(params['dataset'], params['dataFolder']) data = it.loadDataset(params['dataFolder']) maxlen = data['testData'][0].shape[1] print('Parsing input file...') caption, qids, questions, answers = parseInputFile(params['inputFile']) idx = np.array(qids, dtype='int') #inputTestSel = data['testData'][0][idx] #targetTestSel = data['testData'][1][idx] imgids = qids #imgids = inputTestSel[:, 0, 0] inputTest = prep.combine(\ prep.lookupQID(questions, data['questionDict'], maxlen), imgids) targetTest = prep.lookupAnsID(answers, data['ansDict']) questionTypeArray = data['questionTypeArray'][idx] print('Running models...')
for i, flag in enumerate(sys.argv): if flag == '-m' or flag == '-model': modelId = sys.argv[i + 1] elif flag == '-d' or flag == '-data': dataFolder = sys.argv[i + 1] elif flag == '-td' or flag == '-tdata': testDataFolder = sys.argv[i + 1] elif flag == '-reindex': needReindex = True elif flag == '-r' or flag == '-results': resultsFolder = sys.argv[i + 1] elif flag == '-dataset': dataset = sys.argv[i + 1] model = it.loadModel(modelId, resultsFolder) data = it.loadDataset(dataFolder) testdata = it.loadDataset(testDataFolder) if needReindex: testQuestions, testAnswers = reindexDataset( testdata['testData'][0], testdata['testData'][1], testdata['questionIdict'], data['questionDict'], testdata['ansIdict'], data['ansDict']) else: testQuestions = testdata['testData'][0] testAnswers = testdata['testData'][1] outputTest = nn.test(model, testQuestions) rate, correct, total = nn.calcRate(model, outputTest, testAnswers) print 'rate: %.4f' % rate
""" resultsFolder = '../results' modelIds = [] validModelIds = [] for i, flag in enumerate(sys.argv): if flag == '-m' or flag == '-model': modelIds.append(sys.argv[i + 1]) elif flag == '-vm' or flag == '-vmodel': validModelIds.append(sys.argv[i + 1]) elif flag == '-r' or flag == '-results': resultsFolder = sys.argv[i + 1] elif flag == '-d' or flag == '-data': dataFolder = sys.argv[i + 1] elif flag == '-o' or flag == '-output': outputFolder = sys.argv[i + 1] data = it.loadDataset(dataFolder) models = [] validModels = [] for modelId in modelIds: print 'Loading model %s' % modelId models.append(it.loadModel(modelId, resultsFolder)) for modelId in validModelIds: print 'Loading model %s' % modelId validModels.append(it.loadModel(modelId, resultsFolder)) modelOutputs = [] validModelOutputs = [] # for modelId, model in zip(validModelIds, validModels): # print 'Running model %s' % modelId # modelOutput = nn.test(model, data['validData'][0])
for i, flag in enumerate(sys.argv): if flag == '-m' or flag == '-model': modelId = sys.argv[i + 1] elif flag == '-d' or flag == '-data': dataFolder = sys.argv[i + 1] elif flag == '-td' or flag == '-tdata': testDataFolder = sys.argv[i + 1] elif flag == '-reindex': needReindex = True elif flag == '-r' or flag == '-results': resultsFolder = sys.argv[i + 1] elif flag == '-dataset': dataset = sys.argv[i + 1] model = it.loadModel(modelId, resultsFolder) data = it.loadDataset(dataFolder) testdata = it.loadDataset(testDataFolder) if needReindex: testQuestions, testAnswers = reindexDataset(testdata['testData'][0], testdata['testData'][1], testdata['questionIdict'], data['questionDict'], testdata['ansIdict'], data['ansDict']) else: testQuestions = testdata['testData'][0] testAnswers = testdata['testData'][1] outputTest = nn.test(model, testQuestions) rate, correct, total = nn.calcRate(model, outputTest, testAnswers) print 'rate: %.4f' % rate
elif flag == "-mid": mainModelId = sys.argv[i + 1] elif flag == "-bid": boostModelId = sys.argv[i + 1] elif flag == "-vd" or flag == "-vdata": visDataFolder = sys.argv[i + 1] elif flag == "-md" or flag == "-mdata": mainDataFolder = sys.argv[i + 1] elif flag == "-r" or flag == "-results": resultsFolder = sys.argv[i + 1] elif flag == "-qtype": questionType = sys.argv[i + 1] elif flag == "-o" or flag == "-outweights": outputWeightsFolder = sys.argv[i + 1] data = it.loadDataset(visDataFolder) testInput = data["testData"][0] testTarget = data["testData"][1] deltas = [0.000001, 0.000005, 0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] preVisModel = it.loadModel(preVisModelId, resultsFolder) print "Num answer", len(data["ansIdict"]) bestDelta = validDelta( data["trainData"], data["validData"], preVisModel, data["questionDict"], data["questionIdict"], len(data["ansIdict"]), deltas,
elif flag == '-mid': mainModelId = sys.argv[i + 1] elif flag == '-bid': boostModelId = sys.argv[i + 1] elif flag == '-vd' or flag == '-vdata': visDataFolder = sys.argv[i + 1] elif flag == '-md' or flag == '-mdata': mainDataFolder = sys.argv[i + 1] elif flag == '-r' or flag == '-results': resultsFolder = sys.argv[i + 1] elif flag == '-qtype': questionType = sys.argv[i + 1] elif flag == '-o' or flag == '-outweights': outputWeightsFolder = sys.argv[i + 1] data = it.loadDataset(visDataFolder) testInput = data['testData'][0] testTarget = data['testData'][1] deltas = \ [0.000001, 0.000005, 0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5,
def runEnsemblePrior( inputTest, models, dataFolder, classDataFolders, questionTypeArray): """ Similar to "testEnsemble" in imageqa_test. Run visprior on number and color questions. """ data = it.loadDataset(dataFolder) numAns = len(data['ansIdict']) outputTest = np.zeros((inputTest.shape[0], numAns)) count = 0 allOutput = [] ensembleOutputTest = np.zeros((inputTest.shape[0], numAns)) classAnsIdict = [] for i, model in enumerate(models): data_m = it.loadDataset(classDataFolders[i]) classAnsIdict.append(data_m['ansIdict']) tvData_m = ip.combineTrainValid(data_m['trainData'], data_m['validData']) print 'Running test data on model #%d...' % i if i == 0: # Object questions print 'No prior' outputTest = nn.test(model, data_m['testData'][0]) print 'Accuracy:', print ip.calcRate(outputTest, data_m['testData'][1]) elif i == 1 or i == 2 or i == 3: # Number and color and location questions print 'Prior' # Delta is pre-determined if i == 1: delta = 1e-6 questionType = "number" elif i == 2: delta = 5e-4 questionType = "color" elif i == 3: delta = 1.0 questionType = "location" outputTest = ip.runVisPrior( tvData_m, data_m['testData'], questionType, model, data_m['questionDict'], data_m['questionIdict'], len(data_m['ansIdict']), delta) allOutput.append(outputTest) counter = [0, 0, 0, 0] for n in range(inputTest.shape[0]): qtype = questionTypeArray[n] output = allOutput[qtype] for i in range(output.shape[1]): ansId = data['ansDict'][classAnsIdict[qtype][i]] ensembleOutputTest[n, ansId] = output[counter[qtype], i] counter[qtype] += 1 return ensembleOutputTest