def runEnsemble(
                inputTest,
                models, 
                dataFolder, 
                classDataFolders,
                questionTypeArray):
    """
    Run a class specific model on any dataset.
    """
    data = it.loadDataset(dataFolder)
    classAnsIdict = []
    for df in classDataFolders:
        data_c = it.loadDataset(df)
        classAnsIdict.append(data_c['ansIdict'])

    ensembleOutputTest = __runEnsemble(
                                        inputTest, 
                                        models,
                                        data['ansDict'],
                                        classAnsIdict,
                                        questionTypeArray)
    return ensembleOutputTest
def testEnsemble(
                    ensembleId,
                    models,
                    dataFolder,
                    classDataFolders,
                    resultsFolder):
    """
    Test a class specific model in its original dataset.
    """
    data = it.loadDataset(dataFolder)
    inputTest = data['testData'][0]
    targetTest = data['testData'][1]

    ensembleOutputTest = runEnsemble(
                                    inputTest,
                                    models, 
                                    dataFolder, 
                                    classDataFolders,
                                    data['questionTypeArray'])
    ensembleAnswerFile = getAnswerFilename(ensembleId, resultsFolder)
    ensembleTruthFile = getTruthFilename(ensembleId, resultsFolder)

    rate, correct, total = nn.calcRate(
        model, ensembleOutputTest, data['testData'][1])
    print 'rate: %.4f' % rate
    resultsRank, \
    resultsCategory, \
    resultsWups = it.runAllMetrics(
                                inputTest,
                                ensembleOutputTest,
                                targetTest,
                                data['ansIdict'],
                                data['questionTypeArray'],
                                ensembleAnswerFile,
                                ensembleTruthFile)
    it.writeMetricsToFile(
                        ensembleId,
                        rate,
                        resultsRank,
                        resultsCategory,
                        resultsWups,
                        resultsFolder)

    return ensembleOutputTest
Ejemplo n.º 3
0
        -k: Render top-K answers (default 1)
        -p[icture]: Picture folder, only required in LaTeX mode (default "img")
        -r[esults]: Results folder where trained models are stored (default "../results")
        -f[ile]: Output filename, only required in LaTex mode
        -dataset: Use DAQUAR/COCO-QA dataset (default "cocoqa")
        -format: Set output format to HTML/LaTeX (default "html")

    Input question list format:
        QID1,Question1,GroundTruthAnswer1
        QID2,Question2,GroundTruthAnswer2
        ...
    """
    params = ir.parseComparativeParams(sys.argv)

    urlDict = ir.loadImgUrl(params['dataset'], params['dataFolder'])
    data = it.loadDataset(params['dataFolder'])
    maxlen = data['testData'][0].shape[1]

    print('Parsing input file...')
    caption, qids, questions, answers = parseInputFile(params['inputFile'])
    idx = np.array(qids, dtype='int')
    #inputTestSel = data['testData'][0][idx]
    #targetTestSel = data['testData'][1][idx]
    imgids = qids
    #imgids = inputTestSel[:, 0, 0]
    inputTest = prep.combine(\
        prep.lookupQID(questions, data['questionDict'], maxlen), imgids)
    targetTest = prep.lookupAnsID(answers, data['ansDict'])
    questionTypeArray = data['questionTypeArray'][idx]

    print('Running models...')
 for i, flag in enumerate(sys.argv):
     if flag == '-m' or flag == '-model':
         modelId = sys.argv[i + 1]
     elif flag == '-d' or flag == '-data':
         dataFolder = sys.argv[i + 1]
     elif flag == '-td' or flag == '-tdata':
         testDataFolder = sys.argv[i + 1]
     elif flag == '-reindex':
         needReindex = True
     elif flag == '-r' or flag == '-results':
         resultsFolder = sys.argv[i + 1]
     elif flag == '-dataset':
         dataset = sys.argv[i + 1]
 
 model = it.loadModel(modelId, resultsFolder)
 data = it.loadDataset(dataFolder)
 testdata = it.loadDataset(testDataFolder)
 if needReindex:
     testQuestions, testAnswers = reindexDataset(
         testdata['testData'][0],
         testdata['testData'][1],
         testdata['questionIdict'],
         data['questionDict'],
         testdata['ansIdict'],
         data['ansDict'])
 else:
     testQuestions = testdata['testData'][0]
     testAnswers = testdata['testData'][1]
 outputTest = nn.test(model, testQuestions)
 rate, correct, total = nn.calcRate(model, outputTest, testAnswers)
 print 'rate: %.4f' % rate
Ejemplo n.º 5
0
    """
    resultsFolder = '../results'
    modelIds = []
    validModelIds = []
    for i, flag in enumerate(sys.argv):
        if flag == '-m' or flag == '-model':
            modelIds.append(sys.argv[i + 1])
        elif flag == '-vm' or flag == '-vmodel':
            validModelIds.append(sys.argv[i + 1])
        elif flag == '-r' or flag == '-results':
            resultsFolder = sys.argv[i + 1]
        elif flag == '-d' or flag == '-data':
            dataFolder = sys.argv[i + 1]
        elif flag == '-o' or flag == '-output':
            outputFolder = sys.argv[i + 1]
    data = it.loadDataset(dataFolder)
    
    models = []
    validModels = []
    for modelId in modelIds:
        print 'Loading model %s' % modelId
        models.append(it.loadModel(modelId, resultsFolder))
    for modelId in validModelIds:
        print 'Loading model %s' % modelId
        validModels.append(it.loadModel(modelId, resultsFolder))

    modelOutputs = []
    validModelOutputs = []
    # for modelId, model in zip(validModelIds, validModels):
    #     print 'Running model %s' % modelId
    #     modelOutput = nn.test(model, data['validData'][0])
    for i, flag in enumerate(sys.argv):
        if flag == '-m' or flag == '-model':
            modelId = sys.argv[i + 1]
        elif flag == '-d' or flag == '-data':
            dataFolder = sys.argv[i + 1]
        elif flag == '-td' or flag == '-tdata':
            testDataFolder = sys.argv[i + 1]
        elif flag == '-reindex':
            needReindex = True
        elif flag == '-r' or flag == '-results':
            resultsFolder = sys.argv[i + 1]
        elif flag == '-dataset':
            dataset = sys.argv[i + 1]

    model = it.loadModel(modelId, resultsFolder)
    data = it.loadDataset(dataFolder)
    testdata = it.loadDataset(testDataFolder)
    if needReindex:
        testQuestions, testAnswers = reindexDataset(testdata['testData'][0],
                                                    testdata['testData'][1],
                                                    testdata['questionIdict'],
                                                    data['questionDict'],
                                                    testdata['ansIdict'],
                                                    data['ansDict'])
    else:
        testQuestions = testdata['testData'][0]
        testAnswers = testdata['testData'][1]
    outputTest = nn.test(model, testQuestions)
    rate, correct, total = nn.calcRate(model, outputTest, testAnswers)
    print 'rate: %.4f' % rate
        elif flag == "-mid":
            mainModelId = sys.argv[i + 1]
        elif flag == "-bid":
            boostModelId = sys.argv[i + 1]
        elif flag == "-vd" or flag == "-vdata":
            visDataFolder = sys.argv[i + 1]
        elif flag == "-md" or flag == "-mdata":
            mainDataFolder = sys.argv[i + 1]
        elif flag == "-r" or flag == "-results":
            resultsFolder = sys.argv[i + 1]
        elif flag == "-qtype":
            questionType = sys.argv[i + 1]
        elif flag == "-o" or flag == "-outweights":
            outputWeightsFolder = sys.argv[i + 1]

    data = it.loadDataset(visDataFolder)
    testInput = data["testData"][0]
    testTarget = data["testData"][1]
    deltas = [0.000001, 0.000005, 0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0]

    preVisModel = it.loadModel(preVisModelId, resultsFolder)

    print "Num answer", len(data["ansIdict"])
    bestDelta = validDelta(
        data["trainData"],
        data["validData"],
        preVisModel,
        data["questionDict"],
        data["questionIdict"],
        len(data["ansIdict"]),
        deltas,
Ejemplo n.º 8
0
        elif flag == '-mid':
            mainModelId = sys.argv[i + 1]
        elif flag == '-bid':
            boostModelId = sys.argv[i + 1]
        elif flag == '-vd' or flag == '-vdata':
            visDataFolder = sys.argv[i + 1]
        elif flag == '-md' or flag == '-mdata':
            mainDataFolder = sys.argv[i + 1]
        elif flag == '-r' or flag == '-results':
            resultsFolder = sys.argv[i + 1]
        elif flag == '-qtype':
            questionType = sys.argv[i + 1]
        elif flag == '-o' or flag == '-outweights':
            outputWeightsFolder = sys.argv[i + 1]

    data = it.loadDataset(visDataFolder)
    testInput = data['testData'][0]
    testTarget = data['testData'][1]
    deltas = \
        [0.000001, 
        0.000005, 
        0.00001, 
        0.00005, 
        0.0001, 
        0.0005, 
        0.001, 
        0.005, 
        0.01, 
        0.05, 
        0.1, 
        0.5, 
def runEnsemblePrior(
                        inputTest,
                        models, 
                        dataFolder, 
                        classDataFolders,
                        questionTypeArray):
    """
    Similar to "testEnsemble" in imageqa_test.
    Run visprior on number and color questions.
    """
    data = it.loadDataset(dataFolder)
    numAns = len(data['ansIdict'])
    outputTest = np.zeros((inputTest.shape[0], numAns))
    count = 0

    allOutput = []
    ensembleOutputTest = np.zeros((inputTest.shape[0], numAns))
    classAnsIdict = []

    for i, model in enumerate(models):
        data_m = it.loadDataset(classDataFolders[i])
        classAnsIdict.append(data_m['ansIdict'])
        tvData_m = ip.combineTrainValid(data_m['trainData'], data_m['validData'])
        print 'Running test data on model #%d...' % i
        if i == 0:
            # Object questions
            print 'No prior'
            outputTest = nn.test(model, data_m['testData'][0])
            print 'Accuracy:',
            print ip.calcRate(outputTest, data_m['testData'][1])
        elif i == 1 or i == 2 or i == 3:
            # Number and color and location questions
            print 'Prior'
            # Delta is pre-determined
            if i == 1:
                delta = 1e-6
                questionType = "number"
            elif i == 2:
                delta = 5e-4
                questionType = "color"
            elif i == 3:
                delta = 1.0
                questionType = "location"
            outputTest = ip.runVisPrior(
                                tvData_m,
                                data_m['testData'],
                                questionType,
                                model,
                                data_m['questionDict'],
                                data_m['questionIdict'],
                                len(data_m['ansIdict']),
                                delta)
        allOutput.append(outputTest)
    counter = [0, 0, 0, 0]
    for n in range(inputTest.shape[0]):
        qtype = questionTypeArray[n]
        output = allOutput[qtype]
        for i in range(output.shape[1]):
            ansId = data['ansDict'][classAnsIdict[qtype][i]]
            ensembleOutputTest[n, ansId] = output[counter[qtype], i]
        counter[qtype] += 1
    return ensembleOutputTest