def testAvgAll(modelOutputs, mixRatio, data, outputFolder):
    # finalOutput = mixRatio * modelOutputs[0] + \
    #     (1 - mixRatio) * modelOutputs[1]
    finalOutput = np.zeros(modelOutputs[0].shape)
    for output in modelOutputs:
        shape0 = min(finalOutput.shape[0], output.shape[0])
        shape1 = min(finalOutput.shape[1], output.shape[1])
        finalOutput[:shape0, :shape1] += output[:shape0, :shape1] / float(len(modelOutputs))
    testAnswerFile = it.getAnswerFilename(outputFolder, resultsFolder)
    testTruthFile = it.getTruthFilename(outputFolder, resultsFolder)
    resultsRank, \
    resultsCategory, \
    resultsWups = it.runAllMetrics(
        data['testData'][0],
        finalOutput,
        data['testData'][1],
        data['ansIdict'],
        data['questionTypeArray'],
        testAnswerFile,
        testTruthFile)
    it.writeMetricsToFile(
        outputFolder,
        resultsRank,
        resultsCategory,
        resultsWups,
        resultsFolder)
def testEnsemble(
                    ensembleId,
                    models,
                    dataFolder,
                    classDataFolders,
                    resultsFolder):
    """
    Test a class specific model in its original dataset.
    """
    data = it.loadDataset(dataFolder)
    inputTest = data['testData'][0]
    targetTest = data['testData'][1]

    ensembleOutputTest = runEnsemble(
                                    inputTest,
                                    models, 
                                    dataFolder, 
                                    classDataFolders,
                                    data['questionTypeArray'])
    ensembleAnswerFile = getAnswerFilename(ensembleId, resultsFolder)
    ensembleTruthFile = getTruthFilename(ensembleId, resultsFolder)

    rate, correct, total = nn.calcRate(
        model, ensembleOutputTest, data['testData'][1])
    print 'rate: %.4f' % rate
    resultsRank, \
    resultsCategory, \
    resultsWups = it.runAllMetrics(
                                inputTest,
                                ensembleOutputTest,
                                targetTest,
                                data['ansIdict'],
                                data['questionTypeArray'],
                                ensembleAnswerFile,
                                ensembleTruthFile)
    it.writeMetricsToFile(
                        ensembleId,
                        rate,
                        resultsRank,
                        resultsCategory,
                        resultsWups,
                        resultsFolder)

    return ensembleOutputTest