def runTests(activationsPath, indexLabelMappingPath, sourcePath): labels = utility.getLabelStrings(indexLabelMappingPath) activations = utility.arrayFromFile(activationsPath) if sourcePath.endswith('/') == False: sourcePath += '/' clusterGroups = loadClusters(sourcePath) # returns list with all previously generated cluster groups mixedClusterResultsSimple = [] perLabelClusterResultsSimple = [] for clusters in clusterGroups: logger.info("Evaluating clusters at " + clusters['path']) if(clusters['type'] == 'perLabel'): [confusionMatrix, meanAveragePrecision, overallCorrect, overallWrong] = runTest(clusters['data'], activations, labels True) # (clusters, activations, labels) perLabelClusterResultsSimple.append([clusters['k'], meanAveragePrecision, overallCorrect, overallWrong]) saveConfusionMatrix(confusionMatrix, clusters['path'] + "confusion_test.npy") utility.plotConfusionMatrix(confusionMatrix, labels, clusters['path'] + "confusion_test.pdf") else: [confusionMatrix, meanAveragePrecision, overallCorrect, overallWrong] = runTest(clusters['data'], activations, labels False) # (clusters, activations, labels) mixedClusterResultsSimple.append([clusters['k'], meanAveragePrecision, overallCorrect, overallWrong]) saveConfusionMatrix(confusionMatrix, clusters['path'] + "confusion_test.npy") utility.plotConfusionMatrix(confusionMatrix, labels, clusters['path'] + "confusion_test.pdf") overviewPerLabel = np.array(perLabelClusterResultsSimple) overviewMixed = np.array(mixedClusterResultsSimple) saveOverview(overviewPerLabel, sourcePath + 'perLabel_') saveOverview(overviewMixed, sourcePath + 'result_mixed_') utility.plotKMeansOverview(overviewPerLabel, sourcePath + 'perLabel_result.pdf', True) utility.plotKMeansOverview(overviewMixed, sourcePath + 'result_mixed_result.pdf', True)
def runTests(training, test, labels, targetPath, nn): global MAX_K if targetPath.endswith('/') == False: targetPath += '/' if not os.path.exists(os.path.dirname(targetPath)): os.makedirs(os.path.dirname(targetPath)) labelCount = len(labels) MAX_K = training.shape[0] / labelCount neurons = training.shape[1] - labelCount if nn == None: nn = knn.getKNearestNeighbours(training, test, MAX_K, neurons) with open(targetPath + "nn.npy", "w") as outputFile: np.save(outputFile, nn) nnByLabel = [[] for i in range(labelCount)] currentLabelIndex = 0 while currentLabelIndex < labelCount: currentSelection = nn[nn[:, currentLabelIndex] == 1] nnByLabel[currentLabelIndex] = currentSelection currentLabelIndex += 1 allLabels = training[:,neurons:] sumPerLabel = np.sum(allLabels, axis=0) factorPerLabel = np.max(sumPerLabel) / sumPerLabel kCounter = 1 results = [] while kCounter < MAX_K: logger.info("Evaluating K = " + str(kCounter) + ":") overallCorrect = 0 overallWrong = 0 meanAveragePrecision = 0 confusionMatrix = np.zeros((labelCount,labelCount)) for labelIndex, values in enumerate(nnByLabel): confusion = np.zeros((1,len(labels))) activationCounter = 0 while activationCounter < values.shape[0]: currentActivation = values[activationCounter] searchedLabel = np.argwhere(currentActivation[:labelCount] == 1) nIndices = currentActivation[labelCount:labelCount+kCounter].astype(int) neighbourLabels = training[nIndices][:,neurons:] neighbourLabelsSum = np.sum(neighbourLabels, axis=0) neighbourLabelsSum = neighbourLabelsSum * factorPerLabel sortedNeighbourLabels = np.argsort(neighbourLabelsSum)[::-1] if sortedNeighbourLabels[0] == searchedLabel: overallCorrect += 1 else: overallWrong += 1 confusion[0, sortedNeighbourLabels[0]] += 1 averagePrecision = 0 relevant = 0 for idx, value in enumerate(sortedNeighbourLabels): indicator = 0 if(value == searchedLabel and neighbourLabelsSum[value] != 0): indicator = 1 relevant += 1 precision = float(relevant) / (idx + 1) averagePrecision += (precision * indicator) if relevant != 0: averagePrecision = float(averagePrecision) / relevant meanAveragePrecision += averagePrecision activationCounter += 1 confusionMatrix[labelIndex,:] = confusion meanAveragePrecision = float(meanAveragePrecision) / test.shape[0] logger.info(' Accuracy: ' + str(float(overallCorrect)/(overallWrong + overallCorrect))) logger.info(' Mean average precision: '+str(meanAveragePrecision)) currentTargetPath = targetPath + str(kCounter) + "/" if not os.path.exists(os.path.dirname(currentTargetPath)): os.makedirs(os.path.dirname(currentTargetPath)) logger.info('Saving confusion matrix ' + currentTargetPath + "confusion.npy") with open(currentTargetPath + "confusion.npy", "w") as outputFile: np.save(outputFile, confusionMatrix) results.append([kCounter, meanAveragePrecision, overallCorrect, overallWrong]) utility.plotConfusionMatrix(confusionMatrix, labels, currentTargetPath + "confusion.pdf") kCounter += 1 results = np.array(results) logger.info('Writing file ' + targetPath + "overview.csv") np.savetxt( targetPath + "overview.csv", results, delimiter=',') utility.plotKMeansOverview(results, targetPath + "overview.pdf", False)
logger.info("4) Target path for evaluation results.") sys.exit(); labelIndexInfoPath = sys.argv[1] labelIndexMappingPath = sys.argv[2] logFilePath = sys.argv[3] evaluationTargetPath = sys.argv[4] if evaluationTargetPath.endswith('/') == False: evaluationTargetPath += '/' if not os.path.exists(os.path.dirname(evaluationTargetPath)) and os.path.dirname(evaluationTargetPath) != '': os.makedirs(os.path.dirname(evaluationTargetPath)) plotTrainingLossAndAccuracy(logFilePath, evaluationTargetPath) # sys.exit() labels = utility.getLabelStrings(labelIndexMappingPath) [confusionMatrix, meanAveragePrecision, overallCorrect, overallWrong] = testNeuralNet(labels, labelIndexInfoPath, evaluationTargetPath) overviewPath = evaluationTargetPath + "overview.csv" logger.info('Writing file ' + overviewPath) np.savetxt(overviewPath, np.array([0,meanAveragePrecision, overallCorrect, overallWrong]), delimiter=',') confusionMatrixPath = evaluationTargetPath + 'confusionMatrix.npy' logger.info('Writing file ' + confusionMatrixPath) with open(confusionMatrixPath, "w") as outputFile: np.save(outputFile, confusionMatrix) utility.plotConfusionMatrix(confusionMatrix, labels, evaluationTargetPath + 'confusionMatrix.pdf')
def runTests(training, test, labels, targetPath, nn): global MAX_K if targetPath.endswith('/') == False: targetPath += '/' if not os.path.exists(os.path.dirname(targetPath)): os.makedirs(os.path.dirname(targetPath)) labelCount = len(labels) MAX_K = training.shape[0] / labelCount neurons = training.shape[1] - labelCount if nn == None: nn = knn.getKNearestNeighbours(training, test, MAX_K, neurons) with open(targetPath + "nn.npy", "w") as outputFile: np.save(outputFile, nn) nnByLabel = [[] for i in range(labelCount)] currentLabelIndex = 0 while currentLabelIndex < labelCount: currentSelection = nn[nn[:, currentLabelIndex] == 1] nnByLabel[currentLabelIndex] = currentSelection currentLabelIndex += 1 allLabels = training[:, neurons:] sumPerLabel = np.sum(allLabels, axis=0) factorPerLabel = np.max(sumPerLabel) / sumPerLabel kCounter = 1 results = [] while kCounter < MAX_K: logger.info("Evaluating K = " + str(kCounter) + ":") overallCorrect = 0 overallWrong = 0 meanAveragePrecision = 0 confusionMatrix = np.zeros((labelCount, labelCount)) for labelIndex, values in enumerate(nnByLabel): confusion = np.zeros((1, len(labels))) activationCounter = 0 while activationCounter < values.shape[0]: currentActivation = values[activationCounter] searchedLabel = np.argwhere( currentActivation[:labelCount] == 1) nIndices = currentActivation[labelCount:labelCount + kCounter].astype(int) neighbourLabels = training[nIndices][:, neurons:] neighbourLabelsSum = np.sum(neighbourLabels, axis=0) neighbourLabelsSum = neighbourLabelsSum * factorPerLabel sortedNeighbourLabels = np.argsort(neighbourLabelsSum)[::-1] if sortedNeighbourLabels[0] == searchedLabel: overallCorrect += 1 else: overallWrong += 1 confusion[0, sortedNeighbourLabels[0]] += 1 averagePrecision = 0 relevant = 0 for idx, value in enumerate(sortedNeighbourLabels): indicator = 0 if (value == searchedLabel and neighbourLabelsSum[value] != 0): indicator = 1 relevant += 1 precision = float(relevant) / (idx + 1) averagePrecision += (precision * indicator) if relevant != 0: averagePrecision = float(averagePrecision) / relevant meanAveragePrecision += averagePrecision activationCounter += 1 confusionMatrix[labelIndex, :] = confusion meanAveragePrecision = float(meanAveragePrecision) / test.shape[0] logger.info( ' Accuracy: ' + str(float(overallCorrect) / (overallWrong + overallCorrect))) logger.info(' Mean average precision: ' + str(meanAveragePrecision)) currentTargetPath = targetPath + str(kCounter) + "/" if not os.path.exists(os.path.dirname(currentTargetPath)): os.makedirs(os.path.dirname(currentTargetPath)) logger.info('Saving confusion matrix ' + currentTargetPath + "confusion.npy") with open(currentTargetPath + "confusion.npy", "w") as outputFile: np.save(outputFile, confusionMatrix) results.append( [kCounter, meanAveragePrecision, overallCorrect, overallWrong]) utility.plotConfusionMatrix(confusionMatrix, labels, currentTargetPath + "confusion.pdf") kCounter += 1 results = np.array(results) logger.info('Writing file ' + targetPath + "overview.csv") np.savetxt(targetPath + "overview.csv", results, delimiter=',') utility.plotKMeansOverview(results, targetPath + "overview.pdf", False)
def testBayes(training, test, labels, targetPath): if targetPath.endswith('/') == False: targetPath += '/' if not os.path.exists(os.path.dirname(targetPath)): os.makedirs(os.path.dirname(targetPath)) labelCount = len(labels) neurons = training.shape[1] - labelCount minTraining = np.min(training[:, 0:neurons]) minTest = np.min(test[:, 0:neurons]) absoluteMin = 0 if minTraining < minTest: absoluteMin = minTraining else: absoluteMin = minTest training[:, 0:neurons] = training[:, 0:neurons] + np.abs(absoluteMin) test[:, 0:neurons] = test[:, 0:neurons] + np.abs(absoluteMin) PLog = tb.trainBayes(training, labelCount) overallCorrect = 0 overallWrong = 0 meanAveragePrecision = 0 featuresByLabel = utility.splitTestFeaturesByLabel(test, len(labels)) confusionMatrix = np.zeros((labelCount, labelCount)) for labelIndex, activations in enumerate(featuresByLabel): counter = 0 while counter < activations.shape[0]: currentActivation = activations[counter, 0:neurons] searchedLabel = np.argmax(activations[counter, neurons:]) predictions = currentActivation * PLog predictions = np.sum(predictions, axis=1) if np.argmax(predictions) == searchedLabel: overallCorrect += 1 else: overallWrong += 1 confusionMatrix[labelIndex, np.argmax(predictions)] += 1 averagePrecision = 0 relevant = 0 predictedLabelsSorted = np.argsort(predictions)[::-1] for idx, value in enumerate(predictedLabelsSorted): indicator = 0 if (value == searchedLabel): indicator = 1 relevant += 1 precision = float(relevant) / (idx + 1) averagePrecision += (precision * indicator) if relevant != 0: averagePrecision = float(averagePrecision) / relevant meanAveragePrecision += averagePrecision counter += 1 meanAveragePrecision = float(meanAveragePrecision) / test.shape[0] logger.info(' Accuracy: ' + str(float(overallCorrect) / (overallWrong + overallCorrect))) logger.info(' Mean average precision: ' + str(meanAveragePrecision)) utility.plotConfusionMatrix(confusionMatrix, labels, targetPath + "confusion.pdf") results = [0, meanAveragePrecision, overallCorrect, overallWrong] logger.info('Writing file ' + targetPath + "overview.csv") np.savetxt(targetPath + "overview.csv", results, delimiter=',')
evaluationTargetPath = sys.argv[4] if evaluationTargetPath.endswith('/') == False: evaluationTargetPath += '/' if not os.path.exists(os.path.dirname(evaluationTargetPath) ) and os.path.dirname(evaluationTargetPath) != '': os.makedirs(os.path.dirname(evaluationTargetPath)) plotTrainingLossAndAccuracy(logFilePath, evaluationTargetPath) # sys.exit() labels = utility.getLabelStrings(labelIndexMappingPath) [confusionMatrix, meanAveragePrecision, overallCorrect, overallWrong] = testNeuralNet(labels, labelIndexInfoPath, evaluationTargetPath) overviewPath = evaluationTargetPath + "overview.csv" logger.info('Writing file ' + overviewPath) np.savetxt(overviewPath, np.array( [0, meanAveragePrecision, overallCorrect, overallWrong]), delimiter=',') confusionMatrixPath = evaluationTargetPath + 'confusionMatrix.npy' logger.info('Writing file ' + confusionMatrixPath) with open(confusionMatrixPath, "w") as outputFile: np.save(outputFile, confusionMatrix) utility.plotConfusionMatrix(confusionMatrix, labels, evaluationTargetPath + 'confusionMatrix.pdf')
def testBayes(training, test, labels, targetPath): if targetPath.endswith('/') == False: targetPath += '/' if not os.path.exists(os.path.dirname(targetPath)): os.makedirs(os.path.dirname(targetPath)) labelCount = len(labels) neurons = training.shape[1] - labelCount minTraining = np.min(training[:,0:neurons]) minTest = np.min(test[:,0:neurons]) absoluteMin = 0 if minTraining < minTest: absoluteMin = minTraining else: absoluteMin = minTest training[:,0:neurons] = training[:,0:neurons] + np.abs(absoluteMin) test[:,0:neurons] = test[:,0:neurons] + np.abs(absoluteMin) PLog = tb.trainBayes(training, labelCount) overallCorrect = 0 overallWrong = 0 meanAveragePrecision = 0 featuresByLabel = utility.splitTestFeaturesByLabel(test, len(labels)) confusionMatrix = np.zeros((labelCount,labelCount)) for labelIndex, activations in enumerate(featuresByLabel): counter = 0 while counter < activations.shape[0]: currentActivation = activations[counter,0:neurons] searchedLabel = np.argmax(activations[counter,neurons:]) predictions = currentActivation * PLog predictions = np.sum(predictions, axis=1) if np.argmax(predictions) == searchedLabel: overallCorrect += 1 else: overallWrong += 1 confusionMatrix[labelIndex,np.argmax(predictions)] += 1 averagePrecision = 0 relevant = 0 predictedLabelsSorted = np.argsort(predictions)[::-1] for idx, value in enumerate(predictedLabelsSorted): indicator = 0 if(value == searchedLabel): indicator = 1 relevant += 1 precision = float(relevant) / (idx + 1) averagePrecision += (precision * indicator) if relevant != 0: averagePrecision = float(averagePrecision) / relevant meanAveragePrecision += averagePrecision counter += 1 meanAveragePrecision = float(meanAveragePrecision) / test.shape[0] logger.info(' Accuracy: ' + str(float(overallCorrect)/(overallWrong + overallCorrect))) logger.info(' Mean average precision: '+str(meanAveragePrecision)) utility.plotConfusionMatrix(confusionMatrix, labels, targetPath + "confusion.pdf") results = [0, meanAveragePrecision, overallCorrect, overallWrong] logger.info('Writing file ' + targetPath + "overview.csv") np.savetxt( targetPath + "overview.csv", results, delimiter=',')