def test_allTruePos(self): npTestImgTruth1 = np.ones((3, 3)) npTestImgGen1 = np.ones((3, 3)) npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1) npExpectError = [0, 0, 0, 9] self.assertEqual(npExpectError[0], npError[0]) self.assertEqual(npExpectError[1:], npError[1:])
def test_3Ch(self): npTrue = np.array([[[0,0,1],[1,0,0]],[[0,0,1],[0,1,0]]]) npPred = np.array([[[1],[1]],[[1],[1]]]) npColourMap = Error.ColouredPredMap(npPred, npTrue[:,:,1:2]) npExpectedColourMap = np.array([[self.npFP, self.npFP], [self.npFP, self.npTP]]) self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_allTrueNeg(self): npTestImgTruth1 = np.zeros((3, 3)) npTestImgGen1 = np.zeros((3, 3)) npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1) npExpectError = [9, 0, 0, 0] self.assertEqual(npExpectError[0], npError[0]) self.assertEqual(npExpectError[1:], npError[1:])
def test_GenCase(self): npTrue = np.array([[[0],[1]],[[1],[0]]]) npPred = np.array([[[0],[1]],[[0],[1]]]) npColourMap = Error.ColouredPredMap(npPred, npTrue) npExpectedColourMap = np.array([[self.npTN, self.npTP], [self.npFN, self.npFP]], dtype= np.uint8) self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_AllTP(self): npTrue = np.array([[[1],[1]],[[1],[1]]]) npPred = np.array([[[1],[1]],[[1],[1]]]) npColourMap = Error.ColouredPredMap(npPred, npTrue) npExpectedColourMap = np.array([[self.npTP, self.npTP], [self.npTP, self.npTP]]) self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_OneTP(self): """ case where auc is one """ npTruthValues = np.array([1, 1, 1, 1, 1, 0]) npResults = np.array([0.1, 0.9, 0.2, 0.2, 0.2, 0.2]) [auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues) self.assertTrue(auc, 0.5)
def test_AllFN(self): """ case where auc is one """ npTruthValues = np.array([1, 1, 1]) npResults = np.array([0.1, 0.1, 0.2]) [auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues) self.assertTrue(np.isnan(auc))
def test_AllCorrect(self): """ case where auc is one """ npTruthValues = np.array([0, 0, 0, 1, 1, 1]) npResults = np.array([0.1, 0.1, 0.2, 0.9, 0.8, 0.7]) [auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues) self.assertTrue(auc, 1)
def test_AllTN(self): """ case where everthing zero might make things weird """ npTruthValues = np.array([0, 0, 0]) npResults = np.array([0.2, 0.3, 0.4]) [auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues) self.assertTrue(np.isnan(auc))
def test_mixed(self): npTestImgTruth1 = np.array([[0, 0, 1], [0, 0, 1], [1, 0, 1]]) npTestImgGen1 = np.array([[0, 0, 1], [1, 1, 0], [0, 0, 0]]) npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1) #tn, fp, fn, tp npExpectError = [3, 2, 3, 1] self.assertEqual(npExpectError[0], npError[0]) self.assertEqual(npExpectError[1:], npError[1:])
def test_AllGrey(self): lLegend = [self.npTN, self.npFP, self.npFN, self.npTP, self.grey] npExpectedColourMap = np.array( [[self.grey, self.grey], [self.grey, self.grey], [self.grey, self.grey]], dtype=np.uint8) npExpectedCounts = [0, 0, 0, 0, 6] # tn, fp, fn, tp, range self.assertEqual( Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend), npExpectedCounts)
def test_NoGrey(self): lLegend = [self.npTN, self.npFP, self.npFN, self.npTP] npExpectedColourMap = np.array([ [self.npFP, self.npFP, self.npFP, self.npFP, self.npFP, self.npFP], [self.npFP, self.npFP, self.npFP, self.npFP, self.npFP, self.npFP], [self.npTP, self.npTP, self.npTP, self.npFP, self.npFP, self.npFP], [self.npTP, self.npTP, self.npTP, self.npTP, self.npTN, self.npTN], [self.npFN, self.npFN, self.npTP, self.npTP, self.npTN, self.npTN], [self.npFN, self.npFN, self.npTP, self.npTP, self.npTN, self.npTN] ], dtype=np.uint8) npExpectedCounts = [6, 15, 4, 11] # tn, fp, fn, tp, range self.assertEqual( Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend), npExpectedCounts)
def test_TPDiffColours(self): lLegend = [ self.npTN, self.npFP, self.npFN, self.npTP, self.grey, [0, 0, 0] ] npExpectedColourMap = np.array([ [self.npTP, self.npTP, self.grey, self.grey, self.npTN, self.npTN], [self.npTP, self.npTP, self.grey, self.grey, self.npTN, self.npTN], [self.grey, self.grey, self.grey, [0, 0, 0], self.npTN, self.npTN], [self.grey, self.grey, self.grey, self.npTN, self.npTN, self.npTN], [self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN], [self.npTN, self.npTN, self.npTN, [0, 0, 0], [0, 0, 0], self.npTN] ], dtype=np.uint8) npExpectedCounts = [19, 0, 0, 4, 10, 3] # tn, fp, fn, tp, range self.assertEqual( Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend), npExpectedCounts)
def test_AllLarger(self): lLegend = [self.npTN, self.npFP, self.npFN, self.npTP, self.grey] npExpectedColourMap = np.array( [[ self.npFN, self.npFN, self.npFN, self.npFN, self.npFN, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npFN, self.npFN, self.npFN, self.npFN, self.npFN, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npFN, self.npFN, self.npFN, self.npFN, self.npFN, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npTP, self.npTP, self.npTP, self.npTP, self.npTP, self.grey, self.grey, self.npFP, self.npFP, self.npTN, self.npTN, self.npTN ], [ self.npTP, self.npTP, self.npTP, self.npTP, self.npTP, self.grey, self.grey, self.npFP, self.npFP, self.npTN, self.npTN, self.npTN ], [ self.npTP, self.npTP, self.npTP, self.npTP, self.npTP, self.grey, self.grey, self.npFP, self.npFP, self.npTN, self.npTN, self.npTN ], [ self.npFN, self.npFN, self.npFN, self.npFN, self.npFN, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npFN, self.npFN, self.npFN, self.npFN, self.npFN, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.grey, self.grey, self.grey, self.grey, self.grey, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.grey, self.grey, self.grey, self.grey, self.grey, self.grey, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ], [ self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN ]], dtype=np.uint8) npExpectedCounts = [9 * 5 + 6 * 4, 6, 9 + 6 * 2 + 4, 15, 6 * 5 - 1] # tn, fp, fn, tp, range self.assertEqual( Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend), npExpectedCounts)
def TrainAndTestUNet(oConfig, iNumFoldsTotal, kFold, Model, GenTrain, GenVal, np1TrainDir, np1ValDir, np1TestDir, iNumSave=20): """ Trains Model using GenTrain and GenVal data. These are imagedatagenerators Uses TestData, a numpy array, as the prediction values """ # set up training parameters iNumSave = int( oConfig['IO'] ['Number Save']) # number of prediciton maps saved per group lImageSize = [ int(sDim) for sDim in oConfig['Generator']['Image Shape'].split(',') ] lRange = [int(x) for x in oConfig['Setup']['Channels'].split(',')] npFiles = np.genfromtxt( os.path.join('..', oConfig['IO']['Input File']), delimiter=',', dtype=str) # text file that lists directories to file pairs to be read # add escape to suuper directory because this file is in .src sOutDir = os.path.join('..', oConfig['IO']['Output Dir']) iApproxNumImages = sum([ len(files) for directory in npFiles[:, 0] for r, d, files in os.walk( os.path.join(os.path.dirname(__file__), directory)) ]) if iApproxNumImages < 1: raise ValueError('no images found') # this fold factor is used to help determine the steps per epoch # this condition is added so that in the case of only 1 fold, this factor will not # become 0 and break the code. fFoldFactor = (iNumFoldsTotal - 1) / iNumFoldsTotal if fFoldFactor < 1: fFoldFactor = 1 # can add tramsforms to the Fit_args if oConfig['Generator']['Steps per epoch'] == 'Auto': fTestSize = float(oConfig['Generator']['Test Size']) Fit_args = dict( steps_per_epoch=math.ceil(iApproxNumImages // int(oConfig['Generator']['Batch Size']) * (1 - fTestSize) * fFoldFactor), epochs=int(oConfig['Model']['Epochs']), validation_steps=math.ceil( iApproxNumImages // int(oConfig['Generator']['Batch Size']) * fTestSize * fFoldFactor)) else: Fit_args = dict( steps_per_epoch=int(oConfig['Generator']['Steps per epoch']), epochs=int(oConfig['Model']['Epochs']), validation_steps=int(oConfig['Generator']['Steps per epoch'])) # check if error occured in step size calculations if Fit_args['validation_steps'] < 1 or Fit_args['steps_per_epoch'] < 1: raise ValueError( 'validation step size or training step size less than 1. this is an invalid input' ) # train model Model = TrainModel(sOutDir, Model, GenTrain, GenVal, Fit_args, np1TrainDir, np1ValDir, np1TestDir, kFold) #TODO: add search for iNumPred in oConfig object iNumPredict = oConfig['IO']['Number Predict'] if iNumPredict == 'All': # if iNumPredict is none, then it will predict on the entire dataset iNumPredict = None else: iNumPredict = int(iNumPredict) Error.PredictionErrorPerClass(os.path.join(sOutDir, str(kFold) + 'thFold'), np1TestDir, Model, lImageSize, iNumSave=iNumSave, lRange=lRange, iNumPredict=iNumPredict)