def test_allTruePos(self):
npTestImgTruth1 = np.ones((3, 3))
npTestImgGen1 = np.ones((3, 3))
npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1)
npExpectError = [0, 0, 0, 9]
self.assertEqual(npExpectError[0], npError[0])
self.assertEqual(npExpectError[1:], npError[1:])
def test_3Ch(self):
npTrue = np.array([[[0,0,1],[1,0,0]],[[0,0,1],[0,1,0]]])
npPred = np.array([[[1],[1]],[[1],[1]]])
npColourMap = Error.ColouredPredMap(npPred, npTrue[:,:,1:2])
npExpectedColourMap = np.array([[self.npFP, self.npFP],
[self.npFP, self.npTP]])
self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_allTrueNeg(self):
npTestImgTruth1 = np.zeros((3, 3))
npTestImgGen1 = np.zeros((3, 3))
npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1)
npExpectError = [9, 0, 0, 0]
self.assertEqual(npExpectError[0], npError[0])
self.assertEqual(npExpectError[1:], npError[1:])
def test_GenCase(self):
npTrue = np.array([[[0],[1]],[[1],[0]]])
npPred = np.array([[[0],[1]],[[0],[1]]])
npColourMap = Error.ColouredPredMap(npPred, npTrue)
npExpectedColourMap = np.array([[self.npTN, self.npTP],
[self.npFN, self.npFP]], dtype= np.uint8)
self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_AllTP(self):
npTrue = np.array([[[1],[1]],[[1],[1]]])
npPred = np.array([[[1],[1]],[[1],[1]]])
npColourMap = Error.ColouredPredMap(npPred, npTrue)
npExpectedColourMap = np.array([[self.npTP, self.npTP],
[self.npTP, self.npTP]])
self.assertEqual(npColourMap[0].tolist(), npExpectedColourMap.tolist())
def test_OneTP(self):
""" case where auc is one
"""
npTruthValues = np.array([1, 1, 1, 1, 1, 0])
npResults = np.array([0.1, 0.9, 0.2, 0.2, 0.2, 0.2])
[auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues)
self.assertTrue(auc, 0.5)
def test_AllFN(self):
""" case where auc is one
"""
npTruthValues = np.array([1, 1, 1])
npResults = np.array([0.1, 0.1, 0.2])
[auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues)
self.assertTrue(np.isnan(auc))
def test_AllCorrect(self):
""" case where auc is one
"""
npTruthValues = np.array([0, 0, 0, 1, 1, 1])
npResults = np.array([0.1, 0.1, 0.2, 0.9, 0.8, 0.7])
[auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues)
self.assertTrue(auc, 1)
def test_AllTN(self):
""" case where everthing zero might make things weird
"""
npTruthValues = np.array([0, 0, 0])
npResults = np.array([0.2, 0.3, 0.4])
[auc, fpr, tpr, thresholds] = Error.GenAUC(npResults, npTruthValues)
self.assertTrue(np.isnan(auc))
def test_mixed(self):
npTestImgTruth1 = np.array([[0, 0, 1], [0, 0, 1], [1, 0, 1]])
npTestImgGen1 = np.array([[0, 0, 1], [1, 1, 0], [0, 0, 0]])
npError = Error.ConfMatrix(npTestImgTruth1, npTestImgGen1)
#tn, fp, fn, tp
npExpectError = [3, 2, 3, 1]
self.assertEqual(npExpectError[0], npError[0])
self.assertEqual(npExpectError[1:], npError[1:])
def test_AllGrey(self):
lLegend = [self.npTN, self.npFP, self.npFN, self.npTP, self.grey]
npExpectedColourMap = np.array(
[[self.grey, self.grey], [self.grey, self.grey],
[self.grey, self.grey]],
dtype=np.uint8)
npExpectedCounts = [0, 0, 0, 0, 6] # tn, fp, fn, tp, range
self.assertEqual(
Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend),
npExpectedCounts)
def test_NoGrey(self):
lLegend = [self.npTN, self.npFP, self.npFN, self.npTP]
npExpectedColourMap = np.array([
[self.npFP, self.npFP, self.npFP, self.npFP, self.npFP, self.npFP],
[self.npFP, self.npFP, self.npFP, self.npFP, self.npFP, self.npFP],
[self.npTP, self.npTP, self.npTP, self.npFP, self.npFP, self.npFP],
[self.npTP, self.npTP, self.npTP, self.npTP, self.npTN, self.npTN],
[self.npFN, self.npFN, self.npTP, self.npTP, self.npTN, self.npTN],
[self.npFN, self.npFN, self.npTP, self.npTP, self.npTN, self.npTN]
],
dtype=np.uint8)
npExpectedCounts = [6, 15, 4, 11] # tn, fp, fn, tp, range
self.assertEqual(
Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend),
npExpectedCounts)
def test_TPDiffColours(self):
lLegend = [
self.npTN, self.npFP, self.npFN, self.npTP, self.grey, [0, 0, 0]
]
npExpectedColourMap = np.array([
[self.npTP, self.npTP, self.grey, self.grey, self.npTN, self.npTN],
[self.npTP, self.npTP, self.grey, self.grey, self.npTN, self.npTN],
[self.grey, self.grey, self.grey, [0, 0, 0], self.npTN, self.npTN],
[self.grey, self.grey, self.grey, self.npTN, self.npTN, self.npTN],
[self.npTN, self.npTN, self.npTN, self.npTN, self.npTN, self.npTN],
[self.npTN, self.npTN, self.npTN, [0, 0, 0], [0, 0, 0], self.npTN]
],
dtype=np.uint8)
npExpectedCounts = [19, 0, 0, 4, 10, 3] # tn, fp, fn, tp, range
self.assertEqual(
Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend),
npExpectedCounts)
def test_AllLarger(self):
lLegend = [self.npTN, self.npFP, self.npFN, self.npTP, self.grey]
npExpectedColourMap = np.array(
[[
self.npFN, self.npFN, self.npFN, self.npFN, self.npFN,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npFN, self.npFN, self.npFN, self.npFN, self.npFN,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npFN, self.npFN, self.npFN, self.npFN, self.npFN,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npTP, self.npTP, self.npTP, self.npTP, self.npTP,
self.grey, self.grey, self.npFP, self.npFP, self.npTN,
self.npTN, self.npTN
],
[
self.npTP, self.npTP, self.npTP, self.npTP, self.npTP,
self.grey, self.grey, self.npFP, self.npFP, self.npTN,
self.npTN, self.npTN
],
[
self.npTP, self.npTP, self.npTP, self.npTP, self.npTP,
self.grey, self.grey, self.npFP, self.npFP, self.npTN,
self.npTN, self.npTN
],
[
self.npFN, self.npFN, self.npFN, self.npFN, self.npFN,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npFN, self.npFN, self.npFN, self.npFN, self.npFN,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.grey, self.grey, self.grey, self.grey, self.grey,
self.grey, self.grey, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.grey, self.grey, self.grey, self.grey, self.grey,
self.grey, self.npTN, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npTN, self.npTN, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
],
[
self.npTN, self.npTN, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN, self.npTN, self.npTN, self.npTN,
self.npTN, self.npTN
]],
dtype=np.uint8)
npExpectedCounts = [9 * 5 + 6 * 4, 6, 9 + 6 * 2 + 4, 15,
6 * 5 - 1] # tn, fp, fn, tp, range
self.assertEqual(
Error.ConfMatrixFromErrorMap(npExpectedColourMap, lLegend),
npExpectedCounts)
def TrainAndTestUNet(oConfig,
iNumFoldsTotal,
kFold,
Model,
GenTrain,
GenVal,
np1TrainDir,
np1ValDir,
np1TestDir,
iNumSave=20):
"""
Trains Model using GenTrain and GenVal data. These are imagedatagenerators
Uses TestData, a numpy array, as the prediction values
"""
# set up training parameters
iNumSave = int(
oConfig['IO']
['Number Save']) # number of prediciton maps saved per group
lImageSize = [
int(sDim) for sDim in oConfig['Generator']['Image Shape'].split(',')
]
lRange = [int(x) for x in oConfig['Setup']['Channels'].split(',')]
npFiles = np.genfromtxt(
os.path.join('..', oConfig['IO']['Input File']),
delimiter=',',
dtype=str) # text file that lists directories to file pairs to be read
# add escape to suuper directory because this file is in .src
sOutDir = os.path.join('..', oConfig['IO']['Output Dir'])
iApproxNumImages = sum([
len(files) for directory in npFiles[:, 0] for r, d, files in os.walk(
os.path.join(os.path.dirname(__file__), directory))
])
if iApproxNumImages < 1:
raise ValueError('no images found')
# this fold factor is used to help determine the steps per epoch
# this condition is added so that in the case of only 1 fold, this factor will not
# become 0 and break the code.
fFoldFactor = (iNumFoldsTotal - 1) / iNumFoldsTotal
if fFoldFactor < 1:
fFoldFactor = 1
# can add tramsforms to the Fit_args
if oConfig['Generator']['Steps per epoch'] == 'Auto':
fTestSize = float(oConfig['Generator']['Test Size'])
Fit_args = dict(
steps_per_epoch=math.ceil(iApproxNumImages //
int(oConfig['Generator']['Batch Size']) *
(1 - fTestSize) * fFoldFactor),
epochs=int(oConfig['Model']['Epochs']),
validation_steps=math.ceil(
iApproxNumImages // int(oConfig['Generator']['Batch Size']) *
fTestSize * fFoldFactor))
else:
Fit_args = dict(
steps_per_epoch=int(oConfig['Generator']['Steps per epoch']),
epochs=int(oConfig['Model']['Epochs']),
validation_steps=int(oConfig['Generator']['Steps per epoch']))
# check if error occured in step size calculations
if Fit_args['validation_steps'] < 1 or Fit_args['steps_per_epoch'] < 1:
raise ValueError(
'validation step size or training step size less than 1. this is an invalid input'
)
# train model
Model = TrainModel(sOutDir, Model, GenTrain, GenVal, Fit_args, np1TrainDir,
np1ValDir, np1TestDir, kFold)
#TODO: add search for iNumPred in oConfig object
iNumPredict = oConfig['IO']['Number Predict']
if iNumPredict == 'All':
# if iNumPredict is none, then it will predict on the entire dataset
iNumPredict = None
else:
iNumPredict = int(iNumPredict)
Error.PredictionErrorPerClass(os.path.join(sOutDir,
str(kFold) + 'thFold'),
np1TestDir,
Model,
lImageSize,
iNumSave=iNumSave,
lRange=lRange,
iNumPredict=iNumPredict)