def __init__(self):
tk.Tk.__init__(self)
self.title("CV Project")
self.dataHandler = DataHandler(scale_radiographs_by=0.4)
self.frameFactory = FrameFactory(self.dataHandler)
self.statisticalModelTrainer = StatisticalModelTrainer()
self.procrustes = Procrustes()
self.radiographFrameContainer = RadiographFrameContainer(
self, self.frameFactory)
self.procrustesTeethSetImageContainer = None
self.modelFittingContainer = None
self.buttonContainer = ButtonContainer(self)
self.buttonContainer.createImageNavigationButtons(
self.radiographFrameContainer)
self.buttonContainer.createFunctionButtons(self)
self.manualModelPlacementContainer = None
self.alignedTeeth = list()
self.meanModels = list()
self.k_pixels = [13, 7, 2, 2, 1]
self.m_pixels = [25, 15, 4, 3, 2]
self.resolutionLevels = 2
self.filter_settings = [(3, 3, 6), (3, 3, 6), (3, 3, 6), (1, 2, 6),
(1, 2, 3)]
def __init__(self,
completeStatisticalModel,
resolutionLevels=3,
m_pixels=None,
k_pixels=None,
filter_settings=None,
initialPositions=None,
initialRotations=None):
self.completeStatisticalModel = completeStatisticalModel
self.procrustes = Procrustes()
self.pca = PCA()
self.m_pixels = m_pixels
self.k_pixels = k_pixels
self.resolutionLevels = resolutionLevels
self.initialRotations = initialRotations
self.initialPositions = self._prepareScaledInitialPositions(
initialPositions)
self.filter_settings = filter_settings
def fitModel(self, target_tooth, model):
self.procrustes = Procrustes()
self.pca = PCA()
eigenvectors = model.getEigenvectors()
Y = target_tooth
mean = Tooth(model.getMeanModel())
# Init
b = 0
X = mean
i = 0
while i < 20:
i += 1
X, b_new = self.step(Y, X, eigenvectors, mean)
if np.allclose(b, b_new):
break
else:
b = b_new
return X
class ModelFitter:
def fitModel(self, target_tooth, model):
self.procrustes = Procrustes()
self.pca = PCA()
eigenvectors = model.getEigenvectors()
Y = target_tooth
mean = Tooth(model.getMeanModel())
# Init
b = 0
X = mean
i = 0
while i < 20:
i += 1
X, b_new = self.step(Y, X, eigenvectors, mean)
if np.allclose(b, b_new):
break
else:
b = b_new
return X
def step(self, Y, X, eigenvectors, mean):
# Fit Y to X
Y_new = self.procrustes.allignDataToModel(Y, X)
# Project Y into X space and get new b
b = self.pca.project(Y_new.getLandmarks().flatten(), eigenvectors,
mean.getLandmarks().flatten())
# print("b = " + str(b))
# Generate new model points X
X_new = self.pca.reconstruct(b, eigenvectors,
mean.getLandmarks().flatten())
X_new = X_new.reshape((X_new.shape[0] // 2, 2))
return Tooth(X_new), b
class MultiResolutionActiveShapeModel:
def __init__(self,
completeStatisticalModel,
resolutionLevels=3,
m_pixels=None,
k_pixels=None,
filter_settings=None,
initialPositions=None,
initialRotations=None):
self.completeStatisticalModel = completeStatisticalModel
self.procrustes = Procrustes()
self.pca = PCA()
self.m_pixels = m_pixels
self.k_pixels = k_pixels
self.resolutionLevels = resolutionLevels
self.initialRotations = initialRotations
self.initialPositions = self._prepareScaledInitialPositions(
initialPositions)
self.filter_settings = filter_settings
def fitCompleteModel(self, radiograph):
# initialize
downscaledRadiographs = self._prepareScaledRadiographs(radiograph)
return self._fitTeethAtMultiResolutionLevel(downscaledRadiographs)
def _fitTeethAtMultiResolutionLevel(self, downscaledRadiographs):
fittedModels = list()
for i in range(8):
toothIndex = i
X, Y = self._fitToothAtMultiResolutionLevel(
toothIndex, downscaledRadiographs)
# test
Y.scale(0.4)
#print("Fitted Models: ")
#print("Y: " + str(Y.getLandmarks()))
#print("X: " + str(X.getLandmarks()))
fittedModels.append([X, Y])
return np.array(fittedModels)
def _fitToothAtMultiResolutionLevel(self, toothIndex,
downscaledRadiographs):
# Current resolution level
currentResolutionLevel = self.resolutionLevels - 1
# print("Downscaled radiographs: " + str(downscaledRadiographs.shape))
radiograph = downscaledRadiographs[currentResolutionLevel]
# Get initial model rotations
initialModelRotation = self.initialRotations[toothIndex]
# Get the initial model positions
initialModelPosition = self.initialPositions[toothIndex]
if initialModelPosition is None:
# Automatic model initialization not implemented
return None
# Get the mean model for the tooth
statisticalToothModel = self.completeStatisticalModel.getToothModelByIndex(
toothIndex, deepCopy=False)
mean = Tooth(statisticalToothModel.getMeanModel())
eigenvectors = statisticalToothModel.getEigenvectors()
eigenvalues = statisticalToothModel.getEigenvalues()
# Initialize Y, this has to be better
Y = deepcopy(mean)
Y.rotate(initialModelRotation * (math.pi / 180))
Y.scale(500 / (2**(self.resolutionLevels - 1)))
#Y.scale(200)
Y.translate(initialModelPosition)
# print("Initialized Y: " + str(Y.getLandmarks()))
# Init algorithm
b = 0
X = mean
while currentResolutionLevel >= 0:
X, Y = self._fitSingleToothModel(
toothIndex, currentResolutionLevel, radiograph, X, Y, b,
eigenvalues, eigenvectors, mean,
self.filter_settings[currentResolutionLevel])
# print("Upscaling")
# print("Changing resolution level")
# print("X = " + str(X.getLandmarks()))
# print("Y = " + str(Y.getLandmarks()))
currentResolutionLevel -= 1
if currentResolutionLevel >= 0:
Y.scale(2)
radiograph = downscaledRadiographs[currentResolutionLevel]
return X, Y
def _fitSingleToothModel(self, toothIndex, currentResolutionLevel,
radiograph, X, Y, b, eigenvalues, eigenvectors,
mean, filter_settings):
scale = 500 / ((2 * currentResolutionLevel) + 1)
Y_copy = deepcopy(Y)
i = 0
while i < 100:
X, b_new = self._modelFittingStep(Y, X, eigenvalues, eigenvectors,
mean, scale)
# print("X Step: " + str(X.getLandmarks()))
# Calculate new Y from the current X
# I might need to first allign model X to Y, as X could still be at the origin and with no scale and rotation
temp = self.procrustes.allignModelToData(deepcopy(Y_copy),
deepcopy(X))[1]
# print("Temp step: " + str(temp.getLandmarks()))
Y, p = self._getNewYEstimateAtCurrentResolutionLevel(
temp, radiograph, toothIndex, currentResolutionLevel,
filter_settings)
Y = Tooth(Y)
Y_copy = deepcopy(Y)
# if ((np.isclose(b, b_new).all() and i>3) or (i>=(12/(currentResolutionLevel+1)) and currentResolutionLevel > 0)):
# print("Breaking after " + str(i) + " iterations")
# break
# else:
# b = b_new
# i += 1
if (p >= 0.9 and i > 2) or i >= 5:
print("Breaking after " + str(i) + " iterations")
break
else:
b = b_new
i += 1
return X, Y
def _modelFittingStep(self, Y, X, eigenvalues, eigenvectors, mean, scale):
# Fit Y to X
Y_new = self.procrustes.allignDataToModel(Y, X)
# Project Y into X space and get new b
b = self.pca.project(Y_new.getLandmarks().flatten(), eigenvectors,
mean.getLandmarks().flatten())
# print("b = " + str(b))
# Enforce constraint |b_i| < 3*lambda_i
for i in range(len(b)):
if abs(b[i]) > 2 * eigenvalues[i] * scale:
b[i] = 2 * eigenvalues[i] * scale
# print("eigenvalue: " + str(eigenvalues[i]))
# Generate new model points X
X_new = self.pca.reconstruct(b, eigenvectors,
mean.getLandmarks().flatten())
X_new = X_new.reshape((X_new.shape[0] // 2, 2))
return Tooth(X_new), b
def _getNewYEstimateAtCurrentResolutionLevel(self, X, radiograph,
currentTooth,
currentResolutionLevel,
filter_settings):
# Pre process image
img = Filter.process_image(
deepcopy(radiograph.getImage(deepCopy=True)), filter_settings[0],
filter_settings[1], filter_settings[2])
derivate_img = Filter.laplacian(img)
# derivate_img = Filter.histogramEql(Filter.process_image(deepcopy(img), filter_settings[0], filter_settings[1], filter_settings[2]))
# derivate_img = Filter.process_image(deepcopy(derivate_img), median_kernel=3, bilateral_kernel=5)
#cv2.imshow("Test", derivate_img)
#cv2.waitKey(0)
# Get the correct model
multiResModel = self.completeStatisticalModel.getGrayLevelMultiResolutionModel(
deepCopy=True)
singleResModel = multiResModel.getGrayLevelSignleResModelByResolutionLevelIndex(
currentResolutionLevel)
toothModel = singleResModel.getGrayLevelToothModelByToothIndex(
currentTooth, currentResolutionLevel)
# Init X and Y
Y = np.zeros((40, 2))
X = X.getLandmarks()
counter = 0
for i in range(40):
Y[i], is_close = self._getYPointEstimateFromGivenModelPoint(
toothModel, i, X, currentResolutionLevel, currentTooth, img,
derivate_img)
if is_close:
counter += 1
return Y, counter / 40
def _getYPointEstimateFromGivenModelPoint(self, toothModel,
currentPointIndex, X,
currentResolutionLevel,
currentTooth, img, derivate_img):
# Get the points
current_point = X[currentPointIndex]
if currentPointIndex == 0:
previous_point = X[len(X) - 1]
else:
previous_point = X[currentPointIndex - 1]
if currentPointIndex == len(X) - 1:
next_point = X[0]
else:
next_point = X[currentPointIndex + 1]
# Get the correct model
pointModel = toothModel.getGrayLevelPointModelByIndex(
currentPointIndex, currentTooth, currentResolutionLevel)
pointMeanGrayLevels = pointModel.getMeanGrayLevels(
currentTooth, currentPointIndex)
pointCovarianceMatrix = pointModel.getCovarianceOfGrayLevels(
currentTooth, currentPointIndex)
# Get the sample from the image
fullSampleValues, originalPixels, _ = self._getSampleAroundModelPoint(
img, derivate_img, current_point, previous_point, next_point,
currentResolutionLevel)
# print("Value Samples: " + str(fullSampleValues.shape))
# print("original Pixels: " + str(originalPixels.shape))
# print("fullSampleValues: " + str(len(fullSampleValues)))
# print("originalPixels: " + str(len(originalPixels)))
# Check which slice of the sample corresponds best to the model
bestFitValue = math.inf
indexOfBestpoint = 0
for i in range(
len(fullSampleValues) -
(2 * self.k_pixels[currentResolutionLevel])):
j = i + (2 * self.k_pixels[currentResolutionLevel]) + 1
slicedSample = fullSampleValues[i:j]
newFitValue = self.fitFunction(slicedSample, pointMeanGrayLevels,
pointCovarianceMatrix)
if newFitValue < bestFitValue:
bestFitValue = newFitValue
indexOfBestpoint = i + self.k_pixels[currentResolutionLevel]
is_close = False
if indexOfBestpoint > len(fullSampleValues) / 4 and indexOfBestpoint < (
3 * len(fullSampleValues)) / 4:
is_close = True
# Pick the best point
point = originalPixels[indexOfBestpoint][:2]
return point, is_close
def fitFunction(self, sampleSlice, pointMeanGrayLevels,
pointCovarianceMatrix):
diff = (sampleSlice - pointMeanGrayLevels)
diff_T = np.reshape(diff, (1, len(diff)))
# print("Covariance matrix: " + str(pointCovarianceMatrix))
try:
cov_inv = np.linalg.inv(pointCovarianceMatrix)
except Exception:
cov_inv = pointCovarianceMatrix
temp = np.matmul(diff_T, cov_inv)
distance = np.matmul(temp, diff)
return distance
def _getSampleAroundModelPoint(self, img, derivate_img, current_point,
previous_point, next_point,
currentResolutionLevel):
return Utils.getSampleFromImage(img, derivate_img, current_point,
previous_point, next_point,
self.m_pixels[currentResolutionLevel])
def _prepareScaledRadiographs(self, radiograph):
downscaledRadiographs = list()
for i in range(self.resolutionLevels):
temp = deepcopy(radiograph)
for j in range(i):
temp.downScale()
downscaledRadiographs.append(temp)
downscaledRadiographs = np.array(downscaledRadiographs)
return downscaledRadiographs
def _prepareScaledInitialPositions(self, initialPositions):
if initialPositions is None:
print("Automatic position initialisation is not implemented")
return None
scaledInitialPositions = list()
for initialPosition in initialPositions:
[x, y] = initialPosition
for i in range(1, self.resolutionLevels):
[x, y] = [x / 2, y / 2]
scaledInitialPositions.append([x, y])
scaledInitialPositions = np.array(scaledInitialPositions)
return scaledInitialPositions
class MainApp(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
self.title("CV Project")
self.dataHandler = DataHandler(scale_radiographs_by=0.4)
self.frameFactory = FrameFactory(self.dataHandler)
self.statisticalModelTrainer = StatisticalModelTrainer()
self.procrustes = Procrustes()
self.radiographFrameContainer = RadiographFrameContainer(
self, self.frameFactory)
self.procrustesTeethSetImageContainer = None
self.modelFittingContainer = None
self.buttonContainer = ButtonContainer(self)
self.buttonContainer.createImageNavigationButtons(
self.radiographFrameContainer)
self.buttonContainer.createFunctionButtons(self)
self.manualModelPlacementContainer = None
self.alignedTeeth = list()
self.meanModels = list()
self.k_pixels = [13, 7, 2, 2, 1]
self.m_pixels = [25, 15, 4, 3, 2]
self.resolutionLevels = 2
self.filter_settings = [(3, 3, 6), (3, 3, 6), (3, 3, 6), (1, 2, 6),
(1, 2, 3)]
#self.filterTest()
#self.createMeanModelImages = True
def filterTest(self):
radiograph = self.dataHandler.getRadiographs(deepCopy=True)[0]
blurred_img = Filter.process_image(deepcopy(radiograph.getImage()),
median_kernel=3,
bilateral_kernel=10)
cv2.imshow("BlurredImage", blurred_img)
clahe = cv2.equalizeHist(blurred_img)
cv2.imshow("Clahe", clahe)
img_1 = Filter.laplacian(blurred_img)
cv2.imshow("Img1", img_1)
# radiograph.downScale()
# blurred_img = Filter.process_image(deepcopy(radiograph.getImage()), median_kernel=3, bilateral_kernel=10)
# img_2 = Filter.laplacian(deepcopy(blurred_img))
# height = img_1.shape[0]
# width = img_1.shape[1]
# img_2 = cv2.resize(img_1, (int(width*0.5), int(height*0.5)))
# cv2.imshow("Img2", img_2)
# radiograph.downScale()
# blurred_img = Filter.process_image(deepcopy(radiograph.getImage()), median_kernel=3, bilateral_kernel=10)
# img_3 = Filter.laplacian(deepcopy(blurred_img))
# height = img_2.shape[0]
# width = img_2.shape[1]
# img_3 = cv2.resize(img_2, (int(width*0.5), int(height*0.5)))
# cv2.imshow("Img3", img_3)
# radiograph.downScale()
# blurred_img = Filter.process_image(deepcopy(radiograph.getImage()), median_kernel=3, bilateral_kernel=10)
# img_4 = cv2.Canny(deepcopy(blurred_img), 15, 15)
# cv2.imshow("Img4", img_4)
# radiograph.downScale()
# blurred_img = Filter.process_image(deepcopy(radiograph.getImage()), median_kernel=3, bilateral_kernel=10)
# img_5 = cv2.Canny(deepcopy(blurred_img), 15, 15)
# cv2.imshow("Img5", img_5)
def trainCompleteStatisticalModel(self):
self.statisticalModel = self.statisticalModelTrainer.trainCompleteStatisticalModel(
self.k_pixels,
self.resolutionLevels,
self.filter_settings,
leaveOneOut=0)
# for i in range(8):
# maxEig = self.statisticalModel.getToothModelByIndex(i).getEigenvalues()[0]
# print("Tooth " + str(i) + " biggest eigenvalue: " + str(maxEig*10000))
# if(self.createMeanModelImages):
# self.frameFactory.createMeanModelPresentationImages(self.statisticalModel)
def performManualModelPositionInit(self):
self.manualModelPlacementContainer = ManualModelPlacementContainer(
self, self.frameFactory,
self.statisticalModel.getAllToothModels(deepCopy=True))
self.buttonContainer.createImageNavigationButtons(
self.manualModelPlacementContainer)
def acceptModelPosition(self):
self.manualModelPlacementContainer.nextMeanModel()
def changeToothRotation(self, var):
if self.manualModelPlacementContainer is not None:
self.manualModelPlacementContainer.manualRotation(var)
def performAutoModelPositionInit(self):
return None
def performModelFitting(self):
if self.manualModelPlacementContainer is not None:
initialModelPositions = self.manualModelPlacementContainer.getChosenModelPositions(
)
initialModelRotations = self.manualModelPlacementContainer.getChosenModelRotations(
)
initialModelPositions *= 2.5
self.multiResActiveShapeModel = MultiResolutionActiveShapeModel(
self.statisticalModel,
resolutionLevels=self.resolutionLevels,
m_pixels=self.m_pixels,
k_pixels=self.k_pixels,
filter_settings=self.filter_settings,
initialPositions=initialModelPositions,
initialRotations=initialModelRotations)
radiograph = DataHandler().getRadiographs(deepCopy=True)[
self.manualModelPlacementContainer.getChosenRadiograph()]
# Fit the first tooth to the radiograph
fittedModels_Y = self.multiResActiveShapeModel.fitCompleteModel(
radiograph)
self.fittedModels = list()
# allign fitted model to be drawn
for model in fittedModels_Y:
[X, Y] = model
temp = self.procrustes.allignModelToData(Y, X)[1]
self.fittedModels.append(temp)
self.fittedModels = np.array(self.fittedModels)
print("fittedModels" + str(np.array(self.fittedModels).shape))
radiograph.scaleImage(0.4)
self.frameFactory.createFittingErrorPresentationImages(
self.fittedModels, radiograph)
self.modelFittingContainer = ModelFittingContainer(
self, self.frameFactory, [self.fittedModels], radiograph)
self.buttonContainer.createImageNavigationButtons(
self.modelFittingContainer)
def __init__(self):
self.completeDataHandler = DataHandler()
self.procrustes = Procrustes()
self.pca = PCA()
self.modelFitter = ModelFitter()
self.multiResTrainer = MultiResolutionTrainer()
class StatisticalModelTrainer:
def __init__(self):
self.completeDataHandler = DataHandler()
self.procrustes = Procrustes()
self.pca = PCA()
self.modelFitter = ModelFitter()
self.multiResTrainer = MultiResolutionTrainer()
def trainCompleteStatisticalModel(self,
k_pixels,
resolutionLevels,
filter_settings,
doLeaveOneOutTrainingSetTest=False,
leaveOneOut=None):
"""
Performs leave-one-out validation of the training set, then
trains a statistical model from the entire training set.
"""
if doLeaveOneOutTrainingSetTest:
self.doLeaveOneOutTrainingSetValidation()
if leaveOneOut is not None:
self.completeDataHandler = DataHandler(leave_one_out=leaveOneOut)
statisticalToothModels = self.trainModelFromCompleteTrainingSet()
grayLevelMultiResModel = self.multiResTrainer.trainGrayLevelMultiResolutionModel(
k_pixels, resolutionLevels, filter_settings)
completeModel = CompleteStatisticalModel(statisticalToothModels,
grayLevelMultiResModel)
print("Statistical model trained!")
return completeModel
def trainModel(self, dataHandler):
"""
Trains a statistical model from the examples provided in the given dataHandler.
"""
alignedTrainingExamples = self.allignTrainingExamples(dataHandler)
return self.perfromPCA(alignedTrainingExamples)
def allignTrainingExamples(self, dataHandler):
"""
Alligns all training examples, grouped by the same tooth, using procrustes analysis.
"""
alignedTrainingExamples = list()
for i in range(8):
temp = dataHandler.getAllTeethAtIndex(i, deepCopy=True)
alignedTrainingExamples.append(
self.procrustes.performProcrustesAlignment(temp))
return alignedTrainingExamples
def perfromPCA(self, alignedTrainingExamples):
"""
Performs PCA on the given training set, and returns the obtained statistical models.
"""
meanModels = list()
for i in range(8):
[mean, eigenvalues, eigenvectors
] = self.pca.do_pca_and_build_model(alignedTrainingExamples[i])
statisticalToothModel = StatisticalToothModel(
mean, eigenvalues, eigenvectors)
meanModels.append(statisticalToothModel)
meanModels = np.array(meanModels)
return meanModels
def trainModelFromCompleteTrainingSet(self):
"""
Trains a statistical model from the complete training set.
"""
statisticalModel = self.trainModel(self.completeDataHandler)
return statisticalModel
def doLeaveOneOutTrainingSetValidation(self):
"""
Performs a leave-one-out validation of the training set.
If for any subset of the training set the error on the left out example
is too big, the training set is considered insuficient.
"""
# Number of training examples
n = len(self.completeDataHandler.getRadiographs(deepCopy=True))
for i in range(n):
dataHandler = DataHandler(leave_one_out=i)
statisticalModel = self.trainModel(dataHandler)
print("Left out radiograph: " + str(i))
self.testTrainedModelOnLeftOutExample(dataHandler,
statisticalModel)
def testTrainedModelOnLeftOutExample(self, dataHandler, statisticalModel):
"""
Test the learned model by fitting it to the train examples as well as to the
left out exampele and compares the mean error on the training examples to the
error on the left out example.
"""
training_radiographs = dataHandler.getRadiographs(deepCopy=True)
leftOut_radiograph = dataHandler.getLeftOutRadiograph(deepCopy=True)
total_error = 0
for radiograph in training_radiographs:
error = self.fitModelToAnnotatedExampleAndMeasureError(
radiograph, statisticalModel)
total_error += error
errorOnTrainingExamples = total_error / len(training_radiographs)
errorOnLeftOutExample = self.fitModelToAnnotatedExampleAndMeasureError(
leftOut_radiograph, statisticalModel)
errorComparison = errorOnLeftOutExample / errorOnTrainingExamples
if errorComparison > 3:
print("Relative error on left out example = " +
str(errorOnLeftOutExample))
print(
"Error on test image is relativelly big (>3 times the mean error on the train images), "
+ "the training set might not be good enough")
def fitModelToAnnotatedExampleAndMeasureError(self, radiograph,
statisticalModel):
"""
Fits a model to an annotated example and measures the error.
"""
teeth = radiograph.getTeeth(deepCopy=True)
total_error = 0
for i in range(8):
tooth = deepcopy(teeth[i])
model = statisticalModel[i]
# Fit the model to the radiograph
fitted_model = self.modelFitter.fitModel(deepcopy(tooth), model)
# align fitted model to examples
alignedModel = self.procrustes.allignModelToData(
deepcopy(tooth), fitted_model)[1]
# Error is the measured as the norm of the difference of the two matrices
error = (tooth.getLandmarks() - alignedModel.getLandmarks()
) / alignedModel.getLandmarks()
error = np.linalg.norm(error)
total_error += error
return total_error