def apply_normalization(self, image, dtype=np.uint8):
""" Substracts the mean.
"""
imageIndex = image.shape[0] * image.shape[1]
if len(self.mean) == 0 or len(self.std) == 0:
raise Exception("Mean and Std image are not computed")
# test if mean or std have been calculated for this image size
if not imageIndex in self.mean:
#raise Exception("The shape of the mean image and the input image do not match. mean.shape: {0} - image.shape: {1}".format(self.mean.shape, image.shape))
# resize to fit image.
meanImage = (self.mean.values())[0]
stdImage = (self.std.values())[0]
self.mean[imageIndex] = utils.equalize_image_size(meanImage, imageIndex)
self.std[imageIndex] = utils.equalize_image_size(stdImage, imageIndex)
# make sure image type is not uint8
if image.dtype != np.float32:
image = image.astype(np.float32)
# image is scaled to [0, 1] apply scaled values
if np.max(image) <= 1.0:
return image - self.meanScaled[imageIndex]
image -= self.mean[imageIndex]
# don't change dtype if -1
if dtype == -1:
return image
image = image.astype(dtype)
#cv.imshow("Mean", image)
#cv.waitKey(0)
return image
def predict(self, images):
for img in images:
predictions = {class_: 0 for class_ in self.testData.classes}
for clWeight, classifier in self.classifiers:
# copy img in case we need to do some preprocessing
clImg = np.copy(img)
# grayscale or color
if classifier.grayscale:
clImg = cv.cvtColor(clImg, cv.COLOR_BGR2GRAY)
# is there a size constraint for the classifier?
size = classifier.tester.size
if size != (-1, -1):
clImg = utils.crop_to_square(clImg)
desiredArea = size[0] * size[1]
clImg = utils.equalize_image_size(clImg, desiredArea)
# do we need to transform the image for the classifier?
transform = classifier.tester.transformation
if not transform is None:
clImg = transform(clImg)
clPrediction = classifier.predict(
[clImg])[:5] # get top-5 predictions
vote = 5
for class_, pValue in clPrediction:
predictions[class_] += vote * clWeight
vote -= 1
# convert predictions dict back to list and sort it
pList = sorted([(class_, [votes])
for class_, votes in predictions.iteritems()],
key=operator.itemgetter(1),
reverse=True)
return pList
def segment_image(self):
""" Menu for image segmentation."""
choice = utils.menue("[Image Segmentation",
["Segment imgage", "Learn custom marker"],
showBackToMain=True)
if choice == 2:
self.learn_marker()
return
elif choice == 3:
return
im = load_image(utils.get_path_via_file_ui(), 1, 1)
im = utils.equalize_image_size(im, 564000)
seg = ObjectSegmentation()
seg.process_image(im)
im = seg.visulize_regions()
cv.imwrite(utils.get_output_path() + "resultsMarker_regions.jpg", im)
try:
plt.imshow(cv.cvtColor(im, cv.COLOR_BGR2RGB), 'gray')
plt.xticks([]), plt.yticks(
[]) # to hide tick values on X and Y axis
plt.show()
utils.save_plt_figure(plt, "image_segments")
except:
print "Could not plot / show image. Saving instead."
utils.save_plt_figure(plt, "image_segments")
colorSeg = ColorSegmentation()
colorSeg.process_image(seg.get_meal_image())
resultImg, images, titles = colorSeg.visulize_regions(seg)
# if matplotlib does not work use cv
for i in range(len(images)):
cv.imwrite(
"C:/Users/felix/Desktop/Segmentation/results/resultsMarker_regionsArea{0}.jpg"
.format(titles[i]), images[i])
cv.imshow("Area {0}".format(titles[i]), images[i])
cv.imshow("Result", resultImg)
cv.imwrite(
"C:/Users/felix/Desktop/Segmentation/results/resultsMarker_resultsImage.jpg",
resultImg)
cv.waitKey(0)
# matplotlib
utils.display_images([resultImg], titles="Result", columns=1, rows=1)
utils.display_images(images, titles=titles)
def normalize_test_data(self, size, newName="", forceOverwrite=False):
normTestDataRootPath = utils.get_parent_dir(self.get_root_path()) + "/"
if newName == "":
if not forceOverwrite:
overwrite = utils.radio_question("[?]", "Do you really wish to overwrite existing images?", None, ["Yes", "No"], [True, False])
else:
overwrite = True
if not overwrite:
normTestDataRootPath += utils.value_question("", "Provide new foldername:", "s")
else:
normTestDataRootPath = self.get_root_path()
else:
normTestDataRootPath += newName
utils.create_dir_if_necessary(normTestDataRootPath)
print "Saving equalized test data set in path",normTestDataRootPath
# segment data with only one segment
self.segment_test_data({"all": 1})
self.new_segmentation()
numberOfImages = self.testDataSize
numberOfImagesDone = 0
currentClass = ""
print "Starting equalization.\n"
for img, class_, fileName in self.load_data("all", grayscale=False, outputActions=False, yieldFilename=True):
path = normTestDataRootPath + "/" + class_ + "/"
# reset counter if new class
if not currentClass == class_:
currentClass = class_
utils.create_dir_if_necessary(path)
resizedImg = utils.equalize_image_size(img, size)
path += fileName
cv.imwrite(path, resizedImg)
numberOfImagesDone += 1
utils.show_progress(True, numberOfImagesDone, numberOfImages, "Processing class {0}.\tTotal progress:", currentClass)
print "\nEqualization finished."
print "\n"
def predict(self, images, sortResult=True):
"""
runs the prediction for the image or the images.
Keyword arguments:
images -- List of images. Note: Out of laziness the method will only predict the first image regardless of the argument length.
sortResults -- Sort the results based on the response / confidences
Returns:
list of lists (in case more than one image was provided) of predictions. The first entry is the most likely prediction.
"""
if not self.trained:
raise Exception("Classifier is not trained.")
# make sure the images have the desired size if size is set
if not self.imageSize is None and self.imageSize != -1:
images = [
utils.equalize_image_size(image, self.imageSize)
for image in images
]
# get input vector for frame.
inputVectors = [self.create_feature_vector(image) for image in images]
# no keypoints detected
if inputVectors[0] is None:
return None
# get predictions from all SVMs
predictions = {}
reverseSorting = False
for class_, svm in self.svms.iteritems():
prediction = svm.predict(inputVectors)
predictions[class_] = prediction
reverseSorting = svm.reverseSorting
# order predictions
if sortResult:
return sorted(predictions.items(),
key=operator.itemgetter(1),
reverse=reverseSorting)
else:
return predictions
def load_data(self, segmentIndex, numberOfClasses=None, classes=[], grayscale=True, resizeFactor=1, outputActions=True, maxNumberOfImagesPerClass=-1, yieldFilename=False, size=(-1,-1), transformation=None, resolutionSize=-1, forceNormalization=0, forceZNormalization=0, forceZcaWhitening=0):
"""Loads the images of the test data as a generator.
Keyword argument
segmentIndex -- the segment index to load from. If default values were used in segment_test_data use "train" or "test"
numberOfClasses -- the number of classes to load (default None (load all classes))
classes -- explicitly specified classes (default [])
grayscale -- should images be loaded in grayscale or not (default True)
maxNumberOfImagesPerClass -- limits the loading of images (default -1 means no limit)
"""
#Parameter validation
if segmentIndex not in self.segmentRatio and segmentIndex != "all":
raise AttributeError(segmentIndex + " is not in segments")
# test if size is valid
try:
sizeLen = len(size)
if sizeLen != 2:
raise AttributeError("size has to be a 2-element tuple. Use (-1, -1) if you don't care.")
except:
raise AttributeError("size has to be a 2-element tuple. Use (-1, -1) if you don't care.")
desired_h = size[0]
desired_w = size[1]
if (desired_h / desired_w) != 1.0:
raise AttributeError("The current version does not support an aspect ratio other than 1.0.")
if resizeFactor != 1 and resolutionSize != -1:
raise AttributeError("You can't set resizeFactor and resolutionSize at the same time.")
# if forceX parameter is 0 it means do not change. -1 -> force false, +1 -> force true
forceNormalization = (forceNormalization == 0 and Settings.F_APPLY_NORMALIZATION) or forceNormalization == 1
forceZNormalization = (forceZNormalization == 0 and Settings.F_APPLY_ZNORMALIZATION) or forceZNormalization == 1
if (forceNormalization and forceZNormalization):
raise AttributeError("You can't apply normalization (mean substraction) and z-normalization (normalization divided by std) at the same time.")
# calculate mean and std if normalization enabled and needed
if forceNormalization or forceZNormalization:
if desired_h == -1:
if resolutionSize == -1:
raise AttributeError("If F_APPLY_NORMALIZATION or F_APPLY_ZNORMALIZATION is enabled the image must have a fixed size or at least a fixed resolution size.")
else:
desired_h = desired_w = int(sqrt(resolutionSize))
size = (desired_w, desired_h)
if len(self.mean) == 0 or len(self.std) == 0:
self.__calculate_mean_std(size, grayscale)
preprocessImage = forceNormalization or forceZNormalization or Settings.F_APPLY_ZCA_WHITENING or Settings.F_APPLY_CLAHE or Settings.F_APPLY_HISTOGRAM_EQ
if numberOfClasses is None:
numberOfClasses = self.numberOfClasses
else:
numberOfClasses = min(numberOfClasses, self.numberOfClasses)
if not classes:
#Fill classes with numberOfClasses count classes
classes = [key for key in self.__classDictionary]
# only take numberOfClasses classes
classes = classes[:numberOfClasses]
limitImageLoading = True
if maxNumberOfImagesPerClass == -1:
limitImageLoading = False
else:
maxNumberOfImagesPerClass = max(1, maxNumberOfImagesPerClass) # load at least one image
#Load flag for cv.imread.
loadFlag = cv.IMREAD_GRAYSCALE if grayscale else cv.IMREAD_UNCHANGED
if outputActions:
print "Loading dataset {0} {1}. |Classes: {2}| - number of images per segment: {3}".format(segmentIndex, self.__segmentSliceIndex[segmentIndex], numberOfClasses, self.segmentSizeMean[segmentIndex])
print "Loading in grayscale",grayscale
for class_ in classes:
segmentedSamples = self.__classDictionary[class_]
if segmentIndex != "all":
samples = segmentedSamples[segmentIndex]
else:
samples = []
for segId in self.__classDictionary[class_]:
samples.extend(self.__classDictionary[class_][segId])
if limitImageLoading:
samplesToLoad = min(maxNumberOfImagesPerClass, len(samples))
else:
samplesToLoad = len(samples)
if outputActions:
print "\n"
errors = 0
lastError = ""
for i in xrange(samplesToLoad):
filename = self.get_root_path() + class_ + "/" + samples[i]
img = load_image(filename, loadFlag, resizeFactor)
# img is None if the image could not be read. imread cannot read .gif for example
if img is None:
errors += 1
lastError = "Image {0} was None.\nSamples:{1}".format(filename, samples)
continue
# if image has an alpha channel reduce
if len(img.shape) > 2 and img.shape[2] == 4L:
img = cv.cvtColor(img, cv.COLOR_BGRA2BGR)
# do we need to crop and adjust the size
if desired_h != -1 and desired_w != -1:
img = utils.crop_to_square(img)
desiredArea = desired_h * desired_w
img = utils.equalize_image_size(img, desiredArea)
if img is None:
errors += 1
lastError = "Image {0} after eq-1 was None".format(filename)
continue
if desired_w != img.shape[1] or desired_h != img.shape[0]:
img = utils.crop_around_center(img, desired_w, desired_h)
if img is None:
errors += 1
lastError = "Image {0} after cropping was None".format(filename)
continue
# resize image to set size
if resolutionSize != -1 and size[0] == -1 and size[1] == -1:
try:
img = utils.equalize_image_size(img, resolutionSize)
except:
logging.exception("Could not eq image size: img.shape: {0} - new size: {1}".format(img.shape, resolutionSize))
lastError = "Image eq-2 exception".format(filename)
img = None
if img is None:
lastError = "Image {0} after eq-2 was None".format(filename)
errors += 1
continue
if not transformation is None:
try:
img = transformation(img)
except Exception, e:
errors += 1
lastError = "Exception during image transformation:",e.message
continue
if preprocessImage:
try:
img = self.__preprocess_image(img, forceNormalization, forceZNormalization, grayscale)
except Exception, e:
errors += 1
lastError = "Exception during image preprocessing:",e.message
continue
utils.show_progress(outputActions, i+1, samplesToLoad, "Loading {0} images in class {1} (Loss: {2}):", samplesToLoad, class_, errors)
if yieldFilename:
yield (img, class_, samples[i])
else:
yield (img, class_)
def transform(self, Xb, yb):
""" Augments Xb data by applying label preserving changes."""
Xb, yb = super(AugmentingBatchIterator, self).transform(Xb, yb)
# prepare new Xb array because the resulting image size might vary depending on random crop setting
Xb_new = np.empty((Xb.shape[0], Xb.shape[1], Settings.NN_INPUT_SHAPE[0], Settings.NN_INPUT_SHAPE[1]), Xb.dtype)
for i in xrange(Xb.shape[0]):
cropping = False
operationKeys = Settings.NN_AUGMENTATION_PARAMS.keys()
workingImage = Xb[i]
# augment sample with NN_AUGMENT_CHANCE chance for j-th augmentation iteration (j is the len of ops)
ops = []
while random.uniform(0.0, 1.0) < Settings.NN_AUGMENT_CHANCE[len(ops)] and Settings.NN_AUGMENT_DATA:
operation = operationKeys[random.randint(0, len(operationKeys)-1)]
ops.append(operation)
transformation = Settings.NN_AUGMENTATION_PARAMS[operation]
# prevent two cropping augmentation methods (free rotation and random cropping)
if operation == 8 or operation == 6:
operationKeys.remove(6)
operationKeys.remove(8)
# no extra parameter
if transformation[1] is None:
workingImage = transformation[0](workingImage)
# 1 extra parameter
elif len(transformation) == 2:
parameterValue = transformation[1][0](transformation[1][1], transformation[1][2])
workingImage = transformation[0](workingImage, parameterValue)
# 2 extra parameters (rotating)
elif len(transformation) == 3:
parameterValue1 = transformation[1][0](transformation[1][1], transformation[1][2])
parameterValue2 = transformation[2][0](transformation[2][1], transformation[2][2])
workingImage = transformation[0](workingImage, parameterValue1, parameterValue2)
# 3 extra parameters (cropping)
elif len(transformation) == 4:
cropping = True
parameterValue1 = transformation[1][0](transformation[1][1], transformation[1][2])
parameterValue2 = transformation[2][0](transformation[2][1], transformation[2][2])
parameterValue3 = transformation[3][0](transformation[3][1], transformation[3][2])
workingImage = transformation[0](workingImage, parameterValue1, parameterValue2, parameterValue3)
# make sure that image has the right shape
if not cropping:
Xb_new[i] = utils.equalize_image_size(Xb[i], Settings.NN_INPUT_SHAPE[0] * Settings.NN_INPUT_SHAPE[1])
else:
Xb_new[i] = workingImage
# To test the actual output
#print "Applied {0} transformations. Ops: {1}".format(len(ops), ops)
#saveImgs = utils.radio_question("[?]", "Save images", None, ["Yes", "No"], [True, False])
#if saveImgs:
# afterTrans = np.copy(Xb_new[i])
# #print "after trans before:",Xb_new[i].shape
# afterTrans, _ = utils.reshape_to_cv_format(afterTrans, True)
# #beforeTrans, _ = utils.reshape_to_cv_format(beforeTrans, True)
# #print "after trans after:",afterTrans.shape
# cv.imwrite(utils.get_temp_path() + "op_after.jpg", afterTrans)
# #cv.imwrite(utils.get_temp_path() + "op_before.jpg", beforeTrans)
xb = None
return Xb_new, yb