def apply_normalization(self, image, dtype=np.uint8): """ Substracts the mean. """ imageIndex = image.shape[0] * image.shape[1] if len(self.mean) == 0 or len(self.std) == 0: raise Exception("Mean and Std image are not computed") # test if mean or std have been calculated for this image size if not imageIndex in self.mean: #raise Exception("The shape of the mean image and the input image do not match. mean.shape: {0} - image.shape: {1}".format(self.mean.shape, image.shape)) # resize to fit image. meanImage = (self.mean.values())[0] stdImage = (self.std.values())[0] self.mean[imageIndex] = utils.equalize_image_size(meanImage, imageIndex) self.std[imageIndex] = utils.equalize_image_size(stdImage, imageIndex) # make sure image type is not uint8 if image.dtype != np.float32: image = image.astype(np.float32) # image is scaled to [0, 1] apply scaled values if np.max(image) <= 1.0: return image - self.meanScaled[imageIndex] image -= self.mean[imageIndex] # don't change dtype if -1 if dtype == -1: return image image = image.astype(dtype) #cv.imshow("Mean", image) #cv.waitKey(0) return image
def predict(self, images): for img in images: predictions = {class_: 0 for class_ in self.testData.classes} for clWeight, classifier in self.classifiers: # copy img in case we need to do some preprocessing clImg = np.copy(img) # grayscale or color if classifier.grayscale: clImg = cv.cvtColor(clImg, cv.COLOR_BGR2GRAY) # is there a size constraint for the classifier? size = classifier.tester.size if size != (-1, -1): clImg = utils.crop_to_square(clImg) desiredArea = size[0] * size[1] clImg = utils.equalize_image_size(clImg, desiredArea) # do we need to transform the image for the classifier? transform = classifier.tester.transformation if not transform is None: clImg = transform(clImg) clPrediction = classifier.predict( [clImg])[:5] # get top-5 predictions vote = 5 for class_, pValue in clPrediction: predictions[class_] += vote * clWeight vote -= 1 # convert predictions dict back to list and sort it pList = sorted([(class_, [votes]) for class_, votes in predictions.iteritems()], key=operator.itemgetter(1), reverse=True) return pList
def segment_image(self): """ Menu for image segmentation.""" choice = utils.menue("[Image Segmentation", ["Segment imgage", "Learn custom marker"], showBackToMain=True) if choice == 2: self.learn_marker() return elif choice == 3: return im = load_image(utils.get_path_via_file_ui(), 1, 1) im = utils.equalize_image_size(im, 564000) seg = ObjectSegmentation() seg.process_image(im) im = seg.visulize_regions() cv.imwrite(utils.get_output_path() + "resultsMarker_regions.jpg", im) try: plt.imshow(cv.cvtColor(im, cv.COLOR_BGR2RGB), 'gray') plt.xticks([]), plt.yticks( []) # to hide tick values on X and Y axis plt.show() utils.save_plt_figure(plt, "image_segments") except: print "Could not plot / show image. Saving instead." utils.save_plt_figure(plt, "image_segments") colorSeg = ColorSegmentation() colorSeg.process_image(seg.get_meal_image()) resultImg, images, titles = colorSeg.visulize_regions(seg) # if matplotlib does not work use cv for i in range(len(images)): cv.imwrite( "C:/Users/felix/Desktop/Segmentation/results/resultsMarker_regionsArea{0}.jpg" .format(titles[i]), images[i]) cv.imshow("Area {0}".format(titles[i]), images[i]) cv.imshow("Result", resultImg) cv.imwrite( "C:/Users/felix/Desktop/Segmentation/results/resultsMarker_resultsImage.jpg", resultImg) cv.waitKey(0) # matplotlib utils.display_images([resultImg], titles="Result", columns=1, rows=1) utils.display_images(images, titles=titles)
def normalize_test_data(self, size, newName="", forceOverwrite=False): normTestDataRootPath = utils.get_parent_dir(self.get_root_path()) + "/" if newName == "": if not forceOverwrite: overwrite = utils.radio_question("[?]", "Do you really wish to overwrite existing images?", None, ["Yes", "No"], [True, False]) else: overwrite = True if not overwrite: normTestDataRootPath += utils.value_question("", "Provide new foldername:", "s") else: normTestDataRootPath = self.get_root_path() else: normTestDataRootPath += newName utils.create_dir_if_necessary(normTestDataRootPath) print "Saving equalized test data set in path",normTestDataRootPath # segment data with only one segment self.segment_test_data({"all": 1}) self.new_segmentation() numberOfImages = self.testDataSize numberOfImagesDone = 0 currentClass = "" print "Starting equalization.\n" for img, class_, fileName in self.load_data("all", grayscale=False, outputActions=False, yieldFilename=True): path = normTestDataRootPath + "/" + class_ + "/" # reset counter if new class if not currentClass == class_: currentClass = class_ utils.create_dir_if_necessary(path) resizedImg = utils.equalize_image_size(img, size) path += fileName cv.imwrite(path, resizedImg) numberOfImagesDone += 1 utils.show_progress(True, numberOfImagesDone, numberOfImages, "Processing class {0}.\tTotal progress:", currentClass) print "\nEqualization finished." print "\n"
def predict(self, images, sortResult=True): """ runs the prediction for the image or the images. Keyword arguments: images -- List of images. Note: Out of laziness the method will only predict the first image regardless of the argument length. sortResults -- Sort the results based on the response / confidences Returns: list of lists (in case more than one image was provided) of predictions. The first entry is the most likely prediction. """ if not self.trained: raise Exception("Classifier is not trained.") # make sure the images have the desired size if size is set if not self.imageSize is None and self.imageSize != -1: images = [ utils.equalize_image_size(image, self.imageSize) for image in images ] # get input vector for frame. inputVectors = [self.create_feature_vector(image) for image in images] # no keypoints detected if inputVectors[0] is None: return None # get predictions from all SVMs predictions = {} reverseSorting = False for class_, svm in self.svms.iteritems(): prediction = svm.predict(inputVectors) predictions[class_] = prediction reverseSorting = svm.reverseSorting # order predictions if sortResult: return sorted(predictions.items(), key=operator.itemgetter(1), reverse=reverseSorting) else: return predictions
def load_data(self, segmentIndex, numberOfClasses=None, classes=[], grayscale=True, resizeFactor=1, outputActions=True, maxNumberOfImagesPerClass=-1, yieldFilename=False, size=(-1,-1), transformation=None, resolutionSize=-1, forceNormalization=0, forceZNormalization=0, forceZcaWhitening=0): """Loads the images of the test data as a generator. Keyword argument segmentIndex -- the segment index to load from. If default values were used in segment_test_data use "train" or "test" numberOfClasses -- the number of classes to load (default None (load all classes)) classes -- explicitly specified classes (default []) grayscale -- should images be loaded in grayscale or not (default True) maxNumberOfImagesPerClass -- limits the loading of images (default -1 means no limit) """ #Parameter validation if segmentIndex not in self.segmentRatio and segmentIndex != "all": raise AttributeError(segmentIndex + " is not in segments") # test if size is valid try: sizeLen = len(size) if sizeLen != 2: raise AttributeError("size has to be a 2-element tuple. Use (-1, -1) if you don't care.") except: raise AttributeError("size has to be a 2-element tuple. Use (-1, -1) if you don't care.") desired_h = size[0] desired_w = size[1] if (desired_h / desired_w) != 1.0: raise AttributeError("The current version does not support an aspect ratio other than 1.0.") if resizeFactor != 1 and resolutionSize != -1: raise AttributeError("You can't set resizeFactor and resolutionSize at the same time.") # if forceX parameter is 0 it means do not change. -1 -> force false, +1 -> force true forceNormalization = (forceNormalization == 0 and Settings.F_APPLY_NORMALIZATION) or forceNormalization == 1 forceZNormalization = (forceZNormalization == 0 and Settings.F_APPLY_ZNORMALIZATION) or forceZNormalization == 1 if (forceNormalization and forceZNormalization): raise AttributeError("You can't apply normalization (mean substraction) and z-normalization (normalization divided by std) at the same time.") # calculate mean and std if normalization enabled and needed if forceNormalization or forceZNormalization: if desired_h == -1: if resolutionSize == -1: raise AttributeError("If F_APPLY_NORMALIZATION or F_APPLY_ZNORMALIZATION is enabled the image must have a fixed size or at least a fixed resolution size.") else: desired_h = desired_w = int(sqrt(resolutionSize)) size = (desired_w, desired_h) if len(self.mean) == 0 or len(self.std) == 0: self.__calculate_mean_std(size, grayscale) preprocessImage = forceNormalization or forceZNormalization or Settings.F_APPLY_ZCA_WHITENING or Settings.F_APPLY_CLAHE or Settings.F_APPLY_HISTOGRAM_EQ if numberOfClasses is None: numberOfClasses = self.numberOfClasses else: numberOfClasses = min(numberOfClasses, self.numberOfClasses) if not classes: #Fill classes with numberOfClasses count classes classes = [key for key in self.__classDictionary] # only take numberOfClasses classes classes = classes[:numberOfClasses] limitImageLoading = True if maxNumberOfImagesPerClass == -1: limitImageLoading = False else: maxNumberOfImagesPerClass = max(1, maxNumberOfImagesPerClass) # load at least one image #Load flag for cv.imread. loadFlag = cv.IMREAD_GRAYSCALE if grayscale else cv.IMREAD_UNCHANGED if outputActions: print "Loading dataset {0} {1}. |Classes: {2}| - number of images per segment: {3}".format(segmentIndex, self.__segmentSliceIndex[segmentIndex], numberOfClasses, self.segmentSizeMean[segmentIndex]) print "Loading in grayscale",grayscale for class_ in classes: segmentedSamples = self.__classDictionary[class_] if segmentIndex != "all": samples = segmentedSamples[segmentIndex] else: samples = [] for segId in self.__classDictionary[class_]: samples.extend(self.__classDictionary[class_][segId]) if limitImageLoading: samplesToLoad = min(maxNumberOfImagesPerClass, len(samples)) else: samplesToLoad = len(samples) if outputActions: print "\n" errors = 0 lastError = "" for i in xrange(samplesToLoad): filename = self.get_root_path() + class_ + "/" + samples[i] img = load_image(filename, loadFlag, resizeFactor) # img is None if the image could not be read. imread cannot read .gif for example if img is None: errors += 1 lastError = "Image {0} was None.\nSamples:{1}".format(filename, samples) continue # if image has an alpha channel reduce if len(img.shape) > 2 and img.shape[2] == 4L: img = cv.cvtColor(img, cv.COLOR_BGRA2BGR) # do we need to crop and adjust the size if desired_h != -1 and desired_w != -1: img = utils.crop_to_square(img) desiredArea = desired_h * desired_w img = utils.equalize_image_size(img, desiredArea) if img is None: errors += 1 lastError = "Image {0} after eq-1 was None".format(filename) continue if desired_w != img.shape[1] or desired_h != img.shape[0]: img = utils.crop_around_center(img, desired_w, desired_h) if img is None: errors += 1 lastError = "Image {0} after cropping was None".format(filename) continue # resize image to set size if resolutionSize != -1 and size[0] == -1 and size[1] == -1: try: img = utils.equalize_image_size(img, resolutionSize) except: logging.exception("Could not eq image size: img.shape: {0} - new size: {1}".format(img.shape, resolutionSize)) lastError = "Image eq-2 exception".format(filename) img = None if img is None: lastError = "Image {0} after eq-2 was None".format(filename) errors += 1 continue if not transformation is None: try: img = transformation(img) except Exception, e: errors += 1 lastError = "Exception during image transformation:",e.message continue if preprocessImage: try: img = self.__preprocess_image(img, forceNormalization, forceZNormalization, grayscale) except Exception, e: errors += 1 lastError = "Exception during image preprocessing:",e.message continue utils.show_progress(outputActions, i+1, samplesToLoad, "Loading {0} images in class {1} (Loss: {2}):", samplesToLoad, class_, errors) if yieldFilename: yield (img, class_, samples[i]) else: yield (img, class_)
def transform(self, Xb, yb): """ Augments Xb data by applying label preserving changes.""" Xb, yb = super(AugmentingBatchIterator, self).transform(Xb, yb) # prepare new Xb array because the resulting image size might vary depending on random crop setting Xb_new = np.empty((Xb.shape[0], Xb.shape[1], Settings.NN_INPUT_SHAPE[0], Settings.NN_INPUT_SHAPE[1]), Xb.dtype) for i in xrange(Xb.shape[0]): cropping = False operationKeys = Settings.NN_AUGMENTATION_PARAMS.keys() workingImage = Xb[i] # augment sample with NN_AUGMENT_CHANCE chance for j-th augmentation iteration (j is the len of ops) ops = [] while random.uniform(0.0, 1.0) < Settings.NN_AUGMENT_CHANCE[len(ops)] and Settings.NN_AUGMENT_DATA: operation = operationKeys[random.randint(0, len(operationKeys)-1)] ops.append(operation) transformation = Settings.NN_AUGMENTATION_PARAMS[operation] # prevent two cropping augmentation methods (free rotation and random cropping) if operation == 8 or operation == 6: operationKeys.remove(6) operationKeys.remove(8) # no extra parameter if transformation[1] is None: workingImage = transformation[0](workingImage) # 1 extra parameter elif len(transformation) == 2: parameterValue = transformation[1][0](transformation[1][1], transformation[1][2]) workingImage = transformation[0](workingImage, parameterValue) # 2 extra parameters (rotating) elif len(transformation) == 3: parameterValue1 = transformation[1][0](transformation[1][1], transformation[1][2]) parameterValue2 = transformation[2][0](transformation[2][1], transformation[2][2]) workingImage = transformation[0](workingImage, parameterValue1, parameterValue2) # 3 extra parameters (cropping) elif len(transformation) == 4: cropping = True parameterValue1 = transformation[1][0](transformation[1][1], transformation[1][2]) parameterValue2 = transformation[2][0](transformation[2][1], transformation[2][2]) parameterValue3 = transformation[3][0](transformation[3][1], transformation[3][2]) workingImage = transformation[0](workingImage, parameterValue1, parameterValue2, parameterValue3) # make sure that image has the right shape if not cropping: Xb_new[i] = utils.equalize_image_size(Xb[i], Settings.NN_INPUT_SHAPE[0] * Settings.NN_INPUT_SHAPE[1]) else: Xb_new[i] = workingImage # To test the actual output #print "Applied {0} transformations. Ops: {1}".format(len(ops), ops) #saveImgs = utils.radio_question("[?]", "Save images", None, ["Yes", "No"], [True, False]) #if saveImgs: # afterTrans = np.copy(Xb_new[i]) # #print "after trans before:",Xb_new[i].shape # afterTrans, _ = utils.reshape_to_cv_format(afterTrans, True) # #beforeTrans, _ = utils.reshape_to_cv_format(beforeTrans, True) # #print "after trans after:",afterTrans.shape # cv.imwrite(utils.get_temp_path() + "op_after.jpg", afterTrans) # #cv.imwrite(utils.get_temp_path() + "op_before.jpg", beforeTrans) xb = None return Xb_new, yb