def configure_data_gen(config): try: data_gen_train = DataGenerator(config=config, images_path=config.train_images_dir, labels_path=config.train_labels_dir, shuffle=True, using_val_generator=False) data_gen_val = DataGenerator(config=config, images_path=config.val_images_dir, labels_path=config.val_labels_dir, shuffle=True, using_val_generator=True) return data_gen_train, data_gen_val except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Function", "configure_data_gen", "red", "yellow", attrs=['blink']) sys.exit()
def perform_preprocessing_label(label_image, file_name=None, Multi_label=False, image_dimension=None, num_of_multi_label_classes=2, training_classes=None): """ :param label_image: label image as numpy array :return: preprocessed image NOTE-if image is binary it will divide the image with 255 to bring the range between [0,1] -if image is multi labeled it create one hot code for the following label """ try: if Multi_label: height, width, _ = image_dimension label_image = perform_normalization_multi_label(label_image, height, width, file_name, num_of_multi_label_classes, training_classes) else: height, width = image_dimension label_image = perform_binary_label_normalization(label_image, height, width, file_name) return label_image except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) sys.exit()
def convert_to_greyscale(img): try: img_grey = color.rgb2gray(img) * 255 return img_grey except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) sys.exit()
def perform_binary_label_normalization(image, height, width, file_name): try: label_image = np.array(image).reshape((height, width, 1)) label_image = label_image[:, :] / 255 except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink']) sys.exit() return label_image
def perform_normalization(img_array, normalization): """ :param img_array: input image :param normalization: normalization to perform :return: """ try: img_array = getattr(data_normalization, normalization)(img_array) return img_array except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) sys.exit()
def configure_model_complie(model, optimizer, loss, metrics): try: model.compile(optimizer=optimizer, loss=loss, metrics=metrics) return model except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Function", "configure_model_complie", "red", "yellow", attrs=['blink']) sys.exit()
def configure_metrics(list_of_metric): final_metirc = [] try: for value in list_of_metric: final_metirc.append(getattr(metric, value)) return final_metirc except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Function", "configure_metrics", "red", "yellow", attrs=['blink']) sys.exit()
def get_image_label(img_path, label_path, model_input_dimension, input_image_dim, mean_cal_over_function=False): try: img = read_image(img_path) label = read_label(label_path) if mean_cal_over_function: return img, None if model_input_dimension != input_image_dim: img, label = random_crop(img, label, model_input_dimension) return img, label except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Image Path", "{}".format(img_path), "red", "yellow", attrs=['blink']) colored_dual_string_print("Label Path", "{}".format(label_path), "red", "yellow", attrs=['blink']) sys.exit()
def create_feature_dict(features, color_code_for_features, num_of_multi_label_classes): """ ex- input= features = ["EveryThingElse", "Paved", "Unpaved"] color_code_for_features = [(255,255,255), (255,0,0),(0,255,0)] output={0:['EveryThingElse',(255,255,255),0],1:['Paved',(255,0,0),1],2:['Unpaved',(0,255,0),2]} """ class_for_training = {} if len(features) != num_of_multi_label_classes: colored_dual_string_print( "Mismatch", "Mismatch in features and num_of_multi_label_classes", "red", "yellow", attrs=['blink']) return False if len(features) != len(color_code_for_features): colored_dual_string_print( "Mismatch", "Mismatch in features and color_for_features", "red", "yellow", attrs=['blink']) return False for features_count, features in enumerate(features): class_for_training.setdefault(features_count, []) class_for_training[features_count].append(features) class_for_training[features_count].append( color_code_for_features[features_count]) class_for_training[features_count].append(features_count) return class_for_training
def configure_training(model_name, optimizer_to_use, loss_to_use, config): try: model_object = getattr(models, model_name)() model = model_object(config) optimizer = getattr(optimizers, optimizer_to_use) optimizer = optimizer(lr=config.lr) loss = getattr(train_loss_function, loss_to_use) return model, optimizer, loss except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Function", "confiure_training", "red", "yellow", attrs=['blink']) sys.exit()
def perform_normalization_multi_label(image, height, width, file_name, classes, training_classes): try: try: label_image = create_one_hot_code_for_each_image(image, height, width, file_name, classes, training_classes) except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink']) sys.exit() label_image = np.array(label_image).reshape( (height, width, classes)) except Exception as ex: colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink']) colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink']) sys.exit() return label_image
def run_mandatory_check(config): if not hasattr(data_normalization, config.normalization): colored_dual_string_print("Sanity Check Failed", "method {} not implemented in " "data_utility/data_normalization.py".format( config.normalization), "red", "yellow", attrs=['bold']) sys.exit() if not hasattr(train_loss_function, config.loss_function): colored_dual_string_print( "Sanity Check Failed", "method {} not implemented in model_utility/loss_function.py". format(config.loss_function), "red", "yellow", attrs=['bold']) sys.exit() if config.model_input_dimension > config.image_dimension: colored_dual_string_print( "Sanity Check Failed", "Rescaling not supported model dimension should either be" " equal or less than image dimension".format(config.loss_function), "red", "yellow", attrs=['bold']) sys.exit() if not hasattr(models, config.model_name): colored_dual_string_print("Sanity Check Failed", "method {} not implemented in models".format( config.model_name), "red", "yellow", attrs=['bold']) sys.exit() if config.test_loader_batch_size > config.batch_size: config.test_loader_batch_size = config.batch_size
def create_one_hot_code_for_each_image(img, height, width, file_name, num_of_classes, class_for_training): """ https://machinelearningmastery.com/why-one-hot-encode-data-in-machine-learning/ https://en.wikipedia.org/wiki/One-hot ex- [house, car, tooth, car] becomes [[1,0,0,0], [0,1,0,1], [0,0,1,0]] """ if len(img.shape) == 2: colored_dual_string_print("File Name", "{}".format(file_name), "red", "yellow", attrs=['blink']) colored_dual_string_print( "Error", "Found Binary image ,Image Bands should be greater than 2", "red", "yellow", attrs=['blink']) colored_dual_string_print( "Check", "Please check input , or MultiLabel Variable might be true in config file", "red", "yellow", attrs=['blink']) sys.exit() for feature_count in range(0, num_of_classes): img[np.where( (img == [class_for_training[feature_count][1] ]).all(axis=2))] = [class_for_training[feature_count][2]] if np.amax([img > num_of_classes]): if exists("inconsistent_pixel_values.txt"): remove("inconsistent_pixel_values.txt") file = open("inconsistent_pixel_values.txt", 'w') file.write("filename - {}".format(str(file_name)) + "\n") for x in range(0, height): for y in range(0, width): for count in range(0, num_of_classes): if tuple(img[x][y]) == (class_for_training[count][2], class_for_training[count][2], class_for_training[count][2]): break else: file.write( "pixel location - x= {} y={} pixel values-{}". format(str(x), str(y), str(img[x][y])) + "\n") file.close() colored_dual_string_print("File Name", "{}".format(file_name), "red", "yellow", attrs=['blink']) colored_dual_string_print( "Error", "Inconsistent Pixel found please check the provided input label", "red", "yellow", attrs=['blink']) colored_dual_string_print("Check", "Check inconsistent_pixel_values.txt ", "red", "yellow", attrs=['blink']) sys.exit() img = img[:, :, :1] img_one_hot_coded = to_categorical(img, num_of_classes) img_one_hot_coded = np.array(img_one_hot_coded).reshape( height, width, num_of_classes) return img_one_hot_coded
def generate(self): # Generates batches of samples # Infinite loop while 1: images = [] labels = [] file_name_list = [] test_images_list = [] test_images_file_name_list = [] if self.using_val_generator is False: random.shuffle(self.test_images_file_names) for i in range(0, self.config.batch_size): folder_name = self.dir_structure[self.initial_index] if len(self.dir_structure_dict[folder_name][0]) < self.config.batch_size: self.dir_structure_dict[folder_name][0] = self.dir_structure_dict_copy[folder_name][0].copy() file_name = self.dir_structure_dict[folder_name][0][0] if folder_name != "train" and folder_name != "val": image, label = utils.get_image_label(img_path=os.path.join(self.images_path, os.path.join(folder_name, file_name)), label_path=os.path.join(self.labels_path, os.path.join(folder_name, file_name)), model_input_dimension=self.config.model_input_dimension, input_image_dim=self.config.image_dimension) else: image, label = utils.get_image_label(os.path.join(self.images_path, file_name), os.path.join(self.labels_path, file_name), model_input_dimension=self.config.model_input_dimension, input_image_dim=self.config.image_dimension) if self.config.augment and not self.using_val_generator: if self.config.augment_frequency % random.randint(1, 10) == 0: image, label = data_aug.random_augmentation(image, label, self.config.augmentation, self.config.augmentation_type) label = preprocessing.perform_preprocessing_label(label, file_name, image_dimension=label.shape, num_of_multi_label_classes= self.config.num_of_multi_label_classes, Multi_label=self.config.multi_label, training_classes=self.class_for_training) image = preprocessing.perform_normalization(image, self.config.normalization) if self.using_val_generator is False: generate_test_set(self, i, test_images_list, test_images_file_name_list) images.append(image) labels.append(label) file_name_list.append(file_name) self.dir_structure_dict[folder_name][0].remove(file_name) if self.initial_index == len(self.dir_structure) - 1: self.initial_index = 0 else: self.initial_index = self.initial_index + 1 try: yield np.array(images).reshape(self.config.batch_size, self.height, self.width, self.bands), \ np.array(labels).reshape(self.config.batch_size, self.height, self.width, self.config.num_of_multi_label_classes) except ValueError as ex: colored_dual_string_print("ValueError", ex, "red", "yellow", attrs=['blink']) sys.tracebacklimit = None raise SystemExit