def configure_data_gen(config):
try:
data_gen_train = DataGenerator(config=config,
images_path=config.train_images_dir,
labels_path=config.train_labels_dir,
shuffle=True,
using_val_generator=False)
data_gen_val = DataGenerator(config=config,
images_path=config.val_images_dir,
labels_path=config.val_labels_dir,
shuffle=True,
using_val_generator=True)
return data_gen_train, data_gen_val
except Exception as ex:
colored_dual_string_print("Exception",
ex,
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Function",
"configure_data_gen",
"red",
"yellow",
attrs=['blink'])
sys.exit()
def perform_preprocessing_label(label_image, file_name=None, Multi_label=False, image_dimension=None,
num_of_multi_label_classes=2, training_classes=None):
"""
:param label_image: label image as numpy array
:return: preprocessed image
NOTE-if image is binary it will divide the image with 255 to bring the range between [0,1]
-if image is multi labeled it create one hot code for the following label
"""
try:
if Multi_label:
height, width, _ = image_dimension
label_image = perform_normalization_multi_label(label_image, height, width, file_name,
num_of_multi_label_classes, training_classes)
else:
height, width = image_dimension
label_image = perform_binary_label_normalization(label_image, height, width, file_name)
return label_image
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
sys.exit()
def convert_to_greyscale(img):
try:
img_grey = color.rgb2gray(img) * 255
return img_grey
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
sys.exit()
def perform_binary_label_normalization(image, height, width, file_name):
try:
label_image = np.array(image).reshape((height, width, 1))
label_image = label_image[:, :] / 255
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink'])
sys.exit()
return label_image
def perform_normalization(img_array, normalization):
"""
:param img_array: input image
:param normalization: normalization to perform
:return:
"""
try:
img_array = getattr(data_normalization, normalization)(img_array)
return img_array
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
sys.exit()
def configure_model_complie(model, optimizer, loss, metrics):
try:
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
return model
except Exception as ex:
colored_dual_string_print("Exception",
ex,
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Function",
"configure_model_complie",
"red",
"yellow",
attrs=['blink'])
sys.exit()
def configure_metrics(list_of_metric):
final_metirc = []
try:
for value in list_of_metric:
final_metirc.append(getattr(metric, value))
return final_metirc
except Exception as ex:
colored_dual_string_print("Exception",
ex,
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Function",
"configure_metrics",
"red",
"yellow",
attrs=['blink'])
sys.exit()
def get_image_label(img_path,
label_path,
model_input_dimension,
input_image_dim,
mean_cal_over_function=False):
try:
img = read_image(img_path)
label = read_label(label_path)
if mean_cal_over_function:
return img, None
if model_input_dimension != input_image_dim:
img, label = random_crop(img, label, model_input_dimension)
return img, label
except Exception as ex:
colored_dual_string_print("Exception",
ex,
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Image Path",
"{}".format(img_path),
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Label Path",
"{}".format(label_path),
"red",
"yellow",
attrs=['blink'])
sys.exit()
def create_feature_dict(features, color_code_for_features,
num_of_multi_label_classes):
"""
ex-
input=
features = ["EveryThingElse", "Paved", "Unpaved"]
color_code_for_features = [(255,255,255), (255,0,0),(0,255,0)]
output={0:['EveryThingElse',(255,255,255),0],1:['Paved',(255,0,0),1],2:['Unpaved',(0,255,0),2]}
"""
class_for_training = {}
if len(features) != num_of_multi_label_classes:
colored_dual_string_print(
"Mismatch",
"Mismatch in features and num_of_multi_label_classes",
"red",
"yellow",
attrs=['blink'])
return False
if len(features) != len(color_code_for_features):
colored_dual_string_print(
"Mismatch",
"Mismatch in features and color_for_features",
"red",
"yellow",
attrs=['blink'])
return False
for features_count, features in enumerate(features):
class_for_training.setdefault(features_count, [])
class_for_training[features_count].append(features)
class_for_training[features_count].append(
color_code_for_features[features_count])
class_for_training[features_count].append(features_count)
return class_for_training
def configure_training(model_name, optimizer_to_use, loss_to_use, config):
try:
model_object = getattr(models, model_name)()
model = model_object(config)
optimizer = getattr(optimizers, optimizer_to_use)
optimizer = optimizer(lr=config.lr)
loss = getattr(train_loss_function, loss_to_use)
return model, optimizer, loss
except Exception as ex:
colored_dual_string_print("Exception",
ex,
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Function",
"confiure_training",
"red",
"yellow",
attrs=['blink'])
sys.exit()
def perform_normalization_multi_label(image, height, width, file_name, classes, training_classes):
try:
try:
label_image = create_one_hot_code_for_each_image(image, height, width, file_name, classes, training_classes)
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink'])
sys.exit()
label_image = np.array(label_image).reshape(
(height, width, classes))
except Exception as ex:
colored_dual_string_print("Exception", ex, "red", "yellow", attrs=['blink'])
colored_dual_string_print("Infile", "{}".format(file_name), "red", "yellow", attrs=['blink'])
sys.exit()
return label_image
def run_mandatory_check(config):
if not hasattr(data_normalization, config.normalization):
colored_dual_string_print("Sanity Check Failed",
"method {} not implemented in "
"data_utility/data_normalization.py".format(
config.normalization),
"red",
"yellow",
attrs=['bold'])
sys.exit()
if not hasattr(train_loss_function, config.loss_function):
colored_dual_string_print(
"Sanity Check Failed",
"method {} not implemented in model_utility/loss_function.py".
format(config.loss_function),
"red",
"yellow",
attrs=['bold'])
sys.exit()
if config.model_input_dimension > config.image_dimension:
colored_dual_string_print(
"Sanity Check Failed",
"Rescaling not supported model dimension should either be"
" equal or less than image dimension".format(config.loss_function),
"red",
"yellow",
attrs=['bold'])
sys.exit()
if not hasattr(models, config.model_name):
colored_dual_string_print("Sanity Check Failed",
"method {} not implemented in models".format(
config.model_name),
"red",
"yellow",
attrs=['bold'])
sys.exit()
if config.test_loader_batch_size > config.batch_size:
config.test_loader_batch_size = config.batch_size
def create_one_hot_code_for_each_image(img, height, width, file_name,
num_of_classes, class_for_training):
"""
https://machinelearningmastery.com/why-one-hot-encode-data-in-machine-learning/
https://en.wikipedia.org/wiki/One-hot
ex-
[house, car, tooth, car] becomes
[[1,0,0,0],
[0,1,0,1],
[0,0,1,0]]
"""
if len(img.shape) == 2:
colored_dual_string_print("File Name",
"{}".format(file_name),
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print(
"Error",
"Found Binary image ,Image Bands should be greater than 2",
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print(
"Check",
"Please check input , or MultiLabel Variable might be true in config file",
"red",
"yellow",
attrs=['blink'])
sys.exit()
for feature_count in range(0, num_of_classes):
img[np.where(
(img == [class_for_training[feature_count][1]
]).all(axis=2))] = [class_for_training[feature_count][2]]
if np.amax([img > num_of_classes]):
if exists("inconsistent_pixel_values.txt"):
remove("inconsistent_pixel_values.txt")
file = open("inconsistent_pixel_values.txt", 'w')
file.write("filename - {}".format(str(file_name)) + "\n")
for x in range(0, height):
for y in range(0, width):
for count in range(0, num_of_classes):
if tuple(img[x][y]) == (class_for_training[count][2],
class_for_training[count][2],
class_for_training[count][2]):
break
else:
file.write(
"pixel location - x= {} y={} pixel values-{}".
format(str(x), str(y), str(img[x][y])) + "\n")
file.close()
colored_dual_string_print("File Name",
"{}".format(file_name),
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print(
"Error",
"Inconsistent Pixel found please check the provided input label",
"red",
"yellow",
attrs=['blink'])
colored_dual_string_print("Check",
"Check inconsistent_pixel_values.txt ",
"red",
"yellow",
attrs=['blink'])
sys.exit()
img = img[:, :, :1]
img_one_hot_coded = to_categorical(img, num_of_classes)
img_one_hot_coded = np.array(img_one_hot_coded).reshape(
height, width, num_of_classes)
return img_one_hot_coded
def generate(self):
# Generates batches of samples
# Infinite loop
while 1:
images = []
labels = []
file_name_list = []
test_images_list = []
test_images_file_name_list = []
if self.using_val_generator is False:
random.shuffle(self.test_images_file_names)
for i in range(0, self.config.batch_size):
folder_name = self.dir_structure[self.initial_index]
if len(self.dir_structure_dict[folder_name][0]) < self.config.batch_size:
self.dir_structure_dict[folder_name][0] = self.dir_structure_dict_copy[folder_name][0].copy()
file_name = self.dir_structure_dict[folder_name][0][0]
if folder_name != "train" and folder_name != "val":
image, label = utils.get_image_label(img_path=os.path.join(self.images_path,
os.path.join(folder_name, file_name)),
label_path=os.path.join(self.labels_path,
os.path.join(folder_name, file_name)),
model_input_dimension=self.config.model_input_dimension,
input_image_dim=self.config.image_dimension)
else:
image, label = utils.get_image_label(os.path.join(self.images_path, file_name),
os.path.join(self.labels_path, file_name),
model_input_dimension=self.config.model_input_dimension,
input_image_dim=self.config.image_dimension)
if self.config.augment and not self.using_val_generator:
if self.config.augment_frequency % random.randint(1, 10) == 0:
image, label = data_aug.random_augmentation(image, label, self.config.augmentation,
self.config.augmentation_type)
label = preprocessing.perform_preprocessing_label(label, file_name,
image_dimension=label.shape,
num_of_multi_label_classes=
self.config.num_of_multi_label_classes,
Multi_label=self.config.multi_label,
training_classes=self.class_for_training)
image = preprocessing.perform_normalization(image, self.config.normalization)
if self.using_val_generator is False:
generate_test_set(self, i, test_images_list, test_images_file_name_list)
images.append(image)
labels.append(label)
file_name_list.append(file_name)
self.dir_structure_dict[folder_name][0].remove(file_name)
if self.initial_index == len(self.dir_structure) - 1:
self.initial_index = 0
else:
self.initial_index = self.initial_index + 1
try:
yield np.array(images).reshape(self.config.batch_size, self.height, self.width, self.bands), \
np.array(labels).reshape(self.config.batch_size, self.height, self.width,
self.config.num_of_multi_label_classes)
except ValueError as ex:
colored_dual_string_print("ValueError", ex, "red", "yellow", attrs=['blink'])
sys.tracebacklimit = None
raise SystemExit