def create_crop(self, load_extension='.jpg', save_extension='.tiff', removal=True):
"""
This function creates the cropped images if necessary otherwise, it gets the cropped image names directly
from the temporal folders.
:param load_extension:
:param save_extension:
:param removal:
:return:
"""
# If the temporal folders already exist then read and get directly the names.
if exists(self.temp_train) and exists(self.temp_val) and exists(self.temp_test):
print("Cropped data already exists. Obtaining the names from the cropped images")
name_crop_train, name_crop_test, name_crop_val = self.read_crop_names()
return name_crop_train, name_crop_test, name_crop_val
print("Creating the cropped data ...\n")
# Create temporal folders with the cropped data
if exists(self.temp_train):
shutil.rmtree(self.temp_train)
makedirs(self.temp_train)
if exists(self.temp_val):
shutil.rmtree(self.temp_val)
makedirs(self.temp_val)
if exists(self.temp_test):
shutil.rmtree(self.temp_test)
makedirs(self.temp_test)
# Create the cropped data and save it in the appropriate temporal folder
# Train
crops = crop_save_remove_from_list(self.path_name, self.temp_train, self.train_names, self.size,
load_extension=load_extension,
save_extension=save_extension,
remove=removal)
# Produce augmented data if requested
if self.augmentation:
print("Generating augmented data... \n")
generate_complete_flipping(self.temp_train, self.temp_train, data_type=save_extension)
print("Total of training crops created = %d\n" % len(name_list(self.temp_train,extension=save_extension)))
# Validation
crops = crop_save_remove_from_list(self.path_name, self.temp_val, self.val_names, self.size,
load_extension=load_extension,
save_extension=save_extension,
remove=False)
print("Total of validation crops created = %d\n" % crops)
# Test
crops = crop_save_remove_from_list(self.path_name, self.temp_test, self.test_names, self.size,
load_extension=load_extension,
save_extension=save_extension,
remove=False)
print("Total of test crops created = %d\n" % crops)
# Get the names of the cropped images
name_crop_train = name_list(self.temp_train, extension=save_extension)
name_crop_test = name_list(self.temp_test, extension=save_extension)
name_crop_val = name_list(self.temp_val, extension=save_extension)
print('Finished loading data')
return name_crop_train, name_crop_test, name_crop_val
def __init__(self, path_name, data_name,size, train_percentage=0.63, val_percentage=0.07,
test_percentage=0.3,
load_extension='.jpg',augmentation = False):
self.multiple_class = True
self.path_name = path_name # Path were the data is stored
self.data_name = data_name # Name given to the splitted data
self.load_extension = load_extension
self.data_augmentation = int(augmentation)
self.size = size
self.augmentation = augmentation
# Percentages for train validation and test
self.train_percentage = train_percentage # Train percentage
self.val_percentage = val_percentage # Validation percentage
self.test_percentage = test_percentage # Test percentage
self.temp_train = './' + self.data_name + '/Temporal Train Crop Size'+str(self.size[0])+ 'x' + str(size[1])
self.temp_val = './' + self.data_name + '/Temporal Val Crop Size'+str(self.size[0])+ 'x'+str(self.size[1])
self.temp_test = './' + self.data_name + '/Temporal Test Crop Size'+str(self.size[0])+'x'+str(self.size[1])
# Get the train, validation and test names with the data_name given
self.train_names_original, self.val_names_original, self.test_names_original = self.train_val_test_names()
self.get_chip_images(size)
self.train_names = name_list(self.temp_train,load_extension)
self.test_names = name_list(self.temp_test,load_extension)
self.val_names = name_list(self.temp_val,load_extension)
def __init__(self, path_hard_example_files, path_image_files):
self.path_hard_example_files = path_hard_example_files
self.hard_example_texts = name_list(path_hard_example_files,
extension='.txt')
self.path_image_files = path_image_files
self.image_names = name_list(path_image_files)
self.image_extension = get_file_extension(self.image_names[-1])
self.sizes = self.size_hard_example()
print("Existing hard example files are\n")
print(self.hard_example_texts)
def read_crop_names(self):
"""
This function reads the crop names directly from the temporal folders
:return:
"""
save_extension = '.tiff'
name_crop_train = name_list(self.temp_train, extension=save_extension)
name_crop_test = name_list(self.temp_test, extension=save_extension)
name_crop_val = name_list(self.temp_val, extension=save_extension)
print("Total of training crops = %d\n" % len(name_crop_train))
print("Total of validation crops = %d\n" % len(name_crop_val))
print("Total of test crops = %d\n" % len(name_crop_val))
return name_crop_train, name_crop_test, name_crop_val
def train_val_test_names(self):
"""
Separate the data into train,validation and testing.
Saves the train,validation and test names into a single file.
:return:
"""
if exists('./' + self.data_name):
print("The data set already exists. Loading the specified data ...\n")
train_names, val_names, test_names = self.read_data_names()
print('Size of training set: %d\n' % (len(train_names)))
print('Size of validation set: %d\n' % (len(val_names)))
print('Size of test set: %d\n' % (len(test_names)))
return train_names, val_names, test_names
else:
names = name_list(self.path_name,extension=self.load_extension )
# Actions done to light hue images
np.random.shuffle(names)
test_names = names[0:int(len(names) * self.test_percentage)]
val_names = names[
int(len(names) * self.test_percentage):int(len(names) * self.test_percentage) + int(
len(names) * self.val_percentage)]
train_names = names[int(len(names) * self.test_percentage) + int(len(names) * self.val_percentage):]
print('Size of training set: %d\n' % (len(train_names)))
print('Size of validation set: %d\n' % (len(val_names)))
print('Size of test set: %d\n' % (len(test_names)))
makedirs('./' + self.data_name)
save_names(train_names, './' + self.data_name, 'Train Data')
save_names(val_names, './' + self.data_name, 'Validation Data')
save_names(test_names, './' + self.data_name, 'Test Data')
return train_names, val_names, test_names
def save_thick_fibers(read_path, save_path, thickness, write_images=True):
"""
Takes the images in the read path together with the ground truth labels. Produces and saves
new images with thicker ground truth label saved in the save path.
:param read_path: The path were the original ground truths are
:param save_path: The path were the new thicker ground truths will be saved
:param thickness: The thickness for the new ground truth images
:param load_type: The type of the original images saved
:param write_images: This flag determines if the original images will be also copied into the path (the input data)
:return: None
"""
# Get the image names in the read path
names = name_list(read_path)
if not (exists(save_path)):
makedirs(
save_path) # Creates the folder if the save path doesn't exist
# For each of the images produce a new image with the thicker ground truth
for name in names:
print('Converting image %s...\n' % name)
_, label = load_image_label(read_path, name)
thick_label = create_thick_fibers(label, thickness)
cv2.imwrite(
join(
save_path,
name.replace(get_file_extension(name),
'_FIB' + get_file_extension(name))), thick_label)
if write_images:
copyfile(join(read_path, name), join(save_path, name))
def train_val_test_names(self):
"""
Separate the data into train,validation and testing.
Saves the train,validation and test names into a single file.
:return:
"""
if exists('./' + self.data_name):
print("The data set already exists. Loading the specified data ...\n")
train_names, val_names, test_names = self.read_data_names()
return train_names, val_names, test_names
else:
names = name_list(self.path_name)
light_names = np.array([f for f in names if not ('Dark' in f)])
dark_names = np.array([f for f in names if ('Dark' in f)])
# Actions done to light hue images
np.random.shuffle(light_names)
test_light = light_names[0:int(len(light_names) * self.test_percentage)]
val_light = light_names[
int(len(light_names) * self.test_percentage):int(len(light_names) * self.test_percentage) + int(
len(light_names) * self.val_percentage)]
train_light = light_names[
int(len(light_names) * self.test_percentage) + int(len(light_names) * self.val_percentage):]
# Actions done to dark hue images
np.random.shuffle(dark_names)
test_dark = dark_names[0:int(len(dark_names) * self.test_percentage)]
val_dark = dark_names[
int(len(dark_names) * self.test_percentage):int(len(dark_names) * self.test_percentage) + int(
len(dark_names) * self.val_percentage)]
train_dark = dark_names[
int(len(dark_names) * self.test_percentage) + int(len(dark_names) * self.val_percentage):]
train_names = np.concatenate((train_light, train_dark))
val_names = np.concatenate((val_light, val_dark))
test_names = np.concatenate((test_light, test_dark))
print('Size of training set: %d\n' % (len(train_names)))
print('Size of validation set: %d\n' % (len(val_names)))
print('Size of test set: %d\n' % (len(test_names)))
makedirs('./' + self.data_name)
save_names(train_names, './' + self.data_name, 'Train Data')
save_names(val_names, './' + self.data_name, 'Validation Data')
save_names(test_names, './' + self.data_name, 'Test Data')
return train_names, val_names, test_names