class LineDetectorModel(Model):
"""Model to detect lines of text in an image."""
def __init__(
self,
dataset_cls: type = IamParagraphsDataset,
network_fn: Callable = fcn,
dataset_args: Dict = None,
network_args: Dict = None,
):
"""Define the default dataset and network values for this model."""
super().__init__(dataset_cls, network_fn, dataset_args, network_args)
self.data_augmentor = ImageDataGenerator(**_DATA_AUGMENTATION_PARAMS)
self.batch_augment_fn = self.augment_batch
def loss(self): # pylint: disable=no-self-use
return "categorical_crossentropy"
def optimizer(self): # pylint: disable=no-self-use
return Adam(0.001 / 2)
def metrics(self): # pylint: disable=no-self-use
return None
def augment_batch(self, x_batch: np.ndarray,
y_batch: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""Perform different random transformations on the whole batch of x, y samples."""
x_augment, y_augment = zip(
*[self._augment_sample(x, y) for x, y in zip(x_batch, y_batch)])
return np.stack(x_augment, axis=0), np.stack(y_augment, axis=0)
def _augment_sample(self, x: np.ndarray,
y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""
Perform the same random image transformation on both x and y.
x is a 2d image of shape self.image_shape, but self.data_augmentor needs the channel image too.
"""
x_3d = np.expand_dims(x, axis=-1)
transform_parameters = self.data_augmentor.get_random_transform(
x_3d.shape)
x_augment = self.data_augmentor.apply_transform(
x_3d, transform_parameters)
y_augment = self.data_augmentor.apply_transform(
y, transform_parameters)
return np.squeeze(x_augment, axis=-1), y_augment
def predict_on_image(self, x: np.ndarray) -> np.ndarray:
"""Predict on a single input."""
return self.network.predict(np.expand_dims(x, axis=0))[0]
def evaluate(self,
x: np.ndarray,
y: np.ndarray,
batch_size: int = 32,
verbose: bool = False) -> float:
"""Evaluate the model."""
# pylint: disable=unused-argument
return self.network.evaluate(x, y, batch_size=batch_size)
class LineDetectModel(Model):
"""Model to detect lines of text in an image."""
def __init__(self,
network_fn: Callable = lenetFCN,
dataset: type = IAMPara):
"""Define the default dataset and network values for this model."""
super().__init__(network_fn, dataset)
print('[INFO] Arguments passed to data augmentation...',
DATA_AUGMENTATION_PARAMS)
self.data_augmentor = ImageDataGenerator(**DATA_AUGMENTATION_PARAMS)
self.batch_augment_fn = self.augment_batch
def metrics(self):
return None
def augment_batch(self, x_batch: np.ndarray,
y_batch: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""Performs different random transformations on the whole batch of x, y samples."""
x_augment, y_augment = zip(
*[self.augment_sample(x, y) for x, y in zip(x_batch, y_batch)])
return np.stack(x_augment, axis=0), np.stack(y_augment, axis=0)
def augment_sample(self, x: np.ndarray,
y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""
Perform the same random image transformation on both x and y.
x is a 2d image of shape self.image_shape, but self.data_augmentor needs the channel image too.
"""
x_3d = np.expand_dims(x, axis=-1)
transform_parameters = self.data_augmentor.get_random_transform(
x_3d.shape)
x_augment = self.data_augmentor.apply_transform(
x_3d, transform_parameters)
y_augment = self.data_augmentor.apply_transform(
y, transform_parameters)
return np.squeeze(x_augment, axis=-1), y_augment
def predict_on_image(self, x: np.ndarray) -> np.ndarray:
"""Returns the network predictions on x."""
return self.network.predict(np.expand_dims(x, axis=0))[0]
class DataGenerator(tensorflow.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
data_type='train',
dim=(84, 84),
n_channels=3,
way=5,
shot=1,
query=5,
num_batch=500):
'Initialization'
self.type = data_type
# if self.type == 'train':
# self.is_training = np.array([True for _ in range(batch_size)])
# else:
# self.is_training = np.array([False for _ in range(batch_size)])
self.dim = dim
#self.batch_size = batch_size
self.n_channels = n_channels
self.num_per_class = 600
self.num_batch = num_batch
#self.y_target = np.zeros(self.batch_size)
self.build_data(self.type)
self.on_epoch_end()
self.way = way
self.shot = shot
self.query = query
self.transformer = ImageDataGenerator(width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.2,
rotation_range=30,
horizontal_flip=True,
shear_range=0.1)
#TODO!!!!
#self.hard_batch = np.zeros(batch_size, *dim, n_channels)
def build_data(self, data_type):
if data_type == 'train':
self.class_data = np.load('python/mini_train.npy')
elif data_type == 'val':
self.class_data = np.load('python/mini_val.npy')
else:
self.class_data = np.load('python/mini_test.npy')
self.n_classes = len(self.class_data)
def __len__(self):
'Denotes the number of batches per epoch'
return self.num_batch
def __getitem__(self, index):
'Generate one batch of data'
# Generate data
X_sample, X_query, label = self.__data_generation()
#way = np.ones((self.way * self.shot, 1)) * self.way
return [X_sample, X_query], label
def on_epoch_end(self):
'Updates indexes after each epoch'
pass
def __data_generation(self):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X_sample = np.empty((self.way, self.shot, *self.dim, self.n_channels))
X_query = np.empty((self.way, self.query, *self.dim, self.n_channels))
chosen_class = random.sample(range(self.n_classes), self.way)
label = np.empty(self.way * self.query)
# print(pos, neg)
# print(self.class_data[pos][0].shape)
# Generate data
for i in range(self.way):
sample_idx = random.sample(range(self.num_per_class),
self.shot + self.query)
sample_data = self.class_data[chosen_class[i]][sample_idx] / 255.
if True:
#if self.type != 'train':
X_sample[i] = sample_data[:self.shot]
X_query[i] = sample_data[self.shot:self.shot + self.query]
else:
for j in range(self.shot):
params = self.transformer.get_random_transform(
self.dim + (self.n_channels, ))
x = self.transformer.apply_transform(
sample_data[j], params)
X_sample[i][j] = x
for j in range(self.shot, self.shot + self.query):
params = self.transformer.get_random_transform(
self.dim + (self.n_channels, ))
x = self.transformer.apply_transform(
sample_data[j], params)
X_query[i][j - self.shot] = x
label[i * self.query:(i + 1) * self.query] = i
return X_sample, X_query, np_utils.to_categorical(label)
class UnetDataGenerator(Sequence):
"""
Inspired on this thread: https://github.com/keras-team/keras/issues/12120
"""
def __init__(self,
features,
targets,
image_size,
label_size,
classes,
batch_size,
repeats=1,
shuffle=True,
validation=False):
self._features = features
self._targets = targets
self._image_size = image_size
self._label_size = label_size
self._classes = classes
self._batch_size = batch_size
self._shuffle = shuffle
self._repeats = repeats
self._verification = validation
self._ids = list(self._list_files())
self._indexes = np.arange(len(self._ids))
if self._shuffle:
np.random.shuffle(self._indexes)
self._indexes = np.repeat(self._indexes, self._repeats)
self._augmenter = ImageDataGenerator(rotation_range=30,
zoom_range=0.2,
width_shift_range=0.01,
height_shift_range=0.01,
horizontal_flip=True,
vertical_flip=True,
samplewise_std_normalization=True)
def _list_files(self):
feature_files = os.listdir(self._features)
target_files = os.listdir(self._targets)
return zip(feature_files, target_files)
def _encode_one_hot(self, label):
x = np.zeros(
(self._label_size[0] * self._label_size[1], self._classes))
for c in range(1, self._classes + 1):
a = label == c
x[(label == c).squeeze(), c - 1] = 1
return x
def __len__(self):
return int(np.floor(len(self._ids) / self._batch_size)) * self._repeats
def __getitem__(self, item):
indexes = self._indexes[item * self._batch_size:(item + 1) *
self._batch_size]
ids_temp = [self._ids[idx] for idx in indexes]
X = [
cv2.resize(
cv2.imread(os.path.join(self._features, ids_temp[k][0]),
cv2.IMREAD_COLOR), self._image_size,
cv2.INTER_NEAREST).astype(np.float64) / 255.0
for k in range(len(ids_temp))
]
y = [
cv2.resize(
cv2.imread(os.path.join(self._targets, ids_temp[k][1]),
cv2.IMREAD_COLOR), self._image_size,
cv2.INTER_NEAREST) for k in range(len(ids_temp))
]
params = self._augmenter.get_random_transform(self._image_size)
if not self._verification:
X = np.array([
self._augmenter.apply_transform(self._augmenter.standardize(x),
params) for x in X
])
y = np.array([
self._encode_one_hot(
cv2.resize(
self._augmenter.apply_transform(_y, params)[:, :, 0],
self._label_size, cv2.INTER_NEAREST).reshape(-1, 1))
for _y in y
])
else:
X = np.array([self._augmenter.standardize(x) for x in X])
y = np.array([
self._encode_one_hot(
cv2.resize(_y[:, :, 0], self._label_size,
cv2.INTER_NEAREST).reshape(-1, 1)) for _y in y
])
return np.array(X), np.array(y)
def on_epoch_end(self):
self._indexes = np.arange(len(self._ids))
if self._shuffle:
np.random.shuffle(self._indexes)
self._indexes = np.repeat(self._indexes, self._repeats)
def augment_dataset(samples: list, augmented_dir:str, multiplier:int=2, register:bool=False, res:tuple=(120, 160)):
"""
Augments a dataset. Will attempt to balance classes.
Params
------
samples: list
List of sample annotation tuples (rgb path, lwir path, class label)
augmented_dir: str
Directory to store augmented data at
multiplier: int
Multiplication factor for number of new samples
register: bool
Whether to align RGB images with LWIR images
Returns
------
A list containing tuples of (rgb path, lwir path, class name) for new augmented dataset
"""
if os.path.exists(augmented_dir):
shutil.rmtree(augmented_dir)
datagen = ImageDataGenerator(
horizontal_flip=True,
rotation_range=30,
zoom_range=[0.5, 1],
)
classes = {}
# Get classes
for sample in samples:
classes[sample[-1]] = classes.get(sample[-1], 0) + 1
# Get maximum class samples
max_class = max(classes.items(), key=lambda x: x[1])
class_multipliers = {x: max_class[1] * multiplier // y for x, y in classes.items()}
class_remainders = {x: (max_class[1] * multiplier) % y for x, y in classes.items()}
# Generate output directory structure
for label in classes:
os.makedirs(os.path.join(augmented_dir, label, f"{label}_single_1", "lwir"))
os.makedirs(os.path.join(augmented_dir, label, f"{label}_single_1", "rgb"))
# Augment and copy
augmented_samples = []
for rgb_path, lwir_path, label in tqdm(samples, position=0):
rgb = cv2.imread(rgb_path)
if register:
rgb = cv2.resize(rgb, tuple(cfg.large.res))
rgb = cv2.warpAffine(rgb, cfg.large.matrix, tuple(cfg.large.res))
rgb = cv2.resize(rgb, res)
lwir = cv2.imread(lwir_path)
lwir = cv2.resize(lwir, res)
name = (rgb_path.split("/")[-1]).split(".")[0][4:]
out_path = os.path.join(augmented_dir, label, f"{label}_single_1")
# Copy original
augmented_samples.append(save(out_path, name, rgb, lwir, label, extension=""))
# Flipped
if multiplier > 1:
rgb_t = datagen.apply_transform(rgb, {"flip_horizontal": True})
lwir_t = datagen.apply_transform(lwir, {"flip_horizontal": True})
augmented_samples.append(save(out_path, name, rgb_t, lwir_t, label, extension=0))
# Random transformations
remainder = 1 if class_remainders[label] > 0 else 0
class_remainders[label] -= 1
for i in range(2, class_multipliers[label] + remainder):
trans = datagen.get_random_transform(rgb.shape)
rgb_t = datagen.apply_transform(rgb, trans)
lwir_t = datagen.apply_transform(lwir, trans)
augmented_samples.append(save(out_path, name, rgb_t, lwir_t, label, extension=i))
return augmented_samples
class HPatches():
def __init__(self,
train=True,
transform=None,
download=False,
train_fnames=[],
test_fnames=[],
denoise_model=None,
use_clean=False):
self.train = train
self.transform = transform
self.train_fnames = train_fnames
self.test_fnames = test_fnames
self.denoise_model = denoise_model
self.use_clean = use_clean
self.imgaug = ImageDataGenerator(
# featurewise_center=True,
# featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
def set_denoise_model(self, denoise_model):
self.denoise_model = denoise_model
def denoise_patches(self, patches):
batch_size = 100
for i in tqdm(range(int(len(patches) / batch_size)), file=sys.stdout):
batch = patches[i * batch_size:(i + 1) * batch_size]
batch = np.expand_dims(batch, -1)
batch = np.clip(
self.denoise_model.predict(batch).astype(int), 0,
255).astype(np.uint8)[:, :, :, 0]
patches[i * batch_size:(i + 1) * batch_size] = batch
batch = patches[i * batch_size:]
batch = np.expand_dims(batch, -1)
batch = np.clip(self.denoise_model.predict(batch).astype(int), 0,
255).astype(np.uint8)[:, :, :, 0]
patches[i * batch_size:] = batch
return patches
def read_image_file(self, data_dir, train=1):
"""Return a Tensor containing the patches
"""
if self.denoise_model and not self.use_clean:
print('Using denoised patches')
elif not self.denoise_model and not self.use_clean:
print('Using noisy patches')
elif self.use_clean:
print('Using clean patches')
sys.stdout.flush()
patches = []
labels = []
counter = 0
hpatches_sequences = [x[1] for x in os.walk(data_dir)][0]
if train:
list_dirs = self.train_fnames
else:
list_dirs = self.test_fnames
for directory in tqdm(hpatches_sequences, file=sys.stdout):
if (directory in list_dirs):
for tp in tps:
if self.use_clean:
sequence_path = os.path.join(data_dir, directory,
tp) + '.png'
else:
sequence_path = os.path.join(data_dir, directory,
tp) + '_noise.png'
image = cv2.imread(sequence_path, 0)
h, w = image.shape
n_patches = int(h / w)
n_aug = 3
for i in range(n_patches):
patch = image[i * (w):(i + 1) * (w), 0:w]
patch = cv2.resize(patch, (32, 32))
patch = np.array(patch, dtype=np.uint8)
patches.append(patch)
labels.append(i + n_aug * i + counter)
for j in range(0, n_aug):
params = self.imgaug.get_random_transform(
patch.shape)
patch_aug = self.imgaug.apply_transform(
self.imgaug.standardize(patch), params)
patches.append(patch_aug)
labels.append(i + j + counter)
counter += n_patches
patches = np.array(patches, dtype=np.uint8)
if self.denoise_model and not self.use_clean:
print('Denoising patches...')
patches = self.denoise_patches(patches)
return patches, labels
class generator3da(generator3d):
def __init__(self,
list_IDs,
image_path,
mask_path,
to_fit=True,
batch_size=32,
patch_size=8,
dim=(512, 512),
dimy=(512, 512),
n_channels=1,
n_classes=10,
shuffle=True,
data_gen_args=None):
super().__init__(list_IDs,
image_path,
mask_path,
to_fit=to_fit,
batch_size=batch_size,
patch_size=patch_size,
dim=dim,
dimy=dimy,
n_channels=n_channels,
n_classes=n_classes,
shuffle=shuffle)
"""Initialization
:param list_IDs: list of all 'label' ids to use in the generator
:param image_path: path to images location
:param mask_path: path to masks location
:param to_fit: True to return X and y, False to return X only
:param batch_size: batch size at each iteration
:param dim: tuple indicating image dimension of the input
:param dimy: tuple indicating image dimension of the output
:param n_channels: number of image channels
:param n_classes: number of output masks
:param shuffle: True to shuffle label indexes after every epoch
"""
if data_gen_args != None:
self.trans = ImageDataGenerator(**data_gen_args)
def _generate_y(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
Y = np.zeros(
(self.batch_size, *self.dimy, self.patch_size, self.n_channels))
# Generate data
for patch in list_IDs_temp:
for i, ID in enumerate(patch[1]):
path = self.mask_path + '/' + patch[0] + '/' + 'label_' + ID[
6:15] + '.png'
img = self._load_grayscale_image_VTK(path)[:, :, 0]
Y[0, :, :, i, 0] = img
if self.bool:
Y[0, :, :, i, :] = self.trans.apply_transform(
Y[0, :, :, i, :], self.param)
return Y
def _generate_X(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
X = np.zeros(
(self.batch_size, *self.dim, self.patch_size, self.n_channels))
self.param = self.trans.get_random_transform(self.dim)
if random.uniform(0, 1) >= 0.5:
self.bool = True
else:
self.bool = False
# Generate data
for patch in list_IDs_temp:
for i, ID in enumerate(patch[1]):
path = self.image_path + '/' + patch[0] + '/' + ID
img = self._load_grayscale_image_VTK(path)[:, :, 0]
X[0, :, :, i, 0] = img
if self.bool:
X[0, :, :, i, :] = self.trans.apply_transform(
X[0, :, :, i, :], self.param)
return X
class HeadPoseDataGenerator(Sequence):
'''
This class implements a basic Keras data generator overriding methods from its parent class Sequence. The purpose
of this data generator is to deliver in each batch a set of images from the subset used to initalize this generator
containing an equal number of members per class.
'''
def __init__(self, pose_dataframe, img_array, batch_size,
normalize=False, input_norm=None, tilt_norm=None, pan_norm=None,
augment=False, shift_range=None, zoom_range=None, brightness_range=None,
img_rescale=1, out_rescale=1):
'''
Initializes the data generator with the data from a given subset from the original dataset, and the values to
use when doing data augmentation.
Arguments:
pose_dataframe: Dataframe containing a list of each picture in the given subset and its pose values.
img_array: Numpy array containing the images from the given subset.
batch_size: Number of pictures per batch.
normalize: If the data shall be normalized or not.
input_norm: Tuple containing mean and std values for normalizing pictures in the dataset.
tilt_norm: Tuple containing mean and std values for normalizing tilt values in the dataset.
pan_norm: Tuple containing mean and std values for normalizing pan values in the dataset.
augment: If data augmentation shall be applied or not.
shift_range: Value (between 0 and 1) indicating the portion of the length of the side of each picture that
can be used to shift the picture (in both axes).
zoom_range: Tuple containing the minimum and maximum values used to apply zoom to each picture.
brightness_range: Tuple containing the minimum and maximum values used to apply a brightness transformation
to each picture.
img_rescale: Each pixel from every picture in the subset will be multiplied by this value.
out_rescale: Tilt and pan values for every picture in the subset will be multiplied by this value.
'''
# Create empty arrays for pictures and labels from the subset.
self.pics = []
self.labels = []
# Initialize batch size.
self.batch_size = batch_size
# Initialize normalization parameters.
self.normalize = normalize
self.input_norm = input_norm
'''
Initialize the parameter controlling if data augmentation shall be applied or not, and data augmentation
parameters.
'''
self.augment = augment
if self.augment == True:
self.generator = ImageDataGenerator(width_shift_range=shift_range, height_shift_range=shift_range,
brightness_range=brightness_range, zoom_range=zoom_range)
# Initialize scaling parameters.
self.img_rescale = img_rescale
self.out_rescale = out_rescale
'''
Initialize the iterator used to control the position of the next picture from every class that will be included
in a batch.
'''
self.common_iterator = 0
# Sort dataframe by class.
df = pose_dataframe.sort_values('class')
# Initialize the number of pictures in the dataset.
self.total_size = len(df.index)
# Load images and pose values into the previously created arrays.
prev_class = -1
class_index = -1
# For each image in the (ordered) dataset:
for index, row in df.iterrows():
'''
If the class for the current picture is different from the last class recorded, append an empty list for the
new class.
'''
if row['class'] != prev_class:
prev_class = row['class']
self.pics.append([])
self.labels.append([])
class_index = class_index + 1
# Append picture to corresponding class array.
self.pics[class_index].append(np.squeeze(img_array[index]))
# Append labels to corresponding class array (normalized and rescaled).
self.labels[class_index].append([((row['tilt'] * out_rescale) - tilt_norm[0]) / tilt_norm[1] , ((row['pan'] * out_rescale) - pan_norm[0]) / pan_norm[1]])
# Assert batch size is a multiple of the number of classes.
assert(batch_size % len(self.pics) == 0)
def __data_generation(self):
'''
Outputs a batch of pictures.
Returns:
X: Pictures in the batch.
y: Labels for each picture in the batch.
'''
# Create empty lists for pictures and labels.
X = []
y = []
# For each picture-per-class:
for i in range(int(self.batch_size / len(self.pics))):
# For each class:
for j in range(len(self.pics)):
# Select the next picture in the class list (start from beginning after the last picture).
pic = self.pics[j][int(self.common_iterator % len(self.pics[j]))]
pic = np.expand_dims(pic, axis=2)
# Apply data augmentation.
if self.augment == True:
transformation = self.generator.get_random_transform(pic.shape)
transformation['zx'] = transformation['zy']
pic = self.generator.apply_transform(pic, transformation)
# Rescale each pixel value in image.
pic = pic * self.img_rescale
# Normalize image.
if self.normalize == True:
pic = (pic - self.input_norm[0]) / self.input_norm[1]
# Add image and labels to the batch.
X.append(pic)
y.append(self.labels[j][int(self.common_iterator % len(self.labels[j]))])
# Update iterator.
self.common_iterator = self.common_iterator + 1
# Transform lists into Numpy arrays.
X = np.array(X)
y = np.array(y)
# Return images and labels.
return X, y
def __len__(self):
'''
Outputs the length (number of batches) that the data generator can provide.
Returns:
l: The length of the data generator.
'''
'''
Calculate the length of the data generator as the relation between the total number of images and the size of
each batch; in order to function properly with uneven class lengths the number is rounded to the smaller integer
bigger or equal to the obtained result.
'''
l = ceil(self.total_size / self.batch_size)
return l
def __getitem__(self, index):
'''
Outputs a new batch given the batch index.
Returns:
X: Pictures in the batch.
y: Labels for each picture in the batch.
'''
# Set the class iterator in the correct position for obtaining the requested batch.
self.common_iterator = index * int(self.batch_size / len(self.pics))
# Generate the batch.
X, y = self.__data_generation()
# Return images and labels for the requested batch.
return X, y
def reset(self):
'''
Resets the class iterator to its initial position.
'''
self.common_iterator = 0
class DataGenerator2(DataGenerator):
"""Generates data for Keras
Sequence based data generator. Suitable for building data generator for training and prediction.
"""
def __init__(self,
list_IDs,
image_path,
mask_path,
to_fit=True,
batch_size=32,
dim=(512, 512),
dimy=(512, 512),
n_channels=1,
n_classes=10,
shuffle=True,
data_gen_args=None):
super().__init__(list_IDs,
image_path,
mask_path,
to_fit=to_fit,
batch_size=batch_size,
dim=dim,
dimy=dimy,
n_channels=n_channels,
n_classes=n_classes,
shuffle=shuffle)
"""Initialization
:param list_IDs: list of all 'label' ids to use in the generator
:param image_path: path to images location
:param mask_path: path to masks location
:param to_fit: True to return X and y, False to return X only
:param batch_size: batch size at each iteration
:param dim: tuple indicating image dimension
:param n_channels: number of image channels
:param n_classes: number of output masks
:param shuffle: True to shuffle label indexes after every epoch
"""
self.bool = False
if data_gen_args != None:
self.trans = ImageDataGenerator(**data_gen_args)
def __next__(self):
if self.n >= self.max:
self.n = 0
result = self.__getitem__(self.n)
self.n += 1
return result
def _generate_X(self, list_IDs_temp):
"""Generates data containing batch_size images
:param list_IDs_temp: list of label ids to load
:return: batch of images
"""
# Initialization
X = np.empty((self.batch_size, *self.dim, self.n_channels))
self.param = self.trans.get_random_transform(self.dim)
if random.uniform(0, 1) >= 0.5:
self.bool = True
else:
self.bool = False
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
X[i, ] = self._load_dicom_image(self.image_path + '/' + ID)
# X[i,] = self.apply_transform(X[i,],self.get_random_transform((1,512,512)))
if self.bool:
X[i, ] = self.trans.apply_transform(X[i, ], self.param)
# X=np.expand_dims(X, 4)
return X
def _generate_y(self, list_IDs_temp):
"""Generates data containing batch_size masks
:param list_IDs_temp: list of label ids to load
:return: batch if masks
"""
y = np.empty((self.batch_size, *self.dimy, self.n_channels))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
y[i, ] = self._load_grayscale_image_VTK(self.mask_path + '/' +
'label_' + ID[6:15] +
'.png')
if self.bool:
y[i, ] = self.trans.apply_transform(y[i, ], self.param)
return y
class TextImageGenerator:
def __init__(self,
img_dirpath,
labels_path,
img_w,
img_h,
batch_size,
downsample_factor,
idxs,
max_text_len=70,
training=True):
self.img_h = img_h
self.img_w = img_w
self.batch_size = batch_size
self.max_text_len = max_text_len
self.training = training
self.idxs = idxs
self.downsample_factor = downsample_factor
self.img_dirpath = img_dirpath # image dir path
self.labels = json.load(
open(labels_path,
'rb'), encoding="utf8") if labels_path != None else None
self.img_dir = []
for img_file in os.listdir(self.img_dirpath): # images list
if img_file != 'labels.json' and '.ipynb' not in img_file:
self.img_dir.append(img_file)
random.shuffle(self.img_dir)
if self.idxs is not None:
self.img_dir = [self.img_dir[idx] for idx in self.idxs]
self.n = len(self.img_dir) # number of images
self.indexes = list(range(self.n))
self.cur_index = 0
self.imgs = np.ones((self.n, self.img_h, self.img_w, 3),
dtype=np.float16)
self.texts = []
image_datagen_args = {
'shear_range': 0.1,
'zoom_range': 0.01,
'width_shift_range': 0.001,
'height_shift_range': 0.1,
'rotation_range': 1,
'horizontal_flip': False,
'vertical_flip': False
}
self.image_datagen = ImageDataGenerator(**image_datagen_args)
def build_data(self):
print(self.n, " Image Loading start... ", self.img_dirpath)
for i, img_file in enumerate(self.img_dir):
img = image.load_img(self.img_dirpath + img_file,
target_size=SIZE[::-1],
interpolation='bicubic')
img = image.img_to_array(img)
img = preprocess_input(img)
self.imgs[i] = img
if self.labels != None:
self.texts.append(self.labels[img_file][:MAX_LEN])
else:
#valid mode
self.texts.append(img_file)
print("Image Loading finish...")
def next_sample(self):
self.cur_index += 1
if self.cur_index >= self.n:
self.cur_index = 0
random.shuffle(self.indexes)
return self.imgs[self.indexes[self.cur_index]].astype(
np.float32), self.texts[self.indexes[self.cur_index]]
def next_batch(self):
while True:
X_data = np.zeros([self.batch_size, self.img_w, self.img_h, 3],
dtype=np.float32) # (bs,2048, 64, 3)
Y_data = np.zeros([self.batch_size, self.max_text_len],
dtype=np.float32) # (bs, 70)
input_length = np.ones((self.batch_size, 1), dtype=np.float32) * (
self.img_w // self.downsample_factor - 2) # (bs, 1)
label_length = np.zeros((self.batch_size, 1),
dtype=np.float32) # (bs, 1)
for i in range(self.batch_size):
img, text = self.next_sample()
if self.training:
params = self.image_datagen.get_random_transform(img.shape)
img = self.image_datagen.apply_transform(img, params)
img = img.transpose((1, 0, 2))
X_data[i] = img
Y_data[i, :len(text)] = text_to_labels(text)
label_length[i] = len(text)
inputs = {
'the_inputs': X_data, # (bs,2048, 64, 1)
'the_labels': Y_data, # (bs,70)
'input_length': input_length, # (bs, 1)
'label_length': label_length # (bs, 1)
}
outputs = {'ctc': np.zeros([self.batch_size])} # (bs, 1)
yield (inputs, outputs)
class DataGenerator_segmentation(Sequence):
def __init__(self,
list_IDs,
labels,
batch_size=8,
n_classes=3,
shuffle=True,
validation=False):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
if validation:
self.classe_weight = {0: 1.66666667, 1: 1.19047619, 2: 0.64102564}
data_gen_args = dict()
data_gen_args2 = dict()
else:
self.classe_weight = {0: 1.78253119, 1: 2.62467192, 2: 0.48590865}
data_gen_args = dict(
horizontal_flip=True,
vertical_flip=True,
)
data_gen_args2 = dict(
horizontal_flip=True, #rescale=1./255,
vertical_flip=True,
)
self.image_datagen = ImageDataGenerator(**data_gen_args)
self.mask_datagen = ImageDataGenerator(**data_gen_args2)
seed = 1
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
DATAPATH = 'ISIC2017/Tudojunto_r/imagens/'
DATAPATH2 = 'ISIC2017/Tudojunto_r/segm/'
#DATAPATH='ISIC2017/Tudojunto/imagens/'
#DATAPATH2='ISIC2017/Tudojunto/segm/'
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 572, 572, 3))
ys = np.empty((self.batch_size, 388, 388, 1))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Generate Transform
transform_i = self.image_datagen.get_random_transform(
(572, 572, 3), seed=1)
transform_s = self.mask_datagen.get_random_transform((388, 388, 1),
seed=1)
# Store sample
img = imread(DATAPATH + ID + '.jpg')
img = img / 255
mask = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
mask = mask / 255
X[i, ] = self.image_datagen.apply_transform(
img, transform_parameters=transform_i).astype(np.float32)
ys[i, ] = self.mask_datagen.apply_transform(
mask, transform_parameters=transform_s).astype(np.float32)
return X, ys
class DataGenerator(Sequence):
def __init__(self,
list_IDs,
labels,
batch_size=8,
n_classes=3,
shuffle=True,
validation=False):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
if validation:
self.classe_weight = {0: 1.66666667, 1: 1.19047619, 2: 0.64102564}
data_gen_args = dict()
data_gen_args2 = dict()
else:
self.classe_weight = {0: 1.78253119, 1: 2.62467192, 2: 0.48590865}
data_gen_args = dict(
horizontal_flip=True,
vertical_flip=True,
)
data_gen_args2 = dict(
horizontal_flip=True, #rescale=1./255,
vertical_flip=True,
)
self.image_datagen = ImageDataGenerator(**data_gen_args)
self.mask_datagen = ImageDataGenerator(**data_gen_args2)
seed = 1
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index * self.batch_size:(index + 1) *
self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y, sample_weight = self.__data_generation(list_IDs_temp)
return X, y, sample_weight
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
DATAPATH = 'ISIC2017/Tudojunto_r/imagens/'
DATAPATH2 = 'ISIC2017/Tudojunto_r/segm/'
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, 572, 572, 3))
#ys = np.empty((self.batch_size,388, 388, 1 ))
ys = np.empty((self.batch_size, 150544, 1))
yc = np.empty((self.batch_size, 3))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Generate Transform
transform_i = self.image_datagen.get_random_transform(
(572, 572, 3), seed=1)
#print(transform_i)
transform_s = self.mask_datagen.get_random_transform((388, 388, 1),
seed=1)
#print(transform_s)
# Store sample
img = imread(DATAPATH + ID + '.jpg')
img = img / 255
#print(np.amax(img))
mask = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
mask = mask / 255
#img2 = self.image_datagen.apply_transform(img , transform_parameters=transform_i)
#print(np.amax(img2))
X[i, ] = self.image_datagen.apply_transform(
img, transform_parameters=transform_i).astype(np.float32)
#ys[i,] = self.mask_datagen.apply_transform(mask , transform_parameters=transform_s)
mask2 = self.mask_datagen.apply_transform(
mask, transform_parameters=transform_s).astype(np.float32)
ys[i, ] = np.reshape(mask2, (150544, 1))
#print((np.reshape(ys[i,], (150544,1))).shape)
#imshow(img)
#plt.show()
#imshow(img2)
#plt.show()
#print(img.shape)
#X[i,] = img
#ys[i,] = np.expand_dims(imread(DATAPATH2 + ID + '.png'), axis=-1)
# Store class
yc[i] = to_categorical(self.labels[ID], 3)
# Generate Weights
rounded_train = np.argmax(yc, axis=1)
#print("rounded_train:",rounded_train.shape,rounded_train)
#print("yc",yc)
weight_class = np.array(
[self.classe_weight[cls] for cls in rounded_train])
#print("weight_class",weight_class)
#mask_ones = np.ones(388,388,1)
#weight_segm = np.ones(((self.batch_size),388,388))
#tf.expand_dims(weight_mask,axis=0)
#weight_segm = np.ones(((self.batch_size),388,388,1))
#weight_segm = tf.expand_dims(weight_segm,axis=0)
#weight_segm = np.ones(((self.batch_size),1))
# weight_segm = np.array([self.classe_weight[cls] for cls in rounded_train])
weight_segm = np.ones(((self.batch_size), 150544))
return X, [yc, ys], [weight_class, weight_segm]
class TripletGenerator(Sequence):
def __init__(self,
file_path,
image_path,
image_test_path,
nb_classes_batch,
nb_images_per_class_batch,
target_size=TARGET_SIZE,
subset='training',
validation_split=0.0,
seed=42,
is_reptile=False,
shuffle=True,
excluded_classes=['new_whale'],
class_weight_type=None,
**kwargs):
self.file_path = file_path
self.shuffle = shuffle
self.target_size = target_size
self.nb_classes_batch = nb_classes_batch
self.nb_images_per_class_batch = nb_images_per_class_batch
self.batch_size = nb_classes_batch * nb_images_per_class_batch
self.image_path = image_path
self.image_test_path = image_test_path
self.df = pd.read_csv(file_path)
self.index_array = None
self.lock = threading.Lock()
self.is_reptile = is_reptile
logger.info('exclude_class: {}'.format(excluded_classes))
logger.info('class_weight_type: {}'.format(class_weight_type))
blacklisted_class_ix = self.df['Id'].isin(excluded_classes)
logger.info("{} instances excluded".format(
np.sum(blacklisted_class_ix)))
df = self.df[~blacklisted_class_ix]
classes = list(df['Id'].unique())
self.class_indices = dict(zip(classes, range(len(classes))))
self.class_inv_indices = dict(zip(range(len(classes)), classes))
train_classes, test_classes = train_test_split(
classes, test_size=validation_split, random_state=seed)
if subset == 'training':
self.df = df[df['Id'].isin(train_classes)]
else:
self.df = df[df['Id'].isin(test_classes)]
logger.info("data has shape: {}".format(self.df.shape))
classes = self.df['Id'].values
self.classes = np.array([self.class_indices[cls] for cls in classes])
self.filenames = np.array(self.df['Image'])
self.class_indices_subset = {
k: v
for k, v in self.class_inv_indices.items() if k in self.classes
}
kwargs.update({'preprocessing_function': _preprocess_input})
logger.info(kwargs)
self.image_generator = ImageDataGenerator(**kwargs)
# test time generator
self.image_generator_inference = ImageDataGenerator(
preprocessing_function=_preprocess_input)
def get_test_images_and_names(self):
img_names = os.listdir(self.image_test_path)
filepaths = [
os.path.join(self.image_test_path, img_name)
for img_name in img_names
]
_x = np.zeros((len(filepaths), ) + self.target_size, dtype='float32')
for i, _path in enumerate(filepaths):
_img = load_img(_path, target_size=self.target_size)
_x[i] = img_to_array(_img)
if hasattr(_img, 'close'):
_img.close()
return _x, img_names
def __len__(self):
"""
number of steps per epoch
number of classes in subet / nb_classes_batch
:return:
"""
return len(self.class_indices_subset) // self.nb_classes_batch
def get_nb_classes(self):
"""
number of all classes
:return:
"""
return len(self.class_indices)
def _set_index_array(self):
self.index_array = list(self.class_indices_subset.keys())
if self.shuffle:
self.index_array = np.random.permutation(self.index_array)
def on_epoch_end(self):
"""
shuffle at epoch end
"""
self._set_index_array()
def get_train_image_from_class_ix(self, ixs):
return self._get_batches_of_transformed_samples(ixs)
def __getitem__(self, idx):
if self.index_array is None:
self._set_index_array()
# unique classes to pull
index_array = self.index_array[idx * self.nb_classes_batch:self.
nb_classes_batch * (idx + 1)]
return self._get_batches_of_transformed_samples(index_array)
def _get_batches_of_transformed_samples(self, index_array):
"""
validation and training behaves the same way w.r.t. augmentation
:param index_array:
:return:
"""
batch_x = []
for i, class_ix in enumerate(index_array):
_samples = np.zeros(tuple([self.nb_images_per_class_batch] +
list(self.target_size)),
dtype='float32')
filenames = self.filenames[self.classes == class_ix]
if len(filenames) > self.nb_images_per_class_batch:
np.random.shuffle(filenames)
filenames = filenames[:self.nb_images_per_class_batch]
logger.debug("{} files for class {}".format(
len(filenames), class_ix))
for j, filename in enumerate(filenames):
img = load_img(os.path.join(self.image_path, filename),
target_size=self.target_size)
x = img_to_array(img, data_format='channels_last')
# Pillow images should be closed after `load_img`,
# but not PIL images.
if hasattr(img, 'close'):
img.close()
_samples[j] = self.image_generator.standardize(x)
# require at least `nb_images_per_class_batch` per class
nb_missing = _samples.shape[0] - j - 1
# select images to transform. No need to standardize again.
img_ix_transformed = np.random.choice(j + 1, nb_missing)
for img_ix, k in zip(img_ix_transformed, range(nb_missing)):
x = _samples[img_ix]
params = self.image_generator.get_random_transform(
self.target_size)
x = self.image_generator.apply_transform(x, params)
_samples[j + k + 1] = x
logger.debug("{} transformations for class {}".format(
nb_missing, class_ix))
batch_x.append(_samples)
batch_x = np.vstack(batch_x)
# build batch of labels
_labels = range(10) if self.is_reptile else index_array
batch_y = np.repeat(_labels, self.nb_images_per_class_batch)
if self.is_reptile:
batch_y = to_categorical(batch_y, num_classes=10)
assert len(batch_y), len(batch_x)
return batch_x, batch_y
def get_test_generator(self, x):
"""
:param x: test data
:return: NumpyArrayIterator
"""
return self.image_generator_inference.flow(x=x,
shuffle=False,
batch_size=self.batch_size)
